| File: | build/source/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp |
| Warning: | line 3361, column 21 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | ///===- SimpleLoopUnswitch.cpp - Hoist loop-invariant control flow ---------===// | ||||||
| 2 | // | ||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
| 6 | // | ||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||
| 8 | |||||||
| 9 | #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" | ||||||
| 10 | #include "llvm/ADT/DenseMap.h" | ||||||
| 11 | #include "llvm/ADT/STLExtras.h" | ||||||
| 12 | #include "llvm/ADT/Sequence.h" | ||||||
| 13 | #include "llvm/ADT/SetVector.h" | ||||||
| 14 | #include "llvm/ADT/SmallPtrSet.h" | ||||||
| 15 | #include "llvm/ADT/SmallVector.h" | ||||||
| 16 | #include "llvm/ADT/Statistic.h" | ||||||
| 17 | #include "llvm/ADT/Twine.h" | ||||||
| 18 | #include "llvm/Analysis/AssumptionCache.h" | ||||||
| 19 | #include "llvm/Analysis/BlockFrequencyInfo.h" | ||||||
| 20 | #include "llvm/Analysis/CFG.h" | ||||||
| 21 | #include "llvm/Analysis/CodeMetrics.h" | ||||||
| 22 | #include "llvm/Analysis/DomTreeUpdater.h" | ||||||
| 23 | #include "llvm/Analysis/GuardUtils.h" | ||||||
| 24 | #include "llvm/Analysis/LoopAnalysisManager.h" | ||||||
| 25 | #include "llvm/Analysis/LoopInfo.h" | ||||||
| 26 | #include "llvm/Analysis/LoopIterator.h" | ||||||
| 27 | #include "llvm/Analysis/LoopPass.h" | ||||||
| 28 | #include "llvm/Analysis/MemorySSA.h" | ||||||
| 29 | #include "llvm/Analysis/MemorySSAUpdater.h" | ||||||
| 30 | #include "llvm/Analysis/MustExecute.h" | ||||||
| 31 | #include "llvm/Analysis/ProfileSummaryInfo.h" | ||||||
| 32 | #include "llvm/Analysis/ScalarEvolution.h" | ||||||
| 33 | #include "llvm/Analysis/TargetTransformInfo.h" | ||||||
| 34 | #include "llvm/Analysis/ValueTracking.h" | ||||||
| 35 | #include "llvm/IR/BasicBlock.h" | ||||||
| 36 | #include "llvm/IR/Constant.h" | ||||||
| 37 | #include "llvm/IR/Constants.h" | ||||||
| 38 | #include "llvm/IR/Dominators.h" | ||||||
| 39 | #include "llvm/IR/Function.h" | ||||||
| 40 | #include "llvm/IR/IRBuilder.h" | ||||||
| 41 | #include "llvm/IR/InstrTypes.h" | ||||||
| 42 | #include "llvm/IR/Instruction.h" | ||||||
| 43 | #include "llvm/IR/Instructions.h" | ||||||
| 44 | #include "llvm/IR/IntrinsicInst.h" | ||||||
| 45 | #include "llvm/IR/PatternMatch.h" | ||||||
| 46 | #include "llvm/IR/ProfDataUtils.h" | ||||||
| 47 | #include "llvm/IR/Use.h" | ||||||
| 48 | #include "llvm/IR/Value.h" | ||||||
| 49 | #include "llvm/InitializePasses.h" | ||||||
| 50 | #include "llvm/Pass.h" | ||||||
| 51 | #include "llvm/Support/Casting.h" | ||||||
| 52 | #include "llvm/Support/CommandLine.h" | ||||||
| 53 | #include "llvm/Support/Debug.h" | ||||||
| 54 | #include "llvm/Support/ErrorHandling.h" | ||||||
| 55 | #include "llvm/Support/GenericDomTree.h" | ||||||
| 56 | #include "llvm/Support/InstructionCost.h" | ||||||
| 57 | #include "llvm/Support/raw_ostream.h" | ||||||
| 58 | #include "llvm/Transforms/Scalar/LoopPassManager.h" | ||||||
| 59 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | ||||||
| 60 | #include "llvm/Transforms/Utils/Cloning.h" | ||||||
| 61 | #include "llvm/Transforms/Utils/Local.h" | ||||||
| 62 | #include "llvm/Transforms/Utils/LoopUtils.h" | ||||||
| 63 | #include "llvm/Transforms/Utils/ValueMapper.h" | ||||||
| 64 | #include <algorithm> | ||||||
| 65 | #include <cassert> | ||||||
| 66 | #include <iterator> | ||||||
| 67 | #include <numeric> | ||||||
| 68 | #include <optional> | ||||||
| 69 | #include <utility> | ||||||
| 70 | |||||||
| 71 | #define DEBUG_TYPE"simple-loop-unswitch" "simple-loop-unswitch" | ||||||
| 72 | |||||||
| 73 | using namespace llvm; | ||||||
| 74 | using namespace llvm::PatternMatch; | ||||||
| 75 | |||||||
| 76 | STATISTIC(NumBranches, "Number of branches unswitched")static llvm::Statistic NumBranches = {"simple-loop-unswitch", "NumBranches", "Number of branches unswitched"}; | ||||||
| 77 | STATISTIC(NumSwitches, "Number of switches unswitched")static llvm::Statistic NumSwitches = {"simple-loop-unswitch", "NumSwitches", "Number of switches unswitched"}; | ||||||
| 78 | STATISTIC(NumSelects, "Number of selects turned into branches for unswitching")static llvm::Statistic NumSelects = {"simple-loop-unswitch", "NumSelects" , "Number of selects turned into branches for unswitching"}; | ||||||
| 79 | STATISTIC(NumGuards, "Number of guards turned into branches for unswitching")static llvm::Statistic NumGuards = {"simple-loop-unswitch", "NumGuards" , "Number of guards turned into branches for unswitching"}; | ||||||
| 80 | STATISTIC(NumTrivial, "Number of unswitches that are trivial")static llvm::Statistic NumTrivial = {"simple-loop-unswitch", "NumTrivial" , "Number of unswitches that are trivial"}; | ||||||
| 81 | STATISTIC(static llvm::Statistic NumCostMultiplierSkipped = {"simple-loop-unswitch" , "NumCostMultiplierSkipped", "Number of unswitch candidates that had their cost multiplier skipped" } | ||||||
| 82 | NumCostMultiplierSkipped,static llvm::Statistic NumCostMultiplierSkipped = {"simple-loop-unswitch" , "NumCostMultiplierSkipped", "Number of unswitch candidates that had their cost multiplier skipped" } | ||||||
| 83 | "Number of unswitch candidates that had their cost multiplier skipped")static llvm::Statistic NumCostMultiplierSkipped = {"simple-loop-unswitch" , "NumCostMultiplierSkipped", "Number of unswitch candidates that had their cost multiplier skipped" }; | ||||||
| 84 | STATISTIC(NumInvariantConditionsInjected,static llvm::Statistic NumInvariantConditionsInjected = {"simple-loop-unswitch" , "NumInvariantConditionsInjected", "Number of invariant conditions injected and unswitched" } | ||||||
| 85 | "Number of invariant conditions injected and unswitched")static llvm::Statistic NumInvariantConditionsInjected = {"simple-loop-unswitch" , "NumInvariantConditionsInjected", "Number of invariant conditions injected and unswitched" }; | ||||||
| 86 | |||||||
| 87 | static cl::opt<bool> EnableNonTrivialUnswitch( | ||||||
| 88 | "enable-nontrivial-unswitch", cl::init(false), cl::Hidden, | ||||||
| 89 | cl::desc("Forcibly enables non-trivial loop unswitching rather than " | ||||||
| 90 | "following the configuration passed into the pass.")); | ||||||
| 91 | |||||||
| 92 | static cl::opt<int> | ||||||
| 93 | UnswitchThreshold("unswitch-threshold", cl::init(50), cl::Hidden, | ||||||
| 94 | cl::desc("The cost threshold for unswitching a loop.")); | ||||||
| 95 | |||||||
| 96 | static cl::opt<bool> EnableUnswitchCostMultiplier( | ||||||
| 97 | "enable-unswitch-cost-multiplier", cl::init(true), cl::Hidden, | ||||||
| 98 | cl::desc("Enable unswitch cost multiplier that prohibits exponential " | ||||||
| 99 | "explosion in nontrivial unswitch.")); | ||||||
| 100 | static cl::opt<int> UnswitchSiblingsToplevelDiv( | ||||||
| 101 | "unswitch-siblings-toplevel-div", cl::init(2), cl::Hidden, | ||||||
| 102 | cl::desc("Toplevel siblings divisor for cost multiplier.")); | ||||||
| 103 | static cl::opt<int> UnswitchNumInitialUnscaledCandidates( | ||||||
| 104 | "unswitch-num-initial-unscaled-candidates", cl::init(8), cl::Hidden, | ||||||
| 105 | cl::desc("Number of unswitch candidates that are ignored when calculating " | ||||||
| 106 | "cost multiplier.")); | ||||||
| 107 | static cl::opt<bool> UnswitchGuards( | ||||||
| 108 | "simple-loop-unswitch-guards", cl::init(true), cl::Hidden, | ||||||
| 109 | cl::desc("If enabled, simple loop unswitching will also consider " | ||||||
| 110 | "llvm.experimental.guard intrinsics as unswitch candidates.")); | ||||||
| 111 | static cl::opt<bool> DropNonTrivialImplicitNullChecks( | ||||||
| 112 | "simple-loop-unswitch-drop-non-trivial-implicit-null-checks", | ||||||
| 113 | cl::init(false), cl::Hidden, | ||||||
| 114 | cl::desc("If enabled, drop make.implicit metadata in unswitched implicit " | ||||||
| 115 | "null checks to save time analyzing if we can keep it.")); | ||||||
| 116 | static cl::opt<unsigned> | ||||||
| 117 | MSSAThreshold("simple-loop-unswitch-memoryssa-threshold", | ||||||
| 118 | cl::desc("Max number of memory uses to explore during " | ||||||
| 119 | "partial unswitching analysis"), | ||||||
| 120 | cl::init(100), cl::Hidden); | ||||||
| 121 | static cl::opt<bool> FreezeLoopUnswitchCond( | ||||||
| 122 | "freeze-loop-unswitch-cond", cl::init(true), cl::Hidden, | ||||||
| 123 | cl::desc("If enabled, the freeze instruction will be added to condition " | ||||||
| 124 | "of loop unswitch to prevent miscompilation.")); | ||||||
| 125 | |||||||
| 126 | static cl::opt<bool> InjectInvariantConditions( | ||||||
| 127 | "simple-loop-unswitch-inject-invariant-conditions", cl::Hidden, | ||||||
| 128 | cl::desc("Whether we should inject new invariants and unswitch them to " | ||||||
| 129 | "eliminate some existing (non-invariant) conditions."), | ||||||
| 130 | cl::init(true)); | ||||||
| 131 | |||||||
| 132 | static cl::opt<unsigned> InjectInvariantConditionHotnesThreshold( | ||||||
| 133 | "simple-loop-unswitch-inject-invariant-condition-hotness-threshold", | ||||||
| 134 | cl::Hidden, cl::desc("Only try to inject loop invariant conditions and " | ||||||
| 135 | "unswitch on them to eliminate branches that are " | ||||||
| 136 | "not-taken 1/<this option> times or less."), | ||||||
| 137 | cl::init(16)); | ||||||
| 138 | |||||||
| 139 | namespace { | ||||||
| 140 | struct CompareDesc { | ||||||
| 141 | BranchInst *Term; | ||||||
| 142 | Value *Invariant; | ||||||
| 143 | BasicBlock *InLoopSucc; | ||||||
| 144 | |||||||
| 145 | CompareDesc(BranchInst *Term, Value *Invariant, BasicBlock *InLoopSucc) | ||||||
| 146 | : Term(Term), Invariant(Invariant), InLoopSucc(InLoopSucc) {} | ||||||
| 147 | }; | ||||||
| 148 | |||||||
| 149 | struct InjectedInvariant { | ||||||
| 150 | ICmpInst::Predicate Pred; | ||||||
| 151 | Value *LHS; | ||||||
| 152 | Value *RHS; | ||||||
| 153 | BasicBlock *InLoopSucc; | ||||||
| 154 | |||||||
| 155 | InjectedInvariant(ICmpInst::Predicate Pred, Value *LHS, Value *RHS, | ||||||
| 156 | BasicBlock *InLoopSucc) | ||||||
| 157 | : Pred(Pred), LHS(LHS), RHS(RHS), InLoopSucc(InLoopSucc) {} | ||||||
| 158 | }; | ||||||
| 159 | |||||||
| 160 | struct NonTrivialUnswitchCandidate { | ||||||
| 161 | Instruction *TI = nullptr; | ||||||
| 162 | TinyPtrVector<Value *> Invariants; | ||||||
| 163 | std::optional<InstructionCost> Cost; | ||||||
| 164 | std::optional<InjectedInvariant> PendingInjection; | ||||||
| 165 | NonTrivialUnswitchCandidate( | ||||||
| 166 | Instruction *TI, ArrayRef<Value *> Invariants, | ||||||
| 167 | std::optional<InstructionCost> Cost = std::nullopt, | ||||||
| 168 | std::optional<InjectedInvariant> PendingInjection = std::nullopt) | ||||||
| 169 | : TI(TI), Invariants(Invariants), Cost(Cost), | ||||||
| 170 | PendingInjection(PendingInjection) {}; | ||||||
| 171 | |||||||
| 172 | bool hasPendingInjection() const { return PendingInjection.has_value(); } | ||||||
| 173 | }; | ||||||
| 174 | } // end anonymous namespace. | ||||||
| 175 | |||||||
| 176 | // Helper to skip (select x, true, false), which matches both a logical AND and | ||||||
| 177 | // OR and can confuse code that tries to determine if \p Cond is either a | ||||||
| 178 | // logical AND or OR but not both. | ||||||
| 179 | static Value *skipTrivialSelect(Value *Cond) { | ||||||
| 180 | Value *CondNext; | ||||||
| 181 | while (match(Cond, m_Select(m_Value(CondNext), m_One(), m_Zero()))) | ||||||
| 182 | Cond = CondNext; | ||||||
| 183 | return Cond; | ||||||
| 184 | } | ||||||
| 185 | |||||||
| 186 | /// Collect all of the loop invariant input values transitively used by the | ||||||
| 187 | /// homogeneous instruction graph from a given root. | ||||||
| 188 | /// | ||||||
| 189 | /// This essentially walks from a root recursively through loop variant operands | ||||||
| 190 | /// which have perform the same logical operation (AND or OR) and finds all | ||||||
| 191 | /// inputs which are loop invariant. For some operations these can be | ||||||
| 192 | /// re-associated and unswitched out of the loop entirely. | ||||||
| 193 | static TinyPtrVector<Value *> | ||||||
| 194 | collectHomogenousInstGraphLoopInvariants(const Loop &L, Instruction &Root, | ||||||
| 195 | const LoopInfo &LI) { | ||||||
| 196 | assert(!L.isLoopInvariant(&Root) &&(static_cast <bool> (!L.isLoopInvariant(&Root) && "Only need to walk the graph if root itself is not invariant." ) ? void (0) : __assert_fail ("!L.isLoopInvariant(&Root) && \"Only need to walk the graph if root itself is not invariant.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 197, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 197 | "Only need to walk the graph if root itself is not invariant.")(static_cast <bool> (!L.isLoopInvariant(&Root) && "Only need to walk the graph if root itself is not invariant." ) ? void (0) : __assert_fail ("!L.isLoopInvariant(&Root) && \"Only need to walk the graph if root itself is not invariant.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 197, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 198 | TinyPtrVector<Value *> Invariants; | ||||||
| 199 | |||||||
| 200 | bool IsRootAnd = match(&Root, m_LogicalAnd()); | ||||||
| 201 | bool IsRootOr = match(&Root, m_LogicalOr()); | ||||||
| 202 | |||||||
| 203 | // Build a worklist and recurse through operators collecting invariants. | ||||||
| 204 | SmallVector<Instruction *, 4> Worklist; | ||||||
| 205 | SmallPtrSet<Instruction *, 8> Visited; | ||||||
| 206 | Worklist.push_back(&Root); | ||||||
| 207 | Visited.insert(&Root); | ||||||
| 208 | do { | ||||||
| 209 | Instruction &I = *Worklist.pop_back_val(); | ||||||
| 210 | for (Value *OpV : I.operand_values()) { | ||||||
| 211 | // Skip constants as unswitching isn't interesting for them. | ||||||
| 212 | if (isa<Constant>(OpV)) | ||||||
| 213 | continue; | ||||||
| 214 | |||||||
| 215 | // Add it to our result if loop invariant. | ||||||
| 216 | if (L.isLoopInvariant(OpV)) { | ||||||
| 217 | Invariants.push_back(OpV); | ||||||
| 218 | continue; | ||||||
| 219 | } | ||||||
| 220 | |||||||
| 221 | // If not an instruction with the same opcode, nothing we can do. | ||||||
| 222 | Instruction *OpI = dyn_cast<Instruction>(skipTrivialSelect(OpV)); | ||||||
| 223 | |||||||
| 224 | if (OpI && ((IsRootAnd && match(OpI, m_LogicalAnd())) || | ||||||
| 225 | (IsRootOr && match(OpI, m_LogicalOr())))) { | ||||||
| 226 | // Visit this operand. | ||||||
| 227 | if (Visited.insert(OpI).second) | ||||||
| 228 | Worklist.push_back(OpI); | ||||||
| 229 | } | ||||||
| 230 | } | ||||||
| 231 | } while (!Worklist.empty()); | ||||||
| 232 | |||||||
| 233 | return Invariants; | ||||||
| 234 | } | ||||||
| 235 | |||||||
| 236 | static void replaceLoopInvariantUses(const Loop &L, Value *Invariant, | ||||||
| 237 | Constant &Replacement) { | ||||||
| 238 | assert(!isa<Constant>(Invariant) && "Why are we unswitching on a constant?")(static_cast <bool> (!isa<Constant>(Invariant) && "Why are we unswitching on a constant?") ? void (0) : __assert_fail ("!isa<Constant>(Invariant) && \"Why are we unswitching on a constant?\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 238, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 239 | |||||||
| 240 | // Replace uses of LIC in the loop with the given constant. | ||||||
| 241 | // We use make_early_inc_range as set invalidates the iterator. | ||||||
| 242 | for (Use &U : llvm::make_early_inc_range(Invariant->uses())) { | ||||||
| 243 | Instruction *UserI = dyn_cast<Instruction>(U.getUser()); | ||||||
| 244 | |||||||
| 245 | // Replace this use within the loop body. | ||||||
| 246 | if (UserI && L.contains(UserI)) | ||||||
| 247 | U.set(&Replacement); | ||||||
| 248 | } | ||||||
| 249 | } | ||||||
| 250 | |||||||
| 251 | /// Check that all the LCSSA PHI nodes in the loop exit block have trivial | ||||||
| 252 | /// incoming values along this edge. | ||||||
| 253 | static bool areLoopExitPHIsLoopInvariant(const Loop &L, | ||||||
| 254 | const BasicBlock &ExitingBB, | ||||||
| 255 | const BasicBlock &ExitBB) { | ||||||
| 256 | for (const Instruction &I : ExitBB) { | ||||||
| 257 | auto *PN = dyn_cast<PHINode>(&I); | ||||||
| 258 | if (!PN) | ||||||
| 259 | // No more PHIs to check. | ||||||
| 260 | return true; | ||||||
| 261 | |||||||
| 262 | // If the incoming value for this edge isn't loop invariant the unswitch | ||||||
| 263 | // won't be trivial. | ||||||
| 264 | if (!L.isLoopInvariant(PN->getIncomingValueForBlock(&ExitingBB))) | ||||||
| 265 | return false; | ||||||
| 266 | } | ||||||
| 267 | llvm_unreachable("Basic blocks should never be empty!")::llvm::llvm_unreachable_internal("Basic blocks should never be empty!" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 267); | ||||||
| 268 | } | ||||||
| 269 | |||||||
| 270 | /// Copy a set of loop invariant values \p ToDuplicate and insert them at the | ||||||
| 271 | /// end of \p BB and conditionally branch on the copied condition. We only | ||||||
| 272 | /// branch on a single value. | ||||||
| 273 | static void buildPartialUnswitchConditionalBranch( | ||||||
| 274 | BasicBlock &BB, ArrayRef<Value *> Invariants, bool Direction, | ||||||
| 275 | BasicBlock &UnswitchedSucc, BasicBlock &NormalSucc, bool InsertFreeze, | ||||||
| 276 | const Instruction *I, AssumptionCache *AC, const DominatorTree &DT) { | ||||||
| 277 | IRBuilder<> IRB(&BB); | ||||||
| 278 | |||||||
| 279 | SmallVector<Value *> FrozenInvariants; | ||||||
| 280 | for (Value *Inv : Invariants) { | ||||||
| 281 | if (InsertFreeze && !isGuaranteedNotToBeUndefOrPoison(Inv, AC, I, &DT)) | ||||||
| 282 | Inv = IRB.CreateFreeze(Inv, Inv->getName() + ".fr"); | ||||||
| 283 | FrozenInvariants.push_back(Inv); | ||||||
| 284 | } | ||||||
| 285 | |||||||
| 286 | Value *Cond = Direction ? IRB.CreateOr(FrozenInvariants) | ||||||
| 287 | : IRB.CreateAnd(FrozenInvariants); | ||||||
| 288 | IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc, | ||||||
| 289 | Direction ? &NormalSucc : &UnswitchedSucc); | ||||||
| 290 | } | ||||||
| 291 | |||||||
| 292 | /// Copy a set of loop invariant values, and conditionally branch on them. | ||||||
| 293 | static void buildPartialInvariantUnswitchConditionalBranch( | ||||||
| 294 | BasicBlock &BB, ArrayRef<Value *> ToDuplicate, bool Direction, | ||||||
| 295 | BasicBlock &UnswitchedSucc, BasicBlock &NormalSucc, Loop &L, | ||||||
| 296 | MemorySSAUpdater *MSSAU) { | ||||||
| 297 | ValueToValueMapTy VMap; | ||||||
| 298 | for (auto *Val : reverse(ToDuplicate)) { | ||||||
| 299 | Instruction *Inst = cast<Instruction>(Val); | ||||||
| 300 | Instruction *NewInst = Inst->clone(); | ||||||
| 301 | NewInst->insertInto(&BB, BB.end()); | ||||||
| 302 | RemapInstruction(NewInst, VMap, | ||||||
| 303 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | ||||||
| 304 | VMap[Val] = NewInst; | ||||||
| 305 | |||||||
| 306 | if (!MSSAU) | ||||||
| 307 | continue; | ||||||
| 308 | |||||||
| 309 | MemorySSA *MSSA = MSSAU->getMemorySSA(); | ||||||
| 310 | if (auto *MemUse = | ||||||
| 311 | dyn_cast_or_null<MemoryUse>(MSSA->getMemoryAccess(Inst))) { | ||||||
| 312 | auto *DefiningAccess = MemUse->getDefiningAccess(); | ||||||
| 313 | // Get the first defining access before the loop. | ||||||
| 314 | while (L.contains(DefiningAccess->getBlock())) { | ||||||
| 315 | // If the defining access is a MemoryPhi, get the incoming | ||||||
| 316 | // value for the pre-header as defining access. | ||||||
| 317 | if (auto *MemPhi = dyn_cast<MemoryPhi>(DefiningAccess)) | ||||||
| 318 | DefiningAccess = | ||||||
| 319 | MemPhi->getIncomingValueForBlock(L.getLoopPreheader()); | ||||||
| 320 | else | ||||||
| 321 | DefiningAccess = cast<MemoryDef>(DefiningAccess)->getDefiningAccess(); | ||||||
| 322 | } | ||||||
| 323 | MSSAU->createMemoryAccessInBB(NewInst, DefiningAccess, | ||||||
| 324 | NewInst->getParent(), | ||||||
| 325 | MemorySSA::BeforeTerminator); | ||||||
| 326 | } | ||||||
| 327 | } | ||||||
| 328 | |||||||
| 329 | IRBuilder<> IRB(&BB); | ||||||
| 330 | Value *Cond = VMap[ToDuplicate[0]]; | ||||||
| 331 | IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc, | ||||||
| 332 | Direction ? &NormalSucc : &UnswitchedSucc); | ||||||
| 333 | } | ||||||
| 334 | |||||||
| 335 | /// Rewrite the PHI nodes in an unswitched loop exit basic block. | ||||||
| 336 | /// | ||||||
| 337 | /// Requires that the loop exit and unswitched basic block are the same, and | ||||||
| 338 | /// that the exiting block was a unique predecessor of that block. Rewrites the | ||||||
| 339 | /// PHI nodes in that block such that what were LCSSA PHI nodes become trivial | ||||||
| 340 | /// PHI nodes from the old preheader that now contains the unswitched | ||||||
| 341 | /// terminator. | ||||||
| 342 | static void rewritePHINodesForUnswitchedExitBlock(BasicBlock &UnswitchedBB, | ||||||
| 343 | BasicBlock &OldExitingBB, | ||||||
| 344 | BasicBlock &OldPH) { | ||||||
| 345 | for (PHINode &PN : UnswitchedBB.phis()) { | ||||||
| 346 | // When the loop exit is directly unswitched we just need to update the | ||||||
| 347 | // incoming basic block. We loop to handle weird cases with repeated | ||||||
| 348 | // incoming blocks, but expect to typically only have one operand here. | ||||||
| 349 | for (auto i : seq<int>(0, PN.getNumOperands())) { | ||||||
| 350 | assert(PN.getIncomingBlock(i) == &OldExitingBB &&(static_cast <bool> (PN.getIncomingBlock(i) == &OldExitingBB && "Found incoming block different from unique predecessor!" ) ? void (0) : __assert_fail ("PN.getIncomingBlock(i) == &OldExitingBB && \"Found incoming block different from unique predecessor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 351, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 351 | "Found incoming block different from unique predecessor!")(static_cast <bool> (PN.getIncomingBlock(i) == &OldExitingBB && "Found incoming block different from unique predecessor!" ) ? void (0) : __assert_fail ("PN.getIncomingBlock(i) == &OldExitingBB && \"Found incoming block different from unique predecessor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 351, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 352 | PN.setIncomingBlock(i, &OldPH); | ||||||
| 353 | } | ||||||
| 354 | } | ||||||
| 355 | } | ||||||
| 356 | |||||||
| 357 | /// Rewrite the PHI nodes in the loop exit basic block and the split off | ||||||
| 358 | /// unswitched block. | ||||||
| 359 | /// | ||||||
| 360 | /// Because the exit block remains an exit from the loop, this rewrites the | ||||||
| 361 | /// LCSSA PHI nodes in it to remove the unswitched edge and introduces PHI | ||||||
| 362 | /// nodes into the unswitched basic block to select between the value in the | ||||||
| 363 | /// old preheader and the loop exit. | ||||||
| 364 | static void rewritePHINodesForExitAndUnswitchedBlocks(BasicBlock &ExitBB, | ||||||
| 365 | BasicBlock &UnswitchedBB, | ||||||
| 366 | BasicBlock &OldExitingBB, | ||||||
| 367 | BasicBlock &OldPH, | ||||||
| 368 | bool FullUnswitch) { | ||||||
| 369 | assert(&ExitBB != &UnswitchedBB &&(static_cast <bool> (&ExitBB != &UnswitchedBB && "Must have different loop exit and unswitched blocks!") ? void (0) : __assert_fail ("&ExitBB != &UnswitchedBB && \"Must have different loop exit and unswitched blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 370, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 370 | "Must have different loop exit and unswitched blocks!")(static_cast <bool> (&ExitBB != &UnswitchedBB && "Must have different loop exit and unswitched blocks!") ? void (0) : __assert_fail ("&ExitBB != &UnswitchedBB && \"Must have different loop exit and unswitched blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 370, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 371 | Instruction *InsertPt = &*UnswitchedBB.begin(); | ||||||
| 372 | for (PHINode &PN : ExitBB.phis()) { | ||||||
| 373 | auto *NewPN = PHINode::Create(PN.getType(), /*NumReservedValues*/ 2, | ||||||
| 374 | PN.getName() + ".split", InsertPt); | ||||||
| 375 | |||||||
| 376 | // Walk backwards over the old PHI node's inputs to minimize the cost of | ||||||
| 377 | // removing each one. We have to do this weird loop manually so that we | ||||||
| 378 | // create the same number of new incoming edges in the new PHI as we expect | ||||||
| 379 | // each case-based edge to be included in the unswitched switch in some | ||||||
| 380 | // cases. | ||||||
| 381 | // FIXME: This is really, really gross. It would be much cleaner if LLVM | ||||||
| 382 | // allowed us to create a single entry for a predecessor block without | ||||||
| 383 | // having separate entries for each "edge" even though these edges are | ||||||
| 384 | // required to produce identical results. | ||||||
| 385 | for (int i = PN.getNumIncomingValues() - 1; i >= 0; --i) { | ||||||
| 386 | if (PN.getIncomingBlock(i) != &OldExitingBB) | ||||||
| 387 | continue; | ||||||
| 388 | |||||||
| 389 | Value *Incoming = PN.getIncomingValue(i); | ||||||
| 390 | if (FullUnswitch) | ||||||
| 391 | // No more edge from the old exiting block to the exit block. | ||||||
| 392 | PN.removeIncomingValue(i); | ||||||
| 393 | |||||||
| 394 | NewPN->addIncoming(Incoming, &OldPH); | ||||||
| 395 | } | ||||||
| 396 | |||||||
| 397 | // Now replace the old PHI with the new one and wire the old one in as an | ||||||
| 398 | // input to the new one. | ||||||
| 399 | PN.replaceAllUsesWith(NewPN); | ||||||
| 400 | NewPN->addIncoming(&PN, &ExitBB); | ||||||
| 401 | } | ||||||
| 402 | } | ||||||
| 403 | |||||||
| 404 | /// Hoist the current loop up to the innermost loop containing a remaining exit. | ||||||
| 405 | /// | ||||||
| 406 | /// Because we've removed an exit from the loop, we may have changed the set of | ||||||
| 407 | /// loops reachable and need to move the current loop up the loop nest or even | ||||||
| 408 | /// to an entirely separate nest. | ||||||
| 409 | static void hoistLoopToNewParent(Loop &L, BasicBlock &Preheader, | ||||||
| 410 | DominatorTree &DT, LoopInfo &LI, | ||||||
| 411 | MemorySSAUpdater *MSSAU, ScalarEvolution *SE) { | ||||||
| 412 | // If the loop is already at the top level, we can't hoist it anywhere. | ||||||
| 413 | Loop *OldParentL = L.getParentLoop(); | ||||||
| 414 | if (!OldParentL) | ||||||
| 415 | return; | ||||||
| 416 | |||||||
| 417 | SmallVector<BasicBlock *, 4> Exits; | ||||||
| 418 | L.getExitBlocks(Exits); | ||||||
| 419 | Loop *NewParentL = nullptr; | ||||||
| 420 | for (auto *ExitBB : Exits) | ||||||
| 421 | if (Loop *ExitL = LI.getLoopFor(ExitBB)) | ||||||
| 422 | if (!NewParentL || NewParentL->contains(ExitL)) | ||||||
| 423 | NewParentL = ExitL; | ||||||
| 424 | |||||||
| 425 | if (NewParentL == OldParentL) | ||||||
| 426 | return; | ||||||
| 427 | |||||||
| 428 | // The new parent loop (if different) should always contain the old one. | ||||||
| 429 | if (NewParentL) | ||||||
| 430 | assert(NewParentL->contains(OldParentL) &&(static_cast <bool> (NewParentL->contains(OldParentL ) && "Can only hoist this loop up the nest!") ? void ( 0) : __assert_fail ("NewParentL->contains(OldParentL) && \"Can only hoist this loop up the nest!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 431, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 431 | "Can only hoist this loop up the nest!")(static_cast <bool> (NewParentL->contains(OldParentL ) && "Can only hoist this loop up the nest!") ? void ( 0) : __assert_fail ("NewParentL->contains(OldParentL) && \"Can only hoist this loop up the nest!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 431, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 432 | |||||||
| 433 | // The preheader will need to move with the body of this loop. However, | ||||||
| 434 | // because it isn't in this loop we also need to update the primary loop map. | ||||||
| 435 | assert(OldParentL == LI.getLoopFor(&Preheader) &&(static_cast <bool> (OldParentL == LI.getLoopFor(&Preheader ) && "Parent loop of this loop should contain this loop's preheader!" ) ? void (0) : __assert_fail ("OldParentL == LI.getLoopFor(&Preheader) && \"Parent loop of this loop should contain this loop's preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 436, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 436 | "Parent loop of this loop should contain this loop's preheader!")(static_cast <bool> (OldParentL == LI.getLoopFor(&Preheader ) && "Parent loop of this loop should contain this loop's preheader!" ) ? void (0) : __assert_fail ("OldParentL == LI.getLoopFor(&Preheader) && \"Parent loop of this loop should contain this loop's preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 436, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 437 | LI.changeLoopFor(&Preheader, NewParentL); | ||||||
| 438 | |||||||
| 439 | // Remove this loop from its old parent. | ||||||
| 440 | OldParentL->removeChildLoop(&L); | ||||||
| 441 | |||||||
| 442 | // Add the loop either to the new parent or as a top-level loop. | ||||||
| 443 | if (NewParentL) | ||||||
| 444 | NewParentL->addChildLoop(&L); | ||||||
| 445 | else | ||||||
| 446 | LI.addTopLevelLoop(&L); | ||||||
| 447 | |||||||
| 448 | // Remove this loops blocks from the old parent and every other loop up the | ||||||
| 449 | // nest until reaching the new parent. Also update all of these | ||||||
| 450 | // no-longer-containing loops to reflect the nesting change. | ||||||
| 451 | for (Loop *OldContainingL = OldParentL; OldContainingL != NewParentL; | ||||||
| 452 | OldContainingL = OldContainingL->getParentLoop()) { | ||||||
| 453 | llvm::erase_if(OldContainingL->getBlocksVector(), | ||||||
| 454 | [&](const BasicBlock *BB) { | ||||||
| 455 | return BB == &Preheader || L.contains(BB); | ||||||
| 456 | }); | ||||||
| 457 | |||||||
| 458 | OldContainingL->getBlocksSet().erase(&Preheader); | ||||||
| 459 | for (BasicBlock *BB : L.blocks()) | ||||||
| 460 | OldContainingL->getBlocksSet().erase(BB); | ||||||
| 461 | |||||||
| 462 | // Because we just hoisted a loop out of this one, we have essentially | ||||||
| 463 | // created new exit paths from it. That means we need to form LCSSA PHI | ||||||
| 464 | // nodes for values used in the no-longer-nested loop. | ||||||
| 465 | formLCSSA(*OldContainingL, DT, &LI); | ||||||
| 466 | |||||||
| 467 | // We shouldn't need to form dedicated exits because the exit introduced | ||||||
| 468 | // here is the (just split by unswitching) preheader. However, after trivial | ||||||
| 469 | // unswitching it is possible to get new non-dedicated exits out of parent | ||||||
| 470 | // loop so let's conservatively form dedicated exit blocks and figure out | ||||||
| 471 | // if we can optimize later. | ||||||
| 472 | formDedicatedExitBlocks(OldContainingL, &DT, &LI, MSSAU, | ||||||
| 473 | /*PreserveLCSSA*/ true); | ||||||
| 474 | } | ||||||
| 475 | } | ||||||
| 476 | |||||||
| 477 | // Return the top-most loop containing ExitBB and having ExitBB as exiting block | ||||||
| 478 | // or the loop containing ExitBB, if there is no parent loop containing ExitBB | ||||||
| 479 | // as exiting block. | ||||||
| 480 | static Loop *getTopMostExitingLoop(const BasicBlock *ExitBB, | ||||||
| 481 | const LoopInfo &LI) { | ||||||
| 482 | Loop *TopMost = LI.getLoopFor(ExitBB); | ||||||
| 483 | Loop *Current = TopMost; | ||||||
| 484 | while (Current) { | ||||||
| 485 | if (Current->isLoopExiting(ExitBB)) | ||||||
| 486 | TopMost = Current; | ||||||
| 487 | Current = Current->getParentLoop(); | ||||||
| 488 | } | ||||||
| 489 | return TopMost; | ||||||
| 490 | } | ||||||
| 491 | |||||||
| 492 | /// Unswitch a trivial branch if the condition is loop invariant. | ||||||
| 493 | /// | ||||||
| 494 | /// This routine should only be called when loop code leading to the branch has | ||||||
| 495 | /// been validated as trivial (no side effects). This routine checks if the | ||||||
| 496 | /// condition is invariant and one of the successors is a loop exit. This | ||||||
| 497 | /// allows us to unswitch without duplicating the loop, making it trivial. | ||||||
| 498 | /// | ||||||
| 499 | /// If this routine fails to unswitch the branch it returns false. | ||||||
| 500 | /// | ||||||
| 501 | /// If the branch can be unswitched, this routine splits the preheader and | ||||||
| 502 | /// hoists the branch above that split. Preserves loop simplified form | ||||||
| 503 | /// (splitting the exit block as necessary). It simplifies the branch within | ||||||
| 504 | /// the loop to an unconditional branch but doesn't remove it entirely. Further | ||||||
| 505 | /// cleanup can be done with some simplifycfg like pass. | ||||||
| 506 | /// | ||||||
| 507 | /// If `SE` is not null, it will be updated based on the potential loop SCEVs | ||||||
| 508 | /// invalidated by this. | ||||||
| 509 | static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT, | ||||||
| 510 | LoopInfo &LI, ScalarEvolution *SE, | ||||||
| 511 | MemorySSAUpdater *MSSAU) { | ||||||
| 512 | assert(BI.isConditional() && "Can only unswitch a conditional branch!")(static_cast <bool> (BI.isConditional() && "Can only unswitch a conditional branch!" ) ? void (0) : __assert_fail ("BI.isConditional() && \"Can only unswitch a conditional branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 512, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 513 | LLVM_DEBUG(dbgs() << " Trying to unswitch branch: " << BI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Trying to unswitch branch: " << BI << "\n"; } } while (false); | ||||||
| 514 | |||||||
| 515 | // The loop invariant values that we want to unswitch. | ||||||
| 516 | TinyPtrVector<Value *> Invariants; | ||||||
| 517 | |||||||
| 518 | // When true, we're fully unswitching the branch rather than just unswitching | ||||||
| 519 | // some input conditions to the branch. | ||||||
| 520 | bool FullUnswitch = false; | ||||||
| 521 | |||||||
| 522 | Value *Cond = skipTrivialSelect(BI.getCondition()); | ||||||
| 523 | if (L.isLoopInvariant(Cond)) { | ||||||
| 524 | Invariants.push_back(Cond); | ||||||
| 525 | FullUnswitch = true; | ||||||
| 526 | } else { | ||||||
| 527 | if (auto *CondInst = dyn_cast<Instruction>(Cond)) | ||||||
| 528 | Invariants = collectHomogenousInstGraphLoopInvariants(L, *CondInst, LI); | ||||||
| 529 | if (Invariants.empty()) { | ||||||
| 530 | LLVM_DEBUG(dbgs() << " Couldn't find invariant inputs!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Couldn't find invariant inputs!\n" ; } } while (false); | ||||||
| 531 | return false; | ||||||
| 532 | } | ||||||
| 533 | } | ||||||
| 534 | |||||||
| 535 | // Check that one of the branch's successors exits, and which one. | ||||||
| 536 | bool ExitDirection = true; | ||||||
| 537 | int LoopExitSuccIdx = 0; | ||||||
| 538 | auto *LoopExitBB = BI.getSuccessor(0); | ||||||
| 539 | if (L.contains(LoopExitBB)) { | ||||||
| 540 | ExitDirection = false; | ||||||
| 541 | LoopExitSuccIdx = 1; | ||||||
| 542 | LoopExitBB = BI.getSuccessor(1); | ||||||
| 543 | if (L.contains(LoopExitBB)) { | ||||||
| 544 | LLVM_DEBUG(dbgs() << " Branch doesn't exit the loop!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Branch doesn't exit the loop!\n" ; } } while (false); | ||||||
| 545 | return false; | ||||||
| 546 | } | ||||||
| 547 | } | ||||||
| 548 | auto *ContinueBB = BI.getSuccessor(1 - LoopExitSuccIdx); | ||||||
| 549 | auto *ParentBB = BI.getParent(); | ||||||
| 550 | if (!areLoopExitPHIsLoopInvariant(L, *ParentBB, *LoopExitBB)) { | ||||||
| 551 | LLVM_DEBUG(dbgs() << " Loop exit PHI's aren't loop-invariant!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Loop exit PHI's aren't loop-invariant!\n" ; } } while (false); | ||||||
| 552 | return false; | ||||||
| 553 | } | ||||||
| 554 | |||||||
| 555 | // When unswitching only part of the branch's condition, we need the exit | ||||||
| 556 | // block to be reached directly from the partially unswitched input. This can | ||||||
| 557 | // be done when the exit block is along the true edge and the branch condition | ||||||
| 558 | // is a graph of `or` operations, or the exit block is along the false edge | ||||||
| 559 | // and the condition is a graph of `and` operations. | ||||||
| 560 | if (!FullUnswitch) { | ||||||
| 561 | if (ExitDirection ? !match(Cond, m_LogicalOr()) | ||||||
| 562 | : !match(Cond, m_LogicalAnd())) { | ||||||
| 563 | LLVM_DEBUG(dbgs() << " Branch condition is in improper form for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Branch condition is in improper form for " "non-full unswitch!\n"; } } while (false) | ||||||
| 564 | "non-full unswitch!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Branch condition is in improper form for " "non-full unswitch!\n"; } } while (false); | ||||||
| 565 | return false; | ||||||
| 566 | } | ||||||
| 567 | } | ||||||
| 568 | |||||||
| 569 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 570 | dbgs() << " unswitching trivial invariant conditions for: " << BIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 571 | << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 572 | for (Value *Invariant : Invariants) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 573 | dbgs() << " " << *Invariant << " == true";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 574 | if (Invariant != Invariants.back())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 575 | dbgs() << " ||";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 576 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 577 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false) | ||||||
| 578 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { { dbgs() << " unswitching trivial invariant conditions for: " << BI << "\n"; for (Value *Invariant : Invariants ) { dbgs() << " " << *Invariant << " == true" ; if (Invariant != Invariants.back()) dbgs() << " ||"; dbgs () << "\n"; } }; } } while (false); | ||||||
| 579 | |||||||
| 580 | // If we have scalar evolutions, we need to invalidate them including this | ||||||
| 581 | // loop, the loop containing the exit block and the topmost parent loop | ||||||
| 582 | // exiting via LoopExitBB. | ||||||
| 583 | if (SE) { | ||||||
| 584 | if (const Loop *ExitL = getTopMostExitingLoop(LoopExitBB, LI)) | ||||||
| 585 | SE->forgetLoop(ExitL); | ||||||
| 586 | else | ||||||
| 587 | // Forget the entire nest as this exits the entire nest. | ||||||
| 588 | SE->forgetTopmostLoop(&L); | ||||||
| 589 | SE->forgetBlockAndLoopDispositions(); | ||||||
| 590 | } | ||||||
| 591 | |||||||
| 592 | if (MSSAU && VerifyMemorySSA) | ||||||
| 593 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 594 | |||||||
| 595 | // Split the preheader, so that we know that there is a safe place to insert | ||||||
| 596 | // the conditional branch. We will change the preheader to have a conditional | ||||||
| 597 | // branch on LoopCond. | ||||||
| 598 | BasicBlock *OldPH = L.getLoopPreheader(); | ||||||
| 599 | BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI, MSSAU); | ||||||
| 600 | |||||||
| 601 | // Now that we have a place to insert the conditional branch, create a place | ||||||
| 602 | // to branch to: this is the exit block out of the loop that we are | ||||||
| 603 | // unswitching. We need to split this if there are other loop predecessors. | ||||||
| 604 | // Because the loop is in simplified form, *any* other predecessor is enough. | ||||||
| 605 | BasicBlock *UnswitchedBB; | ||||||
| 606 | if (FullUnswitch && LoopExitBB->getUniquePredecessor()) { | ||||||
| 607 | assert(LoopExitBB->getUniquePredecessor() == BI.getParent() &&(static_cast <bool> (LoopExitBB->getUniquePredecessor () == BI.getParent() && "A branch's parent isn't a predecessor!" ) ? void (0) : __assert_fail ("LoopExitBB->getUniquePredecessor() == BI.getParent() && \"A branch's parent isn't a predecessor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 608, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 608 | "A branch's parent isn't a predecessor!")(static_cast <bool> (LoopExitBB->getUniquePredecessor () == BI.getParent() && "A branch's parent isn't a predecessor!" ) ? void (0) : __assert_fail ("LoopExitBB->getUniquePredecessor() == BI.getParent() && \"A branch's parent isn't a predecessor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 608, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 609 | UnswitchedBB = LoopExitBB; | ||||||
| 610 | } else { | ||||||
| 611 | UnswitchedBB = | ||||||
| 612 | SplitBlock(LoopExitBB, &LoopExitBB->front(), &DT, &LI, MSSAU); | ||||||
| 613 | } | ||||||
| 614 | |||||||
| 615 | if (MSSAU && VerifyMemorySSA) | ||||||
| 616 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 617 | |||||||
| 618 | // Actually move the invariant uses into the unswitched position. If possible, | ||||||
| 619 | // we do this by moving the instructions, but when doing partial unswitching | ||||||
| 620 | // we do it by building a new merge of the values in the unswitched position. | ||||||
| 621 | OldPH->getTerminator()->eraseFromParent(); | ||||||
| 622 | if (FullUnswitch) { | ||||||
| 623 | // If fully unswitching, we can use the existing branch instruction. | ||||||
| 624 | // Splice it into the old PH to gate reaching the new preheader and re-point | ||||||
| 625 | // its successors. | ||||||
| 626 | OldPH->splice(OldPH->end(), BI.getParent(), BI.getIterator()); | ||||||
| 627 | BI.setCondition(Cond); | ||||||
| 628 | if (MSSAU) { | ||||||
| 629 | // Temporarily clone the terminator, to make MSSA update cheaper by | ||||||
| 630 | // separating "insert edge" updates from "remove edge" ones. | ||||||
| 631 | BI.clone()->insertInto(ParentBB, ParentBB->end()); | ||||||
| 632 | } else { | ||||||
| 633 | // Create a new unconditional branch that will continue the loop as a new | ||||||
| 634 | // terminator. | ||||||
| 635 | BranchInst::Create(ContinueBB, ParentBB); | ||||||
| 636 | } | ||||||
| 637 | BI.setSuccessor(LoopExitSuccIdx, UnswitchedBB); | ||||||
| 638 | BI.setSuccessor(1 - LoopExitSuccIdx, NewPH); | ||||||
| 639 | } else { | ||||||
| 640 | // Only unswitching a subset of inputs to the condition, so we will need to | ||||||
| 641 | // build a new branch that merges the invariant inputs. | ||||||
| 642 | if (ExitDirection) | ||||||
| 643 | assert(match(skipTrivialSelect(BI.getCondition()), m_LogicalOr()) &&(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalOr()) && "Must have an `or` of `i1`s or `select i1 X, true, Y`s for the " "condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalOr()) && \"Must have an `or` of `i1`s or `select i1 X, true, Y`s for the \" \"condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 645, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 644 | "Must have an `or` of `i1`s or `select i1 X, true, Y`s for the "(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalOr()) && "Must have an `or` of `i1`s or `select i1 X, true, Y`s for the " "condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalOr()) && \"Must have an `or` of `i1`s or `select i1 X, true, Y`s for the \" \"condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 645, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 645 | "condition!")(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalOr()) && "Must have an `or` of `i1`s or `select i1 X, true, Y`s for the " "condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalOr()) && \"Must have an `or` of `i1`s or `select i1 X, true, Y`s for the \" \"condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 645, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 646 | else | ||||||
| 647 | assert(match(skipTrivialSelect(BI.getCondition()), m_LogicalAnd()) &&(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalAnd()) && "Must have an `and` of `i1`s or `select i1 X, Y, false`s for the" " condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalAnd()) && \"Must have an `and` of `i1`s or `select i1 X, Y, false`s for the\" \" condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 649, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 648 | "Must have an `and` of `i1`s or `select i1 X, Y, false`s for the"(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalAnd()) && "Must have an `and` of `i1`s or `select i1 X, Y, false`s for the" " condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalAnd()) && \"Must have an `and` of `i1`s or `select i1 X, Y, false`s for the\" \" condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 649, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 649 | " condition!")(static_cast <bool> (match(skipTrivialSelect(BI.getCondition ()), m_LogicalAnd()) && "Must have an `and` of `i1`s or `select i1 X, Y, false`s for the" " condition!") ? void (0) : __assert_fail ("match(skipTrivialSelect(BI.getCondition()), m_LogicalAnd()) && \"Must have an `and` of `i1`s or `select i1 X, Y, false`s for the\" \" condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 649, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 650 | buildPartialUnswitchConditionalBranch( | ||||||
| 651 | *OldPH, Invariants, ExitDirection, *UnswitchedBB, *NewPH, | ||||||
| 652 | FreezeLoopUnswitchCond, OldPH->getTerminator(), nullptr, DT); | ||||||
| 653 | } | ||||||
| 654 | |||||||
| 655 | // Update the dominator tree with the added edge. | ||||||
| 656 | DT.insertEdge(OldPH, UnswitchedBB); | ||||||
| 657 | |||||||
| 658 | // After the dominator tree was updated with the added edge, update MemorySSA | ||||||
| 659 | // if available. | ||||||
| 660 | if (MSSAU) { | ||||||
| 661 | SmallVector<CFGUpdate, 1> Updates; | ||||||
| 662 | Updates.push_back({cfg::UpdateKind::Insert, OldPH, UnswitchedBB}); | ||||||
| 663 | MSSAU->applyInsertUpdates(Updates, DT); | ||||||
| 664 | } | ||||||
| 665 | |||||||
| 666 | // Finish updating dominator tree and memory ssa for full unswitch. | ||||||
| 667 | if (FullUnswitch) { | ||||||
| 668 | if (MSSAU) { | ||||||
| 669 | // Remove the cloned branch instruction. | ||||||
| 670 | ParentBB->getTerminator()->eraseFromParent(); | ||||||
| 671 | // Create unconditional branch now. | ||||||
| 672 | BranchInst::Create(ContinueBB, ParentBB); | ||||||
| 673 | MSSAU->removeEdge(ParentBB, LoopExitBB); | ||||||
| 674 | } | ||||||
| 675 | DT.deleteEdge(ParentBB, LoopExitBB); | ||||||
| 676 | } | ||||||
| 677 | |||||||
| 678 | if (MSSAU && VerifyMemorySSA) | ||||||
| 679 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 680 | |||||||
| 681 | // Rewrite the relevant PHI nodes. | ||||||
| 682 | if (UnswitchedBB == LoopExitBB) | ||||||
| 683 | rewritePHINodesForUnswitchedExitBlock(*UnswitchedBB, *ParentBB, *OldPH); | ||||||
| 684 | else | ||||||
| 685 | rewritePHINodesForExitAndUnswitchedBlocks(*LoopExitBB, *UnswitchedBB, | ||||||
| 686 | *ParentBB, *OldPH, FullUnswitch); | ||||||
| 687 | |||||||
| 688 | // The constant we can replace all of our invariants with inside the loop | ||||||
| 689 | // body. If any of the invariants have a value other than this the loop won't | ||||||
| 690 | // be entered. | ||||||
| 691 | ConstantInt *Replacement = ExitDirection | ||||||
| 692 | ? ConstantInt::getFalse(BI.getContext()) | ||||||
| 693 | : ConstantInt::getTrue(BI.getContext()); | ||||||
| 694 | |||||||
| 695 | // Since this is an i1 condition we can also trivially replace uses of it | ||||||
| 696 | // within the loop with a constant. | ||||||
| 697 | for (Value *Invariant : Invariants) | ||||||
| 698 | replaceLoopInvariantUses(L, Invariant, *Replacement); | ||||||
| 699 | |||||||
| 700 | // If this was full unswitching, we may have changed the nesting relationship | ||||||
| 701 | // for this loop so hoist it to its correct parent if needed. | ||||||
| 702 | if (FullUnswitch) | ||||||
| 703 | hoistLoopToNewParent(L, *NewPH, DT, LI, MSSAU, SE); | ||||||
| 704 | |||||||
| 705 | if (MSSAU && VerifyMemorySSA) | ||||||
| 706 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 707 | |||||||
| 708 | LLVM_DEBUG(dbgs() << " done: unswitching trivial branch...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " done: unswitching trivial branch...\n" ; } } while (false); | ||||||
| 709 | ++NumTrivial; | ||||||
| 710 | ++NumBranches; | ||||||
| 711 | return true; | ||||||
| 712 | } | ||||||
| 713 | |||||||
| 714 | /// Unswitch a trivial switch if the condition is loop invariant. | ||||||
| 715 | /// | ||||||
| 716 | /// This routine should only be called when loop code leading to the switch has | ||||||
| 717 | /// been validated as trivial (no side effects). This routine checks if the | ||||||
| 718 | /// condition is invariant and that at least one of the successors is a loop | ||||||
| 719 | /// exit. This allows us to unswitch without duplicating the loop, making it | ||||||
| 720 | /// trivial. | ||||||
| 721 | /// | ||||||
| 722 | /// If this routine fails to unswitch the switch it returns false. | ||||||
| 723 | /// | ||||||
| 724 | /// If the switch can be unswitched, this routine splits the preheader and | ||||||
| 725 | /// copies the switch above that split. If the default case is one of the | ||||||
| 726 | /// exiting cases, it copies the non-exiting cases and points them at the new | ||||||
| 727 | /// preheader. If the default case is not exiting, it copies the exiting cases | ||||||
| 728 | /// and points the default at the preheader. It preserves loop simplified form | ||||||
| 729 | /// (splitting the exit blocks as necessary). It simplifies the switch within | ||||||
| 730 | /// the loop by removing now-dead cases. If the default case is one of those | ||||||
| 731 | /// unswitched, it replaces its destination with a new basic block containing | ||||||
| 732 | /// only unreachable. Such basic blocks, while technically loop exits, are not | ||||||
| 733 | /// considered for unswitching so this is a stable transform and the same | ||||||
| 734 | /// switch will not be revisited. If after unswitching there is only a single | ||||||
| 735 | /// in-loop successor, the switch is further simplified to an unconditional | ||||||
| 736 | /// branch. Still more cleanup can be done with some simplifycfg like pass. | ||||||
| 737 | /// | ||||||
| 738 | /// If `SE` is not null, it will be updated based on the potential loop SCEVs | ||||||
| 739 | /// invalidated by this. | ||||||
| 740 | static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT, | ||||||
| 741 | LoopInfo &LI, ScalarEvolution *SE, | ||||||
| 742 | MemorySSAUpdater *MSSAU) { | ||||||
| 743 | LLVM_DEBUG(dbgs() << " Trying to unswitch switch: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Trying to unswitch switch: " << SI << "\n"; } } while (false); | ||||||
| 744 | Value *LoopCond = SI.getCondition(); | ||||||
| 745 | |||||||
| 746 | // If this isn't switching on an invariant condition, we can't unswitch it. | ||||||
| 747 | if (!L.isLoopInvariant(LoopCond)) | ||||||
| 748 | return false; | ||||||
| 749 | |||||||
| 750 | auto *ParentBB = SI.getParent(); | ||||||
| 751 | |||||||
| 752 | // The same check must be used both for the default and the exit cases. We | ||||||
| 753 | // should never leave edges from the switch instruction to a basic block that | ||||||
| 754 | // we are unswitching, hence the condition used to determine the default case | ||||||
| 755 | // needs to also be used to populate ExitCaseIndices, which is then used to | ||||||
| 756 | // remove cases from the switch. | ||||||
| 757 | auto IsTriviallyUnswitchableExitBlock = [&](BasicBlock &BBToCheck) { | ||||||
| 758 | // BBToCheck is not an exit block if it is inside loop L. | ||||||
| 759 | if (L.contains(&BBToCheck)) | ||||||
| 760 | return false; | ||||||
| 761 | // BBToCheck is not trivial to unswitch if its phis aren't loop invariant. | ||||||
| 762 | if (!areLoopExitPHIsLoopInvariant(L, *ParentBB, BBToCheck)) | ||||||
| 763 | return false; | ||||||
| 764 | // We do not unswitch a block that only has an unreachable statement, as | ||||||
| 765 | // it's possible this is a previously unswitched block. Only unswitch if | ||||||
| 766 | // either the terminator is not unreachable, or, if it is, it's not the only | ||||||
| 767 | // instruction in the block. | ||||||
| 768 | auto *TI = BBToCheck.getTerminator(); | ||||||
| 769 | bool isUnreachable = isa<UnreachableInst>(TI); | ||||||
| 770 | return !isUnreachable || | ||||||
| 771 | (isUnreachable && (BBToCheck.getFirstNonPHIOrDbg() != TI)); | ||||||
| 772 | }; | ||||||
| 773 | |||||||
| 774 | SmallVector<int, 4> ExitCaseIndices; | ||||||
| 775 | for (auto Case : SI.cases()) | ||||||
| 776 | if (IsTriviallyUnswitchableExitBlock(*Case.getCaseSuccessor())) | ||||||
| 777 | ExitCaseIndices.push_back(Case.getCaseIndex()); | ||||||
| 778 | BasicBlock *DefaultExitBB = nullptr; | ||||||
| 779 | SwitchInstProfUpdateWrapper::CaseWeightOpt DefaultCaseWeight = | ||||||
| 780 | SwitchInstProfUpdateWrapper::getSuccessorWeight(SI, 0); | ||||||
| 781 | if (IsTriviallyUnswitchableExitBlock(*SI.getDefaultDest())) { | ||||||
| 782 | DefaultExitBB = SI.getDefaultDest(); | ||||||
| 783 | } else if (ExitCaseIndices.empty()) | ||||||
| 784 | return false; | ||||||
| 785 | |||||||
| 786 | LLVM_DEBUG(dbgs() << " unswitching trivial switch...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " unswitching trivial switch...\n" ; } } while (false); | ||||||
| 787 | |||||||
| 788 | if (MSSAU && VerifyMemorySSA) | ||||||
| 789 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 790 | |||||||
| 791 | // We may need to invalidate SCEVs for the outermost loop reached by any of | ||||||
| 792 | // the exits. | ||||||
| 793 | Loop *OuterL = &L; | ||||||
| 794 | |||||||
| 795 | if (DefaultExitBB) { | ||||||
| 796 | // Check the loop containing this exit. | ||||||
| 797 | Loop *ExitL = getTopMostExitingLoop(DefaultExitBB, LI); | ||||||
| 798 | if (!ExitL || ExitL->contains(OuterL)) | ||||||
| 799 | OuterL = ExitL; | ||||||
| 800 | } | ||||||
| 801 | for (unsigned Index : ExitCaseIndices) { | ||||||
| 802 | auto CaseI = SI.case_begin() + Index; | ||||||
| 803 | // Compute the outer loop from this exit. | ||||||
| 804 | Loop *ExitL = getTopMostExitingLoop(CaseI->getCaseSuccessor(), LI); | ||||||
| 805 | if (!ExitL || ExitL->contains(OuterL)) | ||||||
| 806 | OuterL = ExitL; | ||||||
| 807 | } | ||||||
| 808 | |||||||
| 809 | if (SE) { | ||||||
| 810 | if (OuterL) | ||||||
| 811 | SE->forgetLoop(OuterL); | ||||||
| 812 | else | ||||||
| 813 | SE->forgetTopmostLoop(&L); | ||||||
| 814 | } | ||||||
| 815 | |||||||
| 816 | if (DefaultExitBB) { | ||||||
| 817 | // Clear out the default destination temporarily to allow accurate | ||||||
| 818 | // predecessor lists to be examined below. | ||||||
| 819 | SI.setDefaultDest(nullptr); | ||||||
| 820 | } | ||||||
| 821 | |||||||
| 822 | // Store the exit cases into a separate data structure and remove them from | ||||||
| 823 | // the switch. | ||||||
| 824 | SmallVector<std::tuple<ConstantInt *, BasicBlock *, | ||||||
| 825 | SwitchInstProfUpdateWrapper::CaseWeightOpt>, | ||||||
| 826 | 4> ExitCases; | ||||||
| 827 | ExitCases.reserve(ExitCaseIndices.size()); | ||||||
| 828 | SwitchInstProfUpdateWrapper SIW(SI); | ||||||
| 829 | // We walk the case indices backwards so that we remove the last case first | ||||||
| 830 | // and don't disrupt the earlier indices. | ||||||
| 831 | for (unsigned Index : reverse(ExitCaseIndices)) { | ||||||
| 832 | auto CaseI = SI.case_begin() + Index; | ||||||
| 833 | // Save the value of this case. | ||||||
| 834 | auto W = SIW.getSuccessorWeight(CaseI->getSuccessorIndex()); | ||||||
| 835 | ExitCases.emplace_back(CaseI->getCaseValue(), CaseI->getCaseSuccessor(), W); | ||||||
| 836 | // Delete the unswitched cases. | ||||||
| 837 | SIW.removeCase(CaseI); | ||||||
| 838 | } | ||||||
| 839 | |||||||
| 840 | // Check if after this all of the remaining cases point at the same | ||||||
| 841 | // successor. | ||||||
| 842 | BasicBlock *CommonSuccBB = nullptr; | ||||||
| 843 | if (SI.getNumCases() > 0 && | ||||||
| 844 | all_of(drop_begin(SI.cases()), [&SI](const SwitchInst::CaseHandle &Case) { | ||||||
| 845 | return Case.getCaseSuccessor() == SI.case_begin()->getCaseSuccessor(); | ||||||
| 846 | })) | ||||||
| 847 | CommonSuccBB = SI.case_begin()->getCaseSuccessor(); | ||||||
| 848 | if (!DefaultExitBB) { | ||||||
| 849 | // If we're not unswitching the default, we need it to match any cases to | ||||||
| 850 | // have a common successor or if we have no cases it is the common | ||||||
| 851 | // successor. | ||||||
| 852 | if (SI.getNumCases() == 0) | ||||||
| 853 | CommonSuccBB = SI.getDefaultDest(); | ||||||
| 854 | else if (SI.getDefaultDest() != CommonSuccBB) | ||||||
| 855 | CommonSuccBB = nullptr; | ||||||
| 856 | } | ||||||
| 857 | |||||||
| 858 | // Split the preheader, so that we know that there is a safe place to insert | ||||||
| 859 | // the switch. | ||||||
| 860 | BasicBlock *OldPH = L.getLoopPreheader(); | ||||||
| 861 | BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI, MSSAU); | ||||||
| 862 | OldPH->getTerminator()->eraseFromParent(); | ||||||
| 863 | |||||||
| 864 | // Now add the unswitched switch. | ||||||
| 865 | auto *NewSI = SwitchInst::Create(LoopCond, NewPH, ExitCases.size(), OldPH); | ||||||
| 866 | SwitchInstProfUpdateWrapper NewSIW(*NewSI); | ||||||
| 867 | |||||||
| 868 | // Rewrite the IR for the unswitched basic blocks. This requires two steps. | ||||||
| 869 | // First, we split any exit blocks with remaining in-loop predecessors. Then | ||||||
| 870 | // we update the PHIs in one of two ways depending on if there was a split. | ||||||
| 871 | // We walk in reverse so that we split in the same order as the cases | ||||||
| 872 | // appeared. This is purely for convenience of reading the resulting IR, but | ||||||
| 873 | // it doesn't cost anything really. | ||||||
| 874 | SmallPtrSet<BasicBlock *, 2> UnswitchedExitBBs; | ||||||
| 875 | SmallDenseMap<BasicBlock *, BasicBlock *, 2> SplitExitBBMap; | ||||||
| 876 | // Handle the default exit if necessary. | ||||||
| 877 | // FIXME: It'd be great if we could merge this with the loop below but LLVM's | ||||||
| 878 | // ranges aren't quite powerful enough yet. | ||||||
| 879 | if (DefaultExitBB) { | ||||||
| 880 | if (pred_empty(DefaultExitBB)) { | ||||||
| 881 | UnswitchedExitBBs.insert(DefaultExitBB); | ||||||
| 882 | rewritePHINodesForUnswitchedExitBlock(*DefaultExitBB, *ParentBB, *OldPH); | ||||||
| 883 | } else { | ||||||
| 884 | auto *SplitBB = | ||||||
| 885 | SplitBlock(DefaultExitBB, &DefaultExitBB->front(), &DT, &LI, MSSAU); | ||||||
| 886 | rewritePHINodesForExitAndUnswitchedBlocks(*DefaultExitBB, *SplitBB, | ||||||
| 887 | *ParentBB, *OldPH, | ||||||
| 888 | /*FullUnswitch*/ true); | ||||||
| 889 | DefaultExitBB = SplitExitBBMap[DefaultExitBB] = SplitBB; | ||||||
| 890 | } | ||||||
| 891 | } | ||||||
| 892 | // Note that we must use a reference in the for loop so that we update the | ||||||
| 893 | // container. | ||||||
| 894 | for (auto &ExitCase : reverse(ExitCases)) { | ||||||
| 895 | // Grab a reference to the exit block in the pair so that we can update it. | ||||||
| 896 | BasicBlock *ExitBB = std::get<1>(ExitCase); | ||||||
| 897 | |||||||
| 898 | // If this case is the last edge into the exit block, we can simply reuse it | ||||||
| 899 | // as it will no longer be a loop exit. No mapping necessary. | ||||||
| 900 | if (pred_empty(ExitBB)) { | ||||||
| 901 | // Only rewrite once. | ||||||
| 902 | if (UnswitchedExitBBs.insert(ExitBB).second) | ||||||
| 903 | rewritePHINodesForUnswitchedExitBlock(*ExitBB, *ParentBB, *OldPH); | ||||||
| 904 | continue; | ||||||
| 905 | } | ||||||
| 906 | |||||||
| 907 | // Otherwise we need to split the exit block so that we retain an exit | ||||||
| 908 | // block from the loop and a target for the unswitched condition. | ||||||
| 909 | BasicBlock *&SplitExitBB = SplitExitBBMap[ExitBB]; | ||||||
| 910 | if (!SplitExitBB) { | ||||||
| 911 | // If this is the first time we see this, do the split and remember it. | ||||||
| 912 | SplitExitBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI, MSSAU); | ||||||
| 913 | rewritePHINodesForExitAndUnswitchedBlocks(*ExitBB, *SplitExitBB, | ||||||
| 914 | *ParentBB, *OldPH, | ||||||
| 915 | /*FullUnswitch*/ true); | ||||||
| 916 | } | ||||||
| 917 | // Update the case pair to point to the split block. | ||||||
| 918 | std::get<1>(ExitCase) = SplitExitBB; | ||||||
| 919 | } | ||||||
| 920 | |||||||
| 921 | // Now add the unswitched cases. We do this in reverse order as we built them | ||||||
| 922 | // in reverse order. | ||||||
| 923 | for (auto &ExitCase : reverse(ExitCases)) { | ||||||
| 924 | ConstantInt *CaseVal = std::get<0>(ExitCase); | ||||||
| 925 | BasicBlock *UnswitchedBB = std::get<1>(ExitCase); | ||||||
| 926 | |||||||
| 927 | NewSIW.addCase(CaseVal, UnswitchedBB, std::get<2>(ExitCase)); | ||||||
| 928 | } | ||||||
| 929 | |||||||
| 930 | // If the default was unswitched, re-point it and add explicit cases for | ||||||
| 931 | // entering the loop. | ||||||
| 932 | if (DefaultExitBB) { | ||||||
| 933 | NewSIW->setDefaultDest(DefaultExitBB); | ||||||
| 934 | NewSIW.setSuccessorWeight(0, DefaultCaseWeight); | ||||||
| 935 | |||||||
| 936 | // We removed all the exit cases, so we just copy the cases to the | ||||||
| 937 | // unswitched switch. | ||||||
| 938 | for (const auto &Case : SI.cases()) | ||||||
| 939 | NewSIW.addCase(Case.getCaseValue(), NewPH, | ||||||
| 940 | SIW.getSuccessorWeight(Case.getSuccessorIndex())); | ||||||
| 941 | } else if (DefaultCaseWeight) { | ||||||
| 942 | // We have to set branch weight of the default case. | ||||||
| 943 | uint64_t SW = *DefaultCaseWeight; | ||||||
| 944 | for (const auto &Case : SI.cases()) { | ||||||
| 945 | auto W = SIW.getSuccessorWeight(Case.getSuccessorIndex()); | ||||||
| 946 | assert(W &&(static_cast <bool> (W && "case weight must be defined as default case weight is defined" ) ? void (0) : __assert_fail ("W && \"case weight must be defined as default case weight is defined\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 947, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 947 | "case weight must be defined as default case weight is defined")(static_cast <bool> (W && "case weight must be defined as default case weight is defined" ) ? void (0) : __assert_fail ("W && \"case weight must be defined as default case weight is defined\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 947, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 948 | SW += *W; | ||||||
| 949 | } | ||||||
| 950 | NewSIW.setSuccessorWeight(0, SW); | ||||||
| 951 | } | ||||||
| 952 | |||||||
| 953 | // If we ended up with a common successor for every path through the switch | ||||||
| 954 | // after unswitching, rewrite it to an unconditional branch to make it easy | ||||||
| 955 | // to recognize. Otherwise we potentially have to recognize the default case | ||||||
| 956 | // pointing at unreachable and other complexity. | ||||||
| 957 | if (CommonSuccBB) { | ||||||
| 958 | BasicBlock *BB = SI.getParent(); | ||||||
| 959 | // We may have had multiple edges to this common successor block, so remove | ||||||
| 960 | // them as predecessors. We skip the first one, either the default or the | ||||||
| 961 | // actual first case. | ||||||
| 962 | bool SkippedFirst = DefaultExitBB == nullptr; | ||||||
| 963 | for (auto Case : SI.cases()) { | ||||||
| 964 | assert(Case.getCaseSuccessor() == CommonSuccBB &&(static_cast <bool> (Case.getCaseSuccessor() == CommonSuccBB && "Non-common successor!") ? void (0) : __assert_fail ("Case.getCaseSuccessor() == CommonSuccBB && \"Non-common successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 965, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 965 | "Non-common successor!")(static_cast <bool> (Case.getCaseSuccessor() == CommonSuccBB && "Non-common successor!") ? void (0) : __assert_fail ("Case.getCaseSuccessor() == CommonSuccBB && \"Non-common successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 965, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 966 | (void)Case; | ||||||
| 967 | if (!SkippedFirst) { | ||||||
| 968 | SkippedFirst = true; | ||||||
| 969 | continue; | ||||||
| 970 | } | ||||||
| 971 | CommonSuccBB->removePredecessor(BB, | ||||||
| 972 | /*KeepOneInputPHIs*/ true); | ||||||
| 973 | } | ||||||
| 974 | // Now nuke the switch and replace it with a direct branch. | ||||||
| 975 | SIW.eraseFromParent(); | ||||||
| 976 | BranchInst::Create(CommonSuccBB, BB); | ||||||
| 977 | } else if (DefaultExitBB) { | ||||||
| 978 | assert(SI.getNumCases() > 0 &&(static_cast <bool> (SI.getNumCases() > 0 && "If we had no cases we'd have a common successor!") ? void ( 0) : __assert_fail ("SI.getNumCases() > 0 && \"If we had no cases we'd have a common successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 979, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 979 | "If we had no cases we'd have a common successor!")(static_cast <bool> (SI.getNumCases() > 0 && "If we had no cases we'd have a common successor!") ? void ( 0) : __assert_fail ("SI.getNumCases() > 0 && \"If we had no cases we'd have a common successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 979, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 980 | // Move the last case to the default successor. This is valid as if the | ||||||
| 981 | // default got unswitched it cannot be reached. This has the advantage of | ||||||
| 982 | // being simple and keeping the number of edges from this switch to | ||||||
| 983 | // successors the same, and avoiding any PHI update complexity. | ||||||
| 984 | auto LastCaseI = std::prev(SI.case_end()); | ||||||
| 985 | |||||||
| 986 | SI.setDefaultDest(LastCaseI->getCaseSuccessor()); | ||||||
| 987 | SIW.setSuccessorWeight( | ||||||
| 988 | 0, SIW.getSuccessorWeight(LastCaseI->getSuccessorIndex())); | ||||||
| 989 | SIW.removeCase(LastCaseI); | ||||||
| 990 | } | ||||||
| 991 | |||||||
| 992 | // Walk the unswitched exit blocks and the unswitched split blocks and update | ||||||
| 993 | // the dominator tree based on the CFG edits. While we are walking unordered | ||||||
| 994 | // containers here, the API for applyUpdates takes an unordered list of | ||||||
| 995 | // updates and requires them to not contain duplicates. | ||||||
| 996 | SmallVector<DominatorTree::UpdateType, 4> DTUpdates; | ||||||
| 997 | for (auto *UnswitchedExitBB : UnswitchedExitBBs) { | ||||||
| 998 | DTUpdates.push_back({DT.Delete, ParentBB, UnswitchedExitBB}); | ||||||
| 999 | DTUpdates.push_back({DT.Insert, OldPH, UnswitchedExitBB}); | ||||||
| 1000 | } | ||||||
| 1001 | for (auto SplitUnswitchedPair : SplitExitBBMap) { | ||||||
| 1002 | DTUpdates.push_back({DT.Delete, ParentBB, SplitUnswitchedPair.first}); | ||||||
| 1003 | DTUpdates.push_back({DT.Insert, OldPH, SplitUnswitchedPair.second}); | ||||||
| 1004 | } | ||||||
| 1005 | |||||||
| 1006 | if (MSSAU) { | ||||||
| 1007 | MSSAU->applyUpdates(DTUpdates, DT, /*UpdateDT=*/true); | ||||||
| 1008 | if (VerifyMemorySSA) | ||||||
| 1009 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 1010 | } else { | ||||||
| 1011 | DT.applyUpdates(DTUpdates); | ||||||
| 1012 | } | ||||||
| 1013 | |||||||
| 1014 | assert(DT.verify(DominatorTree::VerificationLevel::Fast))(static_cast <bool> (DT.verify(DominatorTree::VerificationLevel ::Fast)) ? void (0) : __assert_fail ("DT.verify(DominatorTree::VerificationLevel::Fast)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1014, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1015 | |||||||
| 1016 | // We may have changed the nesting relationship for this loop so hoist it to | ||||||
| 1017 | // its correct parent if needed. | ||||||
| 1018 | hoistLoopToNewParent(L, *NewPH, DT, LI, MSSAU, SE); | ||||||
| 1019 | |||||||
| 1020 | if (MSSAU && VerifyMemorySSA) | ||||||
| 1021 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 1022 | |||||||
| 1023 | ++NumTrivial; | ||||||
| 1024 | ++NumSwitches; | ||||||
| 1025 | LLVM_DEBUG(dbgs() << " done: unswitching trivial switch...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " done: unswitching trivial switch...\n" ; } } while (false); | ||||||
| 1026 | return true; | ||||||
| 1027 | } | ||||||
| 1028 | |||||||
| 1029 | /// This routine scans the loop to find a branch or switch which occurs before | ||||||
| 1030 | /// any side effects occur. These can potentially be unswitched without | ||||||
| 1031 | /// duplicating the loop. If a branch or switch is successfully unswitched the | ||||||
| 1032 | /// scanning continues to see if subsequent branches or switches have become | ||||||
| 1033 | /// trivial. Once all trivial candidates have been unswitched, this routine | ||||||
| 1034 | /// returns. | ||||||
| 1035 | /// | ||||||
| 1036 | /// The return value indicates whether anything was unswitched (and therefore | ||||||
| 1037 | /// changed). | ||||||
| 1038 | /// | ||||||
| 1039 | /// If `SE` is not null, it will be updated based on the potential loop SCEVs | ||||||
| 1040 | /// invalidated by this. | ||||||
| 1041 | static bool unswitchAllTrivialConditions(Loop &L, DominatorTree &DT, | ||||||
| 1042 | LoopInfo &LI, ScalarEvolution *SE, | ||||||
| 1043 | MemorySSAUpdater *MSSAU) { | ||||||
| 1044 | bool Changed = false; | ||||||
| 1045 | |||||||
| 1046 | // If loop header has only one reachable successor we should keep looking for | ||||||
| 1047 | // trivial condition candidates in the successor as well. An alternative is | ||||||
| 1048 | // to constant fold conditions and merge successors into loop header (then we | ||||||
| 1049 | // only need to check header's terminator). The reason for not doing this in | ||||||
| 1050 | // LoopUnswitch pass is that it could potentially break LoopPassManager's | ||||||
| 1051 | // invariants. Folding dead branches could either eliminate the current loop | ||||||
| 1052 | // or make other loops unreachable. LCSSA form might also not be preserved | ||||||
| 1053 | // after deleting branches. The following code keeps traversing loop header's | ||||||
| 1054 | // successors until it finds the trivial condition candidate (condition that | ||||||
| 1055 | // is not a constant). Since unswitching generates branches with constant | ||||||
| 1056 | // conditions, this scenario could be very common in practice. | ||||||
| 1057 | BasicBlock *CurrentBB = L.getHeader(); | ||||||
| 1058 | SmallPtrSet<BasicBlock *, 8> Visited; | ||||||
| 1059 | Visited.insert(CurrentBB); | ||||||
| 1060 | do { | ||||||
| 1061 | // Check if there are any side-effecting instructions (e.g. stores, calls, | ||||||
| 1062 | // volatile loads) in the part of the loop that the code *would* execute | ||||||
| 1063 | // without unswitching. | ||||||
| 1064 | if (MSSAU) // Possible early exit with MSSA | ||||||
| 1065 | if (auto *Defs = MSSAU->getMemorySSA()->getBlockDefs(CurrentBB)) | ||||||
| 1066 | if (!isa<MemoryPhi>(*Defs->begin()) || (++Defs->begin() != Defs->end())) | ||||||
| 1067 | return Changed; | ||||||
| 1068 | if (llvm::any_of(*CurrentBB, | ||||||
| 1069 | [](Instruction &I) { return I.mayHaveSideEffects(); })) | ||||||
| 1070 | return Changed; | ||||||
| 1071 | |||||||
| 1072 | Instruction *CurrentTerm = CurrentBB->getTerminator(); | ||||||
| 1073 | |||||||
| 1074 | if (auto *SI = dyn_cast<SwitchInst>(CurrentTerm)) { | ||||||
| 1075 | // Don't bother trying to unswitch past a switch with a constant | ||||||
| 1076 | // condition. This should be removed prior to running this pass by | ||||||
| 1077 | // simplifycfg. | ||||||
| 1078 | if (isa<Constant>(SI->getCondition())) | ||||||
| 1079 | return Changed; | ||||||
| 1080 | |||||||
| 1081 | if (!unswitchTrivialSwitch(L, *SI, DT, LI, SE, MSSAU)) | ||||||
| 1082 | // Couldn't unswitch this one so we're done. | ||||||
| 1083 | return Changed; | ||||||
| 1084 | |||||||
| 1085 | // Mark that we managed to unswitch something. | ||||||
| 1086 | Changed = true; | ||||||
| 1087 | |||||||
| 1088 | // If unswitching turned the terminator into an unconditional branch then | ||||||
| 1089 | // we can continue. The unswitching logic specifically works to fold any | ||||||
| 1090 | // cases it can into an unconditional branch to make it easier to | ||||||
| 1091 | // recognize here. | ||||||
| 1092 | auto *BI = dyn_cast<BranchInst>(CurrentBB->getTerminator()); | ||||||
| 1093 | if (!BI || BI->isConditional()) | ||||||
| 1094 | return Changed; | ||||||
| 1095 | |||||||
| 1096 | CurrentBB = BI->getSuccessor(0); | ||||||
| 1097 | continue; | ||||||
| 1098 | } | ||||||
| 1099 | |||||||
| 1100 | auto *BI = dyn_cast<BranchInst>(CurrentTerm); | ||||||
| 1101 | if (!BI) | ||||||
| 1102 | // We do not understand other terminator instructions. | ||||||
| 1103 | return Changed; | ||||||
| 1104 | |||||||
| 1105 | // Don't bother trying to unswitch past an unconditional branch or a branch | ||||||
| 1106 | // with a constant value. These should be removed by simplifycfg prior to | ||||||
| 1107 | // running this pass. | ||||||
| 1108 | if (!BI->isConditional() || | ||||||
| 1109 | isa<Constant>(skipTrivialSelect(BI->getCondition()))) | ||||||
| 1110 | return Changed; | ||||||
| 1111 | |||||||
| 1112 | // Found a trivial condition candidate: non-foldable conditional branch. If | ||||||
| 1113 | // we fail to unswitch this, we can't do anything else that is trivial. | ||||||
| 1114 | if (!unswitchTrivialBranch(L, *BI, DT, LI, SE, MSSAU)) | ||||||
| 1115 | return Changed; | ||||||
| 1116 | |||||||
| 1117 | // Mark that we managed to unswitch something. | ||||||
| 1118 | Changed = true; | ||||||
| 1119 | |||||||
| 1120 | // If we only unswitched some of the conditions feeding the branch, we won't | ||||||
| 1121 | // have collapsed it to a single successor. | ||||||
| 1122 | BI = cast<BranchInst>(CurrentBB->getTerminator()); | ||||||
| 1123 | if (BI->isConditional()) | ||||||
| 1124 | return Changed; | ||||||
| 1125 | |||||||
| 1126 | // Follow the newly unconditional branch into its successor. | ||||||
| 1127 | CurrentBB = BI->getSuccessor(0); | ||||||
| 1128 | |||||||
| 1129 | // When continuing, if we exit the loop or reach a previous visited block, | ||||||
| 1130 | // then we can not reach any trivial condition candidates (unfoldable | ||||||
| 1131 | // branch instructions or switch instructions) and no unswitch can happen. | ||||||
| 1132 | } while (L.contains(CurrentBB) && Visited.insert(CurrentBB).second); | ||||||
| 1133 | |||||||
| 1134 | return Changed; | ||||||
| 1135 | } | ||||||
| 1136 | |||||||
| 1137 | /// Build the cloned blocks for an unswitched copy of the given loop. | ||||||
| 1138 | /// | ||||||
| 1139 | /// The cloned blocks are inserted before the loop preheader (`LoopPH`) and | ||||||
| 1140 | /// after the split block (`SplitBB`) that will be used to select between the | ||||||
| 1141 | /// cloned and original loop. | ||||||
| 1142 | /// | ||||||
| 1143 | /// This routine handles cloning all of the necessary loop blocks and exit | ||||||
| 1144 | /// blocks including rewriting their instructions and the relevant PHI nodes. | ||||||
| 1145 | /// Any loop blocks or exit blocks which are dominated by a different successor | ||||||
| 1146 | /// than the one for this clone of the loop blocks can be trivially skipped. We | ||||||
| 1147 | /// use the `DominatingSucc` map to determine whether a block satisfies that | ||||||
| 1148 | /// property with a simple map lookup. | ||||||
| 1149 | /// | ||||||
| 1150 | /// It also correctly creates the unconditional branch in the cloned | ||||||
| 1151 | /// unswitched parent block to only point at the unswitched successor. | ||||||
| 1152 | /// | ||||||
| 1153 | /// This does not handle most of the necessary updates to `LoopInfo`. Only exit | ||||||
| 1154 | /// block splitting is correctly reflected in `LoopInfo`, essentially all of | ||||||
| 1155 | /// the cloned blocks (and their loops) are left without full `LoopInfo` | ||||||
| 1156 | /// updates. This also doesn't fully update `DominatorTree`. It adds the cloned | ||||||
| 1157 | /// blocks to them but doesn't create the cloned `DominatorTree` structure and | ||||||
| 1158 | /// instead the caller must recompute an accurate DT. It *does* correctly | ||||||
| 1159 | /// update the `AssumptionCache` provided in `AC`. | ||||||
| 1160 | static BasicBlock *buildClonedLoopBlocks( | ||||||
| 1161 | Loop &L, BasicBlock *LoopPH, BasicBlock *SplitBB, | ||||||
| 1162 | ArrayRef<BasicBlock *> ExitBlocks, BasicBlock *ParentBB, | ||||||
| 1163 | BasicBlock *UnswitchedSuccBB, BasicBlock *ContinueSuccBB, | ||||||
| 1164 | const SmallDenseMap<BasicBlock *, BasicBlock *, 16> &DominatingSucc, | ||||||
| 1165 | ValueToValueMapTy &VMap, | ||||||
| 1166 | SmallVectorImpl<DominatorTree::UpdateType> &DTUpdates, AssumptionCache &AC, | ||||||
| 1167 | DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU, | ||||||
| 1168 | ScalarEvolution *SE) { | ||||||
| 1169 | SmallVector<BasicBlock *, 4> NewBlocks; | ||||||
| 1170 | NewBlocks.reserve(L.getNumBlocks() + ExitBlocks.size()); | ||||||
| 1171 | |||||||
| 1172 | // We will need to clone a bunch of blocks, wrap up the clone operation in | ||||||
| 1173 | // a helper. | ||||||
| 1174 | auto CloneBlock = [&](BasicBlock *OldBB) { | ||||||
| 1175 | // Clone the basic block and insert it before the new preheader. | ||||||
| 1176 | BasicBlock *NewBB = CloneBasicBlock(OldBB, VMap, ".us", OldBB->getParent()); | ||||||
| 1177 | NewBB->moveBefore(LoopPH); | ||||||
| 1178 | |||||||
| 1179 | // Record this block and the mapping. | ||||||
| 1180 | NewBlocks.push_back(NewBB); | ||||||
| 1181 | VMap[OldBB] = NewBB; | ||||||
| 1182 | |||||||
| 1183 | return NewBB; | ||||||
| 1184 | }; | ||||||
| 1185 | |||||||
| 1186 | // We skip cloning blocks when they have a dominating succ that is not the | ||||||
| 1187 | // succ we are cloning for. | ||||||
| 1188 | auto SkipBlock = [&](BasicBlock *BB) { | ||||||
| 1189 | auto It = DominatingSucc.find(BB); | ||||||
| 1190 | return It != DominatingSucc.end() && It->second != UnswitchedSuccBB; | ||||||
| 1191 | }; | ||||||
| 1192 | |||||||
| 1193 | // First, clone the preheader. | ||||||
| 1194 | auto *ClonedPH = CloneBlock(LoopPH); | ||||||
| 1195 | |||||||
| 1196 | // Then clone all the loop blocks, skipping the ones that aren't necessary. | ||||||
| 1197 | for (auto *LoopBB : L.blocks()) | ||||||
| 1198 | if (!SkipBlock(LoopBB)) | ||||||
| 1199 | CloneBlock(LoopBB); | ||||||
| 1200 | |||||||
| 1201 | // Split all the loop exit edges so that when we clone the exit blocks, if | ||||||
| 1202 | // any of the exit blocks are *also* a preheader for some other loop, we | ||||||
| 1203 | // don't create multiple predecessors entering the loop header. | ||||||
| 1204 | for (auto *ExitBB : ExitBlocks) { | ||||||
| 1205 | if (SkipBlock(ExitBB)) | ||||||
| 1206 | continue; | ||||||
| 1207 | |||||||
| 1208 | // When we are going to clone an exit, we don't need to clone all the | ||||||
| 1209 | // instructions in the exit block and we want to ensure we have an easy | ||||||
| 1210 | // place to merge the CFG, so split the exit first. This is always safe to | ||||||
| 1211 | // do because there cannot be any non-loop predecessors of a loop exit in | ||||||
| 1212 | // loop simplified form. | ||||||
| 1213 | auto *MergeBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI, MSSAU); | ||||||
| 1214 | |||||||
| 1215 | // Rearrange the names to make it easier to write test cases by having the | ||||||
| 1216 | // exit block carry the suffix rather than the merge block carrying the | ||||||
| 1217 | // suffix. | ||||||
| 1218 | MergeBB->takeName(ExitBB); | ||||||
| 1219 | ExitBB->setName(Twine(MergeBB->getName()) + ".split"); | ||||||
| 1220 | |||||||
| 1221 | // Now clone the original exit block. | ||||||
| 1222 | auto *ClonedExitBB = CloneBlock(ExitBB); | ||||||
| 1223 | assert(ClonedExitBB->getTerminator()->getNumSuccessors() == 1 &&(static_cast <bool> (ClonedExitBB->getTerminator()-> getNumSuccessors() == 1 && "Exit block should have been split to have one successor!" ) ? void (0) : __assert_fail ("ClonedExitBB->getTerminator()->getNumSuccessors() == 1 && \"Exit block should have been split to have one successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1224, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1224 | "Exit block should have been split to have one successor!")(static_cast <bool> (ClonedExitBB->getTerminator()-> getNumSuccessors() == 1 && "Exit block should have been split to have one successor!" ) ? void (0) : __assert_fail ("ClonedExitBB->getTerminator()->getNumSuccessors() == 1 && \"Exit block should have been split to have one successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1224, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1225 | assert(ClonedExitBB->getTerminator()->getSuccessor(0) == MergeBB &&(static_cast <bool> (ClonedExitBB->getTerminator()-> getSuccessor(0) == MergeBB && "Cloned exit block has the wrong successor!" ) ? void (0) : __assert_fail ("ClonedExitBB->getTerminator()->getSuccessor(0) == MergeBB && \"Cloned exit block has the wrong successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1226, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1226 | "Cloned exit block has the wrong successor!")(static_cast <bool> (ClonedExitBB->getTerminator()-> getSuccessor(0) == MergeBB && "Cloned exit block has the wrong successor!" ) ? void (0) : __assert_fail ("ClonedExitBB->getTerminator()->getSuccessor(0) == MergeBB && \"Cloned exit block has the wrong successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1226, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1227 | |||||||
| 1228 | // Remap any cloned instructions and create a merge phi node for them. | ||||||
| 1229 | for (auto ZippedInsts : llvm::zip_first( | ||||||
| 1230 | llvm::make_range(ExitBB->begin(), std::prev(ExitBB->end())), | ||||||
| 1231 | llvm::make_range(ClonedExitBB->begin(), | ||||||
| 1232 | std::prev(ClonedExitBB->end())))) { | ||||||
| 1233 | Instruction &I = std::get<0>(ZippedInsts); | ||||||
| 1234 | Instruction &ClonedI = std::get<1>(ZippedInsts); | ||||||
| 1235 | |||||||
| 1236 | // The only instructions in the exit block should be PHI nodes and | ||||||
| 1237 | // potentially a landing pad. | ||||||
| 1238 | assert((static_cast <bool> ((isa<PHINode>(I) || isa<LandingPadInst >(I) || isa<CatchPadInst>(I)) && "Bad instruction in exit block!" ) ? void (0) : __assert_fail ("(isa<PHINode>(I) || isa<LandingPadInst>(I) || isa<CatchPadInst>(I)) && \"Bad instruction in exit block!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1240, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1239 | (isa<PHINode>(I) || isa<LandingPadInst>(I) || isa<CatchPadInst>(I)) &&(static_cast <bool> ((isa<PHINode>(I) || isa<LandingPadInst >(I) || isa<CatchPadInst>(I)) && "Bad instruction in exit block!" ) ? void (0) : __assert_fail ("(isa<PHINode>(I) || isa<LandingPadInst>(I) || isa<CatchPadInst>(I)) && \"Bad instruction in exit block!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1240, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1240 | "Bad instruction in exit block!")(static_cast <bool> ((isa<PHINode>(I) || isa<LandingPadInst >(I) || isa<CatchPadInst>(I)) && "Bad instruction in exit block!" ) ? void (0) : __assert_fail ("(isa<PHINode>(I) || isa<LandingPadInst>(I) || isa<CatchPadInst>(I)) && \"Bad instruction in exit block!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1240, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1241 | // We should have a value map between the instruction and its clone. | ||||||
| 1242 | assert(VMap.lookup(&I) == &ClonedI && "Mismatch in the value map!")(static_cast <bool> (VMap.lookup(&I) == &ClonedI && "Mismatch in the value map!") ? void (0) : __assert_fail ("VMap.lookup(&I) == &ClonedI && \"Mismatch in the value map!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1242, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1243 | |||||||
| 1244 | // Forget SCEVs based on exit phis in case SCEV looked through the phi. | ||||||
| 1245 | if (SE && isa<PHINode>(I)) | ||||||
| 1246 | SE->forgetValue(&I); | ||||||
| 1247 | |||||||
| 1248 | auto *MergePN = | ||||||
| 1249 | PHINode::Create(I.getType(), /*NumReservedValues*/ 2, ".us-phi", | ||||||
| 1250 | &*MergeBB->getFirstInsertionPt()); | ||||||
| 1251 | I.replaceAllUsesWith(MergePN); | ||||||
| 1252 | MergePN->addIncoming(&I, ExitBB); | ||||||
| 1253 | MergePN->addIncoming(&ClonedI, ClonedExitBB); | ||||||
| 1254 | } | ||||||
| 1255 | } | ||||||
| 1256 | |||||||
| 1257 | // Rewrite the instructions in the cloned blocks to refer to the instructions | ||||||
| 1258 | // in the cloned blocks. We have to do this as a second pass so that we have | ||||||
| 1259 | // everything available. Also, we have inserted new instructions which may | ||||||
| 1260 | // include assume intrinsics, so we update the assumption cache while | ||||||
| 1261 | // processing this. | ||||||
| 1262 | for (auto *ClonedBB : NewBlocks) | ||||||
| 1263 | for (Instruction &I : *ClonedBB) { | ||||||
| 1264 | RemapInstruction(&I, VMap, | ||||||
| 1265 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | ||||||
| 1266 | if (auto *II = dyn_cast<AssumeInst>(&I)) | ||||||
| 1267 | AC.registerAssumption(II); | ||||||
| 1268 | } | ||||||
| 1269 | |||||||
| 1270 | // Update any PHI nodes in the cloned successors of the skipped blocks to not | ||||||
| 1271 | // have spurious incoming values. | ||||||
| 1272 | for (auto *LoopBB : L.blocks()) | ||||||
| 1273 | if (SkipBlock(LoopBB)) | ||||||
| 1274 | for (auto *SuccBB : successors(LoopBB)) | ||||||
| 1275 | if (auto *ClonedSuccBB = cast_or_null<BasicBlock>(VMap.lookup(SuccBB))) | ||||||
| 1276 | for (PHINode &PN : ClonedSuccBB->phis()) | ||||||
| 1277 | PN.removeIncomingValue(LoopBB, /*DeletePHIIfEmpty*/ false); | ||||||
| 1278 | |||||||
| 1279 | // Remove the cloned parent as a predecessor of any successor we ended up | ||||||
| 1280 | // cloning other than the unswitched one. | ||||||
| 1281 | auto *ClonedParentBB = cast<BasicBlock>(VMap.lookup(ParentBB)); | ||||||
| 1282 | for (auto *SuccBB : successors(ParentBB)) { | ||||||
| 1283 | if (SuccBB == UnswitchedSuccBB) | ||||||
| 1284 | continue; | ||||||
| 1285 | |||||||
| 1286 | auto *ClonedSuccBB = cast_or_null<BasicBlock>(VMap.lookup(SuccBB)); | ||||||
| 1287 | if (!ClonedSuccBB) | ||||||
| 1288 | continue; | ||||||
| 1289 | |||||||
| 1290 | ClonedSuccBB->removePredecessor(ClonedParentBB, | ||||||
| 1291 | /*KeepOneInputPHIs*/ true); | ||||||
| 1292 | } | ||||||
| 1293 | |||||||
| 1294 | // Replace the cloned branch with an unconditional branch to the cloned | ||||||
| 1295 | // unswitched successor. | ||||||
| 1296 | auto *ClonedSuccBB = cast<BasicBlock>(VMap.lookup(UnswitchedSuccBB)); | ||||||
| 1297 | Instruction *ClonedTerminator = ClonedParentBB->getTerminator(); | ||||||
| 1298 | // Trivial Simplification. If Terminator is a conditional branch and | ||||||
| 1299 | // condition becomes dead - erase it. | ||||||
| 1300 | Value *ClonedConditionToErase = nullptr; | ||||||
| 1301 | if (auto *BI = dyn_cast<BranchInst>(ClonedTerminator)) | ||||||
| 1302 | ClonedConditionToErase = BI->getCondition(); | ||||||
| 1303 | else if (auto *SI = dyn_cast<SwitchInst>(ClonedTerminator)) | ||||||
| 1304 | ClonedConditionToErase = SI->getCondition(); | ||||||
| 1305 | |||||||
| 1306 | ClonedTerminator->eraseFromParent(); | ||||||
| 1307 | BranchInst::Create(ClonedSuccBB, ClonedParentBB); | ||||||
| 1308 | |||||||
| 1309 | if (ClonedConditionToErase) | ||||||
| 1310 | RecursivelyDeleteTriviallyDeadInstructions(ClonedConditionToErase, nullptr, | ||||||
| 1311 | MSSAU); | ||||||
| 1312 | |||||||
| 1313 | // If there are duplicate entries in the PHI nodes because of multiple edges | ||||||
| 1314 | // to the unswitched successor, we need to nuke all but one as we replaced it | ||||||
| 1315 | // with a direct branch. | ||||||
| 1316 | for (PHINode &PN : ClonedSuccBB->phis()) { | ||||||
| 1317 | bool Found = false; | ||||||
| 1318 | // Loop over the incoming operands backwards so we can easily delete as we | ||||||
| 1319 | // go without invalidating the index. | ||||||
| 1320 | for (int i = PN.getNumOperands() - 1; i >= 0; --i) { | ||||||
| 1321 | if (PN.getIncomingBlock(i) != ClonedParentBB) | ||||||
| 1322 | continue; | ||||||
| 1323 | if (!Found) { | ||||||
| 1324 | Found = true; | ||||||
| 1325 | continue; | ||||||
| 1326 | } | ||||||
| 1327 | PN.removeIncomingValue(i, /*DeletePHIIfEmpty*/ false); | ||||||
| 1328 | } | ||||||
| 1329 | } | ||||||
| 1330 | |||||||
| 1331 | // Record the domtree updates for the new blocks. | ||||||
| 1332 | SmallPtrSet<BasicBlock *, 4> SuccSet; | ||||||
| 1333 | for (auto *ClonedBB : NewBlocks) { | ||||||
| 1334 | for (auto *SuccBB : successors(ClonedBB)) | ||||||
| 1335 | if (SuccSet.insert(SuccBB).second) | ||||||
| 1336 | DTUpdates.push_back({DominatorTree::Insert, ClonedBB, SuccBB}); | ||||||
| 1337 | SuccSet.clear(); | ||||||
| 1338 | } | ||||||
| 1339 | |||||||
| 1340 | return ClonedPH; | ||||||
| 1341 | } | ||||||
| 1342 | |||||||
| 1343 | /// Recursively clone the specified loop and all of its children. | ||||||
| 1344 | /// | ||||||
| 1345 | /// The target parent loop for the clone should be provided, or can be null if | ||||||
| 1346 | /// the clone is a top-level loop. While cloning, all the blocks are mapped | ||||||
| 1347 | /// with the provided value map. The entire original loop must be present in | ||||||
| 1348 | /// the value map. The cloned loop is returned. | ||||||
| 1349 | static Loop *cloneLoopNest(Loop &OrigRootL, Loop *RootParentL, | ||||||
| 1350 | const ValueToValueMapTy &VMap, LoopInfo &LI) { | ||||||
| 1351 | auto AddClonedBlocksToLoop = [&](Loop &OrigL, Loop &ClonedL) { | ||||||
| 1352 | assert(ClonedL.getBlocks().empty() && "Must start with an empty loop!")(static_cast <bool> (ClonedL.getBlocks().empty() && "Must start with an empty loop!") ? void (0) : __assert_fail ("ClonedL.getBlocks().empty() && \"Must start with an empty loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1352, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1353 | ClonedL.reserveBlocks(OrigL.getNumBlocks()); | ||||||
| 1354 | for (auto *BB : OrigL.blocks()) { | ||||||
| 1355 | auto *ClonedBB = cast<BasicBlock>(VMap.lookup(BB)); | ||||||
| 1356 | ClonedL.addBlockEntry(ClonedBB); | ||||||
| 1357 | if (LI.getLoopFor(BB) == &OrigL) | ||||||
| 1358 | LI.changeLoopFor(ClonedBB, &ClonedL); | ||||||
| 1359 | } | ||||||
| 1360 | }; | ||||||
| 1361 | |||||||
| 1362 | // We specially handle the first loop because it may get cloned into | ||||||
| 1363 | // a different parent and because we most commonly are cloning leaf loops. | ||||||
| 1364 | Loop *ClonedRootL = LI.AllocateLoop(); | ||||||
| 1365 | if (RootParentL) | ||||||
| 1366 | RootParentL->addChildLoop(ClonedRootL); | ||||||
| 1367 | else | ||||||
| 1368 | LI.addTopLevelLoop(ClonedRootL); | ||||||
| 1369 | AddClonedBlocksToLoop(OrigRootL, *ClonedRootL); | ||||||
| 1370 | |||||||
| 1371 | if (OrigRootL.isInnermost()) | ||||||
| 1372 | return ClonedRootL; | ||||||
| 1373 | |||||||
| 1374 | // If we have a nest, we can quickly clone the entire loop nest using an | ||||||
| 1375 | // iterative approach because it is a tree. We keep the cloned parent in the | ||||||
| 1376 | // data structure to avoid repeatedly querying through a map to find it. | ||||||
| 1377 | SmallVector<std::pair<Loop *, Loop *>, 16> LoopsToClone; | ||||||
| 1378 | // Build up the loops to clone in reverse order as we'll clone them from the | ||||||
| 1379 | // back. | ||||||
| 1380 | for (Loop *ChildL : llvm::reverse(OrigRootL)) | ||||||
| 1381 | LoopsToClone.push_back({ClonedRootL, ChildL}); | ||||||
| 1382 | do { | ||||||
| 1383 | Loop *ClonedParentL, *L; | ||||||
| 1384 | std::tie(ClonedParentL, L) = LoopsToClone.pop_back_val(); | ||||||
| 1385 | Loop *ClonedL = LI.AllocateLoop(); | ||||||
| 1386 | ClonedParentL->addChildLoop(ClonedL); | ||||||
| 1387 | AddClonedBlocksToLoop(*L, *ClonedL); | ||||||
| 1388 | for (Loop *ChildL : llvm::reverse(*L)) | ||||||
| 1389 | LoopsToClone.push_back({ClonedL, ChildL}); | ||||||
| 1390 | } while (!LoopsToClone.empty()); | ||||||
| 1391 | |||||||
| 1392 | return ClonedRootL; | ||||||
| 1393 | } | ||||||
| 1394 | |||||||
| 1395 | /// Build the cloned loops of an original loop from unswitching. | ||||||
| 1396 | /// | ||||||
| 1397 | /// Because unswitching simplifies the CFG of the loop, this isn't a trivial | ||||||
| 1398 | /// operation. We need to re-verify that there even is a loop (as the backedge | ||||||
| 1399 | /// may not have been cloned), and even if there are remaining backedges the | ||||||
| 1400 | /// backedge set may be different. However, we know that each child loop is | ||||||
| 1401 | /// undisturbed, we only need to find where to place each child loop within | ||||||
| 1402 | /// either any parent loop or within a cloned version of the original loop. | ||||||
| 1403 | /// | ||||||
| 1404 | /// Because child loops may end up cloned outside of any cloned version of the | ||||||
| 1405 | /// original loop, multiple cloned sibling loops may be created. All of them | ||||||
| 1406 | /// are returned so that the newly introduced loop nest roots can be | ||||||
| 1407 | /// identified. | ||||||
| 1408 | static void buildClonedLoops(Loop &OrigL, ArrayRef<BasicBlock *> ExitBlocks, | ||||||
| 1409 | const ValueToValueMapTy &VMap, LoopInfo &LI, | ||||||
| 1410 | SmallVectorImpl<Loop *> &NonChildClonedLoops) { | ||||||
| 1411 | Loop *ClonedL = nullptr; | ||||||
| 1412 | |||||||
| 1413 | auto *OrigPH = OrigL.getLoopPreheader(); | ||||||
| 1414 | auto *OrigHeader = OrigL.getHeader(); | ||||||
| 1415 | |||||||
| 1416 | auto *ClonedPH = cast<BasicBlock>(VMap.lookup(OrigPH)); | ||||||
| 1417 | auto *ClonedHeader = cast<BasicBlock>(VMap.lookup(OrigHeader)); | ||||||
| 1418 | |||||||
| 1419 | // We need to know the loops of the cloned exit blocks to even compute the | ||||||
| 1420 | // accurate parent loop. If we only clone exits to some parent of the | ||||||
| 1421 | // original parent, we want to clone into that outer loop. We also keep track | ||||||
| 1422 | // of the loops that our cloned exit blocks participate in. | ||||||
| 1423 | Loop *ParentL = nullptr; | ||||||
| 1424 | SmallVector<BasicBlock *, 4> ClonedExitsInLoops; | ||||||
| 1425 | SmallDenseMap<BasicBlock *, Loop *, 16> ExitLoopMap; | ||||||
| 1426 | ClonedExitsInLoops.reserve(ExitBlocks.size()); | ||||||
| 1427 | for (auto *ExitBB : ExitBlocks) | ||||||
| 1428 | if (auto *ClonedExitBB = cast_or_null<BasicBlock>(VMap.lookup(ExitBB))) | ||||||
| 1429 | if (Loop *ExitL = LI.getLoopFor(ExitBB)) { | ||||||
| 1430 | ExitLoopMap[ClonedExitBB] = ExitL; | ||||||
| 1431 | ClonedExitsInLoops.push_back(ClonedExitBB); | ||||||
| 1432 | if (!ParentL || (ParentL != ExitL && ParentL->contains(ExitL))) | ||||||
| 1433 | ParentL = ExitL; | ||||||
| 1434 | } | ||||||
| 1435 | assert((!ParentL || ParentL == OrigL.getParentLoop() ||(static_cast <bool> ((!ParentL || ParentL == OrigL.getParentLoop () || ParentL->contains(OrigL.getParentLoop())) && "The computed parent loop should always contain (or be) the parent of " "the original loop.") ? void (0) : __assert_fail ("(!ParentL || ParentL == OrigL.getParentLoop() || ParentL->contains(OrigL.getParentLoop())) && \"The computed parent loop should always contain (or be) the parent of \" \"the original loop.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1438, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1436 | ParentL->contains(OrigL.getParentLoop())) &&(static_cast <bool> ((!ParentL || ParentL == OrigL.getParentLoop () || ParentL->contains(OrigL.getParentLoop())) && "The computed parent loop should always contain (or be) the parent of " "the original loop.") ? void (0) : __assert_fail ("(!ParentL || ParentL == OrigL.getParentLoop() || ParentL->contains(OrigL.getParentLoop())) && \"The computed parent loop should always contain (or be) the parent of \" \"the original loop.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1438, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1437 | "The computed parent loop should always contain (or be) the parent of "(static_cast <bool> ((!ParentL || ParentL == OrigL.getParentLoop () || ParentL->contains(OrigL.getParentLoop())) && "The computed parent loop should always contain (or be) the parent of " "the original loop.") ? void (0) : __assert_fail ("(!ParentL || ParentL == OrigL.getParentLoop() || ParentL->contains(OrigL.getParentLoop())) && \"The computed parent loop should always contain (or be) the parent of \" \"the original loop.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1438, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1438 | "the original loop.")(static_cast <bool> ((!ParentL || ParentL == OrigL.getParentLoop () || ParentL->contains(OrigL.getParentLoop())) && "The computed parent loop should always contain (or be) the parent of " "the original loop.") ? void (0) : __assert_fail ("(!ParentL || ParentL == OrigL.getParentLoop() || ParentL->contains(OrigL.getParentLoop())) && \"The computed parent loop should always contain (or be) the parent of \" \"the original loop.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1438, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1439 | |||||||
| 1440 | // We build the set of blocks dominated by the cloned header from the set of | ||||||
| 1441 | // cloned blocks out of the original loop. While not all of these will | ||||||
| 1442 | // necessarily be in the cloned loop, it is enough to establish that they | ||||||
| 1443 | // aren't in unreachable cycles, etc. | ||||||
| 1444 | SmallSetVector<BasicBlock *, 16> ClonedLoopBlocks; | ||||||
| 1445 | for (auto *BB : OrigL.blocks()) | ||||||
| 1446 | if (auto *ClonedBB = cast_or_null<BasicBlock>(VMap.lookup(BB))) | ||||||
| 1447 | ClonedLoopBlocks.insert(ClonedBB); | ||||||
| 1448 | |||||||
| 1449 | // Rebuild the set of blocks that will end up in the cloned loop. We may have | ||||||
| 1450 | // skipped cloning some region of this loop which can in turn skip some of | ||||||
| 1451 | // the backedges so we have to rebuild the blocks in the loop based on the | ||||||
| 1452 | // backedges that remain after cloning. | ||||||
| 1453 | SmallVector<BasicBlock *, 16> Worklist; | ||||||
| 1454 | SmallPtrSet<BasicBlock *, 16> BlocksInClonedLoop; | ||||||
| 1455 | for (auto *Pred : predecessors(ClonedHeader)) { | ||||||
| 1456 | // The only possible non-loop header predecessor is the preheader because | ||||||
| 1457 | // we know we cloned the loop in simplified form. | ||||||
| 1458 | if (Pred == ClonedPH) | ||||||
| 1459 | continue; | ||||||
| 1460 | |||||||
| 1461 | // Because the loop was in simplified form, the only non-loop predecessor | ||||||
| 1462 | // should be the preheader. | ||||||
| 1463 | assert(ClonedLoopBlocks.count(Pred) && "Found a predecessor of the loop "(static_cast <bool> (ClonedLoopBlocks.count(Pred) && "Found a predecessor of the loop " "header other than the preheader " "that is not part of the loop!") ? void (0) : __assert_fail ( "ClonedLoopBlocks.count(Pred) && \"Found a predecessor of the loop \" \"header other than the preheader \" \"that is not part of the loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1465, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1464 | "header other than the preheader "(static_cast <bool> (ClonedLoopBlocks.count(Pred) && "Found a predecessor of the loop " "header other than the preheader " "that is not part of the loop!") ? void (0) : __assert_fail ( "ClonedLoopBlocks.count(Pred) && \"Found a predecessor of the loop \" \"header other than the preheader \" \"that is not part of the loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1465, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1465 | "that is not part of the loop!")(static_cast <bool> (ClonedLoopBlocks.count(Pred) && "Found a predecessor of the loop " "header other than the preheader " "that is not part of the loop!") ? void (0) : __assert_fail ( "ClonedLoopBlocks.count(Pred) && \"Found a predecessor of the loop \" \"header other than the preheader \" \"that is not part of the loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1465, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1466 | |||||||
| 1467 | // Insert this block into the loop set and on the first visit (and if it | ||||||
| 1468 | // isn't the header we're currently walking) put it into the worklist to | ||||||
| 1469 | // recurse through. | ||||||
| 1470 | if (BlocksInClonedLoop.insert(Pred).second && Pred != ClonedHeader) | ||||||
| 1471 | Worklist.push_back(Pred); | ||||||
| 1472 | } | ||||||
| 1473 | |||||||
| 1474 | // If we had any backedges then there *is* a cloned loop. Put the header into | ||||||
| 1475 | // the loop set and then walk the worklist backwards to find all the blocks | ||||||
| 1476 | // that remain within the loop after cloning. | ||||||
| 1477 | if (!BlocksInClonedLoop.empty()) { | ||||||
| 1478 | BlocksInClonedLoop.insert(ClonedHeader); | ||||||
| 1479 | |||||||
| 1480 | while (!Worklist.empty()) { | ||||||
| 1481 | BasicBlock *BB = Worklist.pop_back_val(); | ||||||
| 1482 | assert(BlocksInClonedLoop.count(BB) &&(static_cast <bool> (BlocksInClonedLoop.count(BB) && "Didn't put block into the loop set!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count(BB) && \"Didn't put block into the loop set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1483, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1483 | "Didn't put block into the loop set!")(static_cast <bool> (BlocksInClonedLoop.count(BB) && "Didn't put block into the loop set!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count(BB) && \"Didn't put block into the loop set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1483, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1484 | |||||||
| 1485 | // Insert any predecessors that are in the possible set into the cloned | ||||||
| 1486 | // set, and if the insert is successful, add them to the worklist. Note | ||||||
| 1487 | // that we filter on the blocks that are definitely reachable via the | ||||||
| 1488 | // backedge to the loop header so we may prune out dead code within the | ||||||
| 1489 | // cloned loop. | ||||||
| 1490 | for (auto *Pred : predecessors(BB)) | ||||||
| 1491 | if (ClonedLoopBlocks.count(Pred) && | ||||||
| 1492 | BlocksInClonedLoop.insert(Pred).second) | ||||||
| 1493 | Worklist.push_back(Pred); | ||||||
| 1494 | } | ||||||
| 1495 | |||||||
| 1496 | ClonedL = LI.AllocateLoop(); | ||||||
| 1497 | if (ParentL) { | ||||||
| 1498 | ParentL->addBasicBlockToLoop(ClonedPH, LI); | ||||||
| 1499 | ParentL->addChildLoop(ClonedL); | ||||||
| 1500 | } else { | ||||||
| 1501 | LI.addTopLevelLoop(ClonedL); | ||||||
| 1502 | } | ||||||
| 1503 | NonChildClonedLoops.push_back(ClonedL); | ||||||
| 1504 | |||||||
| 1505 | ClonedL->reserveBlocks(BlocksInClonedLoop.size()); | ||||||
| 1506 | // We don't want to just add the cloned loop blocks based on how we | ||||||
| 1507 | // discovered them. The original order of blocks was carefully built in | ||||||
| 1508 | // a way that doesn't rely on predecessor ordering. Rather than re-invent | ||||||
| 1509 | // that logic, we just re-walk the original blocks (and those of the child | ||||||
| 1510 | // loops) and filter them as we add them into the cloned loop. | ||||||
| 1511 | for (auto *BB : OrigL.blocks()) { | ||||||
| 1512 | auto *ClonedBB = cast_or_null<BasicBlock>(VMap.lookup(BB)); | ||||||
| 1513 | if (!ClonedBB || !BlocksInClonedLoop.count(ClonedBB)) | ||||||
| 1514 | continue; | ||||||
| 1515 | |||||||
| 1516 | // Directly add the blocks that are only in this loop. | ||||||
| 1517 | if (LI.getLoopFor(BB) == &OrigL) { | ||||||
| 1518 | ClonedL->addBasicBlockToLoop(ClonedBB, LI); | ||||||
| 1519 | continue; | ||||||
| 1520 | } | ||||||
| 1521 | |||||||
| 1522 | // We want to manually add it to this loop and parents. | ||||||
| 1523 | // Registering it with LoopInfo will happen when we clone the top | ||||||
| 1524 | // loop for this block. | ||||||
| 1525 | for (Loop *PL = ClonedL; PL; PL = PL->getParentLoop()) | ||||||
| 1526 | PL->addBlockEntry(ClonedBB); | ||||||
| 1527 | } | ||||||
| 1528 | |||||||
| 1529 | // Now add each child loop whose header remains within the cloned loop. All | ||||||
| 1530 | // of the blocks within the loop must satisfy the same constraints as the | ||||||
| 1531 | // header so once we pass the header checks we can just clone the entire | ||||||
| 1532 | // child loop nest. | ||||||
| 1533 | for (Loop *ChildL : OrigL) { | ||||||
| 1534 | auto *ClonedChildHeader = | ||||||
| 1535 | cast_or_null<BasicBlock>(VMap.lookup(ChildL->getHeader())); | ||||||
| 1536 | if (!ClonedChildHeader || !BlocksInClonedLoop.count(ClonedChildHeader)) | ||||||
| 1537 | continue; | ||||||
| 1538 | |||||||
| 1539 | #ifndef NDEBUG | ||||||
| 1540 | // We should never have a cloned child loop header but fail to have | ||||||
| 1541 | // all of the blocks for that child loop. | ||||||
| 1542 | for (auto *ChildLoopBB : ChildL->blocks()) | ||||||
| 1543 | assert(BlocksInClonedLoop.count((static_cast <bool> (BlocksInClonedLoop.count( cast< BasicBlock>(VMap.lookup(ChildLoopBB))) && "Child cloned loop has a header within the cloned outer " "loop but not all of its blocks!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count( cast<BasicBlock>(VMap.lookup(ChildLoopBB))) && \"Child cloned loop has a header within the cloned outer \" \"loop but not all of its blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1546, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1544 | cast<BasicBlock>(VMap.lookup(ChildLoopBB))) &&(static_cast <bool> (BlocksInClonedLoop.count( cast< BasicBlock>(VMap.lookup(ChildLoopBB))) && "Child cloned loop has a header within the cloned outer " "loop but not all of its blocks!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count( cast<BasicBlock>(VMap.lookup(ChildLoopBB))) && \"Child cloned loop has a header within the cloned outer \" \"loop but not all of its blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1546, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1545 | "Child cloned loop has a header within the cloned outer "(static_cast <bool> (BlocksInClonedLoop.count( cast< BasicBlock>(VMap.lookup(ChildLoopBB))) && "Child cloned loop has a header within the cloned outer " "loop but not all of its blocks!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count( cast<BasicBlock>(VMap.lookup(ChildLoopBB))) && \"Child cloned loop has a header within the cloned outer \" \"loop but not all of its blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1546, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1546 | "loop but not all of its blocks!")(static_cast <bool> (BlocksInClonedLoop.count( cast< BasicBlock>(VMap.lookup(ChildLoopBB))) && "Child cloned loop has a header within the cloned outer " "loop but not all of its blocks!") ? void (0) : __assert_fail ("BlocksInClonedLoop.count( cast<BasicBlock>(VMap.lookup(ChildLoopBB))) && \"Child cloned loop has a header within the cloned outer \" \"loop but not all of its blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1546, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1547 | #endif | ||||||
| 1548 | |||||||
| 1549 | cloneLoopNest(*ChildL, ClonedL, VMap, LI); | ||||||
| 1550 | } | ||||||
| 1551 | } | ||||||
| 1552 | |||||||
| 1553 | // Now that we've handled all the components of the original loop that were | ||||||
| 1554 | // cloned into a new loop, we still need to handle anything from the original | ||||||
| 1555 | // loop that wasn't in a cloned loop. | ||||||
| 1556 | |||||||
| 1557 | // Figure out what blocks are left to place within any loop nest containing | ||||||
| 1558 | // the unswitched loop. If we never formed a loop, the cloned PH is one of | ||||||
| 1559 | // them. | ||||||
| 1560 | SmallPtrSet<BasicBlock *, 16> UnloopedBlockSet; | ||||||
| 1561 | if (BlocksInClonedLoop.empty()) | ||||||
| 1562 | UnloopedBlockSet.insert(ClonedPH); | ||||||
| 1563 | for (auto *ClonedBB : ClonedLoopBlocks) | ||||||
| 1564 | if (!BlocksInClonedLoop.count(ClonedBB)) | ||||||
| 1565 | UnloopedBlockSet.insert(ClonedBB); | ||||||
| 1566 | |||||||
| 1567 | // Copy the cloned exits and sort them in ascending loop depth, we'll work | ||||||
| 1568 | // backwards across these to process them inside out. The order shouldn't | ||||||
| 1569 | // matter as we're just trying to build up the map from inside-out; we use | ||||||
| 1570 | // the map in a more stably ordered way below. | ||||||
| 1571 | auto OrderedClonedExitsInLoops = ClonedExitsInLoops; | ||||||
| 1572 | llvm::sort(OrderedClonedExitsInLoops, [&](BasicBlock *LHS, BasicBlock *RHS) { | ||||||
| 1573 | return ExitLoopMap.lookup(LHS)->getLoopDepth() < | ||||||
| 1574 | ExitLoopMap.lookup(RHS)->getLoopDepth(); | ||||||
| 1575 | }); | ||||||
| 1576 | |||||||
| 1577 | // Populate the existing ExitLoopMap with everything reachable from each | ||||||
| 1578 | // exit, starting from the inner most exit. | ||||||
| 1579 | while (!UnloopedBlockSet.empty() && !OrderedClonedExitsInLoops.empty()) { | ||||||
| 1580 | assert(Worklist.empty() && "Didn't clear worklist!")(static_cast <bool> (Worklist.empty() && "Didn't clear worklist!" ) ? void (0) : __assert_fail ("Worklist.empty() && \"Didn't clear worklist!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1580, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1581 | |||||||
| 1582 | BasicBlock *ExitBB = OrderedClonedExitsInLoops.pop_back_val(); | ||||||
| 1583 | Loop *ExitL = ExitLoopMap.lookup(ExitBB); | ||||||
| 1584 | |||||||
| 1585 | // Walk the CFG back until we hit the cloned PH adding everything reachable | ||||||
| 1586 | // and in the unlooped set to this exit block's loop. | ||||||
| 1587 | Worklist.push_back(ExitBB); | ||||||
| 1588 | do { | ||||||
| 1589 | BasicBlock *BB = Worklist.pop_back_val(); | ||||||
| 1590 | // We can stop recursing at the cloned preheader (if we get there). | ||||||
| 1591 | if (BB == ClonedPH) | ||||||
| 1592 | continue; | ||||||
| 1593 | |||||||
| 1594 | for (BasicBlock *PredBB : predecessors(BB)) { | ||||||
| 1595 | // If this pred has already been moved to our set or is part of some | ||||||
| 1596 | // (inner) loop, no update needed. | ||||||
| 1597 | if (!UnloopedBlockSet.erase(PredBB)) { | ||||||
| 1598 | assert((static_cast <bool> ((BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && "Predecessor not mapped to a loop!" ) ? void (0) : __assert_fail ("(BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && \"Predecessor not mapped to a loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1600, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1599 | (BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) &&(static_cast <bool> ((BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && "Predecessor not mapped to a loop!" ) ? void (0) : __assert_fail ("(BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && \"Predecessor not mapped to a loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1600, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1600 | "Predecessor not mapped to a loop!")(static_cast <bool> ((BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && "Predecessor not mapped to a loop!" ) ? void (0) : __assert_fail ("(BlocksInClonedLoop.count(PredBB) || ExitLoopMap.count(PredBB)) && \"Predecessor not mapped to a loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1600, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1601 | continue; | ||||||
| 1602 | } | ||||||
| 1603 | |||||||
| 1604 | // We just insert into the loop set here. We'll add these blocks to the | ||||||
| 1605 | // exit loop after we build up the set in an order that doesn't rely on | ||||||
| 1606 | // predecessor order (which in turn relies on use list order). | ||||||
| 1607 | bool Inserted = ExitLoopMap.insert({PredBB, ExitL}).second; | ||||||
| 1608 | (void)Inserted; | ||||||
| 1609 | assert(Inserted && "Should only visit an unlooped block once!")(static_cast <bool> (Inserted && "Should only visit an unlooped block once!" ) ? void (0) : __assert_fail ("Inserted && \"Should only visit an unlooped block once!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1609, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1610 | |||||||
| 1611 | // And recurse through to its predecessors. | ||||||
| 1612 | Worklist.push_back(PredBB); | ||||||
| 1613 | } | ||||||
| 1614 | } while (!Worklist.empty()); | ||||||
| 1615 | } | ||||||
| 1616 | |||||||
| 1617 | // Now that the ExitLoopMap gives as mapping for all the non-looping cloned | ||||||
| 1618 | // blocks to their outer loops, walk the cloned blocks and the cloned exits | ||||||
| 1619 | // in their original order adding them to the correct loop. | ||||||
| 1620 | |||||||
| 1621 | // We need a stable insertion order. We use the order of the original loop | ||||||
| 1622 | // order and map into the correct parent loop. | ||||||
| 1623 | for (auto *BB : llvm::concat<BasicBlock *const>( | ||||||
| 1624 | ArrayRef(ClonedPH), ClonedLoopBlocks, ClonedExitsInLoops)) | ||||||
| 1625 | if (Loop *OuterL = ExitLoopMap.lookup(BB)) | ||||||
| 1626 | OuterL->addBasicBlockToLoop(BB, LI); | ||||||
| 1627 | |||||||
| 1628 | #ifndef NDEBUG | ||||||
| 1629 | for (auto &BBAndL : ExitLoopMap) { | ||||||
| 1630 | auto *BB = BBAndL.first; | ||||||
| 1631 | auto *OuterL = BBAndL.second; | ||||||
| 1632 | assert(LI.getLoopFor(BB) == OuterL &&(static_cast <bool> (LI.getLoopFor(BB) == OuterL && "Failed to put all blocks into outer loops!") ? void (0) : __assert_fail ("LI.getLoopFor(BB) == OuterL && \"Failed to put all blocks into outer loops!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1633, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1633 | "Failed to put all blocks into outer loops!")(static_cast <bool> (LI.getLoopFor(BB) == OuterL && "Failed to put all blocks into outer loops!") ? void (0) : __assert_fail ("LI.getLoopFor(BB) == OuterL && \"Failed to put all blocks into outer loops!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1633, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1634 | } | ||||||
| 1635 | #endif | ||||||
| 1636 | |||||||
| 1637 | // Now that all the blocks are placed into the correct containing loop in the | ||||||
| 1638 | // absence of child loops, find all the potentially cloned child loops and | ||||||
| 1639 | // clone them into whatever outer loop we placed their header into. | ||||||
| 1640 | for (Loop *ChildL : OrigL) { | ||||||
| 1641 | auto *ClonedChildHeader = | ||||||
| 1642 | cast_or_null<BasicBlock>(VMap.lookup(ChildL->getHeader())); | ||||||
| 1643 | if (!ClonedChildHeader || BlocksInClonedLoop.count(ClonedChildHeader)) | ||||||
| 1644 | continue; | ||||||
| 1645 | |||||||
| 1646 | #ifndef NDEBUG | ||||||
| 1647 | for (auto *ChildLoopBB : ChildL->blocks()) | ||||||
| 1648 | assert(VMap.count(ChildLoopBB) &&(static_cast <bool> (VMap.count(ChildLoopBB) && "Cloned a child loop header but not all of that loops blocks!" ) ? void (0) : __assert_fail ("VMap.count(ChildLoopBB) && \"Cloned a child loop header but not all of that loops blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1649, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1649 | "Cloned a child loop header but not all of that loops blocks!")(static_cast <bool> (VMap.count(ChildLoopBB) && "Cloned a child loop header but not all of that loops blocks!" ) ? void (0) : __assert_fail ("VMap.count(ChildLoopBB) && \"Cloned a child loop header but not all of that loops blocks!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1649, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1650 | #endif | ||||||
| 1651 | |||||||
| 1652 | NonChildClonedLoops.push_back(cloneLoopNest( | ||||||
| 1653 | *ChildL, ExitLoopMap.lookup(ClonedChildHeader), VMap, LI)); | ||||||
| 1654 | } | ||||||
| 1655 | } | ||||||
| 1656 | |||||||
| 1657 | static void | ||||||
| 1658 | deleteDeadClonedBlocks(Loop &L, ArrayRef<BasicBlock *> ExitBlocks, | ||||||
| 1659 | ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps, | ||||||
| 1660 | DominatorTree &DT, MemorySSAUpdater *MSSAU) { | ||||||
| 1661 | // Find all the dead clones, and remove them from their successors. | ||||||
| 1662 | SmallVector<BasicBlock *, 16> DeadBlocks; | ||||||
| 1663 | for (BasicBlock *BB : llvm::concat<BasicBlock *const>(L.blocks(), ExitBlocks)) | ||||||
| 1664 | for (const auto &VMap : VMaps) | ||||||
| 1665 | if (BasicBlock *ClonedBB = cast_or_null<BasicBlock>(VMap->lookup(BB))) | ||||||
| 1666 | if (!DT.isReachableFromEntry(ClonedBB)) { | ||||||
| 1667 | for (BasicBlock *SuccBB : successors(ClonedBB)) | ||||||
| 1668 | SuccBB->removePredecessor(ClonedBB); | ||||||
| 1669 | DeadBlocks.push_back(ClonedBB); | ||||||
| 1670 | } | ||||||
| 1671 | |||||||
| 1672 | // Remove all MemorySSA in the dead blocks | ||||||
| 1673 | if (MSSAU) { | ||||||
| 1674 | SmallSetVector<BasicBlock *, 8> DeadBlockSet(DeadBlocks.begin(), | ||||||
| 1675 | DeadBlocks.end()); | ||||||
| 1676 | MSSAU->removeBlocks(DeadBlockSet); | ||||||
| 1677 | } | ||||||
| 1678 | |||||||
| 1679 | // Drop any remaining references to break cycles. | ||||||
| 1680 | for (BasicBlock *BB : DeadBlocks) | ||||||
| 1681 | BB->dropAllReferences(); | ||||||
| 1682 | // Erase them from the IR. | ||||||
| 1683 | for (BasicBlock *BB : DeadBlocks) | ||||||
| 1684 | BB->eraseFromParent(); | ||||||
| 1685 | } | ||||||
| 1686 | |||||||
| 1687 | static void | ||||||
| 1688 | deleteDeadBlocksFromLoop(Loop &L, | ||||||
| 1689 | SmallVectorImpl<BasicBlock *> &ExitBlocks, | ||||||
| 1690 | DominatorTree &DT, LoopInfo &LI, | ||||||
| 1691 | MemorySSAUpdater *MSSAU, | ||||||
| 1692 | ScalarEvolution *SE, | ||||||
| 1693 | function_ref<void(Loop &, StringRef)> DestroyLoopCB) { | ||||||
| 1694 | // Find all the dead blocks tied to this loop, and remove them from their | ||||||
| 1695 | // successors. | ||||||
| 1696 | SmallSetVector<BasicBlock *, 8> DeadBlockSet; | ||||||
| 1697 | |||||||
| 1698 | // Start with loop/exit blocks and get a transitive closure of reachable dead | ||||||
| 1699 | // blocks. | ||||||
| 1700 | SmallVector<BasicBlock *, 16> DeathCandidates(ExitBlocks.begin(), | ||||||
| 1701 | ExitBlocks.end()); | ||||||
| 1702 | DeathCandidates.append(L.blocks().begin(), L.blocks().end()); | ||||||
| 1703 | while (!DeathCandidates.empty()) { | ||||||
| 1704 | auto *BB = DeathCandidates.pop_back_val(); | ||||||
| 1705 | if (!DeadBlockSet.count(BB) && !DT.isReachableFromEntry(BB)) { | ||||||
| 1706 | for (BasicBlock *SuccBB : successors(BB)) { | ||||||
| 1707 | SuccBB->removePredecessor(BB); | ||||||
| 1708 | DeathCandidates.push_back(SuccBB); | ||||||
| 1709 | } | ||||||
| 1710 | DeadBlockSet.insert(BB); | ||||||
| 1711 | } | ||||||
| 1712 | } | ||||||
| 1713 | |||||||
| 1714 | // Remove all MemorySSA in the dead blocks | ||||||
| 1715 | if (MSSAU) | ||||||
| 1716 | MSSAU->removeBlocks(DeadBlockSet); | ||||||
| 1717 | |||||||
| 1718 | // Filter out the dead blocks from the exit blocks list so that it can be | ||||||
| 1719 | // used in the caller. | ||||||
| 1720 | llvm::erase_if(ExitBlocks, | ||||||
| 1721 | [&](BasicBlock *BB) { return DeadBlockSet.count(BB); }); | ||||||
| 1722 | |||||||
| 1723 | // Walk from this loop up through its parents removing all of the dead blocks. | ||||||
| 1724 | for (Loop *ParentL = &L; ParentL; ParentL = ParentL->getParentLoop()) { | ||||||
| 1725 | for (auto *BB : DeadBlockSet) | ||||||
| 1726 | ParentL->getBlocksSet().erase(BB); | ||||||
| 1727 | llvm::erase_if(ParentL->getBlocksVector(), | ||||||
| 1728 | [&](BasicBlock *BB) { return DeadBlockSet.count(BB); }); | ||||||
| 1729 | } | ||||||
| 1730 | |||||||
| 1731 | // Now delete the dead child loops. This raw delete will clear them | ||||||
| 1732 | // recursively. | ||||||
| 1733 | llvm::erase_if(L.getSubLoopsVector(), [&](Loop *ChildL) { | ||||||
| 1734 | if (!DeadBlockSet.count(ChildL->getHeader())) | ||||||
| 1735 | return false; | ||||||
| 1736 | |||||||
| 1737 | assert(llvm::all_of(ChildL->blocks(),(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1738 | [&](BasicBlock *ChildBB) {(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1739 | return DeadBlockSet.count(ChildBB);(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1740 | }) &&(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1741 | "If the child loop header is dead all blocks in the child loop must "(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1742 | "be dead as well!")(static_cast <bool> (llvm::all_of(ChildL->blocks(), [ &](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB ); }) && "If the child loop header is dead all blocks in the child loop must " "be dead as well!") ? void (0) : __assert_fail ("llvm::all_of(ChildL->blocks(), [&](BasicBlock *ChildBB) { return DeadBlockSet.count(ChildBB); }) && \"If the child loop header is dead all blocks in the child loop must \" \"be dead as well!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1742, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1743 | DestroyLoopCB(*ChildL, ChildL->getName()); | ||||||
| 1744 | if (SE) | ||||||
| 1745 | SE->forgetBlockAndLoopDispositions(); | ||||||
| 1746 | LI.destroy(ChildL); | ||||||
| 1747 | return true; | ||||||
| 1748 | }); | ||||||
| 1749 | |||||||
| 1750 | // Remove the loop mappings for the dead blocks and drop all the references | ||||||
| 1751 | // from these blocks to others to handle cyclic references as we start | ||||||
| 1752 | // deleting the blocks themselves. | ||||||
| 1753 | for (auto *BB : DeadBlockSet) { | ||||||
| 1754 | // Check that the dominator tree has already been updated. | ||||||
| 1755 | assert(!DT.getNode(BB) && "Should already have cleared domtree!")(static_cast <bool> (!DT.getNode(BB) && "Should already have cleared domtree!" ) ? void (0) : __assert_fail ("!DT.getNode(BB) && \"Should already have cleared domtree!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1755, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1756 | LI.changeLoopFor(BB, nullptr); | ||||||
| 1757 | // Drop all uses of the instructions to make sure we won't have dangling | ||||||
| 1758 | // uses in other blocks. | ||||||
| 1759 | for (auto &I : *BB) | ||||||
| 1760 | if (!I.use_empty()) | ||||||
| 1761 | I.replaceAllUsesWith(PoisonValue::get(I.getType())); | ||||||
| 1762 | BB->dropAllReferences(); | ||||||
| 1763 | } | ||||||
| 1764 | |||||||
| 1765 | // Actually delete the blocks now that they've been fully unhooked from the | ||||||
| 1766 | // IR. | ||||||
| 1767 | for (auto *BB : DeadBlockSet) | ||||||
| 1768 | BB->eraseFromParent(); | ||||||
| 1769 | } | ||||||
| 1770 | |||||||
| 1771 | /// Recompute the set of blocks in a loop after unswitching. | ||||||
| 1772 | /// | ||||||
| 1773 | /// This walks from the original headers predecessors to rebuild the loop. We | ||||||
| 1774 | /// take advantage of the fact that new blocks can't have been added, and so we | ||||||
| 1775 | /// filter by the original loop's blocks. This also handles potentially | ||||||
| 1776 | /// unreachable code that we don't want to explore but might be found examining | ||||||
| 1777 | /// the predecessors of the header. | ||||||
| 1778 | /// | ||||||
| 1779 | /// If the original loop is no longer a loop, this will return an empty set. If | ||||||
| 1780 | /// it remains a loop, all the blocks within it will be added to the set | ||||||
| 1781 | /// (including those blocks in inner loops). | ||||||
| 1782 | static SmallPtrSet<const BasicBlock *, 16> recomputeLoopBlockSet(Loop &L, | ||||||
| 1783 | LoopInfo &LI) { | ||||||
| 1784 | SmallPtrSet<const BasicBlock *, 16> LoopBlockSet; | ||||||
| 1785 | |||||||
| 1786 | auto *PH = L.getLoopPreheader(); | ||||||
| 1787 | auto *Header = L.getHeader(); | ||||||
| 1788 | |||||||
| 1789 | // A worklist to use while walking backwards from the header. | ||||||
| 1790 | SmallVector<BasicBlock *, 16> Worklist; | ||||||
| 1791 | |||||||
| 1792 | // First walk the predecessors of the header to find the backedges. This will | ||||||
| 1793 | // form the basis of our walk. | ||||||
| 1794 | for (auto *Pred : predecessors(Header)) { | ||||||
| 1795 | // Skip the preheader. | ||||||
| 1796 | if (Pred == PH) | ||||||
| 1797 | continue; | ||||||
| 1798 | |||||||
| 1799 | // Because the loop was in simplified form, the only non-loop predecessor | ||||||
| 1800 | // is the preheader. | ||||||
| 1801 | assert(L.contains(Pred) && "Found a predecessor of the loop header other "(static_cast <bool> (L.contains(Pred) && "Found a predecessor of the loop header other " "than the preheader that is not part of the " "loop!") ? void (0) : __assert_fail ("L.contains(Pred) && \"Found a predecessor of the loop header other \" \"than the preheader that is not part of the \" \"loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1803, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1802 | "than the preheader that is not part of the "(static_cast <bool> (L.contains(Pred) && "Found a predecessor of the loop header other " "than the preheader that is not part of the " "loop!") ? void (0) : __assert_fail ("L.contains(Pred) && \"Found a predecessor of the loop header other \" \"than the preheader that is not part of the \" \"loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1803, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1803 | "loop!")(static_cast <bool> (L.contains(Pred) && "Found a predecessor of the loop header other " "than the preheader that is not part of the " "loop!") ? void (0) : __assert_fail ("L.contains(Pred) && \"Found a predecessor of the loop header other \" \"than the preheader that is not part of the \" \"loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1803, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1804 | |||||||
| 1805 | // Insert this block into the loop set and on the first visit and, if it | ||||||
| 1806 | // isn't the header we're currently walking, put it into the worklist to | ||||||
| 1807 | // recurse through. | ||||||
| 1808 | if (LoopBlockSet.insert(Pred).second && Pred != Header) | ||||||
| 1809 | Worklist.push_back(Pred); | ||||||
| 1810 | } | ||||||
| 1811 | |||||||
| 1812 | // If no backedges were found, we're done. | ||||||
| 1813 | if (LoopBlockSet.empty()) | ||||||
| 1814 | return LoopBlockSet; | ||||||
| 1815 | |||||||
| 1816 | // We found backedges, recurse through them to identify the loop blocks. | ||||||
| 1817 | while (!Worklist.empty()) { | ||||||
| 1818 | BasicBlock *BB = Worklist.pop_back_val(); | ||||||
| 1819 | assert(LoopBlockSet.count(BB) && "Didn't put block into the loop set!")(static_cast <bool> (LoopBlockSet.count(BB) && "Didn't put block into the loop set!" ) ? void (0) : __assert_fail ("LoopBlockSet.count(BB) && \"Didn't put block into the loop set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1819, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1820 | |||||||
| 1821 | // No need to walk past the header. | ||||||
| 1822 | if (BB == Header) | ||||||
| 1823 | continue; | ||||||
| 1824 | |||||||
| 1825 | // Because we know the inner loop structure remains valid we can use the | ||||||
| 1826 | // loop structure to jump immediately across the entire nested loop. | ||||||
| 1827 | // Further, because it is in loop simplified form, we can directly jump | ||||||
| 1828 | // to its preheader afterward. | ||||||
| 1829 | if (Loop *InnerL = LI.getLoopFor(BB)) | ||||||
| 1830 | if (InnerL != &L) { | ||||||
| 1831 | assert(L.contains(InnerL) &&(static_cast <bool> (L.contains(InnerL) && "Should not reach a loop *outside* this loop!" ) ? void (0) : __assert_fail ("L.contains(InnerL) && \"Should not reach a loop *outside* this loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1832, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1832 | "Should not reach a loop *outside* this loop!")(static_cast <bool> (L.contains(InnerL) && "Should not reach a loop *outside* this loop!" ) ? void (0) : __assert_fail ("L.contains(InnerL) && \"Should not reach a loop *outside* this loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1832, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1833 | // The preheader is the only possible predecessor of the loop so | ||||||
| 1834 | // insert it into the set and check whether it was already handled. | ||||||
| 1835 | auto *InnerPH = InnerL->getLoopPreheader(); | ||||||
| 1836 | assert(L.contains(InnerPH) && "Cannot contain an inner loop block "(static_cast <bool> (L.contains(InnerPH) && "Cannot contain an inner loop block " "but not contain the inner loop " "preheader!") ? void (0) : __assert_fail ("L.contains(InnerPH) && \"Cannot contain an inner loop block \" \"but not contain the inner loop \" \"preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1838, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1837 | "but not contain the inner loop "(static_cast <bool> (L.contains(InnerPH) && "Cannot contain an inner loop block " "but not contain the inner loop " "preheader!") ? void (0) : __assert_fail ("L.contains(InnerPH) && \"Cannot contain an inner loop block \" \"but not contain the inner loop \" \"preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1838, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1838 | "preheader!")(static_cast <bool> (L.contains(InnerPH) && "Cannot contain an inner loop block " "but not contain the inner loop " "preheader!") ? void (0) : __assert_fail ("L.contains(InnerPH) && \"Cannot contain an inner loop block \" \"but not contain the inner loop \" \"preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1838, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1839 | if (!LoopBlockSet.insert(InnerPH).second) | ||||||
| 1840 | // The only way to reach the preheader is through the loop body | ||||||
| 1841 | // itself so if it has been visited the loop is already handled. | ||||||
| 1842 | continue; | ||||||
| 1843 | |||||||
| 1844 | // Insert all of the blocks (other than those already present) into | ||||||
| 1845 | // the loop set. We expect at least the block that led us to find the | ||||||
| 1846 | // inner loop to be in the block set, but we may also have other loop | ||||||
| 1847 | // blocks if they were already enqueued as predecessors of some other | ||||||
| 1848 | // outer loop block. | ||||||
| 1849 | for (auto *InnerBB : InnerL->blocks()) { | ||||||
| 1850 | if (InnerBB == BB) { | ||||||
| 1851 | assert(LoopBlockSet.count(InnerBB) &&(static_cast <bool> (LoopBlockSet.count(InnerBB) && "Block should already be in the set!") ? void (0) : __assert_fail ("LoopBlockSet.count(InnerBB) && \"Block should already be in the set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1852, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 1852 | "Block should already be in the set!")(static_cast <bool> (LoopBlockSet.count(InnerBB) && "Block should already be in the set!") ? void (0) : __assert_fail ("LoopBlockSet.count(InnerBB) && \"Block should already be in the set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1852, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1853 | continue; | ||||||
| 1854 | } | ||||||
| 1855 | |||||||
| 1856 | LoopBlockSet.insert(InnerBB); | ||||||
| 1857 | } | ||||||
| 1858 | |||||||
| 1859 | // Add the preheader to the worklist so we will continue past the | ||||||
| 1860 | // loop body. | ||||||
| 1861 | Worklist.push_back(InnerPH); | ||||||
| 1862 | continue; | ||||||
| 1863 | } | ||||||
| 1864 | |||||||
| 1865 | // Insert any predecessors that were in the original loop into the new | ||||||
| 1866 | // set, and if the insert is successful, add them to the worklist. | ||||||
| 1867 | for (auto *Pred : predecessors(BB)) | ||||||
| 1868 | if (L.contains(Pred) && LoopBlockSet.insert(Pred).second) | ||||||
| 1869 | Worklist.push_back(Pred); | ||||||
| 1870 | } | ||||||
| 1871 | |||||||
| 1872 | assert(LoopBlockSet.count(Header) && "Cannot fail to add the header!")(static_cast <bool> (LoopBlockSet.count(Header) && "Cannot fail to add the header!") ? void (0) : __assert_fail ("LoopBlockSet.count(Header) && \"Cannot fail to add the header!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1872, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1873 | |||||||
| 1874 | // We've found all the blocks participating in the loop, return our completed | ||||||
| 1875 | // set. | ||||||
| 1876 | return LoopBlockSet; | ||||||
| 1877 | } | ||||||
| 1878 | |||||||
| 1879 | /// Rebuild a loop after unswitching removes some subset of blocks and edges. | ||||||
| 1880 | /// | ||||||
| 1881 | /// The removal may have removed some child loops entirely but cannot have | ||||||
| 1882 | /// disturbed any remaining child loops. However, they may need to be hoisted | ||||||
| 1883 | /// to the parent loop (or to be top-level loops). The original loop may be | ||||||
| 1884 | /// completely removed. | ||||||
| 1885 | /// | ||||||
| 1886 | /// The sibling loops resulting from this update are returned. If the original | ||||||
| 1887 | /// loop remains a valid loop, it will be the first entry in this list with all | ||||||
| 1888 | /// of the newly sibling loops following it. | ||||||
| 1889 | /// | ||||||
| 1890 | /// Returns true if the loop remains a loop after unswitching, and false if it | ||||||
| 1891 | /// is no longer a loop after unswitching (and should not continue to be | ||||||
| 1892 | /// referenced). | ||||||
| 1893 | static bool rebuildLoopAfterUnswitch(Loop &L, ArrayRef<BasicBlock *> ExitBlocks, | ||||||
| 1894 | LoopInfo &LI, | ||||||
| 1895 | SmallVectorImpl<Loop *> &HoistedLoops, | ||||||
| 1896 | ScalarEvolution *SE) { | ||||||
| 1897 | auto *PH = L.getLoopPreheader(); | ||||||
| 1898 | |||||||
| 1899 | // Compute the actual parent loop from the exit blocks. Because we may have | ||||||
| 1900 | // pruned some exits the loop may be different from the original parent. | ||||||
| 1901 | Loop *ParentL = nullptr; | ||||||
| 1902 | SmallVector<Loop *, 4> ExitLoops; | ||||||
| 1903 | SmallVector<BasicBlock *, 4> ExitsInLoops; | ||||||
| 1904 | ExitsInLoops.reserve(ExitBlocks.size()); | ||||||
| 1905 | for (auto *ExitBB : ExitBlocks) | ||||||
| 1906 | if (Loop *ExitL = LI.getLoopFor(ExitBB)) { | ||||||
| 1907 | ExitLoops.push_back(ExitL); | ||||||
| 1908 | ExitsInLoops.push_back(ExitBB); | ||||||
| 1909 | if (!ParentL || (ParentL != ExitL && ParentL->contains(ExitL))) | ||||||
| 1910 | ParentL = ExitL; | ||||||
| 1911 | } | ||||||
| 1912 | |||||||
| 1913 | // Recompute the blocks participating in this loop. This may be empty if it | ||||||
| 1914 | // is no longer a loop. | ||||||
| 1915 | auto LoopBlockSet = recomputeLoopBlockSet(L, LI); | ||||||
| 1916 | |||||||
| 1917 | // If we still have a loop, we need to re-set the loop's parent as the exit | ||||||
| 1918 | // block set changing may have moved it within the loop nest. Note that this | ||||||
| 1919 | // can only happen when this loop has a parent as it can only hoist the loop | ||||||
| 1920 | // *up* the nest. | ||||||
| 1921 | if (!LoopBlockSet.empty() && L.getParentLoop() != ParentL) { | ||||||
| 1922 | // Remove this loop's (original) blocks from all of the intervening loops. | ||||||
| 1923 | for (Loop *IL = L.getParentLoop(); IL != ParentL; | ||||||
| 1924 | IL = IL->getParentLoop()) { | ||||||
| 1925 | IL->getBlocksSet().erase(PH); | ||||||
| 1926 | for (auto *BB : L.blocks()) | ||||||
| 1927 | IL->getBlocksSet().erase(BB); | ||||||
| 1928 | llvm::erase_if(IL->getBlocksVector(), [&](BasicBlock *BB) { | ||||||
| 1929 | return BB == PH || L.contains(BB); | ||||||
| 1930 | }); | ||||||
| 1931 | } | ||||||
| 1932 | |||||||
| 1933 | LI.changeLoopFor(PH, ParentL); | ||||||
| 1934 | L.getParentLoop()->removeChildLoop(&L); | ||||||
| 1935 | if (ParentL) | ||||||
| 1936 | ParentL->addChildLoop(&L); | ||||||
| 1937 | else | ||||||
| 1938 | LI.addTopLevelLoop(&L); | ||||||
| 1939 | } | ||||||
| 1940 | |||||||
| 1941 | // Now we update all the blocks which are no longer within the loop. | ||||||
| 1942 | auto &Blocks = L.getBlocksVector(); | ||||||
| 1943 | auto BlocksSplitI = | ||||||
| 1944 | LoopBlockSet.empty() | ||||||
| 1945 | ? Blocks.begin() | ||||||
| 1946 | : std::stable_partition( | ||||||
| 1947 | Blocks.begin(), Blocks.end(), | ||||||
| 1948 | [&](BasicBlock *BB) { return LoopBlockSet.count(BB); }); | ||||||
| 1949 | |||||||
| 1950 | // Before we erase the list of unlooped blocks, build a set of them. | ||||||
| 1951 | SmallPtrSet<BasicBlock *, 16> UnloopedBlocks(BlocksSplitI, Blocks.end()); | ||||||
| 1952 | if (LoopBlockSet.empty()) | ||||||
| 1953 | UnloopedBlocks.insert(PH); | ||||||
| 1954 | |||||||
| 1955 | // Now erase these blocks from the loop. | ||||||
| 1956 | for (auto *BB : make_range(BlocksSplitI, Blocks.end())) | ||||||
| 1957 | L.getBlocksSet().erase(BB); | ||||||
| 1958 | Blocks.erase(BlocksSplitI, Blocks.end()); | ||||||
| 1959 | |||||||
| 1960 | // Sort the exits in ascending loop depth, we'll work backwards across these | ||||||
| 1961 | // to process them inside out. | ||||||
| 1962 | llvm::stable_sort(ExitsInLoops, [&](BasicBlock *LHS, BasicBlock *RHS) { | ||||||
| 1963 | return LI.getLoopDepth(LHS) < LI.getLoopDepth(RHS); | ||||||
| 1964 | }); | ||||||
| 1965 | |||||||
| 1966 | // We'll build up a set for each exit loop. | ||||||
| 1967 | SmallPtrSet<BasicBlock *, 16> NewExitLoopBlocks; | ||||||
| 1968 | Loop *PrevExitL = L.getParentLoop(); // The deepest possible exit loop. | ||||||
| 1969 | |||||||
| 1970 | auto RemoveUnloopedBlocksFromLoop = | ||||||
| 1971 | [](Loop &L, SmallPtrSetImpl<BasicBlock *> &UnloopedBlocks) { | ||||||
| 1972 | for (auto *BB : UnloopedBlocks) | ||||||
| 1973 | L.getBlocksSet().erase(BB); | ||||||
| 1974 | llvm::erase_if(L.getBlocksVector(), [&](BasicBlock *BB) { | ||||||
| 1975 | return UnloopedBlocks.count(BB); | ||||||
| 1976 | }); | ||||||
| 1977 | }; | ||||||
| 1978 | |||||||
| 1979 | SmallVector<BasicBlock *, 16> Worklist; | ||||||
| 1980 | while (!UnloopedBlocks.empty() && !ExitsInLoops.empty()) { | ||||||
| 1981 | assert(Worklist.empty() && "Didn't clear worklist!")(static_cast <bool> (Worklist.empty() && "Didn't clear worklist!" ) ? void (0) : __assert_fail ("Worklist.empty() && \"Didn't clear worklist!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1981, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1982 | assert(NewExitLoopBlocks.empty() && "Didn't clear loop set!")(static_cast <bool> (NewExitLoopBlocks.empty() && "Didn't clear loop set!") ? void (0) : __assert_fail ("NewExitLoopBlocks.empty() && \"Didn't clear loop set!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1982, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1983 | |||||||
| 1984 | // Grab the next exit block, in decreasing loop depth order. | ||||||
| 1985 | BasicBlock *ExitBB = ExitsInLoops.pop_back_val(); | ||||||
| 1986 | Loop &ExitL = *LI.getLoopFor(ExitBB); | ||||||
| 1987 | assert(ExitL.contains(&L) && "Exit loop must contain the inner loop!")(static_cast <bool> (ExitL.contains(&L) && "Exit loop must contain the inner loop!" ) ? void (0) : __assert_fail ("ExitL.contains(&L) && \"Exit loop must contain the inner loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 1987, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 1988 | |||||||
| 1989 | // Erase all of the unlooped blocks from the loops between the previous | ||||||
| 1990 | // exit loop and this exit loop. This works because the ExitInLoops list is | ||||||
| 1991 | // sorted in increasing order of loop depth and thus we visit loops in | ||||||
| 1992 | // decreasing order of loop depth. | ||||||
| 1993 | for (; PrevExitL != &ExitL; PrevExitL = PrevExitL->getParentLoop()) | ||||||
| 1994 | RemoveUnloopedBlocksFromLoop(*PrevExitL, UnloopedBlocks); | ||||||
| 1995 | |||||||
| 1996 | // Walk the CFG back until we hit the cloned PH adding everything reachable | ||||||
| 1997 | // and in the unlooped set to this exit block's loop. | ||||||
| 1998 | Worklist.push_back(ExitBB); | ||||||
| 1999 | do { | ||||||
| 2000 | BasicBlock *BB = Worklist.pop_back_val(); | ||||||
| 2001 | // We can stop recursing at the cloned preheader (if we get there). | ||||||
| 2002 | if (BB == PH) | ||||||
| 2003 | continue; | ||||||
| 2004 | |||||||
| 2005 | for (BasicBlock *PredBB : predecessors(BB)) { | ||||||
| 2006 | // If this pred has already been moved to our set or is part of some | ||||||
| 2007 | // (inner) loop, no update needed. | ||||||
| 2008 | if (!UnloopedBlocks.erase(PredBB)) { | ||||||
| 2009 | assert((NewExitLoopBlocks.count(PredBB) ||(static_cast <bool> ((NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && "Predecessor not in a nested loop (or already visited)!" ) ? void (0) : __assert_fail ("(NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && \"Predecessor not in a nested loop (or already visited)!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2011, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2010 | ExitL.contains(LI.getLoopFor(PredBB))) &&(static_cast <bool> ((NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && "Predecessor not in a nested loop (or already visited)!" ) ? void (0) : __assert_fail ("(NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && \"Predecessor not in a nested loop (or already visited)!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2011, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2011 | "Predecessor not in a nested loop (or already visited)!")(static_cast <bool> ((NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && "Predecessor not in a nested loop (or already visited)!" ) ? void (0) : __assert_fail ("(NewExitLoopBlocks.count(PredBB) || ExitL.contains(LI.getLoopFor(PredBB))) && \"Predecessor not in a nested loop (or already visited)!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2011, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2012 | continue; | ||||||
| 2013 | } | ||||||
| 2014 | |||||||
| 2015 | // We just insert into the loop set here. We'll add these blocks to the | ||||||
| 2016 | // exit loop after we build up the set in a deterministic order rather | ||||||
| 2017 | // than the predecessor-influenced visit order. | ||||||
| 2018 | bool Inserted = NewExitLoopBlocks.insert(PredBB).second; | ||||||
| 2019 | (void)Inserted; | ||||||
| 2020 | assert(Inserted && "Should only visit an unlooped block once!")(static_cast <bool> (Inserted && "Should only visit an unlooped block once!" ) ? void (0) : __assert_fail ("Inserted && \"Should only visit an unlooped block once!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2020, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2021 | |||||||
| 2022 | // And recurse through to its predecessors. | ||||||
| 2023 | Worklist.push_back(PredBB); | ||||||
| 2024 | } | ||||||
| 2025 | } while (!Worklist.empty()); | ||||||
| 2026 | |||||||
| 2027 | // If blocks in this exit loop were directly part of the original loop (as | ||||||
| 2028 | // opposed to a child loop) update the map to point to this exit loop. This | ||||||
| 2029 | // just updates a map and so the fact that the order is unstable is fine. | ||||||
| 2030 | for (auto *BB : NewExitLoopBlocks) | ||||||
| 2031 | if (Loop *BBL = LI.getLoopFor(BB)) | ||||||
| 2032 | if (BBL == &L || !L.contains(BBL)) | ||||||
| 2033 | LI.changeLoopFor(BB, &ExitL); | ||||||
| 2034 | |||||||
| 2035 | // We will remove the remaining unlooped blocks from this loop in the next | ||||||
| 2036 | // iteration or below. | ||||||
| 2037 | NewExitLoopBlocks.clear(); | ||||||
| 2038 | } | ||||||
| 2039 | |||||||
| 2040 | // Any remaining unlooped blocks are no longer part of any loop unless they | ||||||
| 2041 | // are part of some child loop. | ||||||
| 2042 | for (; PrevExitL; PrevExitL = PrevExitL->getParentLoop()) | ||||||
| 2043 | RemoveUnloopedBlocksFromLoop(*PrevExitL, UnloopedBlocks); | ||||||
| 2044 | for (auto *BB : UnloopedBlocks) | ||||||
| 2045 | if (Loop *BBL = LI.getLoopFor(BB)) | ||||||
| 2046 | if (BBL == &L || !L.contains(BBL)) | ||||||
| 2047 | LI.changeLoopFor(BB, nullptr); | ||||||
| 2048 | |||||||
| 2049 | // Sink all the child loops whose headers are no longer in the loop set to | ||||||
| 2050 | // the parent (or to be top level loops). We reach into the loop and directly | ||||||
| 2051 | // update its subloop vector to make this batch update efficient. | ||||||
| 2052 | auto &SubLoops = L.getSubLoopsVector(); | ||||||
| 2053 | auto SubLoopsSplitI = | ||||||
| 2054 | LoopBlockSet.empty() | ||||||
| 2055 | ? SubLoops.begin() | ||||||
| 2056 | : std::stable_partition( | ||||||
| 2057 | SubLoops.begin(), SubLoops.end(), [&](Loop *SubL) { | ||||||
| 2058 | return LoopBlockSet.count(SubL->getHeader()); | ||||||
| 2059 | }); | ||||||
| 2060 | for (auto *HoistedL : make_range(SubLoopsSplitI, SubLoops.end())) { | ||||||
| 2061 | HoistedLoops.push_back(HoistedL); | ||||||
| 2062 | HoistedL->setParentLoop(nullptr); | ||||||
| 2063 | |||||||
| 2064 | // To compute the new parent of this hoisted loop we look at where we | ||||||
| 2065 | // placed the preheader above. We can't lookup the header itself because we | ||||||
| 2066 | // retained the mapping from the header to the hoisted loop. But the | ||||||
| 2067 | // preheader and header should have the exact same new parent computed | ||||||
| 2068 | // based on the set of exit blocks from the original loop as the preheader | ||||||
| 2069 | // is a predecessor of the header and so reached in the reverse walk. And | ||||||
| 2070 | // because the loops were all in simplified form the preheader of the | ||||||
| 2071 | // hoisted loop can't be part of some *other* loop. | ||||||
| 2072 | if (auto *NewParentL = LI.getLoopFor(HoistedL->getLoopPreheader())) | ||||||
| 2073 | NewParentL->addChildLoop(HoistedL); | ||||||
| 2074 | else | ||||||
| 2075 | LI.addTopLevelLoop(HoistedL); | ||||||
| 2076 | } | ||||||
| 2077 | SubLoops.erase(SubLoopsSplitI, SubLoops.end()); | ||||||
| 2078 | |||||||
| 2079 | // Actually delete the loop if nothing remained within it. | ||||||
| 2080 | if (Blocks.empty()) { | ||||||
| 2081 | assert(SubLoops.empty() &&(static_cast <bool> (SubLoops.empty() && "Failed to remove all subloops from the original loop!" ) ? void (0) : __assert_fail ("SubLoops.empty() && \"Failed to remove all subloops from the original loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2082, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2082 | "Failed to remove all subloops from the original loop!")(static_cast <bool> (SubLoops.empty() && "Failed to remove all subloops from the original loop!" ) ? void (0) : __assert_fail ("SubLoops.empty() && \"Failed to remove all subloops from the original loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2082, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2083 | if (Loop *ParentL = L.getParentLoop()) | ||||||
| 2084 | ParentL->removeChildLoop(llvm::find(*ParentL, &L)); | ||||||
| 2085 | else | ||||||
| 2086 | LI.removeLoop(llvm::find(LI, &L)); | ||||||
| 2087 | // markLoopAsDeleted for L should be triggered by the caller (it is typically | ||||||
| 2088 | // done by using the UnswitchCB callback). | ||||||
| 2089 | if (SE) | ||||||
| 2090 | SE->forgetBlockAndLoopDispositions(); | ||||||
| 2091 | LI.destroy(&L); | ||||||
| 2092 | return false; | ||||||
| 2093 | } | ||||||
| 2094 | |||||||
| 2095 | return true; | ||||||
| 2096 | } | ||||||
| 2097 | |||||||
| 2098 | /// Helper to visit a dominator subtree, invoking a callable on each node. | ||||||
| 2099 | /// | ||||||
| 2100 | /// Returning false at any point will stop walking past that node of the tree. | ||||||
| 2101 | template <typename CallableT> | ||||||
| 2102 | void visitDomSubTree(DominatorTree &DT, BasicBlock *BB, CallableT Callable) { | ||||||
| 2103 | SmallVector<DomTreeNode *, 4> DomWorklist; | ||||||
| 2104 | DomWorklist.push_back(DT[BB]); | ||||||
| 2105 | #ifndef NDEBUG | ||||||
| 2106 | SmallPtrSet<DomTreeNode *, 4> Visited; | ||||||
| 2107 | Visited.insert(DT[BB]); | ||||||
| 2108 | #endif | ||||||
| 2109 | do { | ||||||
| 2110 | DomTreeNode *N = DomWorklist.pop_back_val(); | ||||||
| 2111 | |||||||
| 2112 | // Visit this node. | ||||||
| 2113 | if (!Callable(N->getBlock())) | ||||||
| 2114 | continue; | ||||||
| 2115 | |||||||
| 2116 | // Accumulate the child nodes. | ||||||
| 2117 | for (DomTreeNode *ChildN : *N) { | ||||||
| 2118 | assert(Visited.insert(ChildN).second &&(static_cast <bool> (Visited.insert(ChildN).second && "Cannot visit a node twice when walking a tree!") ? void (0) : __assert_fail ("Visited.insert(ChildN).second && \"Cannot visit a node twice when walking a tree!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2119, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2119 | "Cannot visit a node twice when walking a tree!")(static_cast <bool> (Visited.insert(ChildN).second && "Cannot visit a node twice when walking a tree!") ? void (0) : __assert_fail ("Visited.insert(ChildN).second && \"Cannot visit a node twice when walking a tree!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2119, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2120 | DomWorklist.push_back(ChildN); | ||||||
| 2121 | } | ||||||
| 2122 | } while (!DomWorklist.empty()); | ||||||
| 2123 | } | ||||||
| 2124 | |||||||
| 2125 | static void unswitchNontrivialInvariants( | ||||||
| 2126 | Loop &L, Instruction &TI, ArrayRef<Value *> Invariants, | ||||||
| 2127 | IVConditionInfo &PartialIVInfo, DominatorTree &DT, LoopInfo &LI, | ||||||
| 2128 | AssumptionCache &AC, | ||||||
| 2129 | function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB, | ||||||
| 2130 | ScalarEvolution *SE, MemorySSAUpdater *MSSAU, | ||||||
| 2131 | function_ref<void(Loop &, StringRef)> DestroyLoopCB, bool InsertFreeze) { | ||||||
| 2132 | auto *ParentBB = TI.getParent(); | ||||||
| 2133 | BranchInst *BI = dyn_cast<BranchInst>(&TI); | ||||||
| 2134 | SwitchInst *SI = BI ? nullptr : cast<SwitchInst>(&TI); | ||||||
| 2135 | |||||||
| 2136 | // We can only unswitch switches, conditional branches with an invariant | ||||||
| 2137 | // condition, or combining invariant conditions with an instruction or | ||||||
| 2138 | // partially invariant instructions. | ||||||
| 2139 | assert((SI || (BI && BI->isConditional())) &&(static_cast <bool> ((SI || (BI && BI->isConditional ())) && "Can only unswitch switches and conditional branch!" ) ? void (0) : __assert_fail ("(SI || (BI && BI->isConditional())) && \"Can only unswitch switches and conditional branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2140, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2140 | "Can only unswitch switches and conditional branch!")(static_cast <bool> ((SI || (BI && BI->isConditional ())) && "Can only unswitch switches and conditional branch!" ) ? void (0) : __assert_fail ("(SI || (BI && BI->isConditional())) && \"Can only unswitch switches and conditional branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2140, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2141 | bool PartiallyInvariant = !PartialIVInfo.InstToDuplicate.empty(); | ||||||
| 2142 | bool FullUnswitch = | ||||||
| 2143 | SI || (skipTrivialSelect(BI->getCondition()) == Invariants[0] && | ||||||
| 2144 | !PartiallyInvariant); | ||||||
| 2145 | if (FullUnswitch) | ||||||
| 2146 | assert(Invariants.size() == 1 &&(static_cast <bool> (Invariants.size() == 1 && "Cannot have other invariants with full unswitching!" ) ? void (0) : __assert_fail ("Invariants.size() == 1 && \"Cannot have other invariants with full unswitching!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2147, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2147 | "Cannot have other invariants with full unswitching!")(static_cast <bool> (Invariants.size() == 1 && "Cannot have other invariants with full unswitching!" ) ? void (0) : __assert_fail ("Invariants.size() == 1 && \"Cannot have other invariants with full unswitching!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2147, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2148 | else | ||||||
| 2149 | assert(isa<Instruction>(skipTrivialSelect(BI->getCondition())) &&(static_cast <bool> (isa<Instruction>(skipTrivialSelect (BI->getCondition())) && "Partial unswitching requires an instruction as the condition!" ) ? void (0) : __assert_fail ("isa<Instruction>(skipTrivialSelect(BI->getCondition())) && \"Partial unswitching requires an instruction as the condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2150, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2150 | "Partial unswitching requires an instruction as the condition!")(static_cast <bool> (isa<Instruction>(skipTrivialSelect (BI->getCondition())) && "Partial unswitching requires an instruction as the condition!" ) ? void (0) : __assert_fail ("isa<Instruction>(skipTrivialSelect(BI->getCondition())) && \"Partial unswitching requires an instruction as the condition!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2150, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2151 | |||||||
| 2152 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2153 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2154 | |||||||
| 2155 | // Constant and BBs tracking the cloned and continuing successor. When we are | ||||||
| 2156 | // unswitching the entire condition, this can just be trivially chosen to | ||||||
| 2157 | // unswitch towards `true`. However, when we are unswitching a set of | ||||||
| 2158 | // invariants combined with `and` or `or` or partially invariant instructions, | ||||||
| 2159 | // the combining operation determines the best direction to unswitch: we want | ||||||
| 2160 | // to unswitch the direction that will collapse the branch. | ||||||
| 2161 | bool Direction = true; | ||||||
| 2162 | int ClonedSucc = 0; | ||||||
| 2163 | if (!FullUnswitch) { | ||||||
| 2164 | Value *Cond = skipTrivialSelect(BI->getCondition()); | ||||||
| 2165 | (void)Cond; | ||||||
| 2166 | assert(((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) ||(static_cast <bool> (((match(Cond, m_LogicalAnd()) ^ match (Cond, m_LogicalOr())) || PartiallyInvariant) && "Only `or`, `and`, an `select`, partially invariant instructions " "can combine invariants being unswitched.") ? void (0) : __assert_fail ("((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) || PartiallyInvariant) && \"Only `or`, `and`, an `select`, partially invariant instructions \" \"can combine invariants being unswitched.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2169, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2167 | PartiallyInvariant) &&(static_cast <bool> (((match(Cond, m_LogicalAnd()) ^ match (Cond, m_LogicalOr())) || PartiallyInvariant) && "Only `or`, `and`, an `select`, partially invariant instructions " "can combine invariants being unswitched.") ? void (0) : __assert_fail ("((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) || PartiallyInvariant) && \"Only `or`, `and`, an `select`, partially invariant instructions \" \"can combine invariants being unswitched.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2169, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2168 | "Only `or`, `and`, an `select`, partially invariant instructions "(static_cast <bool> (((match(Cond, m_LogicalAnd()) ^ match (Cond, m_LogicalOr())) || PartiallyInvariant) && "Only `or`, `and`, an `select`, partially invariant instructions " "can combine invariants being unswitched.") ? void (0) : __assert_fail ("((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) || PartiallyInvariant) && \"Only `or`, `and`, an `select`, partially invariant instructions \" \"can combine invariants being unswitched.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2169, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2169 | "can combine invariants being unswitched.")(static_cast <bool> (((match(Cond, m_LogicalAnd()) ^ match (Cond, m_LogicalOr())) || PartiallyInvariant) && "Only `or`, `and`, an `select`, partially invariant instructions " "can combine invariants being unswitched.") ? void (0) : __assert_fail ("((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) || PartiallyInvariant) && \"Only `or`, `and`, an `select`, partially invariant instructions \" \"can combine invariants being unswitched.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2169, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2170 | if (!match(Cond, m_LogicalOr())) { | ||||||
| 2171 | if (match(Cond, m_LogicalAnd()) || | ||||||
| 2172 | (PartiallyInvariant && !PartialIVInfo.KnownValue->isOneValue())) { | ||||||
| 2173 | Direction = false; | ||||||
| 2174 | ClonedSucc = 1; | ||||||
| 2175 | } | ||||||
| 2176 | } | ||||||
| 2177 | } | ||||||
| 2178 | |||||||
| 2179 | BasicBlock *RetainedSuccBB = | ||||||
| 2180 | BI ? BI->getSuccessor(1 - ClonedSucc) : SI->getDefaultDest(); | ||||||
| 2181 | SmallSetVector<BasicBlock *, 4> UnswitchedSuccBBs; | ||||||
| 2182 | if (BI) | ||||||
| 2183 | UnswitchedSuccBBs.insert(BI->getSuccessor(ClonedSucc)); | ||||||
| 2184 | else | ||||||
| 2185 | for (auto Case : SI->cases()) | ||||||
| 2186 | if (Case.getCaseSuccessor() != RetainedSuccBB) | ||||||
| 2187 | UnswitchedSuccBBs.insert(Case.getCaseSuccessor()); | ||||||
| 2188 | |||||||
| 2189 | assert(!UnswitchedSuccBBs.count(RetainedSuccBB) &&(static_cast <bool> (!UnswitchedSuccBBs.count(RetainedSuccBB ) && "Should not unswitch the same successor we are retaining!" ) ? void (0) : __assert_fail ("!UnswitchedSuccBBs.count(RetainedSuccBB) && \"Should not unswitch the same successor we are retaining!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2190, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2190 | "Should not unswitch the same successor we are retaining!")(static_cast <bool> (!UnswitchedSuccBBs.count(RetainedSuccBB ) && "Should not unswitch the same successor we are retaining!" ) ? void (0) : __assert_fail ("!UnswitchedSuccBBs.count(RetainedSuccBB) && \"Should not unswitch the same successor we are retaining!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2190, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2191 | |||||||
| 2192 | // The branch should be in this exact loop. Any inner loop's invariant branch | ||||||
| 2193 | // should be handled by unswitching that inner loop. The caller of this | ||||||
| 2194 | // routine should filter out any candidates that remain (but were skipped for | ||||||
| 2195 | // whatever reason). | ||||||
| 2196 | assert(LI.getLoopFor(ParentBB) == &L && "Branch in an inner loop!")(static_cast <bool> (LI.getLoopFor(ParentBB) == &L && "Branch in an inner loop!") ? void (0) : __assert_fail ("LI.getLoopFor(ParentBB) == &L && \"Branch in an inner loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2196, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2197 | |||||||
| 2198 | // Compute the parent loop now before we start hacking on things. | ||||||
| 2199 | Loop *ParentL = L.getParentLoop(); | ||||||
| 2200 | // Get blocks in RPO order for MSSA update, before changing the CFG. | ||||||
| 2201 | LoopBlocksRPO LBRPO(&L); | ||||||
| 2202 | if (MSSAU) | ||||||
| 2203 | LBRPO.perform(&LI); | ||||||
| 2204 | |||||||
| 2205 | // Compute the outer-most loop containing one of our exit blocks. This is the | ||||||
| 2206 | // furthest up our loopnest which can be mutated, which we will use below to | ||||||
| 2207 | // update things. | ||||||
| 2208 | Loop *OuterExitL = &L; | ||||||
| 2209 | SmallVector<BasicBlock *, 4> ExitBlocks; | ||||||
| 2210 | L.getUniqueExitBlocks(ExitBlocks); | ||||||
| 2211 | for (auto *ExitBB : ExitBlocks) { | ||||||
| 2212 | // ExitBB can be an exit block for several levels in the loop nest. Make | ||||||
| 2213 | // sure we find the top most. | ||||||
| 2214 | Loop *NewOuterExitL = getTopMostExitingLoop(ExitBB, LI); | ||||||
| 2215 | if (!NewOuterExitL) { | ||||||
| 2216 | // We exited the entire nest with this block, so we're done. | ||||||
| 2217 | OuterExitL = nullptr; | ||||||
| 2218 | break; | ||||||
| 2219 | } | ||||||
| 2220 | if (NewOuterExitL != OuterExitL && NewOuterExitL->contains(OuterExitL)) | ||||||
| 2221 | OuterExitL = NewOuterExitL; | ||||||
| 2222 | } | ||||||
| 2223 | |||||||
| 2224 | // At this point, we're definitely going to unswitch something so invalidate | ||||||
| 2225 | // any cached information in ScalarEvolution for the outer most loop | ||||||
| 2226 | // containing an exit block and all nested loops. | ||||||
| 2227 | if (SE) { | ||||||
| 2228 | if (OuterExitL) | ||||||
| 2229 | SE->forgetLoop(OuterExitL); | ||||||
| 2230 | else | ||||||
| 2231 | SE->forgetTopmostLoop(&L); | ||||||
| 2232 | SE->forgetBlockAndLoopDispositions(); | ||||||
| 2233 | } | ||||||
| 2234 | |||||||
| 2235 | // If the edge from this terminator to a successor dominates that successor, | ||||||
| 2236 | // store a map from each block in its dominator subtree to it. This lets us | ||||||
| 2237 | // tell when cloning for a particular successor if a block is dominated by | ||||||
| 2238 | // some *other* successor with a single data structure. We use this to | ||||||
| 2239 | // significantly reduce cloning. | ||||||
| 2240 | SmallDenseMap<BasicBlock *, BasicBlock *, 16> DominatingSucc; | ||||||
| 2241 | for (auto *SuccBB : llvm::concat<BasicBlock *const>(ArrayRef(RetainedSuccBB), | ||||||
| 2242 | UnswitchedSuccBBs)) | ||||||
| 2243 | if (SuccBB->getUniquePredecessor() || | ||||||
| 2244 | llvm::all_of(predecessors(SuccBB), [&](BasicBlock *PredBB) { | ||||||
| 2245 | return PredBB == ParentBB || DT.dominates(SuccBB, PredBB); | ||||||
| 2246 | })) | ||||||
| 2247 | visitDomSubTree(DT, SuccBB, [&](BasicBlock *BB) { | ||||||
| 2248 | DominatingSucc[BB] = SuccBB; | ||||||
| 2249 | return true; | ||||||
| 2250 | }); | ||||||
| 2251 | |||||||
| 2252 | // Split the preheader, so that we know that there is a safe place to insert | ||||||
| 2253 | // the conditional branch. We will change the preheader to have a conditional | ||||||
| 2254 | // branch on LoopCond. The original preheader will become the split point | ||||||
| 2255 | // between the unswitched versions, and we will have a new preheader for the | ||||||
| 2256 | // original loop. | ||||||
| 2257 | BasicBlock *SplitBB = L.getLoopPreheader(); | ||||||
| 2258 | BasicBlock *LoopPH = SplitEdge(SplitBB, L.getHeader(), &DT, &LI, MSSAU); | ||||||
| 2259 | |||||||
| 2260 | // Keep track of the dominator tree updates needed. | ||||||
| 2261 | SmallVector<DominatorTree::UpdateType, 4> DTUpdates; | ||||||
| 2262 | |||||||
| 2263 | // Clone the loop for each unswitched successor. | ||||||
| 2264 | SmallVector<std::unique_ptr<ValueToValueMapTy>, 4> VMaps; | ||||||
| 2265 | VMaps.reserve(UnswitchedSuccBBs.size()); | ||||||
| 2266 | SmallDenseMap<BasicBlock *, BasicBlock *, 4> ClonedPHs; | ||||||
| 2267 | for (auto *SuccBB : UnswitchedSuccBBs) { | ||||||
| 2268 | VMaps.emplace_back(new ValueToValueMapTy()); | ||||||
| 2269 | ClonedPHs[SuccBB] = buildClonedLoopBlocks( | ||||||
| 2270 | L, LoopPH, SplitBB, ExitBlocks, ParentBB, SuccBB, RetainedSuccBB, | ||||||
| 2271 | DominatingSucc, *VMaps.back(), DTUpdates, AC, DT, LI, MSSAU, SE); | ||||||
| 2272 | } | ||||||
| 2273 | |||||||
| 2274 | // Drop metadata if we may break its semantics by moving this instr into the | ||||||
| 2275 | // split block. | ||||||
| 2276 | if (TI.getMetadata(LLVMContext::MD_make_implicit)) { | ||||||
| 2277 | if (DropNonTrivialImplicitNullChecks) | ||||||
| 2278 | // Do not spend time trying to understand if we can keep it, just drop it | ||||||
| 2279 | // to save compile time. | ||||||
| 2280 | TI.setMetadata(LLVMContext::MD_make_implicit, nullptr); | ||||||
| 2281 | else { | ||||||
| 2282 | // It is only legal to preserve make.implicit metadata if we are | ||||||
| 2283 | // guaranteed no reach implicit null check after following this branch. | ||||||
| 2284 | ICFLoopSafetyInfo SafetyInfo; | ||||||
| 2285 | SafetyInfo.computeLoopSafetyInfo(&L); | ||||||
| 2286 | if (!SafetyInfo.isGuaranteedToExecute(TI, &DT, &L)) | ||||||
| 2287 | TI.setMetadata(LLVMContext::MD_make_implicit, nullptr); | ||||||
| 2288 | } | ||||||
| 2289 | } | ||||||
| 2290 | |||||||
| 2291 | // The stitching of the branched code back together depends on whether we're | ||||||
| 2292 | // doing full unswitching or not with the exception that we always want to | ||||||
| 2293 | // nuke the initial terminator placed in the split block. | ||||||
| 2294 | SplitBB->getTerminator()->eraseFromParent(); | ||||||
| 2295 | if (FullUnswitch) { | ||||||
| 2296 | // Splice the terminator from the original loop and rewrite its | ||||||
| 2297 | // successors. | ||||||
| 2298 | SplitBB->splice(SplitBB->end(), ParentBB, TI.getIterator()); | ||||||
| 2299 | |||||||
| 2300 | // Keep a clone of the terminator for MSSA updates. | ||||||
| 2301 | Instruction *NewTI = TI.clone(); | ||||||
| 2302 | NewTI->insertInto(ParentBB, ParentBB->end()); | ||||||
| 2303 | |||||||
| 2304 | // First wire up the moved terminator to the preheaders. | ||||||
| 2305 | if (BI) { | ||||||
| 2306 | BasicBlock *ClonedPH = ClonedPHs.begin()->second; | ||||||
| 2307 | BI->setSuccessor(ClonedSucc, ClonedPH); | ||||||
| 2308 | BI->setSuccessor(1 - ClonedSucc, LoopPH); | ||||||
| 2309 | Value *Cond = skipTrivialSelect(BI->getCondition()); | ||||||
| 2310 | if (InsertFreeze) | ||||||
| 2311 | Cond = new FreezeInst( | ||||||
| 2312 | Cond, Cond->getName() + ".fr", BI); | ||||||
| 2313 | BI->setCondition(Cond); | ||||||
| 2314 | DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH}); | ||||||
| 2315 | } else { | ||||||
| 2316 | assert(SI && "Must either be a branch or switch!")(static_cast <bool> (SI && "Must either be a branch or switch!" ) ? void (0) : __assert_fail ("SI && \"Must either be a branch or switch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2316, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2317 | |||||||
| 2318 | // Walk the cases and directly update their successors. | ||||||
| 2319 | assert(SI->getDefaultDest() == RetainedSuccBB &&(static_cast <bool> (SI->getDefaultDest() == RetainedSuccBB && "Not retaining default successor!") ? void (0) : __assert_fail ("SI->getDefaultDest() == RetainedSuccBB && \"Not retaining default successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2320, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2320 | "Not retaining default successor!")(static_cast <bool> (SI->getDefaultDest() == RetainedSuccBB && "Not retaining default successor!") ? void (0) : __assert_fail ("SI->getDefaultDest() == RetainedSuccBB && \"Not retaining default successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2320, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2321 | SI->setDefaultDest(LoopPH); | ||||||
| 2322 | for (const auto &Case : SI->cases()) | ||||||
| 2323 | if (Case.getCaseSuccessor() == RetainedSuccBB) | ||||||
| 2324 | Case.setSuccessor(LoopPH); | ||||||
| 2325 | else | ||||||
| 2326 | Case.setSuccessor(ClonedPHs.find(Case.getCaseSuccessor())->second); | ||||||
| 2327 | |||||||
| 2328 | if (InsertFreeze) | ||||||
| 2329 | SI->setCondition(new FreezeInst( | ||||||
| 2330 | SI->getCondition(), SI->getCondition()->getName() + ".fr", SI)); | ||||||
| 2331 | |||||||
| 2332 | // We need to use the set to populate domtree updates as even when there | ||||||
| 2333 | // are multiple cases pointing at the same successor we only want to | ||||||
| 2334 | // remove and insert one edge in the domtree. | ||||||
| 2335 | for (BasicBlock *SuccBB : UnswitchedSuccBBs) | ||||||
| 2336 | DTUpdates.push_back( | ||||||
| 2337 | {DominatorTree::Insert, SplitBB, ClonedPHs.find(SuccBB)->second}); | ||||||
| 2338 | } | ||||||
| 2339 | |||||||
| 2340 | if (MSSAU) { | ||||||
| 2341 | DT.applyUpdates(DTUpdates); | ||||||
| 2342 | DTUpdates.clear(); | ||||||
| 2343 | |||||||
| 2344 | // Remove all but one edge to the retained block and all unswitched | ||||||
| 2345 | // blocks. This is to avoid having duplicate entries in the cloned Phis, | ||||||
| 2346 | // when we know we only keep a single edge for each case. | ||||||
| 2347 | MSSAU->removeDuplicatePhiEdgesBetween(ParentBB, RetainedSuccBB); | ||||||
| 2348 | for (BasicBlock *SuccBB : UnswitchedSuccBBs) | ||||||
| 2349 | MSSAU->removeDuplicatePhiEdgesBetween(ParentBB, SuccBB); | ||||||
| 2350 | |||||||
| 2351 | for (auto &VMap : VMaps) | ||||||
| 2352 | MSSAU->updateForClonedLoop(LBRPO, ExitBlocks, *VMap, | ||||||
| 2353 | /*IgnoreIncomingWithNoClones=*/true); | ||||||
| 2354 | MSSAU->updateExitBlocksForClonedLoop(ExitBlocks, VMaps, DT); | ||||||
| 2355 | |||||||
| 2356 | // Remove all edges to unswitched blocks. | ||||||
| 2357 | for (BasicBlock *SuccBB : UnswitchedSuccBBs) | ||||||
| 2358 | MSSAU->removeEdge(ParentBB, SuccBB); | ||||||
| 2359 | } | ||||||
| 2360 | |||||||
| 2361 | // Now unhook the successor relationship as we'll be replacing | ||||||
| 2362 | // the terminator with a direct branch. This is much simpler for branches | ||||||
| 2363 | // than switches so we handle those first. | ||||||
| 2364 | if (BI) { | ||||||
| 2365 | // Remove the parent as a predecessor of the unswitched successor. | ||||||
| 2366 | assert(UnswitchedSuccBBs.size() == 1 &&(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2367, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2367 | "Only one possible unswitched block for a branch!")(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2367, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2368 | BasicBlock *UnswitchedSuccBB = *UnswitchedSuccBBs.begin(); | ||||||
| 2369 | UnswitchedSuccBB->removePredecessor(ParentBB, | ||||||
| 2370 | /*KeepOneInputPHIs*/ true); | ||||||
| 2371 | DTUpdates.push_back({DominatorTree::Delete, ParentBB, UnswitchedSuccBB}); | ||||||
| 2372 | } else { | ||||||
| 2373 | // Note that we actually want to remove the parent block as a predecessor | ||||||
| 2374 | // of *every* case successor. The case successor is either unswitched, | ||||||
| 2375 | // completely eliminating an edge from the parent to that successor, or it | ||||||
| 2376 | // is a duplicate edge to the retained successor as the retained successor | ||||||
| 2377 | // is always the default successor and as we'll replace this with a direct | ||||||
| 2378 | // branch we no longer need the duplicate entries in the PHI nodes. | ||||||
| 2379 | SwitchInst *NewSI = cast<SwitchInst>(NewTI); | ||||||
| 2380 | assert(NewSI->getDefaultDest() == RetainedSuccBB &&(static_cast <bool> (NewSI->getDefaultDest() == RetainedSuccBB && "Not retaining default successor!") ? void (0) : __assert_fail ("NewSI->getDefaultDest() == RetainedSuccBB && \"Not retaining default successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2381, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2381 | "Not retaining default successor!")(static_cast <bool> (NewSI->getDefaultDest() == RetainedSuccBB && "Not retaining default successor!") ? void (0) : __assert_fail ("NewSI->getDefaultDest() == RetainedSuccBB && \"Not retaining default successor!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2381, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2382 | for (const auto &Case : NewSI->cases()) | ||||||
| 2383 | Case.getCaseSuccessor()->removePredecessor( | ||||||
| 2384 | ParentBB, | ||||||
| 2385 | /*KeepOneInputPHIs*/ true); | ||||||
| 2386 | |||||||
| 2387 | // We need to use the set to populate domtree updates as even when there | ||||||
| 2388 | // are multiple cases pointing at the same successor we only want to | ||||||
| 2389 | // remove and insert one edge in the domtree. | ||||||
| 2390 | for (BasicBlock *SuccBB : UnswitchedSuccBBs) | ||||||
| 2391 | DTUpdates.push_back({DominatorTree::Delete, ParentBB, SuccBB}); | ||||||
| 2392 | } | ||||||
| 2393 | |||||||
| 2394 | // After MSSAU update, remove the cloned terminator instruction NewTI. | ||||||
| 2395 | ParentBB->getTerminator()->eraseFromParent(); | ||||||
| 2396 | |||||||
| 2397 | // Create a new unconditional branch to the continuing block (as opposed to | ||||||
| 2398 | // the one cloned). | ||||||
| 2399 | BranchInst::Create(RetainedSuccBB, ParentBB); | ||||||
| 2400 | } else { | ||||||
| 2401 | assert(BI && "Only branches have partial unswitching.")(static_cast <bool> (BI && "Only branches have partial unswitching." ) ? void (0) : __assert_fail ("BI && \"Only branches have partial unswitching.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2401, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2402 | assert(UnswitchedSuccBBs.size() == 1 &&(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2403, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2403 | "Only one possible unswitched block for a branch!")(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2403, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2404 | BasicBlock *ClonedPH = ClonedPHs.begin()->second; | ||||||
| 2405 | // When doing a partial unswitch, we have to do a bit more work to build up | ||||||
| 2406 | // the branch in the split block. | ||||||
| 2407 | if (PartiallyInvariant) | ||||||
| 2408 | buildPartialInvariantUnswitchConditionalBranch( | ||||||
| 2409 | *SplitBB, Invariants, Direction, *ClonedPH, *LoopPH, L, MSSAU); | ||||||
| 2410 | else { | ||||||
| 2411 | buildPartialUnswitchConditionalBranch( | ||||||
| 2412 | *SplitBB, Invariants, Direction, *ClonedPH, *LoopPH, | ||||||
| 2413 | FreezeLoopUnswitchCond, BI, &AC, DT); | ||||||
| 2414 | } | ||||||
| 2415 | DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH}); | ||||||
| 2416 | |||||||
| 2417 | if (MSSAU) { | ||||||
| 2418 | DT.applyUpdates(DTUpdates); | ||||||
| 2419 | DTUpdates.clear(); | ||||||
| 2420 | |||||||
| 2421 | // Perform MSSA cloning updates. | ||||||
| 2422 | for (auto &VMap : VMaps) | ||||||
| 2423 | MSSAU->updateForClonedLoop(LBRPO, ExitBlocks, *VMap, | ||||||
| 2424 | /*IgnoreIncomingWithNoClones=*/true); | ||||||
| 2425 | MSSAU->updateExitBlocksForClonedLoop(ExitBlocks, VMaps, DT); | ||||||
| 2426 | } | ||||||
| 2427 | } | ||||||
| 2428 | |||||||
| 2429 | // Apply the updates accumulated above to get an up-to-date dominator tree. | ||||||
| 2430 | DT.applyUpdates(DTUpdates); | ||||||
| 2431 | |||||||
| 2432 | // Now that we have an accurate dominator tree, first delete the dead cloned | ||||||
| 2433 | // blocks so that we can accurately build any cloned loops. It is important to | ||||||
| 2434 | // not delete the blocks from the original loop yet because we still want to | ||||||
| 2435 | // reference the original loop to understand the cloned loop's structure. | ||||||
| 2436 | deleteDeadClonedBlocks(L, ExitBlocks, VMaps, DT, MSSAU); | ||||||
| 2437 | |||||||
| 2438 | // Build the cloned loop structure itself. This may be substantially | ||||||
| 2439 | // different from the original structure due to the simplified CFG. This also | ||||||
| 2440 | // handles inserting all the cloned blocks into the correct loops. | ||||||
| 2441 | SmallVector<Loop *, 4> NonChildClonedLoops; | ||||||
| 2442 | for (std::unique_ptr<ValueToValueMapTy> &VMap : VMaps) | ||||||
| 2443 | buildClonedLoops(L, ExitBlocks, *VMap, LI, NonChildClonedLoops); | ||||||
| 2444 | |||||||
| 2445 | // Now that our cloned loops have been built, we can update the original loop. | ||||||
| 2446 | // First we delete the dead blocks from it and then we rebuild the loop | ||||||
| 2447 | // structure taking these deletions into account. | ||||||
| 2448 | deleteDeadBlocksFromLoop(L, ExitBlocks, DT, LI, MSSAU, SE,DestroyLoopCB); | ||||||
| 2449 | |||||||
| 2450 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2451 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2452 | |||||||
| 2453 | SmallVector<Loop *, 4> HoistedLoops; | ||||||
| 2454 | bool IsStillLoop = | ||||||
| 2455 | rebuildLoopAfterUnswitch(L, ExitBlocks, LI, HoistedLoops, SE); | ||||||
| 2456 | |||||||
| 2457 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2458 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2459 | |||||||
| 2460 | // This transformation has a high risk of corrupting the dominator tree, and | ||||||
| 2461 | // the below steps to rebuild loop structures will result in hard to debug | ||||||
| 2462 | // errors in that case so verify that the dominator tree is sane first. | ||||||
| 2463 | // FIXME: Remove this when the bugs stop showing up and rely on existing | ||||||
| 2464 | // verification steps. | ||||||
| 2465 | assert(DT.verify(DominatorTree::VerificationLevel::Fast))(static_cast <bool> (DT.verify(DominatorTree::VerificationLevel ::Fast)) ? void (0) : __assert_fail ("DT.verify(DominatorTree::VerificationLevel::Fast)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2465, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2466 | |||||||
| 2467 | if (BI && !PartiallyInvariant) { | ||||||
| 2468 | // If we unswitched a branch which collapses the condition to a known | ||||||
| 2469 | // constant we want to replace all the uses of the invariants within both | ||||||
| 2470 | // the original and cloned blocks. We do this here so that we can use the | ||||||
| 2471 | // now updated dominator tree to identify which side the users are on. | ||||||
| 2472 | assert(UnswitchedSuccBBs.size() == 1 &&(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2473, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2473 | "Only one possible unswitched block for a branch!")(static_cast <bool> (UnswitchedSuccBBs.size() == 1 && "Only one possible unswitched block for a branch!") ? void ( 0) : __assert_fail ("UnswitchedSuccBBs.size() == 1 && \"Only one possible unswitched block for a branch!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2473, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2474 | BasicBlock *ClonedPH = ClonedPHs.begin()->second; | ||||||
| 2475 | |||||||
| 2476 | // When considering multiple partially-unswitched invariants | ||||||
| 2477 | // we cant just go replace them with constants in both branches. | ||||||
| 2478 | // | ||||||
| 2479 | // For 'AND' we infer that true branch ("continue") means true | ||||||
| 2480 | // for each invariant operand. | ||||||
| 2481 | // For 'OR' we can infer that false branch ("continue") means false | ||||||
| 2482 | // for each invariant operand. | ||||||
| 2483 | // So it happens that for multiple-partial case we dont replace | ||||||
| 2484 | // in the unswitched branch. | ||||||
| 2485 | bool ReplaceUnswitched = | ||||||
| 2486 | FullUnswitch || (Invariants.size() == 1) || PartiallyInvariant; | ||||||
| 2487 | |||||||
| 2488 | ConstantInt *UnswitchedReplacement = | ||||||
| 2489 | Direction ? ConstantInt::getTrue(BI->getContext()) | ||||||
| 2490 | : ConstantInt::getFalse(BI->getContext()); | ||||||
| 2491 | ConstantInt *ContinueReplacement = | ||||||
| 2492 | Direction ? ConstantInt::getFalse(BI->getContext()) | ||||||
| 2493 | : ConstantInt::getTrue(BI->getContext()); | ||||||
| 2494 | for (Value *Invariant : Invariants) { | ||||||
| 2495 | assert(!isa<Constant>(Invariant) &&(static_cast <bool> (!isa<Constant>(Invariant) && "Should not be replacing constant values!") ? void (0) : __assert_fail ("!isa<Constant>(Invariant) && \"Should not be replacing constant values!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2496, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2496 | "Should not be replacing constant values!")(static_cast <bool> (!isa<Constant>(Invariant) && "Should not be replacing constant values!") ? void (0) : __assert_fail ("!isa<Constant>(Invariant) && \"Should not be replacing constant values!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2496, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2497 | // Use make_early_inc_range here as set invalidates the iterator. | ||||||
| 2498 | for (Use &U : llvm::make_early_inc_range(Invariant->uses())) { | ||||||
| 2499 | Instruction *UserI = dyn_cast<Instruction>(U.getUser()); | ||||||
| 2500 | if (!UserI) | ||||||
| 2501 | continue; | ||||||
| 2502 | |||||||
| 2503 | // Replace it with the 'continue' side if in the main loop body, and the | ||||||
| 2504 | // unswitched if in the cloned blocks. | ||||||
| 2505 | if (DT.dominates(LoopPH, UserI->getParent())) | ||||||
| 2506 | U.set(ContinueReplacement); | ||||||
| 2507 | else if (ReplaceUnswitched && | ||||||
| 2508 | DT.dominates(ClonedPH, UserI->getParent())) | ||||||
| 2509 | U.set(UnswitchedReplacement); | ||||||
| 2510 | } | ||||||
| 2511 | } | ||||||
| 2512 | } | ||||||
| 2513 | |||||||
| 2514 | // We can change which blocks are exit blocks of all the cloned sibling | ||||||
| 2515 | // loops, the current loop, and any parent loops which shared exit blocks | ||||||
| 2516 | // with the current loop. As a consequence, we need to re-form LCSSA for | ||||||
| 2517 | // them. But we shouldn't need to re-form LCSSA for any child loops. | ||||||
| 2518 | // FIXME: This could be made more efficient by tracking which exit blocks are | ||||||
| 2519 | // new, and focusing on them, but that isn't likely to be necessary. | ||||||
| 2520 | // | ||||||
| 2521 | // In order to reasonably rebuild LCSSA we need to walk inside-out across the | ||||||
| 2522 | // loop nest and update every loop that could have had its exits changed. We | ||||||
| 2523 | // also need to cover any intervening loops. We add all of these loops to | ||||||
| 2524 | // a list and sort them by loop depth to achieve this without updating | ||||||
| 2525 | // unnecessary loops. | ||||||
| 2526 | auto UpdateLoop = [&](Loop &UpdateL) { | ||||||
| 2527 | #ifndef NDEBUG | ||||||
| 2528 | UpdateL.verifyLoop(); | ||||||
| 2529 | for (Loop *ChildL : UpdateL) { | ||||||
| 2530 | ChildL->verifyLoop(); | ||||||
| 2531 | assert(ChildL->isRecursivelyLCSSAForm(DT, LI) &&(static_cast <bool> (ChildL->isRecursivelyLCSSAForm( DT, LI) && "Perturbed a child loop's LCSSA form!") ? void (0) : __assert_fail ("ChildL->isRecursivelyLCSSAForm(DT, LI) && \"Perturbed a child loop's LCSSA form!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2532, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 2532 | "Perturbed a child loop's LCSSA form!")(static_cast <bool> (ChildL->isRecursivelyLCSSAForm( DT, LI) && "Perturbed a child loop's LCSSA form!") ? void (0) : __assert_fail ("ChildL->isRecursivelyLCSSAForm(DT, LI) && \"Perturbed a child loop's LCSSA form!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2532, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2533 | } | ||||||
| 2534 | #endif | ||||||
| 2535 | // First build LCSSA for this loop so that we can preserve it when | ||||||
| 2536 | // forming dedicated exits. We don't want to perturb some other loop's | ||||||
| 2537 | // LCSSA while doing that CFG edit. | ||||||
| 2538 | formLCSSA(UpdateL, DT, &LI); | ||||||
| 2539 | |||||||
| 2540 | // For loops reached by this loop's original exit blocks we may | ||||||
| 2541 | // introduced new, non-dedicated exits. At least try to re-form dedicated | ||||||
| 2542 | // exits for these loops. This may fail if they couldn't have dedicated | ||||||
| 2543 | // exits to start with. | ||||||
| 2544 | formDedicatedExitBlocks(&UpdateL, &DT, &LI, MSSAU, /*PreserveLCSSA*/ true); | ||||||
| 2545 | }; | ||||||
| 2546 | |||||||
| 2547 | // For non-child cloned loops and hoisted loops, we just need to update LCSSA | ||||||
| 2548 | // and we can do it in any order as they don't nest relative to each other. | ||||||
| 2549 | // | ||||||
| 2550 | // Also check if any of the loops we have updated have become top-level loops | ||||||
| 2551 | // as that will necessitate widening the outer loop scope. | ||||||
| 2552 | for (Loop *UpdatedL : | ||||||
| 2553 | llvm::concat<Loop *>(NonChildClonedLoops, HoistedLoops)) { | ||||||
| 2554 | UpdateLoop(*UpdatedL); | ||||||
| 2555 | if (UpdatedL->isOutermost()) | ||||||
| 2556 | OuterExitL = nullptr; | ||||||
| 2557 | } | ||||||
| 2558 | if (IsStillLoop) { | ||||||
| 2559 | UpdateLoop(L); | ||||||
| 2560 | if (L.isOutermost()) | ||||||
| 2561 | OuterExitL = nullptr; | ||||||
| 2562 | } | ||||||
| 2563 | |||||||
| 2564 | // If the original loop had exit blocks, walk up through the outer most loop | ||||||
| 2565 | // of those exit blocks to update LCSSA and form updated dedicated exits. | ||||||
| 2566 | if (OuterExitL != &L) | ||||||
| 2567 | for (Loop *OuterL = ParentL; OuterL != OuterExitL; | ||||||
| 2568 | OuterL = OuterL->getParentLoop()) | ||||||
| 2569 | UpdateLoop(*OuterL); | ||||||
| 2570 | |||||||
| 2571 | #ifndef NDEBUG | ||||||
| 2572 | // Verify the entire loop structure to catch any incorrect updates before we | ||||||
| 2573 | // progress in the pass pipeline. | ||||||
| 2574 | LI.verify(DT); | ||||||
| 2575 | #endif | ||||||
| 2576 | |||||||
| 2577 | // Now that we've unswitched something, make callbacks to report the changes. | ||||||
| 2578 | // For that we need to merge together the updated loops and the cloned loops | ||||||
| 2579 | // and check whether the original loop survived. | ||||||
| 2580 | SmallVector<Loop *, 4> SibLoops; | ||||||
| 2581 | for (Loop *UpdatedL : llvm::concat<Loop *>(NonChildClonedLoops, HoistedLoops)) | ||||||
| 2582 | if (UpdatedL->getParentLoop() == ParentL) | ||||||
| 2583 | SibLoops.push_back(UpdatedL); | ||||||
| 2584 | UnswitchCB(IsStillLoop, PartiallyInvariant, SibLoops); | ||||||
| 2585 | |||||||
| 2586 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2587 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2588 | |||||||
| 2589 | if (BI) | ||||||
| 2590 | ++NumBranches; | ||||||
| 2591 | else | ||||||
| 2592 | ++NumSwitches; | ||||||
| 2593 | } | ||||||
| 2594 | |||||||
| 2595 | /// Recursively compute the cost of a dominator subtree based on the per-block | ||||||
| 2596 | /// cost map provided. | ||||||
| 2597 | /// | ||||||
| 2598 | /// The recursive computation is memozied into the provided DT-indexed cost map | ||||||
| 2599 | /// to allow querying it for most nodes in the domtree without it becoming | ||||||
| 2600 | /// quadratic. | ||||||
| 2601 | static InstructionCost computeDomSubtreeCost( | ||||||
| 2602 | DomTreeNode &N, | ||||||
| 2603 | const SmallDenseMap<BasicBlock *, InstructionCost, 4> &BBCostMap, | ||||||
| 2604 | SmallDenseMap<DomTreeNode *, InstructionCost, 4> &DTCostMap) { | ||||||
| 2605 | // Don't accumulate cost (or recurse through) blocks not in our block cost | ||||||
| 2606 | // map and thus not part of the duplication cost being considered. | ||||||
| 2607 | auto BBCostIt = BBCostMap.find(N.getBlock()); | ||||||
| 2608 | if (BBCostIt == BBCostMap.end()) | ||||||
| 2609 | return 0; | ||||||
| 2610 | |||||||
| 2611 | // Lookup this node to see if we already computed its cost. | ||||||
| 2612 | auto DTCostIt = DTCostMap.find(&N); | ||||||
| 2613 | if (DTCostIt != DTCostMap.end()) | ||||||
| 2614 | return DTCostIt->second; | ||||||
| 2615 | |||||||
| 2616 | // If not, we have to compute it. We can't use insert above and update | ||||||
| 2617 | // because computing the cost may insert more things into the map. | ||||||
| 2618 | InstructionCost Cost = std::accumulate( | ||||||
| 2619 | N.begin(), N.end(), BBCostIt->second, | ||||||
| 2620 | [&](InstructionCost Sum, DomTreeNode *ChildN) -> InstructionCost { | ||||||
| 2621 | return Sum + computeDomSubtreeCost(*ChildN, BBCostMap, DTCostMap); | ||||||
| 2622 | }); | ||||||
| 2623 | bool Inserted = DTCostMap.insert({&N, Cost}).second; | ||||||
| 2624 | (void)Inserted; | ||||||
| 2625 | assert(Inserted && "Should not insert a node while visiting children!")(static_cast <bool> (Inserted && "Should not insert a node while visiting children!" ) ? void (0) : __assert_fail ("Inserted && \"Should not insert a node while visiting children!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2625, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2626 | return Cost; | ||||||
| 2627 | } | ||||||
| 2628 | |||||||
| 2629 | /// Turns a select instruction into implicit control flow branch, | ||||||
| 2630 | /// making the following replacement: | ||||||
| 2631 | /// | ||||||
| 2632 | /// head: | ||||||
| 2633 | /// --code before select-- | ||||||
| 2634 | /// select %cond, %trueval, %falseval | ||||||
| 2635 | /// --code after select-- | ||||||
| 2636 | /// | ||||||
| 2637 | /// into | ||||||
| 2638 | /// | ||||||
| 2639 | /// head: | ||||||
| 2640 | /// --code before select-- | ||||||
| 2641 | /// br i1 %cond, label %then, label %tail | ||||||
| 2642 | /// | ||||||
| 2643 | /// then: | ||||||
| 2644 | /// br %tail | ||||||
| 2645 | /// | ||||||
| 2646 | /// tail: | ||||||
| 2647 | /// phi [ %trueval, %then ], [ %falseval, %head] | ||||||
| 2648 | /// unreachable | ||||||
| 2649 | /// | ||||||
| 2650 | /// It also makes all relevant DT and LI updates, so that all structures are in | ||||||
| 2651 | /// valid state after this transform. | ||||||
| 2652 | static BranchInst *turnSelectIntoBranch(SelectInst *SI, DominatorTree &DT, | ||||||
| 2653 | LoopInfo &LI, MemorySSAUpdater *MSSAU, | ||||||
| 2654 | AssumptionCache *AC) { | ||||||
| 2655 | LLVM_DEBUG(dbgs() << "Turning " << *SI << " into a branch.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Turning " << *SI << " into a branch.\n"; } } while (false); | ||||||
| 2656 | BasicBlock *HeadBB = SI->getParent(); | ||||||
| 2657 | |||||||
| 2658 | DomTreeUpdater DTU = | ||||||
| 2659 | DomTreeUpdater(DT, DomTreeUpdater::UpdateStrategy::Eager); | ||||||
| 2660 | SplitBlockAndInsertIfThen(SI->getCondition(), SI, false, | ||||||
| 2661 | SI->getMetadata(LLVMContext::MD_prof), &DTU, &LI); | ||||||
| 2662 | auto *CondBr = cast<BranchInst>(HeadBB->getTerminator()); | ||||||
| 2663 | BasicBlock *ThenBB = CondBr->getSuccessor(0), | ||||||
| 2664 | *TailBB = CondBr->getSuccessor(1); | ||||||
| 2665 | if (MSSAU) | ||||||
| 2666 | MSSAU->moveAllAfterSpliceBlocks(HeadBB, TailBB, SI); | ||||||
| 2667 | |||||||
| 2668 | PHINode *Phi = PHINode::Create(SI->getType(), 2, "unswitched.select", SI); | ||||||
| 2669 | Phi->addIncoming(SI->getTrueValue(), ThenBB); | ||||||
| 2670 | Phi->addIncoming(SI->getFalseValue(), HeadBB); | ||||||
| 2671 | SI->replaceAllUsesWith(Phi); | ||||||
| 2672 | SI->eraseFromParent(); | ||||||
| 2673 | |||||||
| 2674 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2675 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2676 | |||||||
| 2677 | ++NumSelects; | ||||||
| 2678 | return CondBr; | ||||||
| 2679 | } | ||||||
| 2680 | |||||||
| 2681 | /// Turns a llvm.experimental.guard intrinsic into implicit control flow branch, | ||||||
| 2682 | /// making the following replacement: | ||||||
| 2683 | /// | ||||||
| 2684 | /// --code before guard-- | ||||||
| 2685 | /// call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ] | ||||||
| 2686 | /// --code after guard-- | ||||||
| 2687 | /// | ||||||
| 2688 | /// into | ||||||
| 2689 | /// | ||||||
| 2690 | /// --code before guard-- | ||||||
| 2691 | /// br i1 %cond, label %guarded, label %deopt | ||||||
| 2692 | /// | ||||||
| 2693 | /// guarded: | ||||||
| 2694 | /// --code after guard-- | ||||||
| 2695 | /// | ||||||
| 2696 | /// deopt: | ||||||
| 2697 | /// call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] | ||||||
| 2698 | /// unreachable | ||||||
| 2699 | /// | ||||||
| 2700 | /// It also makes all relevant DT and LI updates, so that all structures are in | ||||||
| 2701 | /// valid state after this transform. | ||||||
| 2702 | static BranchInst *turnGuardIntoBranch(IntrinsicInst *GI, Loop &L, | ||||||
| 2703 | DominatorTree &DT, LoopInfo &LI, | ||||||
| 2704 | MemorySSAUpdater *MSSAU) { | ||||||
| 2705 | SmallVector<DominatorTree::UpdateType, 4> DTUpdates; | ||||||
| 2706 | LLVM_DEBUG(dbgs() << "Turning " << *GI << " into a branch.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Turning " << *GI << " into a branch.\n"; } } while (false); | ||||||
| 2707 | BasicBlock *CheckBB = GI->getParent(); | ||||||
| 2708 | |||||||
| 2709 | if (MSSAU && VerifyMemorySSA) | ||||||
| 2710 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2711 | |||||||
| 2712 | // Remove all CheckBB's successors from DomTree. A block can be seen among | ||||||
| 2713 | // successors more than once, but for DomTree it should be added only once. | ||||||
| 2714 | SmallPtrSet<BasicBlock *, 4> Successors; | ||||||
| 2715 | for (auto *Succ : successors(CheckBB)) | ||||||
| 2716 | if (Successors.insert(Succ).second) | ||||||
| 2717 | DTUpdates.push_back({DominatorTree::Delete, CheckBB, Succ}); | ||||||
| 2718 | |||||||
| 2719 | Instruction *DeoptBlockTerm = | ||||||
| 2720 | SplitBlockAndInsertIfThen(GI->getArgOperand(0), GI, true); | ||||||
| 2721 | BranchInst *CheckBI = cast<BranchInst>(CheckBB->getTerminator()); | ||||||
| 2722 | // SplitBlockAndInsertIfThen inserts control flow that branches to | ||||||
| 2723 | // DeoptBlockTerm if the condition is true. We want the opposite. | ||||||
| 2724 | CheckBI->swapSuccessors(); | ||||||
| 2725 | |||||||
| 2726 | BasicBlock *GuardedBlock = CheckBI->getSuccessor(0); | ||||||
| 2727 | GuardedBlock->setName("guarded"); | ||||||
| 2728 | CheckBI->getSuccessor(1)->setName("deopt"); | ||||||
| 2729 | BasicBlock *DeoptBlock = CheckBI->getSuccessor(1); | ||||||
| 2730 | |||||||
| 2731 | if (MSSAU) | ||||||
| 2732 | MSSAU->moveAllAfterSpliceBlocks(CheckBB, GuardedBlock, GI); | ||||||
| 2733 | |||||||
| 2734 | GI->moveBefore(DeoptBlockTerm); | ||||||
| 2735 | GI->setArgOperand(0, ConstantInt::getFalse(GI->getContext())); | ||||||
| 2736 | |||||||
| 2737 | // Add new successors of CheckBB into DomTree. | ||||||
| 2738 | for (auto *Succ : successors(CheckBB)) | ||||||
| 2739 | DTUpdates.push_back({DominatorTree::Insert, CheckBB, Succ}); | ||||||
| 2740 | |||||||
| 2741 | // Now the blocks that used to be CheckBB's successors are GuardedBlock's | ||||||
| 2742 | // successors. | ||||||
| 2743 | for (auto *Succ : Successors) | ||||||
| 2744 | DTUpdates.push_back({DominatorTree::Insert, GuardedBlock, Succ}); | ||||||
| 2745 | |||||||
| 2746 | // Make proper changes to DT. | ||||||
| 2747 | DT.applyUpdates(DTUpdates); | ||||||
| 2748 | // Inform LI of a new loop block. | ||||||
| 2749 | L.addBasicBlockToLoop(GuardedBlock, LI); | ||||||
| 2750 | |||||||
| 2751 | if (MSSAU) { | ||||||
| 2752 | MemoryDef *MD = cast<MemoryDef>(MSSAU->getMemorySSA()->getMemoryAccess(GI)); | ||||||
| 2753 | MSSAU->moveToPlace(MD, DeoptBlock, MemorySSA::BeforeTerminator); | ||||||
| 2754 | if (VerifyMemorySSA) | ||||||
| 2755 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 2756 | } | ||||||
| 2757 | |||||||
| 2758 | ++NumGuards; | ||||||
| 2759 | return CheckBI; | ||||||
| 2760 | } | ||||||
| 2761 | |||||||
| 2762 | /// Cost multiplier is a way to limit potentially exponential behavior | ||||||
| 2763 | /// of loop-unswitch. Cost is multipied in proportion of 2^number of unswitch | ||||||
| 2764 | /// candidates available. Also accounting for the number of "sibling" loops with | ||||||
| 2765 | /// the idea to account for previous unswitches that already happened on this | ||||||
| 2766 | /// cluster of loops. There was an attempt to keep this formula simple, | ||||||
| 2767 | /// just enough to limit the worst case behavior. Even if it is not that simple | ||||||
| 2768 | /// now it is still not an attempt to provide a detailed heuristic size | ||||||
| 2769 | /// prediction. | ||||||
| 2770 | /// | ||||||
| 2771 | /// TODO: Make a proper accounting of "explosion" effect for all kinds of | ||||||
| 2772 | /// unswitch candidates, making adequate predictions instead of wild guesses. | ||||||
| 2773 | /// That requires knowing not just the number of "remaining" candidates but | ||||||
| 2774 | /// also costs of unswitching for each of these candidates. | ||||||
| 2775 | static int CalculateUnswitchCostMultiplier( | ||||||
| 2776 | const Instruction &TI, const Loop &L, const LoopInfo &LI, | ||||||
| 2777 | const DominatorTree &DT, | ||||||
| 2778 | ArrayRef<NonTrivialUnswitchCandidate> UnswitchCandidates) { | ||||||
| 2779 | |||||||
| 2780 | // Guards and other exiting conditions do not contribute to exponential | ||||||
| 2781 | // explosion as soon as they dominate the latch (otherwise there might be | ||||||
| 2782 | // another path to the latch remaining that does not allow to eliminate the | ||||||
| 2783 | // loop copy on unswitch). | ||||||
| 2784 | const BasicBlock *Latch = L.getLoopLatch(); | ||||||
| 2785 | const BasicBlock *CondBlock = TI.getParent(); | ||||||
| 2786 | if (DT.dominates(CondBlock, Latch) && | ||||||
| 2787 | (isGuard(&TI) || | ||||||
| 2788 | (TI.isTerminator() && | ||||||
| 2789 | llvm::count_if(successors(&TI), [&L](const BasicBlock *SuccBB) { | ||||||
| 2790 | return L.contains(SuccBB); | ||||||
| 2791 | }) <= 1))) { | ||||||
| 2792 | NumCostMultiplierSkipped++; | ||||||
| 2793 | return 1; | ||||||
| 2794 | } | ||||||
| 2795 | |||||||
| 2796 | auto *ParentL = L.getParentLoop(); | ||||||
| 2797 | int SiblingsCount = (ParentL ? ParentL->getSubLoopsVector().size() | ||||||
| 2798 | : std::distance(LI.begin(), LI.end())); | ||||||
| 2799 | // Count amount of clones that all the candidates might cause during | ||||||
| 2800 | // unswitching. Branch/guard/select counts as 1, switch counts as log2 of its | ||||||
| 2801 | // cases. | ||||||
| 2802 | int UnswitchedClones = 0; | ||||||
| 2803 | for (const auto &Candidate : UnswitchCandidates) { | ||||||
| 2804 | const Instruction *CI = Candidate.TI; | ||||||
| 2805 | const BasicBlock *CondBlock = CI->getParent(); | ||||||
| 2806 | bool SkipExitingSuccessors = DT.dominates(CondBlock, Latch); | ||||||
| 2807 | if (isa<SelectInst>(CI)) { | ||||||
| 2808 | UnswitchedClones++; | ||||||
| 2809 | continue; | ||||||
| 2810 | } | ||||||
| 2811 | if (isGuard(CI)) { | ||||||
| 2812 | if (!SkipExitingSuccessors) | ||||||
| 2813 | UnswitchedClones++; | ||||||
| 2814 | continue; | ||||||
| 2815 | } | ||||||
| 2816 | int NonExitingSuccessors = | ||||||
| 2817 | llvm::count_if(successors(CondBlock), | ||||||
| 2818 | [SkipExitingSuccessors, &L](const BasicBlock *SuccBB) { | ||||||
| 2819 | return !SkipExitingSuccessors || L.contains(SuccBB); | ||||||
| 2820 | }); | ||||||
| 2821 | UnswitchedClones += Log2_32(NonExitingSuccessors); | ||||||
| 2822 | } | ||||||
| 2823 | |||||||
| 2824 | // Ignore up to the "unscaled candidates" number of unswitch candidates | ||||||
| 2825 | // when calculating the power-of-two scaling of the cost. The main idea | ||||||
| 2826 | // with this control is to allow a small number of unswitches to happen | ||||||
| 2827 | // and rely more on siblings multiplier (see below) when the number | ||||||
| 2828 | // of candidates is small. | ||||||
| 2829 | unsigned ClonesPower = | ||||||
| 2830 | std::max(UnswitchedClones - (int)UnswitchNumInitialUnscaledCandidates, 0); | ||||||
| 2831 | |||||||
| 2832 | // Allowing top-level loops to spread a bit more than nested ones. | ||||||
| 2833 | int SiblingsMultiplier = | ||||||
| 2834 | std::max((ParentL ? SiblingsCount | ||||||
| 2835 | : SiblingsCount / (int)UnswitchSiblingsToplevelDiv), | ||||||
| 2836 | 1); | ||||||
| 2837 | // Compute the cost multiplier in a way that won't overflow by saturating | ||||||
| 2838 | // at an upper bound. | ||||||
| 2839 | int CostMultiplier; | ||||||
| 2840 | if (ClonesPower > Log2_32(UnswitchThreshold) || | ||||||
| 2841 | SiblingsMultiplier > UnswitchThreshold) | ||||||
| 2842 | CostMultiplier = UnswitchThreshold; | ||||||
| 2843 | else | ||||||
| 2844 | CostMultiplier = std::min(SiblingsMultiplier * (1 << ClonesPower), | ||||||
| 2845 | (int)UnswitchThreshold); | ||||||
| 2846 | |||||||
| 2847 | LLVM_DEBUG(dbgs() << " Computed multiplier " << CostMultiplierdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed multiplier " << CostMultiplier << " (siblings " << SiblingsMultiplier << " * clones " << (1 << ClonesPower) << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 2848 | << " (siblings " << SiblingsMultiplier << " * clones "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed multiplier " << CostMultiplier << " (siblings " << SiblingsMultiplier << " * clones " << (1 << ClonesPower) << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 2849 | << (1 << ClonesPower) << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed multiplier " << CostMultiplier << " (siblings " << SiblingsMultiplier << " * clones " << (1 << ClonesPower) << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 2850 | << " for unswitch candidate: " << TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed multiplier " << CostMultiplier << " (siblings " << SiblingsMultiplier << " * clones " << (1 << ClonesPower) << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false); | ||||||
| 2851 | return CostMultiplier; | ||||||
| 2852 | } | ||||||
| 2853 | |||||||
| 2854 | static bool collectUnswitchCandidates( | ||||||
| 2855 | SmallVectorImpl<NonTrivialUnswitchCandidate> &UnswitchCandidates, | ||||||
| 2856 | IVConditionInfo &PartialIVInfo, Instruction *&PartialIVCondBranch, | ||||||
| 2857 | const Loop &L, const LoopInfo &LI, AAResults &AA, | ||||||
| 2858 | const MemorySSAUpdater *MSSAU) { | ||||||
| 2859 | assert(UnswitchCandidates.empty() && "Should be!")(static_cast <bool> (UnswitchCandidates.empty() && "Should be!") ? void (0) : __assert_fail ("UnswitchCandidates.empty() && \"Should be!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 2859, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 2860 | // Whether or not we should also collect guards in the loop. | ||||||
| 2861 | bool CollectGuards = false; | ||||||
| 2862 | if (UnswitchGuards) { | ||||||
| 2863 | auto *GuardDecl = L.getHeader()->getParent()->getParent()->getFunction( | ||||||
| 2864 | Intrinsic::getName(Intrinsic::experimental_guard)); | ||||||
| 2865 | if (GuardDecl && !GuardDecl->use_empty()) | ||||||
| 2866 | CollectGuards = true; | ||||||
| 2867 | } | ||||||
| 2868 | |||||||
| 2869 | for (auto *BB : L.blocks()) { | ||||||
| 2870 | if (LI.getLoopFor(BB) != &L) | ||||||
| 2871 | continue; | ||||||
| 2872 | |||||||
| 2873 | for (auto &I : *BB) { | ||||||
| 2874 | if (auto *SI = dyn_cast<SelectInst>(&I)) { | ||||||
| 2875 | auto *Cond = SI->getCondition(); | ||||||
| 2876 | if (!isa<Constant>(Cond) && L.isLoopInvariant(Cond)) | ||||||
| 2877 | UnswitchCandidates.push_back({&I, {Cond}}); | ||||||
| 2878 | } else if (CollectGuards && isGuard(&I)) { | ||||||
| 2879 | auto *Cond = | ||||||
| 2880 | skipTrivialSelect(cast<IntrinsicInst>(&I)->getArgOperand(0)); | ||||||
| 2881 | // TODO: Support AND, OR conditions and partial unswitching. | ||||||
| 2882 | if (!isa<Constant>(Cond) && L.isLoopInvariant(Cond)) | ||||||
| 2883 | UnswitchCandidates.push_back({&I, {Cond}}); | ||||||
| 2884 | } | ||||||
| 2885 | } | ||||||
| 2886 | |||||||
| 2887 | if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { | ||||||
| 2888 | // We can only consider fully loop-invariant switch conditions as we need | ||||||
| 2889 | // to completely eliminate the switch after unswitching. | ||||||
| 2890 | if (!isa<Constant>(SI->getCondition()) && | ||||||
| 2891 | L.isLoopInvariant(SI->getCondition()) && !BB->getUniqueSuccessor()) | ||||||
| 2892 | UnswitchCandidates.push_back({SI, {SI->getCondition()}}); | ||||||
| 2893 | continue; | ||||||
| 2894 | } | ||||||
| 2895 | |||||||
| 2896 | auto *BI = dyn_cast<BranchInst>(BB->getTerminator()); | ||||||
| 2897 | if (!BI || !BI->isConditional() || isa<Constant>(BI->getCondition()) || | ||||||
| 2898 | BI->getSuccessor(0) == BI->getSuccessor(1)) | ||||||
| 2899 | continue; | ||||||
| 2900 | |||||||
| 2901 | Value *Cond = skipTrivialSelect(BI->getCondition()); | ||||||
| 2902 | if (isa<Constant>(Cond)) | ||||||
| 2903 | continue; | ||||||
| 2904 | |||||||
| 2905 | if (L.isLoopInvariant(Cond)) { | ||||||
| 2906 | UnswitchCandidates.push_back({BI, {Cond}}); | ||||||
| 2907 | continue; | ||||||
| 2908 | } | ||||||
| 2909 | |||||||
| 2910 | Instruction &CondI = *cast<Instruction>(Cond); | ||||||
| 2911 | if (match(&CondI, m_CombineOr(m_LogicalAnd(), m_LogicalOr()))) { | ||||||
| 2912 | TinyPtrVector<Value *> Invariants = | ||||||
| 2913 | collectHomogenousInstGraphLoopInvariants(L, CondI, LI); | ||||||
| 2914 | if (Invariants.empty()) | ||||||
| 2915 | continue; | ||||||
| 2916 | |||||||
| 2917 | UnswitchCandidates.push_back({BI, std::move(Invariants)}); | ||||||
| 2918 | continue; | ||||||
| 2919 | } | ||||||
| 2920 | } | ||||||
| 2921 | |||||||
| 2922 | if (MSSAU
| ||||||
| 2923 | !any_of(UnswitchCandidates, [&L](auto &TerminatorAndInvariants) { | ||||||
| 2924 | return TerminatorAndInvariants.TI == L.getHeader()->getTerminator(); | ||||||
| 2925 | })) { | ||||||
| 2926 | MemorySSA *MSSA = MSSAU->getMemorySSA(); | ||||||
| 2927 | if (auto Info = hasPartialIVCondition(L, MSSAThreshold, *MSSA, AA)) { | ||||||
| 2928 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "simple-loop-unswitch: Found partially invariant condition " << *Info->InstToDuplicate[0] << "\n"; } } while (false) | ||||||
| 2929 | dbgs() << "simple-loop-unswitch: Found partially invariant condition "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "simple-loop-unswitch: Found partially invariant condition " << *Info->InstToDuplicate[0] << "\n"; } } while (false) | ||||||
| 2930 | << *Info->InstToDuplicate[0] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "simple-loop-unswitch: Found partially invariant condition " << *Info->InstToDuplicate[0] << "\n"; } } while (false); | ||||||
| 2931 | PartialIVInfo = *Info; | ||||||
| 2932 | PartialIVCondBranch = L.getHeader()->getTerminator(); | ||||||
| 2933 | TinyPtrVector<Value *> ValsToDuplicate; | ||||||
| 2934 | llvm::append_range(ValsToDuplicate, Info->InstToDuplicate); | ||||||
| 2935 | UnswitchCandidates.push_back( | ||||||
| 2936 | {L.getHeader()->getTerminator(), std::move(ValsToDuplicate)}); | ||||||
| 2937 | } | ||||||
| 2938 | } | ||||||
| 2939 | return !UnswitchCandidates.empty(); | ||||||
| 2940 | } | ||||||
| 2941 | |||||||
| 2942 | /// Tries to canonicalize condition described by: | ||||||
| 2943 | /// | ||||||
| 2944 | /// br (LHS pred RHS), label IfTrue, label IfFalse | ||||||
| 2945 | /// | ||||||
| 2946 | /// into its equivalent where `Pred` is something that we support for injected | ||||||
| 2947 | /// invariants (so far it is limited to ult), LHS in canonicalized form is | ||||||
| 2948 | /// non-invariant and RHS is an invariant. | ||||||
| 2949 | static void canonicalizeForInvariantConditionInjection( | ||||||
| 2950 | ICmpInst::Predicate &Pred, Value *&LHS, Value *&RHS, BasicBlock *&IfTrue, | ||||||
| 2951 | BasicBlock *&IfFalse, const Loop &L) { | ||||||
| 2952 | if (!L.contains(IfTrue)) { | ||||||
| 2953 | Pred = ICmpInst::getInversePredicate(Pred); | ||||||
| 2954 | std::swap(IfTrue, IfFalse); | ||||||
| 2955 | } | ||||||
| 2956 | |||||||
| 2957 | // Move loop-invariant argument to RHS position. | ||||||
| 2958 | if (L.isLoopInvariant(LHS)) { | ||||||
| 2959 | Pred = ICmpInst::getSwappedPredicate(Pred); | ||||||
| 2960 | std::swap(LHS, RHS); | ||||||
| 2961 | } | ||||||
| 2962 | |||||||
| 2963 | if (Pred == ICmpInst::ICMP_SGE && match(RHS, m_Zero())) { | ||||||
| 2964 | // Turn "x >=s 0" into "x <u UMIN_INT" | ||||||
| 2965 | Pred = ICmpInst::ICMP_ULT; | ||||||
| 2966 | RHS = ConstantInt::get( | ||||||
| 2967 | RHS->getContext(), | ||||||
| 2968 | APInt::getSignedMinValue(RHS->getType()->getIntegerBitWidth())); | ||||||
| 2969 | } | ||||||
| 2970 | } | ||||||
| 2971 | |||||||
| 2972 | /// Returns true, if predicate described by ( \p Pred, \p LHS, \p RHS ) | ||||||
| 2973 | /// succeeding into blocks ( \p IfTrue, \p IfFalse) can be optimized by | ||||||
| 2974 | /// injecting a loop-invariant condition. | ||||||
| 2975 | static bool shouldTryInjectInvariantCondition( | ||||||
| 2976 | const ICmpInst::Predicate Pred, const Value *LHS, const Value *RHS, | ||||||
| 2977 | const BasicBlock *IfTrue, const BasicBlock *IfFalse, const Loop &L) { | ||||||
| 2978 | if (L.isLoopInvariant(LHS) || !L.isLoopInvariant(RHS)) | ||||||
| 2979 | return false; | ||||||
| 2980 | // TODO: Support other predicates. | ||||||
| 2981 | if (Pred != ICmpInst::ICMP_ULT) | ||||||
| 2982 | return false; | ||||||
| 2983 | // TODO: Support non-loop-exiting branches? | ||||||
| 2984 | if (!L.contains(IfTrue) || L.contains(IfFalse)) | ||||||
| 2985 | return false; | ||||||
| 2986 | // FIXME: For some reason this causes problems with MSSA updates, need to | ||||||
| 2987 | // investigate why. So far, just don't unswitch latch. | ||||||
| 2988 | if (L.getHeader() == IfTrue) | ||||||
| 2989 | return false; | ||||||
| 2990 | return true; | ||||||
| 2991 | } | ||||||
| 2992 | |||||||
| 2993 | /// Returns true, if metadata on \p BI allows us to optimize branching into \p | ||||||
| 2994 | /// TakenSucc via injection of invariant conditions. The branch should be not | ||||||
| 2995 | /// enough and not previously unswitched, the information about this comes from | ||||||
| 2996 | /// the metadata. | ||||||
| 2997 | bool shouldTryInjectBasingOnMetadata(const BranchInst *BI, | ||||||
| 2998 | const BasicBlock *TakenSucc) { | ||||||
| 2999 | // Skip branches that have already been unswithed this way. After successful | ||||||
| 3000 | // unswitching of injected condition, we will still have a copy of this loop | ||||||
| 3001 | // which looks exactly the same as original one. To prevent the 2nd attempt | ||||||
| 3002 | // of unswitching it in the same pass, mark this branch as "nothing to do | ||||||
| 3003 | // here". | ||||||
| 3004 | if (BI->hasMetadata("llvm.invariant.condition.injection.disabled")) | ||||||
| 3005 | return false; | ||||||
| 3006 | SmallVector<uint32_t> Weights; | ||||||
| 3007 | if (!extractBranchWeights(*BI, Weights)) | ||||||
| 3008 | return false; | ||||||
| 3009 | unsigned T = InjectInvariantConditionHotnesThreshold; | ||||||
| 3010 | BranchProbability LikelyTaken(T - 1, T); | ||||||
| 3011 | |||||||
| 3012 | assert(Weights.size() == 2 && "Unexpected profile data!")(static_cast <bool> (Weights.size() == 2 && "Unexpected profile data!" ) ? void (0) : __assert_fail ("Weights.size() == 2 && \"Unexpected profile data!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3012, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3013 | size_t Idx = BI->getSuccessor(0) == TakenSucc ? 0 : 1; | ||||||
| 3014 | auto Num = Weights[Idx]; | ||||||
| 3015 | auto Denom = Weights[0] + Weights[1]; | ||||||
| 3016 | // Degenerate or overflowed metadata. | ||||||
| 3017 | if (Denom == 0 || Num > Denom) | ||||||
| 3018 | return false; | ||||||
| 3019 | BranchProbability ActualTaken(Num, Denom); | ||||||
| 3020 | if (LikelyTaken > ActualTaken) | ||||||
| 3021 | return false; | ||||||
| 3022 | return true; | ||||||
| 3023 | } | ||||||
| 3024 | |||||||
| 3025 | /// Materialize pending invariant condition of the given candidate into IR. The | ||||||
| 3026 | /// injected loop-invariant condition implies the original loop-variant branch | ||||||
| 3027 | /// condition, so the materialization turns | ||||||
| 3028 | /// | ||||||
| 3029 | /// loop_block: | ||||||
| 3030 | /// ... | ||||||
| 3031 | /// br i1 %variant_cond, label InLoopSucc, label OutOfLoopSucc | ||||||
| 3032 | /// | ||||||
| 3033 | /// into | ||||||
| 3034 | /// | ||||||
| 3035 | /// preheader: | ||||||
| 3036 | /// %invariant_cond = LHS pred RHS | ||||||
| 3037 | /// ... | ||||||
| 3038 | /// loop_block: | ||||||
| 3039 | /// br i1 %invariant_cond, label InLoopSucc, label OriginalCheck | ||||||
| 3040 | /// OriginalCheck: | ||||||
| 3041 | /// br i1 %variant_cond, label InLoopSucc, label OutOfLoopSucc | ||||||
| 3042 | /// ... | ||||||
| 3043 | static NonTrivialUnswitchCandidate | ||||||
| 3044 | injectPendingInvariantConditions(NonTrivialUnswitchCandidate Candidate, Loop &L, | ||||||
| 3045 | DominatorTree &DT, LoopInfo &LI, | ||||||
| 3046 | AssumptionCache &AC, MemorySSAUpdater *MSSAU) { | ||||||
| 3047 | assert(Candidate.hasPendingInjection() && "Nothing to inject!")(static_cast <bool> (Candidate.hasPendingInjection() && "Nothing to inject!") ? void (0) : __assert_fail ("Candidate.hasPendingInjection() && \"Nothing to inject!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3047, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3048 | BasicBlock *Preheader = L.getLoopPreheader(); | ||||||
| 3049 | assert(Preheader && "Loop is not in simplified form?")(static_cast <bool> (Preheader && "Loop is not in simplified form?" ) ? void (0) : __assert_fail ("Preheader && \"Loop is not in simplified form?\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3049, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3050 | assert(LI.getLoopFor(Candidate.TI->getParent()) == &L &&(static_cast <bool> (LI.getLoopFor(Candidate.TI->getParent ()) == &L && "Unswitching branch of inner loop!") ? void (0) : __assert_fail ("LI.getLoopFor(Candidate.TI->getParent()) == &L && \"Unswitching branch of inner loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3051, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3051 | "Unswitching branch of inner loop!")(static_cast <bool> (LI.getLoopFor(Candidate.TI->getParent ()) == &L && "Unswitching branch of inner loop!") ? void (0) : __assert_fail ("LI.getLoopFor(Candidate.TI->getParent()) == &L && \"Unswitching branch of inner loop!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3051, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3052 | |||||||
| 3053 | auto Pred = Candidate.PendingInjection->Pred; | ||||||
| 3054 | auto *LHS = Candidate.PendingInjection->LHS; | ||||||
| 3055 | auto *RHS = Candidate.PendingInjection->RHS; | ||||||
| 3056 | auto *InLoopSucc = Candidate.PendingInjection->InLoopSucc; | ||||||
| 3057 | auto *TI = cast<BranchInst>(Candidate.TI); | ||||||
| 3058 | auto *BB = Candidate.TI->getParent(); | ||||||
| 3059 | auto *OutOfLoopSucc = InLoopSucc == TI->getSuccessor(0) ? TI->getSuccessor(1) | ||||||
| 3060 | : TI->getSuccessor(0); | ||||||
| 3061 | // FIXME: Remove this once limitation on successors is lifted. | ||||||
| 3062 | assert(L.contains(InLoopSucc) && "Not supported yet!")(static_cast <bool> (L.contains(InLoopSucc) && "Not supported yet!" ) ? void (0) : __assert_fail ("L.contains(InLoopSucc) && \"Not supported yet!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3062, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3063 | assert(!L.contains(OutOfLoopSucc) && "Not supported yet!")(static_cast <bool> (!L.contains(OutOfLoopSucc) && "Not supported yet!") ? void (0) : __assert_fail ("!L.contains(OutOfLoopSucc) && \"Not supported yet!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3063, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3064 | auto &Ctx = BB->getContext(); | ||||||
| 3065 | |||||||
| 3066 | IRBuilder<> Builder(Preheader->getTerminator()); | ||||||
| 3067 | assert(ICmpInst::isUnsigned(Pred) && "Not supported yet!")(static_cast <bool> (ICmpInst::isUnsigned(Pred) && "Not supported yet!") ? void (0) : __assert_fail ("ICmpInst::isUnsigned(Pred) && \"Not supported yet!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3067, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3068 | if (LHS->getType() != RHS->getType()) { | ||||||
| 3069 | if (LHS->getType()->getIntegerBitWidth() < | ||||||
| 3070 | RHS->getType()->getIntegerBitWidth()) | ||||||
| 3071 | LHS = Builder.CreateZExt(LHS, RHS->getType(), LHS->getName() + ".wide"); | ||||||
| 3072 | else | ||||||
| 3073 | RHS = Builder.CreateZExt(RHS, LHS->getType(), RHS->getName() + ".wide"); | ||||||
| 3074 | } | ||||||
| 3075 | // Do not use builder here: CreateICmp may simplify this into a constant and | ||||||
| 3076 | // unswitching will break. Better optimize it away later. | ||||||
| 3077 | auto *InjectedCond = | ||||||
| 3078 | ICmpInst::Create(Instruction::ICmp, Pred, LHS, RHS, "injected.cond", | ||||||
| 3079 | Preheader->getTerminator()); | ||||||
| 3080 | auto *OldCond = TI->getCondition(); | ||||||
| 3081 | |||||||
| 3082 | BasicBlock *CheckBlock = BasicBlock::Create(Ctx, BB->getName() + ".check", | ||||||
| 3083 | BB->getParent(), InLoopSucc); | ||||||
| 3084 | Builder.SetInsertPoint(TI); | ||||||
| 3085 | auto *InvariantBr = | ||||||
| 3086 | Builder.CreateCondBr(InjectedCond, InLoopSucc, CheckBlock); | ||||||
| 3087 | |||||||
| 3088 | Builder.SetInsertPoint(CheckBlock); | ||||||
| 3089 | auto *NewTerm = Builder.CreateCondBr(OldCond, InLoopSucc, OutOfLoopSucc); | ||||||
| 3090 | |||||||
| 3091 | TI->eraseFromParent(); | ||||||
| 3092 | // Prevent infinite unswitching. | ||||||
| 3093 | NewTerm->setMetadata("llvm.invariant.condition.injection.disabled", | ||||||
| 3094 | MDNode::get(BB->getContext(), {})); | ||||||
| 3095 | |||||||
| 3096 | // Fixup phis. | ||||||
| 3097 | for (auto &I : *InLoopSucc) { | ||||||
| 3098 | auto *PN = dyn_cast<PHINode>(&I); | ||||||
| 3099 | if (!PN) | ||||||
| 3100 | break; | ||||||
| 3101 | auto *Inc = PN->getIncomingValueForBlock(BB); | ||||||
| 3102 | PN->addIncoming(Inc, CheckBlock); | ||||||
| 3103 | } | ||||||
| 3104 | OutOfLoopSucc->replacePhiUsesWith(BB, CheckBlock); | ||||||
| 3105 | |||||||
| 3106 | SmallVector<DominatorTree::UpdateType, 4> DTUpdates = { | ||||||
| 3107 | { DominatorTree::Insert, BB, CheckBlock }, | ||||||
| 3108 | { DominatorTree::Insert, CheckBlock, InLoopSucc }, | ||||||
| 3109 | { DominatorTree::Insert, CheckBlock, OutOfLoopSucc }, | ||||||
| 3110 | { DominatorTree::Delete, BB, OutOfLoopSucc } | ||||||
| 3111 | }; | ||||||
| 3112 | |||||||
| 3113 | DT.applyUpdates(DTUpdates); | ||||||
| 3114 | if (MSSAU) | ||||||
| 3115 | MSSAU->applyUpdates(DTUpdates, DT); | ||||||
| 3116 | L.addBasicBlockToLoop(CheckBlock, LI); | ||||||
| 3117 | |||||||
| 3118 | #ifndef NDEBUG | ||||||
| 3119 | DT.verify(); | ||||||
| 3120 | LI.verify(DT); | ||||||
| 3121 | if (MSSAU && VerifyMemorySSA) | ||||||
| 3122 | MSSAU->getMemorySSA()->verifyMemorySSA(); | ||||||
| 3123 | #endif | ||||||
| 3124 | |||||||
| 3125 | // TODO: In fact, cost of unswitching a new invariant candidate is *slightly* | ||||||
| 3126 | // higher because we have just inserted a new block. Need to think how to | ||||||
| 3127 | // adjust the cost of injected candidates when it was first computed. | ||||||
| 3128 | LLVM_DEBUG(dbgs() << "Injected a new loop-invariant branch " << *InvariantBrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Injected a new loop-invariant branch " << *InvariantBr << " and considering it for unswitching." ; } } while (false) | ||||||
| 3129 | << " and considering it for unswitching.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Injected a new loop-invariant branch " << *InvariantBr << " and considering it for unswitching." ; } } while (false); | ||||||
| 3130 | ++NumInvariantConditionsInjected; | ||||||
| 3131 | return NonTrivialUnswitchCandidate(InvariantBr, { InjectedCond }, | ||||||
| 3132 | Candidate.Cost); | ||||||
| 3133 | } | ||||||
| 3134 | |||||||
| 3135 | /// Given chain of loop branch conditions looking like: | ||||||
| 3136 | /// br (Variant < Invariant1) | ||||||
| 3137 | /// br (Variant < Invariant2) | ||||||
| 3138 | /// br (Variant < Invariant3) | ||||||
| 3139 | /// ... | ||||||
| 3140 | /// collect set of invariant conditions on which we want to unswitch, which | ||||||
| 3141 | /// look like: | ||||||
| 3142 | /// Invariant1 <= Invariant2 | ||||||
| 3143 | /// Invariant2 <= Invariant3 | ||||||
| 3144 | /// ... | ||||||
| 3145 | /// Though they might not immediately exist in the IR, we can still inject them. | ||||||
| 3146 | static bool insertCandidatesWithPendingInjections( | ||||||
| 3147 | SmallVectorImpl<NonTrivialUnswitchCandidate> &UnswitchCandidates, Loop &L, | ||||||
| 3148 | ICmpInst::Predicate Pred, ArrayRef<CompareDesc> Compares, | ||||||
| 3149 | const DominatorTree &DT) { | ||||||
| 3150 | |||||||
| 3151 | assert(ICmpInst::isRelational(Pred))(static_cast <bool> (ICmpInst::isRelational(Pred)) ? void (0) : __assert_fail ("ICmpInst::isRelational(Pred)", "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp" , 3151, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3152 | assert(ICmpInst::isStrictPredicate(Pred))(static_cast <bool> (ICmpInst::isStrictPredicate(Pred)) ? void (0) : __assert_fail ("ICmpInst::isStrictPredicate(Pred)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3152, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3153 | if (Compares.size() < 2) | ||||||
| 3154 | return false; | ||||||
| 3155 | ICmpInst::Predicate NonStrictPred = ICmpInst::getNonStrictPredicate(Pred); | ||||||
| 3156 | for (auto Prev = Compares.begin(), Next = Compares.begin() + 1; | ||||||
| 3157 | Next != Compares.end(); ++Prev, ++Next) { | ||||||
| 3158 | Value *LHS = Next->Invariant; | ||||||
| 3159 | Value *RHS = Prev->Invariant; | ||||||
| 3160 | BasicBlock *InLoopSucc = Prev->InLoopSucc; | ||||||
| 3161 | InjectedInvariant ToInject(NonStrictPred, LHS, RHS, InLoopSucc); | ||||||
| 3162 | NonTrivialUnswitchCandidate Candidate(Prev->Term, { LHS, RHS }, | ||||||
| 3163 | std::nullopt, std::move(ToInject)); | ||||||
| 3164 | UnswitchCandidates.push_back(std::move(Candidate)); | ||||||
| 3165 | } | ||||||
| 3166 | return true; | ||||||
| 3167 | } | ||||||
| 3168 | |||||||
| 3169 | /// Collect unswitch candidates by invariant conditions that are not immediately | ||||||
| 3170 | /// present in the loop. However, they can be injected into the code if we | ||||||
| 3171 | /// decide it's profitable. | ||||||
| 3172 | /// An example of such conditions is following: | ||||||
| 3173 | /// | ||||||
| 3174 | /// for (...) { | ||||||
| 3175 | /// x = load ... | ||||||
| 3176 | /// if (! x <u C1) break; | ||||||
| 3177 | /// if (! x <u C2) break; | ||||||
| 3178 | /// <do something> | ||||||
| 3179 | /// } | ||||||
| 3180 | /// | ||||||
| 3181 | /// We can unswitch by condition "C1 <=u C2". If that is true, then "x <u C1 <= | ||||||
| 3182 | /// C2" automatically implies "x <u C2", so we can get rid of one of | ||||||
| 3183 | /// loop-variant checks in unswitched loop version. | ||||||
| 3184 | static bool collectUnswitchCandidatesWithInjections( | ||||||
| 3185 | SmallVectorImpl<NonTrivialUnswitchCandidate> &UnswitchCandidates, | ||||||
| 3186 | IVConditionInfo &PartialIVInfo, Instruction *&PartialIVCondBranch, Loop &L, | ||||||
| 3187 | const DominatorTree &DT, const LoopInfo &LI, AAResults &AA, | ||||||
| 3188 | const MemorySSAUpdater *MSSAU) { | ||||||
| 3189 | if (!InjectInvariantConditions) | ||||||
| 3190 | return false; | ||||||
| 3191 | |||||||
| 3192 | if (!DT.isReachableFromEntry(L.getHeader())) | ||||||
| 3193 | return false; | ||||||
| 3194 | auto *Latch = L.getLoopLatch(); | ||||||
| 3195 | // Need to have a single latch and a preheader. | ||||||
| 3196 | if (!Latch) | ||||||
| 3197 | return false; | ||||||
| 3198 | assert(L.getLoopPreheader() && "Must have a preheader!")(static_cast <bool> (L.getLoopPreheader() && "Must have a preheader!" ) ? void (0) : __assert_fail ("L.getLoopPreheader() && \"Must have a preheader!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3198, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3199 | |||||||
| 3200 | DenseMap<Value *, SmallVector<CompareDesc, 4> > CandidatesULT; | ||||||
| 3201 | // Traverse the conditions that dominate latch (and therefore dominate each | ||||||
| 3202 | // other). | ||||||
| 3203 | for (auto *DTN = DT.getNode(Latch); L.contains(DTN->getBlock()); | ||||||
| 3204 | DTN = DTN->getIDom()) { | ||||||
| 3205 | ICmpInst::Predicate Pred; | ||||||
| 3206 | Value *LHS = nullptr, *RHS = nullptr; | ||||||
| 3207 | BasicBlock *IfTrue = nullptr, *IfFalse = nullptr; | ||||||
| 3208 | auto *BB = DTN->getBlock(); | ||||||
| 3209 | // Ignore inner loops. | ||||||
| 3210 | if (LI.getLoopFor(BB) != &L) | ||||||
| 3211 | continue; | ||||||
| 3212 | auto *Term = BB->getTerminator(); | ||||||
| 3213 | if (!match(Term, m_Br(m_ICmp(Pred, m_Value(LHS), m_Value(RHS)), | ||||||
| 3214 | m_BasicBlock(IfTrue), m_BasicBlock(IfFalse)))) | ||||||
| 3215 | continue; | ||||||
| 3216 | if (!LHS->getType()->isIntegerTy()) | ||||||
| 3217 | continue; | ||||||
| 3218 | canonicalizeForInvariantConditionInjection(Pred, LHS, RHS, IfTrue, IfFalse, | ||||||
| 3219 | L); | ||||||
| 3220 | if (!shouldTryInjectInvariantCondition(Pred, LHS, RHS, IfTrue, IfFalse, L)) | ||||||
| 3221 | continue; | ||||||
| 3222 | if (!shouldTryInjectBasingOnMetadata(cast<BranchInst>(Term), IfTrue)) | ||||||
| 3223 | continue; | ||||||
| 3224 | // Strip ZEXT for unsigned predicate. | ||||||
| 3225 | // TODO: once signed predicates are supported, also strip SEXT. | ||||||
| 3226 | CompareDesc Desc(cast<BranchInst>(Term), RHS, IfTrue); | ||||||
| 3227 | while (auto *Zext = dyn_cast<ZExtInst>(LHS)) | ||||||
| 3228 | LHS = Zext->getOperand(0); | ||||||
| 3229 | CandidatesULT[LHS].push_back(Desc); | ||||||
| 3230 | } | ||||||
| 3231 | |||||||
| 3232 | bool Found = false; | ||||||
| 3233 | for (auto &It : CandidatesULT) | ||||||
| 3234 | Found |= insertCandidatesWithPendingInjections( | ||||||
| 3235 | UnswitchCandidates, L, ICmpInst::ICMP_ULT, It.second, DT); | ||||||
| 3236 | return Found; | ||||||
| 3237 | } | ||||||
| 3238 | |||||||
| 3239 | static bool isSafeForNoNTrivialUnswitching(Loop &L, LoopInfo &LI) { | ||||||
| 3240 | if (!L.isSafeToClone()) | ||||||
| 3241 | return false; | ||||||
| 3242 | for (auto *BB : L.blocks()) | ||||||
| 3243 | for (auto &I : *BB) { | ||||||
| 3244 | if (I.getType()->isTokenTy() && I.isUsedOutsideOfBlock(BB)) | ||||||
| 3245 | return false; | ||||||
| 3246 | if (auto *CB = dyn_cast<CallBase>(&I)) { | ||||||
| 3247 | assert(!CB->cannotDuplicate() && "Checked by L.isSafeToClone().")(static_cast <bool> (!CB->cannotDuplicate() && "Checked by L.isSafeToClone().") ? void (0) : __assert_fail ( "!CB->cannotDuplicate() && \"Checked by L.isSafeToClone().\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3247, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3248 | if (CB->isConvergent()) | ||||||
| 3249 | return false; | ||||||
| 3250 | } | ||||||
| 3251 | } | ||||||
| 3252 | |||||||
| 3253 | // Check if there are irreducible CFG cycles in this loop. If so, we cannot | ||||||
| 3254 | // easily unswitch non-trivial edges out of the loop. Doing so might turn the | ||||||
| 3255 | // irreducible control flow into reducible control flow and introduce new | ||||||
| 3256 | // loops "out of thin air". If we ever discover important use cases for doing | ||||||
| 3257 | // this, we can add support to loop unswitch, but it is a lot of complexity | ||||||
| 3258 | // for what seems little or no real world benefit. | ||||||
| 3259 | LoopBlocksRPO RPOT(&L); | ||||||
| 3260 | RPOT.perform(&LI); | ||||||
| 3261 | if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI)) | ||||||
| 3262 | return false; | ||||||
| 3263 | |||||||
| 3264 | SmallVector<BasicBlock *, 4> ExitBlocks; | ||||||
| 3265 | L.getUniqueExitBlocks(ExitBlocks); | ||||||
| 3266 | // We cannot unswitch if exit blocks contain a cleanuppad/catchswitch | ||||||
| 3267 | // instruction as we don't know how to split those exit blocks. | ||||||
| 3268 | // FIXME: We should teach SplitBlock to handle this and remove this | ||||||
| 3269 | // restriction. | ||||||
| 3270 | for (auto *ExitBB : ExitBlocks) { | ||||||
| 3271 | auto *I = ExitBB->getFirstNonPHI(); | ||||||
| 3272 | if (isa<CleanupPadInst>(I) || isa<CatchSwitchInst>(I)) { | ||||||
| 3273 | LLVM_DEBUG(dbgs() << "Cannot unswitch because of cleanuppad/catchswitch "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Cannot unswitch because of cleanuppad/catchswitch " "in exit block\n"; } } while (false) | ||||||
| 3274 | "in exit block\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Cannot unswitch because of cleanuppad/catchswitch " "in exit block\n"; } } while (false); | ||||||
| 3275 | return false; | ||||||
| 3276 | } | ||||||
| 3277 | } | ||||||
| 3278 | |||||||
| 3279 | return true; | ||||||
| 3280 | } | ||||||
| 3281 | |||||||
| 3282 | static NonTrivialUnswitchCandidate findBestNonTrivialUnswitchCandidate( | ||||||
| 3283 | ArrayRef<NonTrivialUnswitchCandidate> UnswitchCandidates, const Loop &L, | ||||||
| 3284 | const DominatorTree &DT, const LoopInfo &LI, AssumptionCache &AC, | ||||||
| 3285 | const TargetTransformInfo &TTI, const IVConditionInfo &PartialIVInfo) { | ||||||
| 3286 | // Given that unswitching these terminators will require duplicating parts of | ||||||
| 3287 | // the loop, so we need to be able to model that cost. Compute the ephemeral | ||||||
| 3288 | // values and set up a data structure to hold per-BB costs. We cache each | ||||||
| 3289 | // block's cost so that we don't recompute this when considering different | ||||||
| 3290 | // subsets of the loop for duplication during unswitching. | ||||||
| 3291 | SmallPtrSet<const Value *, 4> EphValues; | ||||||
| 3292 | CodeMetrics::collectEphemeralValues(&L, &AC, EphValues); | ||||||
| 3293 | SmallDenseMap<BasicBlock *, InstructionCost, 4> BBCostMap; | ||||||
| 3294 | |||||||
| 3295 | // Compute the cost of each block, as well as the total loop cost. Also, bail | ||||||
| 3296 | // out if we see instructions which are incompatible with loop unswitching | ||||||
| 3297 | // (convergent, noduplicate, or cross-basic-block tokens). | ||||||
| 3298 | // FIXME: We might be able to safely handle some of these in non-duplicated | ||||||
| 3299 | // regions. | ||||||
| 3300 | TargetTransformInfo::TargetCostKind CostKind = | ||||||
| 3301 | L.getHeader()->getParent()->hasMinSize() | ||||||
| 3302 | ? TargetTransformInfo::TCK_CodeSize | ||||||
| 3303 | : TargetTransformInfo::TCK_SizeAndLatency; | ||||||
| 3304 | InstructionCost LoopCost = 0; | ||||||
| 3305 | for (auto *BB : L.blocks()) { | ||||||
| 3306 | InstructionCost Cost = 0; | ||||||
| 3307 | for (auto &I : *BB) { | ||||||
| 3308 | if (EphValues.count(&I)) | ||||||
| 3309 | continue; | ||||||
| 3310 | Cost += TTI.getInstructionCost(&I, CostKind); | ||||||
| 3311 | } | ||||||
| 3312 | assert(Cost >= 0 && "Must not have negative costs!")(static_cast <bool> (Cost >= 0 && "Must not have negative costs!" ) ? void (0) : __assert_fail ("Cost >= 0 && \"Must not have negative costs!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3312, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3313 | LoopCost += Cost; | ||||||
| 3314 | assert(LoopCost >= 0 && "Must not have negative loop costs!")(static_cast <bool> (LoopCost >= 0 && "Must not have negative loop costs!" ) ? void (0) : __assert_fail ("LoopCost >= 0 && \"Must not have negative loop costs!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3314, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3315 | BBCostMap[BB] = Cost; | ||||||
| 3316 | } | ||||||
| 3317 | LLVM_DEBUG(dbgs() << " Total loop cost: " << LoopCost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Total loop cost: " << LoopCost << "\n"; } } while (false); | ||||||
| 3318 | |||||||
| 3319 | // Now we find the best candidate by searching for the one with the following | ||||||
| 3320 | // properties in order: | ||||||
| 3321 | // | ||||||
| 3322 | // 1) An unswitching cost below the threshold | ||||||
| 3323 | // 2) The smallest number of duplicated unswitch candidates (to avoid | ||||||
| 3324 | // creating redundant subsequent unswitching) | ||||||
| 3325 | // 3) The smallest cost after unswitching. | ||||||
| 3326 | // | ||||||
| 3327 | // We prioritize reducing fanout of unswitch candidates provided the cost | ||||||
| 3328 | // remains below the threshold because this has a multiplicative effect. | ||||||
| 3329 | // | ||||||
| 3330 | // This requires memoizing each dominator subtree to avoid redundant work. | ||||||
| 3331 | // | ||||||
| 3332 | // FIXME: Need to actually do the number of candidates part above. | ||||||
| 3333 | SmallDenseMap<DomTreeNode *, InstructionCost, 4> DTCostMap; | ||||||
| 3334 | // Given a terminator which might be unswitched, computes the non-duplicated | ||||||
| 3335 | // cost for that terminator. | ||||||
| 3336 | auto ComputeUnswitchedCost = [&](Instruction &TI, | ||||||
| 3337 | bool FullUnswitch) -> InstructionCost { | ||||||
| 3338 | BasicBlock &BB = *TI.getParent(); | ||||||
| 3339 | SmallPtrSet<BasicBlock *, 4> Visited; | ||||||
| 3340 | |||||||
| 3341 | InstructionCost Cost = 0; | ||||||
| 3342 | for (BasicBlock *SuccBB : successors(&BB)) { | ||||||
| 3343 | // Don't count successors more than once. | ||||||
| 3344 | if (!Visited.insert(SuccBB).second) | ||||||
| 3345 | continue; | ||||||
| 3346 | |||||||
| 3347 | // If this is a partial unswitch candidate, then it must be a conditional | ||||||
| 3348 | // branch with a condition of either `or`, `and`, their corresponding | ||||||
| 3349 | // select forms or partially invariant instructions. In that case, one of | ||||||
| 3350 | // the successors is necessarily duplicated, so don't even try to remove | ||||||
| 3351 | // its cost. | ||||||
| 3352 | if (!FullUnswitch
| ||||||
| 3353 | auto &BI = cast<BranchInst>(TI); | ||||||
| 3354 | Value *Cond = skipTrivialSelect(BI.getCondition()); | ||||||
| 3355 | if (match(Cond, m_LogicalAnd())) { | ||||||
| 3356 | if (SuccBB == BI.getSuccessor(1)) | ||||||
| 3357 | continue; | ||||||
| 3358 | } else if (match(Cond, m_LogicalOr())) { | ||||||
| 3359 | if (SuccBB == BI.getSuccessor(0)) | ||||||
| 3360 | continue; | ||||||
| 3361 | } else if ((PartialIVInfo.KnownValue->isOneValue() && | ||||||
| |||||||
| 3362 | SuccBB == BI.getSuccessor(0)) || | ||||||
| 3363 | (!PartialIVInfo.KnownValue->isOneValue() && | ||||||
| 3364 | SuccBB == BI.getSuccessor(1))) | ||||||
| 3365 | continue; | ||||||
| 3366 | } | ||||||
| 3367 | |||||||
| 3368 | // This successor's domtree will not need to be duplicated after | ||||||
| 3369 | // unswitching if the edge to the successor dominates it (and thus the | ||||||
| 3370 | // entire tree). This essentially means there is no other path into this | ||||||
| 3371 | // subtree and so it will end up live in only one clone of the loop. | ||||||
| 3372 | if (SuccBB->getUniquePredecessor() || | ||||||
| 3373 | llvm::all_of(predecessors(SuccBB), [&](BasicBlock *PredBB) { | ||||||
| 3374 | return PredBB == &BB || DT.dominates(SuccBB, PredBB); | ||||||
| 3375 | })) { | ||||||
| 3376 | Cost += computeDomSubtreeCost(*DT[SuccBB], BBCostMap, DTCostMap); | ||||||
| 3377 | assert(Cost <= LoopCost &&(static_cast <bool> (Cost <= LoopCost && "Non-duplicated cost should never exceed total loop cost!" ) ? void (0) : __assert_fail ("Cost <= LoopCost && \"Non-duplicated cost should never exceed total loop cost!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3378, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3378 | "Non-duplicated cost should never exceed total loop cost!")(static_cast <bool> (Cost <= LoopCost && "Non-duplicated cost should never exceed total loop cost!" ) ? void (0) : __assert_fail ("Cost <= LoopCost && \"Non-duplicated cost should never exceed total loop cost!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3378, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3379 | } | ||||||
| 3380 | } | ||||||
| 3381 | |||||||
| 3382 | // Now scale the cost by the number of unique successors minus one. We | ||||||
| 3383 | // subtract one because there is already at least one copy of the entire | ||||||
| 3384 | // loop. This is computing the new cost of unswitching a condition. | ||||||
| 3385 | // Note that guards always have 2 unique successors that are implicit and | ||||||
| 3386 | // will be materialized if we decide to unswitch it. | ||||||
| 3387 | int SuccessorsCount = | ||||||
| 3388 | isGuard(&TI) || isa<SelectInst>(TI) ? 2 : Visited.size(); | ||||||
| 3389 | assert(SuccessorsCount > 1 &&(static_cast <bool> (SuccessorsCount > 1 && "Cannot unswitch a condition without multiple distinct successors!" ) ? void (0) : __assert_fail ("SuccessorsCount > 1 && \"Cannot unswitch a condition without multiple distinct successors!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3390, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3390 | "Cannot unswitch a condition without multiple distinct successors!")(static_cast <bool> (SuccessorsCount > 1 && "Cannot unswitch a condition without multiple distinct successors!" ) ? void (0) : __assert_fail ("SuccessorsCount > 1 && \"Cannot unswitch a condition without multiple distinct successors!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3390, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3391 | return (LoopCost - Cost) * (SuccessorsCount - 1); | ||||||
| 3392 | }; | ||||||
| 3393 | |||||||
| 3394 | std::optional<NonTrivialUnswitchCandidate> Best; | ||||||
| 3395 | for (auto &Candidate : UnswitchCandidates) { | ||||||
| 3396 | Instruction &TI = *Candidate.TI; | ||||||
| 3397 | ArrayRef<Value *> Invariants = Candidate.Invariants; | ||||||
| 3398 | BranchInst *BI = dyn_cast<BranchInst>(&TI); | ||||||
| 3399 | bool FullUnswitch = | ||||||
| 3400 | !BI
| ||||||
| 3401 | (Invariants.size() == 1 && | ||||||
| 3402 | Invariants[0] == skipTrivialSelect(BI->getCondition())); | ||||||
| 3403 | InstructionCost CandidateCost = ComputeUnswitchedCost(TI, FullUnswitch); | ||||||
| 3404 | // Calculate cost multiplier which is a tool to limit potentially | ||||||
| 3405 | // exponential behavior of loop-unswitch. | ||||||
| 3406 | if (EnableUnswitchCostMultiplier) { | ||||||
| 3407 | int CostMultiplier = | ||||||
| 3408 | CalculateUnswitchCostMultiplier(TI, L, LI, DT, UnswitchCandidates); | ||||||
| 3409 | assert((static_cast <bool> ((CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && "cost multiplier needs to be in the range of 1..UnswitchThreshold" ) ? void (0) : __assert_fail ("(CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && \"cost multiplier needs to be in the range of 1..UnswitchThreshold\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3411, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3410 | (CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) &&(static_cast <bool> ((CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && "cost multiplier needs to be in the range of 1..UnswitchThreshold" ) ? void (0) : __assert_fail ("(CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && \"cost multiplier needs to be in the range of 1..UnswitchThreshold\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3411, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3411 | "cost multiplier needs to be in the range of 1..UnswitchThreshold")(static_cast <bool> ((CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && "cost multiplier needs to be in the range of 1..UnswitchThreshold" ) ? void (0) : __assert_fail ("(CostMultiplier > 0 && CostMultiplier <= UnswitchThreshold) && \"cost multiplier needs to be in the range of 1..UnswitchThreshold\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3411, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3412 | CandidateCost *= CostMultiplier; | ||||||
| 3413 | LLVM_DEBUG(dbgs() << " Computed cost of " << CandidateCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed cost of " << CandidateCost << " (multiplier: " << CostMultiplier << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 3414 | << " (multiplier: " << CostMultiplier << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed cost of " << CandidateCost << " (multiplier: " << CostMultiplier << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 3415 | << " for unswitch candidate: " << TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed cost of " << CandidateCost << " (multiplier: " << CostMultiplier << ")" << " for unswitch candidate: " << TI << "\n"; } } while (false); | ||||||
| 3416 | } else { | ||||||
| 3417 | LLVM_DEBUG(dbgs() << " Computed cost of " << CandidateCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed cost of " << CandidateCost << " for unswitch candidate: " << TI << "\n"; } } while (false) | ||||||
| 3418 | << " for unswitch candidate: " << TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Computed cost of " << CandidateCost << " for unswitch candidate: " << TI << "\n"; } } while (false); | ||||||
| 3419 | } | ||||||
| 3420 | |||||||
| 3421 | if (!Best || CandidateCost < Best->Cost) { | ||||||
| 3422 | Best = Candidate; | ||||||
| 3423 | Best->Cost = CandidateCost; | ||||||
| 3424 | } | ||||||
| 3425 | } | ||||||
| 3426 | assert(Best && "Must be!")(static_cast <bool> (Best && "Must be!") ? void (0) : __assert_fail ("Best && \"Must be!\"", "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp" , 3426, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3427 | return *Best; | ||||||
| 3428 | } | ||||||
| 3429 | |||||||
| 3430 | // Insert a freeze on an unswitched branch if all is true: | ||||||
| 3431 | // 1. freeze-loop-unswitch-cond option is true | ||||||
| 3432 | // 2. The branch may not execute in the loop pre-transformation. If a branch may | ||||||
| 3433 | // not execute and could cause UB, it would always cause UB if it is hoisted outside | ||||||
| 3434 | // of the loop. Insert a freeze to prevent this case. | ||||||
| 3435 | // 3. The branch condition may be poison or undef | ||||||
| 3436 | static bool shouldInsertFreeze(Loop &L, Instruction &TI, DominatorTree &DT, | ||||||
| 3437 | AssumptionCache &AC) { | ||||||
| 3438 | assert(isa<BranchInst>(TI) || isa<SwitchInst>(TI))(static_cast <bool> (isa<BranchInst>(TI) || isa< SwitchInst>(TI)) ? void (0) : __assert_fail ("isa<BranchInst>(TI) || isa<SwitchInst>(TI)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3438, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3439 | if (!FreezeLoopUnswitchCond) | ||||||
| 3440 | return false; | ||||||
| 3441 | |||||||
| 3442 | ICFLoopSafetyInfo SafetyInfo; | ||||||
| 3443 | SafetyInfo.computeLoopSafetyInfo(&L); | ||||||
| 3444 | if (SafetyInfo.isGuaranteedToExecute(TI, &DT, &L)) | ||||||
| 3445 | return false; | ||||||
| 3446 | |||||||
| 3447 | Value *Cond; | ||||||
| 3448 | if (BranchInst *BI = dyn_cast<BranchInst>(&TI)) | ||||||
| 3449 | Cond = skipTrivialSelect(BI->getCondition()); | ||||||
| 3450 | else | ||||||
| 3451 | Cond = skipTrivialSelect(cast<SwitchInst>(&TI)->getCondition()); | ||||||
| 3452 | return !isGuaranteedNotToBeUndefOrPoison( | ||||||
| 3453 | Cond, &AC, L.getLoopPreheader()->getTerminator(), &DT); | ||||||
| 3454 | } | ||||||
| 3455 | |||||||
| 3456 | static bool unswitchBestCondition( | ||||||
| 3457 | Loop &L, DominatorTree &DT, LoopInfo &LI, AssumptionCache &AC, | ||||||
| 3458 | AAResults &AA, TargetTransformInfo &TTI, | ||||||
| 3459 | function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB, | ||||||
| 3460 | ScalarEvolution *SE, MemorySSAUpdater *MSSAU, | ||||||
| 3461 | function_ref<void(Loop &, StringRef)> DestroyLoopCB) { | ||||||
| 3462 | // Collect all invariant conditions within this loop (as opposed to an inner | ||||||
| 3463 | // loop which would be handled when visiting that inner loop). | ||||||
| 3464 | SmallVector<NonTrivialUnswitchCandidate, 4> UnswitchCandidates; | ||||||
| 3465 | IVConditionInfo PartialIVInfo; | ||||||
| 3466 | Instruction *PartialIVCondBranch = nullptr; | ||||||
| 3467 | collectUnswitchCandidates(UnswitchCandidates, PartialIVInfo, | ||||||
| 3468 | PartialIVCondBranch, L, LI, AA, MSSAU); | ||||||
| 3469 | collectUnswitchCandidatesWithInjections(UnswitchCandidates, PartialIVInfo, | ||||||
| 3470 | PartialIVCondBranch, L, DT, LI, AA, | ||||||
| 3471 | MSSAU); | ||||||
| 3472 | // If we didn't find any candidates, we're done. | ||||||
| 3473 | if (UnswitchCandidates.empty()) | ||||||
| 3474 | return false; | ||||||
| 3475 | |||||||
| 3476 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Considering " << UnswitchCandidates.size() << " non-trivial loop invariant conditions for unswitching.\n" ; } } while (false) | ||||||
| 3477 | dbgs() << "Considering " << UnswitchCandidates.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Considering " << UnswitchCandidates.size() << " non-trivial loop invariant conditions for unswitching.\n" ; } } while (false) | ||||||
| 3478 | << " non-trivial loop invariant conditions for unswitching.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Considering " << UnswitchCandidates.size() << " non-trivial loop invariant conditions for unswitching.\n" ; } } while (false); | ||||||
| 3479 | |||||||
| 3480 | NonTrivialUnswitchCandidate Best = findBestNonTrivialUnswitchCandidate( | ||||||
| 3481 | UnswitchCandidates, L, DT, LI, AC, TTI, PartialIVInfo); | ||||||
| 3482 | |||||||
| 3483 | assert(Best.TI && "Failed to find loop unswitch candidate")(static_cast <bool> (Best.TI && "Failed to find loop unswitch candidate" ) ? void (0) : __assert_fail ("Best.TI && \"Failed to find loop unswitch candidate\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3483, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3484 | assert(Best.Cost && "Failed to compute cost")(static_cast <bool> (Best.Cost && "Failed to compute cost" ) ? void (0) : __assert_fail ("Best.Cost && \"Failed to compute cost\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3484, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3485 | |||||||
| 3486 | if (*Best.Cost >= UnswitchThreshold) { | ||||||
| 3487 | LLVM_DEBUG(dbgs() << "Cannot unswitch, lowest cost found: " << *Best.Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Cannot unswitch, lowest cost found: " << *Best.Cost << "\n"; } } while (false) | ||||||
| 3488 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Cannot unswitch, lowest cost found: " << *Best.Cost << "\n"; } } while (false); | ||||||
| 3489 | return false; | ||||||
| 3490 | } | ||||||
| 3491 | |||||||
| 3492 | if (Best.hasPendingInjection()) | ||||||
| 3493 | Best = injectPendingInvariantConditions(Best, L, DT, LI, AC, MSSAU); | ||||||
| 3494 | assert(!Best.hasPendingInjection() &&(static_cast <bool> (!Best.hasPendingInjection() && "All injections should have been done by now!") ? void (0) : __assert_fail ("!Best.hasPendingInjection() && \"All injections should have been done by now!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3495, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3495 | "All injections should have been done by now!")(static_cast <bool> (!Best.hasPendingInjection() && "All injections should have been done by now!") ? void (0) : __assert_fail ("!Best.hasPendingInjection() && \"All injections should have been done by now!\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3495, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3496 | |||||||
| 3497 | if (Best.TI != PartialIVCondBranch) | ||||||
| 3498 | PartialIVInfo.InstToDuplicate.clear(); | ||||||
| 3499 | |||||||
| 3500 | bool InsertFreeze; | ||||||
| 3501 | if (auto *SI = dyn_cast<SelectInst>(Best.TI)) { | ||||||
| 3502 | // If the best candidate is a select, turn it into a branch. Select | ||||||
| 3503 | // instructions with a poison conditional do not propagate poison, but | ||||||
| 3504 | // branching on poison causes UB. Insert a freeze on the select | ||||||
| 3505 | // conditional to prevent UB after turning the select into a branch. | ||||||
| 3506 | InsertFreeze = !isGuaranteedNotToBeUndefOrPoison( | ||||||
| 3507 | SI->getCondition(), &AC, L.getLoopPreheader()->getTerminator(), &DT); | ||||||
| 3508 | Best.TI = turnSelectIntoBranch(SI, DT, LI, MSSAU, &AC); | ||||||
| 3509 | } else { | ||||||
| 3510 | // If the best candidate is a guard, turn it into a branch. | ||||||
| 3511 | if (isGuard(Best.TI)) | ||||||
| 3512 | Best.TI = | ||||||
| 3513 | turnGuardIntoBranch(cast<IntrinsicInst>(Best.TI), L, DT, LI, MSSAU); | ||||||
| 3514 | InsertFreeze = shouldInsertFreeze(L, *Best.TI, DT, AC); | ||||||
| 3515 | } | ||||||
| 3516 | |||||||
| 3517 | LLVM_DEBUG(dbgs() << " Unswitching non-trivial (cost = " << Best.Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Unswitching non-trivial (cost = " << Best.Cost << ") terminator: " << *Best. TI << "\n"; } } while (false) | ||||||
| 3518 | << ") terminator: " << *Best.TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Unswitching non-trivial (cost = " << Best.Cost << ") terminator: " << *Best. TI << "\n"; } } while (false); | ||||||
| 3519 | unswitchNontrivialInvariants(L, *Best.TI, Best.Invariants, PartialIVInfo, DT, | ||||||
| 3520 | LI, AC, UnswitchCB, SE, MSSAU, DestroyLoopCB, | ||||||
| 3521 | InsertFreeze); | ||||||
| 3522 | return true; | ||||||
| 3523 | } | ||||||
| 3524 | |||||||
| 3525 | /// Unswitch control flow predicated on loop invariant conditions. | ||||||
| 3526 | /// | ||||||
| 3527 | /// This first hoists all branches or switches which are trivial (IE, do not | ||||||
| 3528 | /// require duplicating any part of the loop) out of the loop body. It then | ||||||
| 3529 | /// looks at other loop invariant control flows and tries to unswitch those as | ||||||
| 3530 | /// well by cloning the loop if the result is small enough. | ||||||
| 3531 | /// | ||||||
| 3532 | /// The `DT`, `LI`, `AC`, `AA`, `TTI` parameters are required analyses that are | ||||||
| 3533 | /// also updated based on the unswitch. The `MSSA` analysis is also updated if | ||||||
| 3534 | /// valid (i.e. its use is enabled). | ||||||
| 3535 | /// | ||||||
| 3536 | /// If either `NonTrivial` is true or the flag `EnableNonTrivialUnswitch` is | ||||||
| 3537 | /// true, we will attempt to do non-trivial unswitching as well as trivial | ||||||
| 3538 | /// unswitching. | ||||||
| 3539 | /// | ||||||
| 3540 | /// The `UnswitchCB` callback provided will be run after unswitching is | ||||||
| 3541 | /// complete, with the first parameter set to `true` if the provided loop | ||||||
| 3542 | /// remains a loop, and a list of new sibling loops created. | ||||||
| 3543 | /// | ||||||
| 3544 | /// If `SE` is non-null, we will update that analysis based on the unswitching | ||||||
| 3545 | /// done. | ||||||
| 3546 | static bool | ||||||
| 3547 | unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI, AssumptionCache &AC, | ||||||
| 3548 | AAResults &AA, TargetTransformInfo &TTI, bool Trivial, | ||||||
| 3549 | bool NonTrivial, | ||||||
| 3550 | function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB, | ||||||
| 3551 | ScalarEvolution *SE, MemorySSAUpdater *MSSAU, | ||||||
| 3552 | ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, | ||||||
| 3553 | function_ref<void(Loop &, StringRef)> DestroyLoopCB) { | ||||||
| 3554 | assert(L.isRecursivelyLCSSAForm(DT, LI) &&(static_cast <bool> (L.isRecursivelyLCSSAForm(DT, LI) && "Loops must be in LCSSA form before unswitching.") ? void (0 ) : __assert_fail ("L.isRecursivelyLCSSAForm(DT, LI) && \"Loops must be in LCSSA form before unswitching.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3555, __extension__ __PRETTY_FUNCTION__)) | ||||||
| 3555 | "Loops must be in LCSSA form before unswitching.")(static_cast <bool> (L.isRecursivelyLCSSAForm(DT, LI) && "Loops must be in LCSSA form before unswitching.") ? void (0 ) : __assert_fail ("L.isRecursivelyLCSSAForm(DT, LI) && \"Loops must be in LCSSA form before unswitching.\"" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3555, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3556 | |||||||
| 3557 | // Must be in loop simplified form: we need a preheader and dedicated exits. | ||||||
| 3558 | if (!L.isLoopSimplifyForm()) | ||||||
| 3559 | return false; | ||||||
| 3560 | |||||||
| 3561 | // Try trivial unswitch first before loop over other basic blocks in the loop. | ||||||
| 3562 | if (Trivial && unswitchAllTrivialConditions(L, DT, LI, SE, MSSAU)) { | ||||||
| 3563 | // If we unswitched successfully we will want to clean up the loop before | ||||||
| 3564 | // processing it further so just mark it as unswitched and return. | ||||||
| 3565 | UnswitchCB(/*CurrentLoopValid*/ true, false, {}); | ||||||
| 3566 | return true; | ||||||
| 3567 | } | ||||||
| 3568 | |||||||
| 3569 | // Check whether we should continue with non-trivial conditions. | ||||||
| 3570 | // EnableNonTrivialUnswitch: Global variable that forces non-trivial | ||||||
| 3571 | // unswitching for testing and debugging. | ||||||
| 3572 | // NonTrivial: Parameter that enables non-trivial unswitching for this | ||||||
| 3573 | // invocation of the transform. But this should be allowed only | ||||||
| 3574 | // for targets without branch divergence. | ||||||
| 3575 | // | ||||||
| 3576 | // FIXME: If divergence analysis becomes available to a loop | ||||||
| 3577 | // transform, we should allow unswitching for non-trivial uniform | ||||||
| 3578 | // branches even on targets that have divergence. | ||||||
| 3579 | // https://bugs.llvm.org/show_bug.cgi?id=48819 | ||||||
| 3580 | bool ContinueWithNonTrivial = | ||||||
| 3581 | EnableNonTrivialUnswitch || (NonTrivial && !TTI.hasBranchDivergence()); | ||||||
| 3582 | if (!ContinueWithNonTrivial
| ||||||
| 3583 | return false; | ||||||
| 3584 | |||||||
| 3585 | // Skip non-trivial unswitching for optsize functions. | ||||||
| 3586 | if (L.getHeader()->getParent()->hasOptSize()) | ||||||
| 3587 | return false; | ||||||
| 3588 | |||||||
| 3589 | // Returns true if Loop L's loop nest is cold, i.e. if the headers of L, | ||||||
| 3590 | // of the loops L is nested in, and of the loops nested in L are all cold. | ||||||
| 3591 | auto IsLoopNestCold = [&](const Loop *L) { | ||||||
| 3592 | // Check L and all of its parent loops. | ||||||
| 3593 | auto *Parent = L; | ||||||
| 3594 | while (Parent) { | ||||||
| 3595 | if (!PSI->isColdBlock(Parent->getHeader(), BFI)) | ||||||
| 3596 | return false; | ||||||
| 3597 | Parent = Parent->getParentLoop(); | ||||||
| 3598 | } | ||||||
| 3599 | // Next check all loops nested within L. | ||||||
| 3600 | SmallVector<const Loop *, 4> Worklist; | ||||||
| 3601 | Worklist.insert(Worklist.end(), L->getSubLoops().begin(), | ||||||
| 3602 | L->getSubLoops().end()); | ||||||
| 3603 | while (!Worklist.empty()) { | ||||||
| 3604 | auto *CurLoop = Worklist.pop_back_val(); | ||||||
| 3605 | if (!PSI->isColdBlock(CurLoop->getHeader(), BFI)) | ||||||
| 3606 | return false; | ||||||
| 3607 | Worklist.insert(Worklist.end(), CurLoop->getSubLoops().begin(), | ||||||
| 3608 | CurLoop->getSubLoops().end()); | ||||||
| 3609 | } | ||||||
| 3610 | return true; | ||||||
| 3611 | }; | ||||||
| 3612 | |||||||
| 3613 | // Skip cold loops in cold loop nests, as unswitching them brings little | ||||||
| 3614 | // benefit but increases the code size | ||||||
| 3615 | if (PSI
| ||||||
| 3616 | LLVM_DEBUG(dbgs() << " Skip cold loop: " << L << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << " Skip cold loop: " << L << "\n"; } } while (false); | ||||||
| 3617 | return false; | ||||||
| 3618 | } | ||||||
| 3619 | |||||||
| 3620 | // Perform legality checks. | ||||||
| 3621 | if (!isSafeForNoNTrivialUnswitching(L, LI)) | ||||||
| 3622 | return false; | ||||||
| 3623 | |||||||
| 3624 | // For non-trivial unswitching, because it often creates new loops, we rely on | ||||||
| 3625 | // the pass manager to iterate on the loops rather than trying to immediately | ||||||
| 3626 | // reach a fixed point. There is no substantial advantage to iterating | ||||||
| 3627 | // internally, and if any of the new loops are simplified enough to contain | ||||||
| 3628 | // trivial unswitching we want to prefer those. | ||||||
| 3629 | |||||||
| 3630 | // Try to unswitch the best invariant condition. We prefer this full unswitch to | ||||||
| 3631 | // a partial unswitch when possible below the threshold. | ||||||
| 3632 | if (unswitchBestCondition(L, DT, LI, AC, AA, TTI, UnswitchCB, SE, MSSAU, | ||||||
| 3633 | DestroyLoopCB)) | ||||||
| 3634 | return true; | ||||||
| 3635 | |||||||
| 3636 | // No other opportunities to unswitch. | ||||||
| 3637 | return false; | ||||||
| 3638 | } | ||||||
| 3639 | |||||||
| 3640 | PreservedAnalyses SimpleLoopUnswitchPass::run(Loop &L, LoopAnalysisManager &AM, | ||||||
| 3641 | LoopStandardAnalysisResults &AR, | ||||||
| 3642 | LPMUpdater &U) { | ||||||
| 3643 | Function &F = *L.getHeader()->getParent(); | ||||||
| 3644 | (void)F; | ||||||
| 3645 | ProfileSummaryInfo *PSI = nullptr; | ||||||
| 3646 | if (auto OuterProxy
| ||||||
| 3647 | AM.getResult<FunctionAnalysisManagerLoopProxy>(L, AR) | ||||||
| 3648 | .getCachedResult<ModuleAnalysisManagerFunctionProxy>(F)) | ||||||
| 3649 | PSI = OuterProxy->getCachedResult<ProfileSummaryAnalysis>(*F.getParent()); | ||||||
| 3650 | LLVM_DEBUG(dbgs() << "Unswitching loop in " << F.getName() << ": " << Ldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Unswitching loop in " << F.getName() << ": " << L << "\n"; } } while (false) | ||||||
| |||||||
| 3651 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Unswitching loop in " << F.getName() << ": " << L << "\n"; } } while (false); | ||||||
| 3652 | |||||||
| 3653 | // Save the current loop name in a variable so that we can report it even | ||||||
| 3654 | // after it has been deleted. | ||||||
| 3655 | std::string LoopName = std::string(L.getName()); | ||||||
| 3656 | |||||||
| 3657 | auto UnswitchCB = [&L, &U, &LoopName](bool CurrentLoopValid, | ||||||
| 3658 | bool PartiallyInvariant, | ||||||
| 3659 | ArrayRef<Loop *> NewLoops) { | ||||||
| 3660 | // If we did a non-trivial unswitch, we have added new (cloned) loops. | ||||||
| 3661 | if (!NewLoops.empty()) | ||||||
| 3662 | U.addSiblingLoops(NewLoops); | ||||||
| 3663 | |||||||
| 3664 | // If the current loop remains valid, we should revisit it to catch any | ||||||
| 3665 | // other unswitch opportunities. Otherwise, we need to mark it as deleted. | ||||||
| 3666 | if (CurrentLoopValid) { | ||||||
| 3667 | if (PartiallyInvariant) { | ||||||
| 3668 | // Mark the new loop as partially unswitched, to avoid unswitching on | ||||||
| 3669 | // the same condition again. | ||||||
| 3670 | auto &Context = L.getHeader()->getContext(); | ||||||
| 3671 | MDNode *DisableUnswitchMD = MDNode::get( | ||||||
| 3672 | Context, | ||||||
| 3673 | MDString::get(Context, "llvm.loop.unswitch.partial.disable")); | ||||||
| 3674 | MDNode *NewLoopID = makePostTransformationMetadata( | ||||||
| 3675 | Context, L.getLoopID(), {"llvm.loop.unswitch.partial"}, | ||||||
| 3676 | {DisableUnswitchMD}); | ||||||
| 3677 | L.setLoopID(NewLoopID); | ||||||
| 3678 | } else | ||||||
| 3679 | U.revisitCurrentLoop(); | ||||||
| 3680 | } else | ||||||
| 3681 | U.markLoopAsDeleted(L, LoopName); | ||||||
| 3682 | }; | ||||||
| 3683 | |||||||
| 3684 | auto DestroyLoopCB = [&U](Loop &L, StringRef Name) { | ||||||
| 3685 | U.markLoopAsDeleted(L, Name); | ||||||
| 3686 | }; | ||||||
| 3687 | |||||||
| 3688 | std::optional<MemorySSAUpdater> MSSAU; | ||||||
| 3689 | if (AR.MSSA) { | ||||||
| 3690 | MSSAU = MemorySSAUpdater(AR.MSSA); | ||||||
| 3691 | if (VerifyMemorySSA) | ||||||
| 3692 | AR.MSSA->verifyMemorySSA(); | ||||||
| 3693 | } | ||||||
| 3694 | if (!unswitchLoop(L, AR.DT, AR.LI, AR.AC, AR.AA, AR.TTI, Trivial, NonTrivial, | ||||||
| 3695 | UnswitchCB, &AR.SE, MSSAU ? &*MSSAU : nullptr, PSI, AR.BFI, | ||||||
| 3696 | DestroyLoopCB)) | ||||||
| 3697 | return PreservedAnalyses::all(); | ||||||
| 3698 | |||||||
| 3699 | if (AR.MSSA && VerifyMemorySSA) | ||||||
| 3700 | AR.MSSA->verifyMemorySSA(); | ||||||
| 3701 | |||||||
| 3702 | // Historically this pass has had issues with the dominator tree so verify it | ||||||
| 3703 | // in asserts builds. | ||||||
| 3704 | assert(AR.DT.verify(DominatorTree::VerificationLevel::Fast))(static_cast <bool> (AR.DT.verify(DominatorTree::VerificationLevel ::Fast)) ? void (0) : __assert_fail ("AR.DT.verify(DominatorTree::VerificationLevel::Fast)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3704, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3705 | |||||||
| 3706 | auto PA = getLoopPassPreservedAnalyses(); | ||||||
| 3707 | if (AR.MSSA) | ||||||
| 3708 | PA.preserve<MemorySSAAnalysis>(); | ||||||
| 3709 | return PA; | ||||||
| 3710 | } | ||||||
| 3711 | |||||||
| 3712 | void SimpleLoopUnswitchPass::printPipeline( | ||||||
| 3713 | raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) { | ||||||
| 3714 | static_cast<PassInfoMixin<SimpleLoopUnswitchPass> *>(this)->printPipeline( | ||||||
| 3715 | OS, MapClassName2PassName); | ||||||
| 3716 | |||||||
| 3717 | OS << '<'; | ||||||
| 3718 | OS << (NonTrivial ? "" : "no-") << "nontrivial;"; | ||||||
| 3719 | OS << (Trivial ? "" : "no-") << "trivial"; | ||||||
| 3720 | OS << '>'; | ||||||
| 3721 | } | ||||||
| 3722 | |||||||
| 3723 | namespace { | ||||||
| 3724 | |||||||
| 3725 | class SimpleLoopUnswitchLegacyPass : public LoopPass { | ||||||
| 3726 | bool NonTrivial; | ||||||
| 3727 | |||||||
| 3728 | public: | ||||||
| 3729 | static char ID; // Pass ID, replacement for typeid | ||||||
| 3730 | |||||||
| 3731 | explicit SimpleLoopUnswitchLegacyPass(bool NonTrivial = false) | ||||||
| 3732 | : LoopPass(ID), NonTrivial(NonTrivial) { | ||||||
| 3733 | initializeSimpleLoopUnswitchLegacyPassPass( | ||||||
| 3734 | *PassRegistry::getPassRegistry()); | ||||||
| 3735 | } | ||||||
| 3736 | |||||||
| 3737 | bool runOnLoop(Loop *L, LPPassManager &LPM) override; | ||||||
| 3738 | |||||||
| 3739 | void getAnalysisUsage(AnalysisUsage &AU) const override { | ||||||
| 3740 | AU.addRequired<AssumptionCacheTracker>(); | ||||||
| 3741 | AU.addRequired<TargetTransformInfoWrapperPass>(); | ||||||
| 3742 | AU.addRequired<MemorySSAWrapperPass>(); | ||||||
| 3743 | AU.addPreserved<MemorySSAWrapperPass>(); | ||||||
| 3744 | getLoopAnalysisUsage(AU); | ||||||
| 3745 | } | ||||||
| 3746 | }; | ||||||
| 3747 | |||||||
| 3748 | } // end anonymous namespace | ||||||
| 3749 | |||||||
| 3750 | bool SimpleLoopUnswitchLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) { | ||||||
| 3751 | if (skipLoop(L)) | ||||||
| 3752 | return false; | ||||||
| 3753 | |||||||
| 3754 | Function &F = *L->getHeader()->getParent(); | ||||||
| 3755 | |||||||
| 3756 | LLVM_DEBUG(dbgs() << "Unswitching loop in " << F.getName() << ": " << *Ldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Unswitching loop in " << F.getName() << ": " << *L << "\n" ; } } while (false) | ||||||
| 3757 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simple-loop-unswitch")) { dbgs() << "Unswitching loop in " << F.getName() << ": " << *L << "\n" ; } } while (false); | ||||||
| 3758 | auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); | ||||||
| 3759 | auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); | ||||||
| 3760 | auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); | ||||||
| 3761 | auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults(); | ||||||
| 3762 | auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); | ||||||
| 3763 | MemorySSA *MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA(); | ||||||
| 3764 | MemorySSAUpdater MSSAU(MSSA); | ||||||
| 3765 | |||||||
| 3766 | auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>(); | ||||||
| 3767 | auto *SE = SEWP ? &SEWP->getSE() : nullptr; | ||||||
| 3768 | |||||||
| 3769 | auto UnswitchCB = [&L, &LPM](bool CurrentLoopValid, bool PartiallyInvariant, | ||||||
| 3770 | ArrayRef<Loop *> NewLoops) { | ||||||
| 3771 | // If we did a non-trivial unswitch, we have added new (cloned) loops. | ||||||
| 3772 | for (auto *NewL : NewLoops) | ||||||
| 3773 | LPM.addLoop(*NewL); | ||||||
| 3774 | |||||||
| 3775 | // If the current loop remains valid, re-add it to the queue. This is | ||||||
| 3776 | // a little wasteful as we'll finish processing the current loop as well, | ||||||
| 3777 | // but it is the best we can do in the old PM. | ||||||
| 3778 | if (CurrentLoopValid) { | ||||||
| 3779 | // If the current loop has been unswitched using a partially invariant | ||||||
| 3780 | // condition, we should not re-add the current loop to avoid unswitching | ||||||
| 3781 | // on the same condition again. | ||||||
| 3782 | if (!PartiallyInvariant) | ||||||
| 3783 | LPM.addLoop(*L); | ||||||
| 3784 | } else | ||||||
| 3785 | LPM.markLoopAsDeleted(*L); | ||||||
| 3786 | }; | ||||||
| 3787 | |||||||
| 3788 | auto DestroyLoopCB = [&LPM](Loop &L, StringRef /* Name */) { | ||||||
| 3789 | LPM.markLoopAsDeleted(L); | ||||||
| 3790 | }; | ||||||
| 3791 | |||||||
| 3792 | if (VerifyMemorySSA) | ||||||
| 3793 | MSSA->verifyMemorySSA(); | ||||||
| 3794 | bool Changed = | ||||||
| 3795 | unswitchLoop(*L, DT, LI, AC, AA, TTI, true, NonTrivial, UnswitchCB, SE, | ||||||
| 3796 | &MSSAU, nullptr, nullptr, DestroyLoopCB); | ||||||
| 3797 | |||||||
| 3798 | if (VerifyMemorySSA) | ||||||
| 3799 | MSSA->verifyMemorySSA(); | ||||||
| 3800 | |||||||
| 3801 | // Historically this pass has had issues with the dominator tree so verify it | ||||||
| 3802 | // in asserts builds. | ||||||
| 3803 | assert(DT.verify(DominatorTree::VerificationLevel::Fast))(static_cast <bool> (DT.verify(DominatorTree::VerificationLevel ::Fast)) ? void (0) : __assert_fail ("DT.verify(DominatorTree::VerificationLevel::Fast)" , "llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp", 3803, __extension__ __PRETTY_FUNCTION__)); | ||||||
| 3804 | |||||||
| 3805 | return Changed; | ||||||
| 3806 | } | ||||||
| 3807 | |||||||
| 3808 | char SimpleLoopUnswitchLegacyPass::ID = 0; | ||||||
| 3809 | INITIALIZE_PASS_BEGIN(SimpleLoopUnswitchLegacyPass, "simple-loop-unswitch",static void *initializeSimpleLoopUnswitchLegacyPassPassOnce(PassRegistry &Registry) { | ||||||
| 3810 | "Simple unswitch loops", false, false)static void *initializeSimpleLoopUnswitchLegacyPassPassOnce(PassRegistry &Registry) { | ||||||
| 3811 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | ||||||
| 3812 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | ||||||
| 3813 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry); | ||||||
| 3814 | INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry); | ||||||
| 3815 | INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry); | ||||||
| 3816 | INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry); | ||||||
| 3817 | INITIALIZE_PASS_END(SimpleLoopUnswitchLegacyPass, "simple-loop-unswitch",PassInfo *PI = new PassInfo( "Simple unswitch loops", "simple-loop-unswitch" , &SimpleLoopUnswitchLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<SimpleLoopUnswitchLegacyPass>), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeSimpleLoopUnswitchLegacyPassPassFlag ; void llvm::initializeSimpleLoopUnswitchLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeSimpleLoopUnswitchLegacyPassPassFlag , initializeSimpleLoopUnswitchLegacyPassPassOnce, std::ref(Registry )); } | ||||||
| 3818 | "Simple unswitch loops", false, false)PassInfo *PI = new PassInfo( "Simple unswitch loops", "simple-loop-unswitch" , &SimpleLoopUnswitchLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<SimpleLoopUnswitchLegacyPass>), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeSimpleLoopUnswitchLegacyPassPassFlag ; void llvm::initializeSimpleLoopUnswitchLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeSimpleLoopUnswitchLegacyPassPassFlag , initializeSimpleLoopUnswitchLegacyPassPassOnce, std::ref(Registry )); } | ||||||
| 3819 | |||||||
| 3820 | Pass *llvm::createSimpleLoopUnswitchLegacyPass(bool NonTrivial) { | ||||||
| 3821 | return new SimpleLoopUnswitchLegacyPass(NonTrivial); | ||||||
| 3822 | } |
| 1 | // Components for manipulating sequences of characters -*- C++ -*- |
| 2 | |
| 3 | // Copyright (C) 1997-2020 Free Software Foundation, Inc. |
| 4 | // |
| 5 | // This file is part of the GNU ISO C++ Library. This library is free |
| 6 | // software; you can redistribute it and/or modify it under the |
| 7 | // terms of the GNU General Public License as published by the |
| 8 | // Free Software Foundation; either version 3, or (at your option) |
| 9 | // any later version. |
| 10 | |
| 11 | // This library is distributed in the hope that it will be useful, |
| 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | // GNU General Public License for more details. |
| 15 | |
| 16 | // Under Section 7 of GPL version 3, you are granted additional |
| 17 | // permissions described in the GCC Runtime Library Exception, version |
| 18 | // 3.1, as published by the Free Software Foundation. |
| 19 | |
| 20 | // You should have received a copy of the GNU General Public License and |
| 21 | // a copy of the GCC Runtime Library Exception along with this program; |
| 22 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| 23 | // <http://www.gnu.org/licenses/>. |
| 24 | |
| 25 | /** @file bits/basic_string.h |
| 26 | * This is an internal header file, included by other library headers. |
| 27 | * Do not attempt to use it directly. @headername{string} |
| 28 | */ |
| 29 | |
| 30 | // |
| 31 | // ISO C++ 14882: 21 Strings library |
| 32 | // |
| 33 | |
| 34 | #ifndef _BASIC_STRING_H1 |
| 35 | #define _BASIC_STRING_H1 1 |
| 36 | |
| 37 | #pragma GCC system_header |
| 38 | |
| 39 | #include <ext/atomicity.h> |
| 40 | #include <ext/alloc_traits.h> |
| 41 | #include <debug/debug.h> |
| 42 | |
| 43 | #if __cplusplus201703L >= 201103L |
| 44 | #include <initializer_list> |
| 45 | #endif |
| 46 | |
| 47 | #if __cplusplus201703L >= 201703L |
| 48 | # include <string_view> |
| 49 | #endif |
| 50 | |
| 51 | |
| 52 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 53 | { |
| 54 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 55 | |
| 56 | #if _GLIBCXX_USE_CXX11_ABI1 |
| 57 | _GLIBCXX_BEGIN_NAMESPACE_CXX11namespace __cxx11 { |
| 58 | /** |
| 59 | * @class basic_string basic_string.h <string> |
| 60 | * @brief Managing sequences of characters and character-like objects. |
| 61 | * |
| 62 | * @ingroup strings |
| 63 | * @ingroup sequences |
| 64 | * |
| 65 | * @tparam _CharT Type of character |
| 66 | * @tparam _Traits Traits for character type, defaults to |
| 67 | * char_traits<_CharT>. |
| 68 | * @tparam _Alloc Allocator type, defaults to allocator<_CharT>. |
| 69 | * |
| 70 | * Meets the requirements of a <a href="tables.html#65">container</a>, a |
| 71 | * <a href="tables.html#66">reversible container</a>, and a |
| 72 | * <a href="tables.html#67">sequence</a>. Of the |
| 73 | * <a href="tables.html#68">optional sequence requirements</a>, only |
| 74 | * @c push_back, @c at, and @c %array access are supported. |
| 75 | */ |
| 76 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 77 | class basic_string |
| 78 | { |
| 79 | typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template |
| 80 | rebind<_CharT>::other _Char_alloc_type; |
| 81 | typedef __gnu_cxx::__alloc_traits<_Char_alloc_type> _Alloc_traits; |
| 82 | |
| 83 | // Types: |
| 84 | public: |
| 85 | typedef _Traits traits_type; |
| 86 | typedef typename _Traits::char_type value_type; |
| 87 | typedef _Char_alloc_type allocator_type; |
| 88 | typedef typename _Alloc_traits::size_type size_type; |
| 89 | typedef typename _Alloc_traits::difference_type difference_type; |
| 90 | typedef typename _Alloc_traits::reference reference; |
| 91 | typedef typename _Alloc_traits::const_reference const_reference; |
| 92 | typedef typename _Alloc_traits::pointer pointer; |
| 93 | typedef typename _Alloc_traits::const_pointer const_pointer; |
| 94 | typedef __gnu_cxx::__normal_iterator<pointer, basic_string> iterator; |
| 95 | typedef __gnu_cxx::__normal_iterator<const_pointer, basic_string> |
| 96 | const_iterator; |
| 97 | typedef std::reverse_iterator<const_iterator> const_reverse_iterator; |
| 98 | typedef std::reverse_iterator<iterator> reverse_iterator; |
| 99 | |
| 100 | /// Value returned by various member functions when they fail. |
| 101 | static const size_type npos = static_cast<size_type>(-1); |
| 102 | |
| 103 | protected: |
| 104 | // type used for positions in insert, erase etc. |
| 105 | #if __cplusplus201703L < 201103L |
| 106 | typedef iterator __const_iterator; |
| 107 | #else |
| 108 | typedef const_iterator __const_iterator; |
| 109 | #endif |
| 110 | |
| 111 | private: |
| 112 | #if __cplusplus201703L >= 201703L |
| 113 | // A helper type for avoiding boiler-plate. |
| 114 | typedef basic_string_view<_CharT, _Traits> __sv_type; |
| 115 | |
| 116 | template<typename _Tp, typename _Res> |
| 117 | using _If_sv = enable_if_t< |
| 118 | __and_<is_convertible<const _Tp&, __sv_type>, |
| 119 | __not_<is_convertible<const _Tp*, const basic_string*>>, |
| 120 | __not_<is_convertible<const _Tp&, const _CharT*>>>::value, |
| 121 | _Res>; |
| 122 | |
| 123 | // Allows an implicit conversion to __sv_type. |
| 124 | static __sv_type |
| 125 | _S_to_string_view(__sv_type __svt) noexcept |
| 126 | { return __svt; } |
| 127 | |
| 128 | // Wraps a string_view by explicit conversion and thus |
| 129 | // allows to add an internal constructor that does not |
| 130 | // participate in overload resolution when a string_view |
| 131 | // is provided. |
| 132 | struct __sv_wrapper |
| 133 | { |
| 134 | explicit __sv_wrapper(__sv_type __sv) noexcept : _M_sv(__sv) { } |
| 135 | __sv_type _M_sv; |
| 136 | }; |
| 137 | |
| 138 | /** |
| 139 | * @brief Only internally used: Construct string from a string view |
| 140 | * wrapper. |
| 141 | * @param __svw string view wrapper. |
| 142 | * @param __a Allocator to use. |
| 143 | */ |
| 144 | explicit |
| 145 | basic_string(__sv_wrapper __svw, const _Alloc& __a) |
| 146 | : basic_string(__svw._M_sv.data(), __svw._M_sv.size(), __a) { } |
| 147 | #endif |
| 148 | |
| 149 | // Use empty-base optimization: http://www.cantrip.org/emptyopt.html |
| 150 | struct _Alloc_hider : allocator_type // TODO check __is_final |
| 151 | { |
| 152 | #if __cplusplus201703L < 201103L |
| 153 | _Alloc_hider(pointer __dat, const _Alloc& __a = _Alloc()) |
| 154 | : allocator_type(__a), _M_p(__dat) { } |
| 155 | #else |
| 156 | _Alloc_hider(pointer __dat, const _Alloc& __a) |
| 157 | : allocator_type(__a), _M_p(__dat) { } |
| 158 | |
| 159 | _Alloc_hider(pointer __dat, _Alloc&& __a = _Alloc()) |
| 160 | : allocator_type(std::move(__a)), _M_p(__dat) { } |
| 161 | #endif |
| 162 | |
| 163 | pointer _M_p; // The actual data. |
| 164 | }; |
| 165 | |
| 166 | _Alloc_hider _M_dataplus; |
| 167 | size_type _M_string_length; |
| 168 | |
| 169 | enum { _S_local_capacity = 15 / sizeof(_CharT) }; |
| 170 | |
| 171 | union |
| 172 | { |
| 173 | _CharT _M_local_buf[_S_local_capacity + 1]; |
| 174 | size_type _M_allocated_capacity; |
| 175 | }; |
| 176 | |
| 177 | void |
| 178 | _M_data(pointer __p) |
| 179 | { _M_dataplus._M_p = __p; } |
| 180 | |
| 181 | void |
| 182 | _M_length(size_type __length) |
| 183 | { _M_string_length = __length; } |
| 184 | |
| 185 | pointer |
| 186 | _M_data() const |
| 187 | { return _M_dataplus._M_p; } |
| 188 | |
| 189 | pointer |
| 190 | _M_local_data() |
| 191 | { |
| 192 | #if __cplusplus201703L >= 201103L |
| 193 | return std::pointer_traits<pointer>::pointer_to(*_M_local_buf); |
| 194 | #else |
| 195 | return pointer(_M_local_buf); |
| 196 | #endif |
| 197 | } |
| 198 | |
| 199 | const_pointer |
| 200 | _M_local_data() const |
| 201 | { |
| 202 | #if __cplusplus201703L >= 201103L |
| 203 | return std::pointer_traits<const_pointer>::pointer_to(*_M_local_buf); |
| 204 | #else |
| 205 | return const_pointer(_M_local_buf); |
| 206 | #endif |
| 207 | } |
| 208 | |
| 209 | void |
| 210 | _M_capacity(size_type __capacity) |
| 211 | { _M_allocated_capacity = __capacity; } |
| 212 | |
| 213 | void |
| 214 | _M_set_length(size_type __n) |
| 215 | { |
| 216 | _M_length(__n); |
| 217 | traits_type::assign(_M_data()[__n], _CharT()); |
| 218 | } |
| 219 | |
| 220 | bool |
| 221 | _M_is_local() const |
| 222 | { return _M_data() == _M_local_data(); } |
| 223 | |
| 224 | // Create & Destroy |
| 225 | pointer |
| 226 | _M_create(size_type&, size_type); |
| 227 | |
| 228 | void |
| 229 | _M_dispose() |
| 230 | { |
| 231 | if (!_M_is_local()) |
| 232 | _M_destroy(_M_allocated_capacity); |
| 233 | } |
| 234 | |
| 235 | void |
| 236 | _M_destroy(size_type __size) throw() |
| 237 | { _Alloc_traits::deallocate(_M_get_allocator(), _M_data(), __size + 1); } |
| 238 | |
| 239 | // _M_construct_aux is used to implement the 21.3.1 para 15 which |
| 240 | // requires special behaviour if _InIterator is an integral type |
| 241 | template<typename _InIterator> |
| 242 | void |
| 243 | _M_construct_aux(_InIterator __beg, _InIterator __end, |
| 244 | std::__false_type) |
| 245 | { |
| 246 | typedef typename iterator_traits<_InIterator>::iterator_category _Tag; |
| 247 | _M_construct(__beg, __end, _Tag()); |
| 248 | } |
| 249 | |
| 250 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 251 | // 438. Ambiguity in the "do the right thing" clause |
| 252 | template<typename _Integer> |
| 253 | void |
| 254 | _M_construct_aux(_Integer __beg, _Integer __end, std::__true_type) |
| 255 | { _M_construct_aux_2(static_cast<size_type>(__beg), __end); } |
| 256 | |
| 257 | void |
| 258 | _M_construct_aux_2(size_type __req, _CharT __c) |
| 259 | { _M_construct(__req, __c); } |
| 260 | |
| 261 | template<typename _InIterator> |
| 262 | void |
| 263 | _M_construct(_InIterator __beg, _InIterator __end) |
| 264 | { |
| 265 | typedef typename std::__is_integer<_InIterator>::__type _Integral; |
| 266 | _M_construct_aux(__beg, __end, _Integral()); |
| 267 | } |
| 268 | |
| 269 | // For Input Iterators, used in istreambuf_iterators, etc. |
| 270 | template<typename _InIterator> |
| 271 | void |
| 272 | _M_construct(_InIterator __beg, _InIterator __end, |
| 273 | std::input_iterator_tag); |
| 274 | |
| 275 | // For forward_iterators up to random_access_iterators, used for |
| 276 | // string::iterator, _CharT*, etc. |
| 277 | template<typename _FwdIterator> |
| 278 | void |
| 279 | _M_construct(_FwdIterator __beg, _FwdIterator __end, |
| 280 | std::forward_iterator_tag); |
| 281 | |
| 282 | void |
| 283 | _M_construct(size_type __req, _CharT __c); |
| 284 | |
| 285 | allocator_type& |
| 286 | _M_get_allocator() |
| 287 | { return _M_dataplus; } |
| 288 | |
| 289 | const allocator_type& |
| 290 | _M_get_allocator() const |
| 291 | { return _M_dataplus; } |
| 292 | |
| 293 | private: |
| 294 | |
| 295 | #ifdef _GLIBCXX_DISAMBIGUATE_REPLACE_INST |
| 296 | // The explicit instantiations in misc-inst.cc require this due to |
| 297 | // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64063 |
| 298 | template<typename _Tp, bool _Requires = |
| 299 | !__are_same<_Tp, _CharT*>::__value |
| 300 | && !__are_same<_Tp, const _CharT*>::__value |
| 301 | && !__are_same<_Tp, iterator>::__value |
| 302 | && !__are_same<_Tp, const_iterator>::__value> |
| 303 | struct __enable_if_not_native_iterator |
| 304 | { typedef basic_string& __type; }; |
| 305 | template<typename _Tp> |
| 306 | struct __enable_if_not_native_iterator<_Tp, false> { }; |
| 307 | #endif |
| 308 | |
| 309 | size_type |
| 310 | _M_check(size_type __pos, const char* __s) const |
| 311 | { |
| 312 | if (__pos > this->size()) |
| 313 | __throw_out_of_range_fmt(__N("%s: __pos (which is %zu) > "("%s: __pos (which is %zu) > " "this->size() (which is %zu)" ) |
| 314 | "this->size() (which is %zu)")("%s: __pos (which is %zu) > " "this->size() (which is %zu)" ), |
| 315 | __s, __pos, this->size()); |
| 316 | return __pos; |
| 317 | } |
| 318 | |
| 319 | void |
| 320 | _M_check_length(size_type __n1, size_type __n2, const char* __s) const |
| 321 | { |
| 322 | if (this->max_size() - (this->size() - __n1) < __n2) |
| 323 | __throw_length_error(__N(__s)(__s)); |
| 324 | } |
| 325 | |
| 326 | |
| 327 | // NB: _M_limit doesn't check for a bad __pos value. |
| 328 | size_type |
| 329 | _M_limit(size_type __pos, size_type __off) const _GLIBCXX_NOEXCEPTnoexcept |
| 330 | { |
| 331 | const bool __testoff = __off < this->size() - __pos; |
| 332 | return __testoff ? __off : this->size() - __pos; |
| 333 | } |
| 334 | |
| 335 | // True if _Rep and source do not overlap. |
| 336 | bool |
| 337 | _M_disjunct(const _CharT* __s) const _GLIBCXX_NOEXCEPTnoexcept |
| 338 | { |
| 339 | return (less<const _CharT*>()(__s, _M_data()) |
| 340 | || less<const _CharT*>()(_M_data() + this->size(), __s)); |
| 341 | } |
| 342 | |
| 343 | // When __n = 1 way faster than the general multichar |
| 344 | // traits_type::copy/move/assign. |
| 345 | static void |
| 346 | _S_copy(_CharT* __d, const _CharT* __s, size_type __n) |
| 347 | { |
| 348 | if (__n == 1) |
| 349 | traits_type::assign(*__d, *__s); |
| 350 | else |
| 351 | traits_type::copy(__d, __s, __n); |
| 352 | } |
| 353 | |
| 354 | static void |
| 355 | _S_move(_CharT* __d, const _CharT* __s, size_type __n) |
| 356 | { |
| 357 | if (__n == 1) |
| 358 | traits_type::assign(*__d, *__s); |
| 359 | else |
| 360 | traits_type::move(__d, __s, __n); |
| 361 | } |
| 362 | |
| 363 | static void |
| 364 | _S_assign(_CharT* __d, size_type __n, _CharT __c) |
| 365 | { |
| 366 | if (__n == 1) |
| 367 | traits_type::assign(*__d, __c); |
| 368 | else |
| 369 | traits_type::assign(__d, __n, __c); |
| 370 | } |
| 371 | |
| 372 | // _S_copy_chars is a separate template to permit specialization |
| 373 | // to optimize for the common case of pointers as iterators. |
| 374 | template<class _Iterator> |
| 375 | static void |
| 376 | _S_copy_chars(_CharT* __p, _Iterator __k1, _Iterator __k2) |
| 377 | { |
| 378 | for (; __k1 != __k2; ++__k1, (void)++__p) |
| 379 | traits_type::assign(*__p, *__k1); // These types are off. |
| 380 | } |
| 381 | |
| 382 | static void |
| 383 | _S_copy_chars(_CharT* __p, iterator __k1, iterator __k2) _GLIBCXX_NOEXCEPTnoexcept |
| 384 | { _S_copy_chars(__p, __k1.base(), __k2.base()); } |
| 385 | |
| 386 | static void |
| 387 | _S_copy_chars(_CharT* __p, const_iterator __k1, const_iterator __k2) |
| 388 | _GLIBCXX_NOEXCEPTnoexcept |
| 389 | { _S_copy_chars(__p, __k1.base(), __k2.base()); } |
| 390 | |
| 391 | static void |
| 392 | _S_copy_chars(_CharT* __p, _CharT* __k1, _CharT* __k2) _GLIBCXX_NOEXCEPTnoexcept |
| 393 | { _S_copy(__p, __k1, __k2 - __k1); } |
| 394 | |
| 395 | static void |
| 396 | _S_copy_chars(_CharT* __p, const _CharT* __k1, const _CharT* __k2) |
| 397 | _GLIBCXX_NOEXCEPTnoexcept |
| 398 | { _S_copy(__p, __k1, __k2 - __k1); } |
| 399 | |
| 400 | static int |
| 401 | _S_compare(size_type __n1, size_type __n2) _GLIBCXX_NOEXCEPTnoexcept |
| 402 | { |
| 403 | const difference_type __d = difference_type(__n1 - __n2); |
| 404 | |
| 405 | if (__d > __gnu_cxx::__numeric_traits<int>::__max) |
| 406 | return __gnu_cxx::__numeric_traits<int>::__max; |
| 407 | else if (__d < __gnu_cxx::__numeric_traits<int>::__min) |
| 408 | return __gnu_cxx::__numeric_traits<int>::__min; |
| 409 | else |
| 410 | return int(__d); |
| 411 | } |
| 412 | |
| 413 | void |
| 414 | _M_assign(const basic_string&); |
| 415 | |
| 416 | void |
| 417 | _M_mutate(size_type __pos, size_type __len1, const _CharT* __s, |
| 418 | size_type __len2); |
| 419 | |
| 420 | void |
| 421 | _M_erase(size_type __pos, size_type __n); |
| 422 | |
| 423 | public: |
| 424 | // Construct/copy/destroy: |
| 425 | // NB: We overload ctors in some cases instead of using default |
| 426 | // arguments, per 17.4.4.4 para. 2 item 2. |
| 427 | |
| 428 | /** |
| 429 | * @brief Default constructor creates an empty string. |
| 430 | */ |
| 431 | basic_string() |
| 432 | _GLIBCXX_NOEXCEPT_IF(is_nothrow_default_constructible<_Alloc>::value)noexcept(is_nothrow_default_constructible<_Alloc>::value ) |
| 433 | : _M_dataplus(_M_local_data()) |
| 434 | { _M_set_length(0); } |
| 435 | |
| 436 | /** |
| 437 | * @brief Construct an empty string using allocator @a a. |
| 438 | */ |
| 439 | explicit |
| 440 | basic_string(const _Alloc& __a) _GLIBCXX_NOEXCEPTnoexcept |
| 441 | : _M_dataplus(_M_local_data(), __a) |
| 442 | { _M_set_length(0); } |
| 443 | |
| 444 | /** |
| 445 | * @brief Construct string with copy of value of @a __str. |
| 446 | * @param __str Source string. |
| 447 | */ |
| 448 | basic_string(const basic_string& __str) |
| 449 | : _M_dataplus(_M_local_data(), |
| 450 | _Alloc_traits::_S_select_on_copy(__str._M_get_allocator())) |
| 451 | { _M_construct(__str._M_data(), __str._M_data() + __str.length()); } |
| 452 | |
| 453 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 454 | // 2583. no way to supply an allocator for basic_string(str, pos) |
| 455 | /** |
| 456 | * @brief Construct string as copy of a substring. |
| 457 | * @param __str Source string. |
| 458 | * @param __pos Index of first character to copy from. |
| 459 | * @param __a Allocator to use. |
| 460 | */ |
| 461 | basic_string(const basic_string& __str, size_type __pos, |
| 462 | const _Alloc& __a = _Alloc()) |
| 463 | : _M_dataplus(_M_local_data(), __a) |
| 464 | { |
| 465 | const _CharT* __start = __str._M_data() |
| 466 | + __str._M_check(__pos, "basic_string::basic_string"); |
| 467 | _M_construct(__start, __start + __str._M_limit(__pos, npos)); |
| 468 | } |
| 469 | |
| 470 | /** |
| 471 | * @brief Construct string as copy of a substring. |
| 472 | * @param __str Source string. |
| 473 | * @param __pos Index of first character to copy from. |
| 474 | * @param __n Number of characters to copy. |
| 475 | */ |
| 476 | basic_string(const basic_string& __str, size_type __pos, |
| 477 | size_type __n) |
| 478 | : _M_dataplus(_M_local_data()) |
| 479 | { |
| 480 | const _CharT* __start = __str._M_data() |
| 481 | + __str._M_check(__pos, "basic_string::basic_string"); |
| 482 | _M_construct(__start, __start + __str._M_limit(__pos, __n)); |
| 483 | } |
| 484 | |
| 485 | /** |
| 486 | * @brief Construct string as copy of a substring. |
| 487 | * @param __str Source string. |
| 488 | * @param __pos Index of first character to copy from. |
| 489 | * @param __n Number of characters to copy. |
| 490 | * @param __a Allocator to use. |
| 491 | */ |
| 492 | basic_string(const basic_string& __str, size_type __pos, |
| 493 | size_type __n, const _Alloc& __a) |
| 494 | : _M_dataplus(_M_local_data(), __a) |
| 495 | { |
| 496 | const _CharT* __start |
| 497 | = __str._M_data() + __str._M_check(__pos, "string::string"); |
| 498 | _M_construct(__start, __start + __str._M_limit(__pos, __n)); |
| 499 | } |
| 500 | |
| 501 | /** |
| 502 | * @brief Construct string initialized by a character %array. |
| 503 | * @param __s Source character %array. |
| 504 | * @param __n Number of characters to copy. |
| 505 | * @param __a Allocator to use (default is default allocator). |
| 506 | * |
| 507 | * NB: @a __s must have at least @a __n characters, '\\0' |
| 508 | * has no special meaning. |
| 509 | */ |
| 510 | basic_string(const _CharT* __s, size_type __n, |
| 511 | const _Alloc& __a = _Alloc()) |
| 512 | : _M_dataplus(_M_local_data(), __a) |
| 513 | { _M_construct(__s, __s + __n); } |
| 514 | |
| 515 | /** |
| 516 | * @brief Construct string as copy of a C string. |
| 517 | * @param __s Source C string. |
| 518 | * @param __a Allocator to use (default is default allocator). |
| 519 | */ |
| 520 | #if __cpp_deduction_guides201703L && ! defined _GLIBCXX_DEFINING_STRING_INSTANTIATIONS |
| 521 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 522 | // 3076. basic_string CTAD ambiguity |
| 523 | template<typename = _RequireAllocator<_Alloc>> |
| 524 | #endif |
| 525 | basic_string(const _CharT* __s, const _Alloc& __a = _Alloc()) |
| 526 | : _M_dataplus(_M_local_data(), __a) |
| 527 | { _M_construct(__s, __s ? __s + traits_type::length(__s) : __s+npos); } |
| 528 | |
| 529 | /** |
| 530 | * @brief Construct string as multiple characters. |
| 531 | * @param __n Number of characters. |
| 532 | * @param __c Character to use. |
| 533 | * @param __a Allocator to use (default is default allocator). |
| 534 | */ |
| 535 | #if __cpp_deduction_guides201703L && ! defined _GLIBCXX_DEFINING_STRING_INSTANTIATIONS |
| 536 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 537 | // 3076. basic_string CTAD ambiguity |
| 538 | template<typename = _RequireAllocator<_Alloc>> |
| 539 | #endif |
| 540 | basic_string(size_type __n, _CharT __c, const _Alloc& __a = _Alloc()) |
| 541 | : _M_dataplus(_M_local_data(), __a) |
| 542 | { _M_construct(__n, __c); } |
| 543 | |
| 544 | #if __cplusplus201703L >= 201103L |
| 545 | /** |
| 546 | * @brief Move construct string. |
| 547 | * @param __str Source string. |
| 548 | * |
| 549 | * The newly-created string contains the exact contents of @a __str. |
| 550 | * @a __str is a valid, but unspecified string. |
| 551 | **/ |
| 552 | basic_string(basic_string&& __str) noexcept |
| 553 | : _M_dataplus(_M_local_data(), std::move(__str._M_get_allocator())) |
| 554 | { |
| 555 | if (__str._M_is_local()) |
| 556 | { |
| 557 | traits_type::copy(_M_local_buf, __str._M_local_buf, |
| 558 | _S_local_capacity + 1); |
| 559 | } |
| 560 | else |
| 561 | { |
| 562 | _M_data(__str._M_data()); |
| 563 | _M_capacity(__str._M_allocated_capacity); |
| 564 | } |
| 565 | |
| 566 | // Must use _M_length() here not _M_set_length() because |
| 567 | // basic_stringbuf relies on writing into unallocated capacity so |
| 568 | // we mess up the contents if we put a '\0' in the string. |
| 569 | _M_length(__str.length()); |
| 570 | __str._M_data(__str._M_local_data()); |
| 571 | __str._M_set_length(0); |
| 572 | } |
| 573 | |
| 574 | /** |
| 575 | * @brief Construct string from an initializer %list. |
| 576 | * @param __l std::initializer_list of characters. |
| 577 | * @param __a Allocator to use (default is default allocator). |
| 578 | */ |
| 579 | basic_string(initializer_list<_CharT> __l, const _Alloc& __a = _Alloc()) |
| 580 | : _M_dataplus(_M_local_data(), __a) |
| 581 | { _M_construct(__l.begin(), __l.end()); } |
| 582 | |
| 583 | basic_string(const basic_string& __str, const _Alloc& __a) |
| 584 | : _M_dataplus(_M_local_data(), __a) |
| 585 | { _M_construct(__str.begin(), __str.end()); } |
| 586 | |
| 587 | basic_string(basic_string&& __str, const _Alloc& __a) |
| 588 | noexcept(_Alloc_traits::_S_always_equal()) |
| 589 | : _M_dataplus(_M_local_data(), __a) |
| 590 | { |
| 591 | if (__str._M_is_local()) |
| 592 | { |
| 593 | traits_type::copy(_M_local_buf, __str._M_local_buf, |
| 594 | _S_local_capacity + 1); |
| 595 | _M_length(__str.length()); |
| 596 | __str._M_set_length(0); |
| 597 | } |
| 598 | else if (_Alloc_traits::_S_always_equal() |
| 599 | || __str.get_allocator() == __a) |
| 600 | { |
| 601 | _M_data(__str._M_data()); |
| 602 | _M_length(__str.length()); |
| 603 | _M_capacity(__str._M_allocated_capacity); |
| 604 | __str._M_data(__str._M_local_buf); |
| 605 | __str._M_set_length(0); |
| 606 | } |
| 607 | else |
| 608 | _M_construct(__str.begin(), __str.end()); |
| 609 | } |
| 610 | |
| 611 | #endif // C++11 |
| 612 | |
| 613 | /** |
| 614 | * @brief Construct string as copy of a range. |
| 615 | * @param __beg Start of range. |
| 616 | * @param __end End of range. |
| 617 | * @param __a Allocator to use (default is default allocator). |
| 618 | */ |
| 619 | #if __cplusplus201703L >= 201103L |
| 620 | template<typename _InputIterator, |
| 621 | typename = std::_RequireInputIter<_InputIterator>> |
| 622 | #else |
| 623 | template<typename _InputIterator> |
| 624 | #endif |
| 625 | basic_string(_InputIterator __beg, _InputIterator __end, |
| 626 | const _Alloc& __a = _Alloc()) |
| 627 | : _M_dataplus(_M_local_data(), __a) |
| 628 | { _M_construct(__beg, __end); } |
| 629 | |
| 630 | #if __cplusplus201703L >= 201703L |
| 631 | /** |
| 632 | * @brief Construct string from a substring of a string_view. |
| 633 | * @param __t Source object convertible to string view. |
| 634 | * @param __pos The index of the first character to copy from __t. |
| 635 | * @param __n The number of characters to copy from __t. |
| 636 | * @param __a Allocator to use. |
| 637 | */ |
| 638 | template<typename _Tp, typename = _If_sv<_Tp, void>> |
| 639 | basic_string(const _Tp& __t, size_type __pos, size_type __n, |
| 640 | const _Alloc& __a = _Alloc()) |
| 641 | : basic_string(_S_to_string_view(__t).substr(__pos, __n), __a) { } |
| 642 | |
| 643 | /** |
| 644 | * @brief Construct string from a string_view. |
| 645 | * @param __t Source object convertible to string view. |
| 646 | * @param __a Allocator to use (default is default allocator). |
| 647 | */ |
| 648 | template<typename _Tp, typename = _If_sv<_Tp, void>> |
| 649 | explicit |
| 650 | basic_string(const _Tp& __t, const _Alloc& __a = _Alloc()) |
| 651 | : basic_string(__sv_wrapper(_S_to_string_view(__t)), __a) { } |
| 652 | #endif // C++17 |
| 653 | |
| 654 | /** |
| 655 | * @brief Destroy the string instance. |
| 656 | */ |
| 657 | ~basic_string() |
| 658 | { _M_dispose(); } |
| 659 | |
| 660 | /** |
| 661 | * @brief Assign the value of @a str to this string. |
| 662 | * @param __str Source string. |
| 663 | */ |
| 664 | basic_string& |
| 665 | operator=(const basic_string& __str) |
| 666 | { |
| 667 | return this->assign(__str); |
| 668 | } |
| 669 | |
| 670 | /** |
| 671 | * @brief Copy contents of @a s into this string. |
| 672 | * @param __s Source null-terminated string. |
| 673 | */ |
| 674 | basic_string& |
| 675 | operator=(const _CharT* __s) |
| 676 | { return this->assign(__s); } |
| 677 | |
| 678 | /** |
| 679 | * @brief Set value to string of length 1. |
| 680 | * @param __c Source character. |
| 681 | * |
| 682 | * Assigning to a character makes this string length 1 and |
| 683 | * (*this)[0] == @a c. |
| 684 | */ |
| 685 | basic_string& |
| 686 | operator=(_CharT __c) |
| 687 | { |
| 688 | this->assign(1, __c); |
| 689 | return *this; |
| 690 | } |
| 691 | |
| 692 | #if __cplusplus201703L >= 201103L |
| 693 | /** |
| 694 | * @brief Move assign the value of @a str to this string. |
| 695 | * @param __str Source string. |
| 696 | * |
| 697 | * The contents of @a str are moved into this string (without copying). |
| 698 | * @a str is a valid, but unspecified string. |
| 699 | **/ |
| 700 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 701 | // 2063. Contradictory requirements for string move assignment |
| 702 | basic_string& |
| 703 | operator=(basic_string&& __str) |
| 704 | noexcept(_Alloc_traits::_S_nothrow_move()) |
| 705 | { |
| 706 | if (!_M_is_local() && _Alloc_traits::_S_propagate_on_move_assign() |
| 707 | && !_Alloc_traits::_S_always_equal() |
| 708 | && _M_get_allocator() != __str._M_get_allocator()) |
| 709 | { |
| 710 | // Destroy existing storage before replacing allocator. |
| 711 | _M_destroy(_M_allocated_capacity); |
| 712 | _M_data(_M_local_data()); |
| 713 | _M_set_length(0); |
| 714 | } |
| 715 | // Replace allocator if POCMA is true. |
| 716 | std::__alloc_on_move(_M_get_allocator(), __str._M_get_allocator()); |
| 717 | |
| 718 | if (__str._M_is_local()) |
| 719 | { |
| 720 | // We've always got room for a short string, just copy it. |
| 721 | if (__str.size()) |
| 722 | this->_S_copy(_M_data(), __str._M_data(), __str.size()); |
| 723 | _M_set_length(__str.size()); |
| 724 | } |
| 725 | else if (_Alloc_traits::_S_propagate_on_move_assign() |
| 726 | || _Alloc_traits::_S_always_equal() |
| 727 | || _M_get_allocator() == __str._M_get_allocator()) |
| 728 | { |
| 729 | // Just move the allocated pointer, our allocator can free it. |
| 730 | pointer __data = nullptr; |
| 731 | size_type __capacity; |
| 732 | if (!_M_is_local()) |
| 733 | { |
| 734 | if (_Alloc_traits::_S_always_equal()) |
| 735 | { |
| 736 | // __str can reuse our existing storage. |
| 737 | __data = _M_data(); |
| 738 | __capacity = _M_allocated_capacity; |
| 739 | } |
| 740 | else // __str can't use it, so free it. |
| 741 | _M_destroy(_M_allocated_capacity); |
| 742 | } |
| 743 | |
| 744 | _M_data(__str._M_data()); |
| 745 | _M_length(__str.length()); |
| 746 | _M_capacity(__str._M_allocated_capacity); |
| 747 | if (__data) |
| 748 | { |
| 749 | __str._M_data(__data); |
| 750 | __str._M_capacity(__capacity); |
| 751 | } |
| 752 | else |
| 753 | __str._M_data(__str._M_local_buf); |
| 754 | } |
| 755 | else // Need to do a deep copy |
| 756 | assign(__str); |
| 757 | __str.clear(); |
| 758 | return *this; |
| 759 | } |
| 760 | |
| 761 | /** |
| 762 | * @brief Set value to string constructed from initializer %list. |
| 763 | * @param __l std::initializer_list. |
| 764 | */ |
| 765 | basic_string& |
| 766 | operator=(initializer_list<_CharT> __l) |
| 767 | { |
| 768 | this->assign(__l.begin(), __l.size()); |
| 769 | return *this; |
| 770 | } |
| 771 | #endif // C++11 |
| 772 | |
| 773 | #if __cplusplus201703L >= 201703L |
| 774 | /** |
| 775 | * @brief Set value to string constructed from a string_view. |
| 776 | * @param __svt An object convertible to string_view. |
| 777 | */ |
| 778 | template<typename _Tp> |
| 779 | _If_sv<_Tp, basic_string&> |
| 780 | operator=(const _Tp& __svt) |
| 781 | { return this->assign(__svt); } |
| 782 | |
| 783 | /** |
| 784 | * @brief Convert to a string_view. |
| 785 | * @return A string_view. |
| 786 | */ |
| 787 | operator __sv_type() const noexcept |
| 788 | { return __sv_type(data(), size()); } |
| 789 | #endif // C++17 |
| 790 | |
| 791 | // Iterators: |
| 792 | /** |
| 793 | * Returns a read/write iterator that points to the first character in |
| 794 | * the %string. |
| 795 | */ |
| 796 | iterator |
| 797 | begin() _GLIBCXX_NOEXCEPTnoexcept |
| 798 | { return iterator(_M_data()); } |
| 799 | |
| 800 | /** |
| 801 | * Returns a read-only (constant) iterator that points to the first |
| 802 | * character in the %string. |
| 803 | */ |
| 804 | const_iterator |
| 805 | begin() const _GLIBCXX_NOEXCEPTnoexcept |
| 806 | { return const_iterator(_M_data()); } |
| 807 | |
| 808 | /** |
| 809 | * Returns a read/write iterator that points one past the last |
| 810 | * character in the %string. |
| 811 | */ |
| 812 | iterator |
| 813 | end() _GLIBCXX_NOEXCEPTnoexcept |
| 814 | { return iterator(_M_data() + this->size()); } |
| 815 | |
| 816 | /** |
| 817 | * Returns a read-only (constant) iterator that points one past the |
| 818 | * last character in the %string. |
| 819 | */ |
| 820 | const_iterator |
| 821 | end() const _GLIBCXX_NOEXCEPTnoexcept |
| 822 | { return const_iterator(_M_data() + this->size()); } |
| 823 | |
| 824 | /** |
| 825 | * Returns a read/write reverse iterator that points to the last |
| 826 | * character in the %string. Iteration is done in reverse element |
| 827 | * order. |
| 828 | */ |
| 829 | reverse_iterator |
| 830 | rbegin() _GLIBCXX_NOEXCEPTnoexcept |
| 831 | { return reverse_iterator(this->end()); } |
| 832 | |
| 833 | /** |
| 834 | * Returns a read-only (constant) reverse iterator that points |
| 835 | * to the last character in the %string. Iteration is done in |
| 836 | * reverse element order. |
| 837 | */ |
| 838 | const_reverse_iterator |
| 839 | rbegin() const _GLIBCXX_NOEXCEPTnoexcept |
| 840 | { return const_reverse_iterator(this->end()); } |
| 841 | |
| 842 | /** |
| 843 | * Returns a read/write reverse iterator that points to one before the |
| 844 | * first character in the %string. Iteration is done in reverse |
| 845 | * element order. |
| 846 | */ |
| 847 | reverse_iterator |
| 848 | rend() _GLIBCXX_NOEXCEPTnoexcept |
| 849 | { return reverse_iterator(this->begin()); } |
| 850 | |
| 851 | /** |
| 852 | * Returns a read-only (constant) reverse iterator that points |
| 853 | * to one before the first character in the %string. Iteration |
| 854 | * is done in reverse element order. |
| 855 | */ |
| 856 | const_reverse_iterator |
| 857 | rend() const _GLIBCXX_NOEXCEPTnoexcept |
| 858 | { return const_reverse_iterator(this->begin()); } |
| 859 | |
| 860 | #if __cplusplus201703L >= 201103L |
| 861 | /** |
| 862 | * Returns a read-only (constant) iterator that points to the first |
| 863 | * character in the %string. |
| 864 | */ |
| 865 | const_iterator |
| 866 | cbegin() const noexcept |
| 867 | { return const_iterator(this->_M_data()); } |
| 868 | |
| 869 | /** |
| 870 | * Returns a read-only (constant) iterator that points one past the |
| 871 | * last character in the %string. |
| 872 | */ |
| 873 | const_iterator |
| 874 | cend() const noexcept |
| 875 | { return const_iterator(this->_M_data() + this->size()); } |
| 876 | |
| 877 | /** |
| 878 | * Returns a read-only (constant) reverse iterator that points |
| 879 | * to the last character in the %string. Iteration is done in |
| 880 | * reverse element order. |
| 881 | */ |
| 882 | const_reverse_iterator |
| 883 | crbegin() const noexcept |
| 884 | { return const_reverse_iterator(this->end()); } |
| 885 | |
| 886 | /** |
| 887 | * Returns a read-only (constant) reverse iterator that points |
| 888 | * to one before the first character in the %string. Iteration |
| 889 | * is done in reverse element order. |
| 890 | */ |
| 891 | const_reverse_iterator |
| 892 | crend() const noexcept |
| 893 | { return const_reverse_iterator(this->begin()); } |
| 894 | #endif |
| 895 | |
| 896 | public: |
| 897 | // Capacity: |
| 898 | /// Returns the number of characters in the string, not including any |
| 899 | /// null-termination. |
| 900 | size_type |
| 901 | size() const _GLIBCXX_NOEXCEPTnoexcept |
| 902 | { return _M_string_length; } |
| 903 | |
| 904 | /// Returns the number of characters in the string, not including any |
| 905 | /// null-termination. |
| 906 | size_type |
| 907 | length() const _GLIBCXX_NOEXCEPTnoexcept |
| 908 | { return _M_string_length; } |
| 909 | |
| 910 | /// Returns the size() of the largest possible %string. |
| 911 | size_type |
| 912 | max_size() const _GLIBCXX_NOEXCEPTnoexcept |
| 913 | { return (_Alloc_traits::max_size(_M_get_allocator()) - 1) / 2; } |
| 914 | |
| 915 | /** |
| 916 | * @brief Resizes the %string to the specified number of characters. |
| 917 | * @param __n Number of characters the %string should contain. |
| 918 | * @param __c Character to fill any new elements. |
| 919 | * |
| 920 | * This function will %resize the %string to the specified |
| 921 | * number of characters. If the number is smaller than the |
| 922 | * %string's current size the %string is truncated, otherwise |
| 923 | * the %string is extended and new elements are %set to @a __c. |
| 924 | */ |
| 925 | void |
| 926 | resize(size_type __n, _CharT __c); |
| 927 | |
| 928 | /** |
| 929 | * @brief Resizes the %string to the specified number of characters. |
| 930 | * @param __n Number of characters the %string should contain. |
| 931 | * |
| 932 | * This function will resize the %string to the specified length. If |
| 933 | * the new size is smaller than the %string's current size the %string |
| 934 | * is truncated, otherwise the %string is extended and new characters |
| 935 | * are default-constructed. For basic types such as char, this means |
| 936 | * setting them to 0. |
| 937 | */ |
| 938 | void |
| 939 | resize(size_type __n) |
| 940 | { this->resize(__n, _CharT()); } |
| 941 | |
| 942 | #if __cplusplus201703L >= 201103L |
| 943 | /// A non-binding request to reduce capacity() to size(). |
| 944 | void |
| 945 | shrink_to_fit() noexcept |
| 946 | { |
| 947 | #if __cpp_exceptions |
| 948 | if (capacity() > size()) |
| 949 | { |
| 950 | try |
| 951 | { reserve(0); } |
| 952 | catch(...) |
| 953 | { } |
| 954 | } |
| 955 | #endif |
| 956 | } |
| 957 | #endif |
| 958 | |
| 959 | /** |
| 960 | * Returns the total number of characters that the %string can hold |
| 961 | * before needing to allocate more memory. |
| 962 | */ |
| 963 | size_type |
| 964 | capacity() const _GLIBCXX_NOEXCEPTnoexcept |
| 965 | { |
| 966 | return _M_is_local() ? size_type(_S_local_capacity) |
| 967 | : _M_allocated_capacity; |
| 968 | } |
| 969 | |
| 970 | /** |
| 971 | * @brief Attempt to preallocate enough memory for specified number of |
| 972 | * characters. |
| 973 | * @param __res_arg Number of characters required. |
| 974 | * @throw std::length_error If @a __res_arg exceeds @c max_size(). |
| 975 | * |
| 976 | * This function attempts to reserve enough memory for the |
| 977 | * %string to hold the specified number of characters. If the |
| 978 | * number requested is more than max_size(), length_error is |
| 979 | * thrown. |
| 980 | * |
| 981 | * The advantage of this function is that if optimal code is a |
| 982 | * necessity and the user can determine the string length that will be |
| 983 | * required, the user can reserve the memory in %advance, and thus |
| 984 | * prevent a possible reallocation of memory and copying of %string |
| 985 | * data. |
| 986 | */ |
| 987 | void |
| 988 | reserve(size_type __res_arg = 0); |
| 989 | |
| 990 | /** |
| 991 | * Erases the string, making it empty. |
| 992 | */ |
| 993 | void |
| 994 | clear() _GLIBCXX_NOEXCEPTnoexcept |
| 995 | { _M_set_length(0); } |
| 996 | |
| 997 | /** |
| 998 | * Returns true if the %string is empty. Equivalent to |
| 999 | * <code>*this == ""</code>. |
| 1000 | */ |
| 1001 | _GLIBCXX_NODISCARD[[__nodiscard__]] bool |
| 1002 | empty() const _GLIBCXX_NOEXCEPTnoexcept |
| 1003 | { return this->size() == 0; } |
| 1004 | |
| 1005 | // Element access: |
| 1006 | /** |
| 1007 | * @brief Subscript access to the data contained in the %string. |
| 1008 | * @param __pos The index of the character to access. |
| 1009 | * @return Read-only (constant) reference to the character. |
| 1010 | * |
| 1011 | * This operator allows for easy, array-style, data access. |
| 1012 | * Note that data access with this operator is unchecked and |
| 1013 | * out_of_range lookups are not defined. (For checked lookups |
| 1014 | * see at().) |
| 1015 | */ |
| 1016 | const_reference |
| 1017 | operator[] (size_type __pos) const _GLIBCXX_NOEXCEPTnoexcept |
| 1018 | { |
| 1019 | __glibcxx_assert(__pos <= size())do { if (! (__pos <= size())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1019, __PRETTY_FUNCTION__, "__pos <= size()"); } while ( false); |
| 1020 | return _M_data()[__pos]; |
| 1021 | } |
| 1022 | |
| 1023 | /** |
| 1024 | * @brief Subscript access to the data contained in the %string. |
| 1025 | * @param __pos The index of the character to access. |
| 1026 | * @return Read/write reference to the character. |
| 1027 | * |
| 1028 | * This operator allows for easy, array-style, data access. |
| 1029 | * Note that data access with this operator is unchecked and |
| 1030 | * out_of_range lookups are not defined. (For checked lookups |
| 1031 | * see at().) |
| 1032 | */ |
| 1033 | reference |
| 1034 | operator[](size_type __pos) |
| 1035 | { |
| 1036 | // Allow pos == size() both in C++98 mode, as v3 extension, |
| 1037 | // and in C++11 mode. |
| 1038 | __glibcxx_assert(__pos <= size())do { if (! (__pos <= size())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1038, __PRETTY_FUNCTION__, "__pos <= size()"); } while ( false); |
| 1039 | // In pedantic mode be strict in C++98 mode. |
| 1040 | _GLIBCXX_DEBUG_PEDASSERT(__cplusplus >= 201103L || __pos < size()); |
| 1041 | return _M_data()[__pos]; |
| 1042 | } |
| 1043 | |
| 1044 | /** |
| 1045 | * @brief Provides access to the data contained in the %string. |
| 1046 | * @param __n The index of the character to access. |
| 1047 | * @return Read-only (const) reference to the character. |
| 1048 | * @throw std::out_of_range If @a n is an invalid index. |
| 1049 | * |
| 1050 | * This function provides for safer data access. The parameter is |
| 1051 | * first checked that it is in the range of the string. The function |
| 1052 | * throws out_of_range if the check fails. |
| 1053 | */ |
| 1054 | const_reference |
| 1055 | at(size_type __n) const |
| 1056 | { |
| 1057 | if (__n >= this->size()) |
| 1058 | __throw_out_of_range_fmt(__N("basic_string::at: __n "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 1059 | "(which is %zu) >= this->size() "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 1060 | "(which is %zu)")("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)"), |
| 1061 | __n, this->size()); |
| 1062 | return _M_data()[__n]; |
| 1063 | } |
| 1064 | |
| 1065 | /** |
| 1066 | * @brief Provides access to the data contained in the %string. |
| 1067 | * @param __n The index of the character to access. |
| 1068 | * @return Read/write reference to the character. |
| 1069 | * @throw std::out_of_range If @a n is an invalid index. |
| 1070 | * |
| 1071 | * This function provides for safer data access. The parameter is |
| 1072 | * first checked that it is in the range of the string. The function |
| 1073 | * throws out_of_range if the check fails. |
| 1074 | */ |
| 1075 | reference |
| 1076 | at(size_type __n) |
| 1077 | { |
| 1078 | if (__n >= size()) |
| 1079 | __throw_out_of_range_fmt(__N("basic_string::at: __n "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 1080 | "(which is %zu) >= this->size() "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 1081 | "(which is %zu)")("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)"), |
| 1082 | __n, this->size()); |
| 1083 | return _M_data()[__n]; |
| 1084 | } |
| 1085 | |
| 1086 | #if __cplusplus201703L >= 201103L |
| 1087 | /** |
| 1088 | * Returns a read/write reference to the data at the first |
| 1089 | * element of the %string. |
| 1090 | */ |
| 1091 | reference |
| 1092 | front() noexcept |
| 1093 | { |
| 1094 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1094, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 1095 | return operator[](0); |
| 1096 | } |
| 1097 | |
| 1098 | /** |
| 1099 | * Returns a read-only (constant) reference to the data at the first |
| 1100 | * element of the %string. |
| 1101 | */ |
| 1102 | const_reference |
| 1103 | front() const noexcept |
| 1104 | { |
| 1105 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1105, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 1106 | return operator[](0); |
| 1107 | } |
| 1108 | |
| 1109 | /** |
| 1110 | * Returns a read/write reference to the data at the last |
| 1111 | * element of the %string. |
| 1112 | */ |
| 1113 | reference |
| 1114 | back() noexcept |
| 1115 | { |
| 1116 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1116, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 1117 | return operator[](this->size() - 1); |
| 1118 | } |
| 1119 | |
| 1120 | /** |
| 1121 | * Returns a read-only (constant) reference to the data at the |
| 1122 | * last element of the %string. |
| 1123 | */ |
| 1124 | const_reference |
| 1125 | back() const noexcept |
| 1126 | { |
| 1127 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1127, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 1128 | return operator[](this->size() - 1); |
| 1129 | } |
| 1130 | #endif |
| 1131 | |
| 1132 | // Modifiers: |
| 1133 | /** |
| 1134 | * @brief Append a string to this string. |
| 1135 | * @param __str The string to append. |
| 1136 | * @return Reference to this string. |
| 1137 | */ |
| 1138 | basic_string& |
| 1139 | operator+=(const basic_string& __str) |
| 1140 | { return this->append(__str); } |
| 1141 | |
| 1142 | /** |
| 1143 | * @brief Append a C string. |
| 1144 | * @param __s The C string to append. |
| 1145 | * @return Reference to this string. |
| 1146 | */ |
| 1147 | basic_string& |
| 1148 | operator+=(const _CharT* __s) |
| 1149 | { return this->append(__s); } |
| 1150 | |
| 1151 | /** |
| 1152 | * @brief Append a character. |
| 1153 | * @param __c The character to append. |
| 1154 | * @return Reference to this string. |
| 1155 | */ |
| 1156 | basic_string& |
| 1157 | operator+=(_CharT __c) |
| 1158 | { |
| 1159 | this->push_back(__c); |
| 1160 | return *this; |
| 1161 | } |
| 1162 | |
| 1163 | #if __cplusplus201703L >= 201103L |
| 1164 | /** |
| 1165 | * @brief Append an initializer_list of characters. |
| 1166 | * @param __l The initializer_list of characters to be appended. |
| 1167 | * @return Reference to this string. |
| 1168 | */ |
| 1169 | basic_string& |
| 1170 | operator+=(initializer_list<_CharT> __l) |
| 1171 | { return this->append(__l.begin(), __l.size()); } |
| 1172 | #endif // C++11 |
| 1173 | |
| 1174 | #if __cplusplus201703L >= 201703L |
| 1175 | /** |
| 1176 | * @brief Append a string_view. |
| 1177 | * @param __svt An object convertible to string_view to be appended. |
| 1178 | * @return Reference to this string. |
| 1179 | */ |
| 1180 | template<typename _Tp> |
| 1181 | _If_sv<_Tp, basic_string&> |
| 1182 | operator+=(const _Tp& __svt) |
| 1183 | { return this->append(__svt); } |
| 1184 | #endif // C++17 |
| 1185 | |
| 1186 | /** |
| 1187 | * @brief Append a string to this string. |
| 1188 | * @param __str The string to append. |
| 1189 | * @return Reference to this string. |
| 1190 | */ |
| 1191 | basic_string& |
| 1192 | append(const basic_string& __str) |
| 1193 | { return _M_append(__str._M_data(), __str.size()); } |
| 1194 | |
| 1195 | /** |
| 1196 | * @brief Append a substring. |
| 1197 | * @param __str The string to append. |
| 1198 | * @param __pos Index of the first character of str to append. |
| 1199 | * @param __n The number of characters to append. |
| 1200 | * @return Reference to this string. |
| 1201 | * @throw std::out_of_range if @a __pos is not a valid index. |
| 1202 | * |
| 1203 | * This function appends @a __n characters from @a __str |
| 1204 | * starting at @a __pos to this string. If @a __n is is larger |
| 1205 | * than the number of available characters in @a __str, the |
| 1206 | * remainder of @a __str is appended. |
| 1207 | */ |
| 1208 | basic_string& |
| 1209 | append(const basic_string& __str, size_type __pos, size_type __n = npos) |
| 1210 | { return _M_append(__str._M_data() |
| 1211 | + __str._M_check(__pos, "basic_string::append"), |
| 1212 | __str._M_limit(__pos, __n)); } |
| 1213 | |
| 1214 | /** |
| 1215 | * @brief Append a C substring. |
| 1216 | * @param __s The C string to append. |
| 1217 | * @param __n The number of characters to append. |
| 1218 | * @return Reference to this string. |
| 1219 | */ |
| 1220 | basic_string& |
| 1221 | append(const _CharT* __s, size_type __n) |
| 1222 | { |
| 1223 | __glibcxx_requires_string_len(__s, __n); |
| 1224 | _M_check_length(size_type(0), __n, "basic_string::append"); |
| 1225 | return _M_append(__s, __n); |
| 1226 | } |
| 1227 | |
| 1228 | /** |
| 1229 | * @brief Append a C string. |
| 1230 | * @param __s The C string to append. |
| 1231 | * @return Reference to this string. |
| 1232 | */ |
| 1233 | basic_string& |
| 1234 | append(const _CharT* __s) |
| 1235 | { |
| 1236 | __glibcxx_requires_string(__s); |
| 1237 | const size_type __n = traits_type::length(__s); |
| 1238 | _M_check_length(size_type(0), __n, "basic_string::append"); |
| 1239 | return _M_append(__s, __n); |
| 1240 | } |
| 1241 | |
| 1242 | /** |
| 1243 | * @brief Append multiple characters. |
| 1244 | * @param __n The number of characters to append. |
| 1245 | * @param __c The character to use. |
| 1246 | * @return Reference to this string. |
| 1247 | * |
| 1248 | * Appends __n copies of __c to this string. |
| 1249 | */ |
| 1250 | basic_string& |
| 1251 | append(size_type __n, _CharT __c) |
| 1252 | { return _M_replace_aux(this->size(), size_type(0), __n, __c); } |
| 1253 | |
| 1254 | #if __cplusplus201703L >= 201103L |
| 1255 | /** |
| 1256 | * @brief Append an initializer_list of characters. |
| 1257 | * @param __l The initializer_list of characters to append. |
| 1258 | * @return Reference to this string. |
| 1259 | */ |
| 1260 | basic_string& |
| 1261 | append(initializer_list<_CharT> __l) |
| 1262 | { return this->append(__l.begin(), __l.size()); } |
| 1263 | #endif // C++11 |
| 1264 | |
| 1265 | /** |
| 1266 | * @brief Append a range of characters. |
| 1267 | * @param __first Iterator referencing the first character to append. |
| 1268 | * @param __last Iterator marking the end of the range. |
| 1269 | * @return Reference to this string. |
| 1270 | * |
| 1271 | * Appends characters in the range [__first,__last) to this string. |
| 1272 | */ |
| 1273 | #if __cplusplus201703L >= 201103L |
| 1274 | template<class _InputIterator, |
| 1275 | typename = std::_RequireInputIter<_InputIterator>> |
| 1276 | #else |
| 1277 | template<class _InputIterator> |
| 1278 | #endif |
| 1279 | basic_string& |
| 1280 | append(_InputIterator __first, _InputIterator __last) |
| 1281 | { return this->replace(end(), end(), __first, __last); } |
| 1282 | |
| 1283 | #if __cplusplus201703L >= 201703L |
| 1284 | /** |
| 1285 | * @brief Append a string_view. |
| 1286 | * @param __svt An object convertible to string_view to be appended. |
| 1287 | * @return Reference to this string. |
| 1288 | */ |
| 1289 | template<typename _Tp> |
| 1290 | _If_sv<_Tp, basic_string&> |
| 1291 | append(const _Tp& __svt) |
| 1292 | { |
| 1293 | __sv_type __sv = __svt; |
| 1294 | return this->append(__sv.data(), __sv.size()); |
| 1295 | } |
| 1296 | |
| 1297 | /** |
| 1298 | * @brief Append a range of characters from a string_view. |
| 1299 | * @param __svt An object convertible to string_view to be appended from. |
| 1300 | * @param __pos The position in the string_view to append from. |
| 1301 | * @param __n The number of characters to append from the string_view. |
| 1302 | * @return Reference to this string. |
| 1303 | */ |
| 1304 | template<typename _Tp> |
| 1305 | _If_sv<_Tp, basic_string&> |
| 1306 | append(const _Tp& __svt, size_type __pos, size_type __n = npos) |
| 1307 | { |
| 1308 | __sv_type __sv = __svt; |
| 1309 | return _M_append(__sv.data() |
| 1310 | + std::__sv_check(__sv.size(), __pos, "basic_string::append"), |
| 1311 | std::__sv_limit(__sv.size(), __pos, __n)); |
| 1312 | } |
| 1313 | #endif // C++17 |
| 1314 | |
| 1315 | /** |
| 1316 | * @brief Append a single character. |
| 1317 | * @param __c Character to append. |
| 1318 | */ |
| 1319 | void |
| 1320 | push_back(_CharT __c) |
| 1321 | { |
| 1322 | const size_type __size = this->size(); |
| 1323 | if (__size + 1 > this->capacity()) |
| 1324 | this->_M_mutate(__size, size_type(0), 0, size_type(1)); |
| 1325 | traits_type::assign(this->_M_data()[__size], __c); |
| 1326 | this->_M_set_length(__size + 1); |
| 1327 | } |
| 1328 | |
| 1329 | /** |
| 1330 | * @brief Set value to contents of another string. |
| 1331 | * @param __str Source string to use. |
| 1332 | * @return Reference to this string. |
| 1333 | */ |
| 1334 | basic_string& |
| 1335 | assign(const basic_string& __str) |
| 1336 | { |
| 1337 | #if __cplusplus201703L >= 201103L |
| 1338 | if (_Alloc_traits::_S_propagate_on_copy_assign()) |
| 1339 | { |
| 1340 | if (!_Alloc_traits::_S_always_equal() && !_M_is_local() |
| 1341 | && _M_get_allocator() != __str._M_get_allocator()) |
| 1342 | { |
| 1343 | // Propagating allocator cannot free existing storage so must |
| 1344 | // deallocate it before replacing current allocator. |
| 1345 | if (__str.size() <= _S_local_capacity) |
| 1346 | { |
| 1347 | _M_destroy(_M_allocated_capacity); |
| 1348 | _M_data(_M_local_data()); |
| 1349 | _M_set_length(0); |
| 1350 | } |
| 1351 | else |
| 1352 | { |
| 1353 | const auto __len = __str.size(); |
| 1354 | auto __alloc = __str._M_get_allocator(); |
| 1355 | // If this allocation throws there are no effects: |
| 1356 | auto __ptr = _Alloc_traits::allocate(__alloc, __len + 1); |
| 1357 | _M_destroy(_M_allocated_capacity); |
| 1358 | _M_data(__ptr); |
| 1359 | _M_capacity(__len); |
| 1360 | _M_set_length(__len); |
| 1361 | } |
| 1362 | } |
| 1363 | std::__alloc_on_copy(_M_get_allocator(), __str._M_get_allocator()); |
| 1364 | } |
| 1365 | #endif |
| 1366 | this->_M_assign(__str); |
| 1367 | return *this; |
| 1368 | } |
| 1369 | |
| 1370 | #if __cplusplus201703L >= 201103L |
| 1371 | /** |
| 1372 | * @brief Set value to contents of another string. |
| 1373 | * @param __str Source string to use. |
| 1374 | * @return Reference to this string. |
| 1375 | * |
| 1376 | * This function sets this string to the exact contents of @a __str. |
| 1377 | * @a __str is a valid, but unspecified string. |
| 1378 | */ |
| 1379 | basic_string& |
| 1380 | assign(basic_string&& __str) |
| 1381 | noexcept(_Alloc_traits::_S_nothrow_move()) |
| 1382 | { |
| 1383 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 1384 | // 2063. Contradictory requirements for string move assignment |
| 1385 | return *this = std::move(__str); |
| 1386 | } |
| 1387 | #endif // C++11 |
| 1388 | |
| 1389 | /** |
| 1390 | * @brief Set value to a substring of a string. |
| 1391 | * @param __str The string to use. |
| 1392 | * @param __pos Index of the first character of str. |
| 1393 | * @param __n Number of characters to use. |
| 1394 | * @return Reference to this string. |
| 1395 | * @throw std::out_of_range if @a pos is not a valid index. |
| 1396 | * |
| 1397 | * This function sets this string to the substring of @a __str |
| 1398 | * consisting of @a __n characters at @a __pos. If @a __n is |
| 1399 | * is larger than the number of available characters in @a |
| 1400 | * __str, the remainder of @a __str is used. |
| 1401 | */ |
| 1402 | basic_string& |
| 1403 | assign(const basic_string& __str, size_type __pos, size_type __n = npos) |
| 1404 | { return _M_replace(size_type(0), this->size(), __str._M_data() |
| 1405 | + __str._M_check(__pos, "basic_string::assign"), |
| 1406 | __str._M_limit(__pos, __n)); } |
| 1407 | |
| 1408 | /** |
| 1409 | * @brief Set value to a C substring. |
| 1410 | * @param __s The C string to use. |
| 1411 | * @param __n Number of characters to use. |
| 1412 | * @return Reference to this string. |
| 1413 | * |
| 1414 | * This function sets the value of this string to the first @a __n |
| 1415 | * characters of @a __s. If @a __n is is larger than the number of |
| 1416 | * available characters in @a __s, the remainder of @a __s is used. |
| 1417 | */ |
| 1418 | basic_string& |
| 1419 | assign(const _CharT* __s, size_type __n) |
| 1420 | { |
| 1421 | __glibcxx_requires_string_len(__s, __n); |
| 1422 | return _M_replace(size_type(0), this->size(), __s, __n); |
| 1423 | } |
| 1424 | |
| 1425 | /** |
| 1426 | * @brief Set value to contents of a C string. |
| 1427 | * @param __s The C string to use. |
| 1428 | * @return Reference to this string. |
| 1429 | * |
| 1430 | * This function sets the value of this string to the value of @a __s. |
| 1431 | * The data is copied, so there is no dependence on @a __s once the |
| 1432 | * function returns. |
| 1433 | */ |
| 1434 | basic_string& |
| 1435 | assign(const _CharT* __s) |
| 1436 | { |
| 1437 | __glibcxx_requires_string(__s); |
| 1438 | return _M_replace(size_type(0), this->size(), __s, |
| 1439 | traits_type::length(__s)); |
| 1440 | } |
| 1441 | |
| 1442 | /** |
| 1443 | * @brief Set value to multiple characters. |
| 1444 | * @param __n Length of the resulting string. |
| 1445 | * @param __c The character to use. |
| 1446 | * @return Reference to this string. |
| 1447 | * |
| 1448 | * This function sets the value of this string to @a __n copies of |
| 1449 | * character @a __c. |
| 1450 | */ |
| 1451 | basic_string& |
| 1452 | assign(size_type __n, _CharT __c) |
| 1453 | { return _M_replace_aux(size_type(0), this->size(), __n, __c); } |
| 1454 | |
| 1455 | /** |
| 1456 | * @brief Set value to a range of characters. |
| 1457 | * @param __first Iterator referencing the first character to append. |
| 1458 | * @param __last Iterator marking the end of the range. |
| 1459 | * @return Reference to this string. |
| 1460 | * |
| 1461 | * Sets value of string to characters in the range [__first,__last). |
| 1462 | */ |
| 1463 | #if __cplusplus201703L >= 201103L |
| 1464 | template<class _InputIterator, |
| 1465 | typename = std::_RequireInputIter<_InputIterator>> |
| 1466 | #else |
| 1467 | template<class _InputIterator> |
| 1468 | #endif |
| 1469 | basic_string& |
| 1470 | assign(_InputIterator __first, _InputIterator __last) |
| 1471 | { return this->replace(begin(), end(), __first, __last); } |
| 1472 | |
| 1473 | #if __cplusplus201703L >= 201103L |
| 1474 | /** |
| 1475 | * @brief Set value to an initializer_list of characters. |
| 1476 | * @param __l The initializer_list of characters to assign. |
| 1477 | * @return Reference to this string. |
| 1478 | */ |
| 1479 | basic_string& |
| 1480 | assign(initializer_list<_CharT> __l) |
| 1481 | { return this->assign(__l.begin(), __l.size()); } |
| 1482 | #endif // C++11 |
| 1483 | |
| 1484 | #if __cplusplus201703L >= 201703L |
| 1485 | /** |
| 1486 | * @brief Set value from a string_view. |
| 1487 | * @param __svt The source object convertible to string_view. |
| 1488 | * @return Reference to this string. |
| 1489 | */ |
| 1490 | template<typename _Tp> |
| 1491 | _If_sv<_Tp, basic_string&> |
| 1492 | assign(const _Tp& __svt) |
| 1493 | { |
| 1494 | __sv_type __sv = __svt; |
| 1495 | return this->assign(__sv.data(), __sv.size()); |
| 1496 | } |
| 1497 | |
| 1498 | /** |
| 1499 | * @brief Set value from a range of characters in a string_view. |
| 1500 | * @param __svt The source object convertible to string_view. |
| 1501 | * @param __pos The position in the string_view to assign from. |
| 1502 | * @param __n The number of characters to assign. |
| 1503 | * @return Reference to this string. |
| 1504 | */ |
| 1505 | template<typename _Tp> |
| 1506 | _If_sv<_Tp, basic_string&> |
| 1507 | assign(const _Tp& __svt, size_type __pos, size_type __n = npos) |
| 1508 | { |
| 1509 | __sv_type __sv = __svt; |
| 1510 | return _M_replace(size_type(0), this->size(), |
| 1511 | __sv.data() |
| 1512 | + std::__sv_check(__sv.size(), __pos, "basic_string::assign"), |
| 1513 | std::__sv_limit(__sv.size(), __pos, __n)); |
| 1514 | } |
| 1515 | #endif // C++17 |
| 1516 | |
| 1517 | #if __cplusplus201703L >= 201103L |
| 1518 | /** |
| 1519 | * @brief Insert multiple characters. |
| 1520 | * @param __p Const_iterator referencing location in string to |
| 1521 | * insert at. |
| 1522 | * @param __n Number of characters to insert |
| 1523 | * @param __c The character to insert. |
| 1524 | * @return Iterator referencing the first inserted char. |
| 1525 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1526 | * |
| 1527 | * Inserts @a __n copies of character @a __c starting at the |
| 1528 | * position referenced by iterator @a __p. If adding |
| 1529 | * characters causes the length to exceed max_size(), |
| 1530 | * length_error is thrown. The value of the string doesn't |
| 1531 | * change if an error is thrown. |
| 1532 | */ |
| 1533 | iterator |
| 1534 | insert(const_iterator __p, size_type __n, _CharT __c) |
| 1535 | { |
| 1536 | _GLIBCXX_DEBUG_PEDASSERT(__p >= begin() && __p <= end()); |
| 1537 | const size_type __pos = __p - begin(); |
| 1538 | this->replace(__p, __p, __n, __c); |
| 1539 | return iterator(this->_M_data() + __pos); |
| 1540 | } |
| 1541 | #else |
| 1542 | /** |
| 1543 | * @brief Insert multiple characters. |
| 1544 | * @param __p Iterator referencing location in string to insert at. |
| 1545 | * @param __n Number of characters to insert |
| 1546 | * @param __c The character to insert. |
| 1547 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1548 | * |
| 1549 | * Inserts @a __n copies of character @a __c starting at the |
| 1550 | * position referenced by iterator @a __p. If adding |
| 1551 | * characters causes the length to exceed max_size(), |
| 1552 | * length_error is thrown. The value of the string doesn't |
| 1553 | * change if an error is thrown. |
| 1554 | */ |
| 1555 | void |
| 1556 | insert(iterator __p, size_type __n, _CharT __c) |
| 1557 | { this->replace(__p, __p, __n, __c); } |
| 1558 | #endif |
| 1559 | |
| 1560 | #if __cplusplus201703L >= 201103L |
| 1561 | /** |
| 1562 | * @brief Insert a range of characters. |
| 1563 | * @param __p Const_iterator referencing location in string to |
| 1564 | * insert at. |
| 1565 | * @param __beg Start of range. |
| 1566 | * @param __end End of range. |
| 1567 | * @return Iterator referencing the first inserted char. |
| 1568 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1569 | * |
| 1570 | * Inserts characters in range [beg,end). If adding characters |
| 1571 | * causes the length to exceed max_size(), length_error is |
| 1572 | * thrown. The value of the string doesn't change if an error |
| 1573 | * is thrown. |
| 1574 | */ |
| 1575 | template<class _InputIterator, |
| 1576 | typename = std::_RequireInputIter<_InputIterator>> |
| 1577 | iterator |
| 1578 | insert(const_iterator __p, _InputIterator __beg, _InputIterator __end) |
| 1579 | { |
| 1580 | _GLIBCXX_DEBUG_PEDASSERT(__p >= begin() && __p <= end()); |
| 1581 | const size_type __pos = __p - begin(); |
| 1582 | this->replace(__p, __p, __beg, __end); |
| 1583 | return iterator(this->_M_data() + __pos); |
| 1584 | } |
| 1585 | #else |
| 1586 | /** |
| 1587 | * @brief Insert a range of characters. |
| 1588 | * @param __p Iterator referencing location in string to insert at. |
| 1589 | * @param __beg Start of range. |
| 1590 | * @param __end End of range. |
| 1591 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1592 | * |
| 1593 | * Inserts characters in range [__beg,__end). If adding |
| 1594 | * characters causes the length to exceed max_size(), |
| 1595 | * length_error is thrown. The value of the string doesn't |
| 1596 | * change if an error is thrown. |
| 1597 | */ |
| 1598 | template<class _InputIterator> |
| 1599 | void |
| 1600 | insert(iterator __p, _InputIterator __beg, _InputIterator __end) |
| 1601 | { this->replace(__p, __p, __beg, __end); } |
| 1602 | #endif |
| 1603 | |
| 1604 | #if __cplusplus201703L >= 201103L |
| 1605 | /** |
| 1606 | * @brief Insert an initializer_list of characters. |
| 1607 | * @param __p Iterator referencing location in string to insert at. |
| 1608 | * @param __l The initializer_list of characters to insert. |
| 1609 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1610 | */ |
| 1611 | iterator |
| 1612 | insert(const_iterator __p, initializer_list<_CharT> __l) |
| 1613 | { return this->insert(__p, __l.begin(), __l.end()); } |
| 1614 | |
| 1615 | #ifdef _GLIBCXX_DEFINING_STRING_INSTANTIATIONS |
| 1616 | // See PR libstdc++/83328 |
| 1617 | void |
| 1618 | insert(iterator __p, initializer_list<_CharT> __l) |
| 1619 | { |
| 1620 | _GLIBCXX_DEBUG_PEDASSERT(__p >= begin() && __p <= end()); |
| 1621 | this->insert(__p - begin(), __l.begin(), __l.size()); |
| 1622 | } |
| 1623 | #endif |
| 1624 | #endif // C++11 |
| 1625 | |
| 1626 | /** |
| 1627 | * @brief Insert value of a string. |
| 1628 | * @param __pos1 Position in string to insert at. |
| 1629 | * @param __str The string to insert. |
| 1630 | * @return Reference to this string. |
| 1631 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1632 | * |
| 1633 | * Inserts value of @a __str starting at @a __pos1. If adding |
| 1634 | * characters causes the length to exceed max_size(), |
| 1635 | * length_error is thrown. The value of the string doesn't |
| 1636 | * change if an error is thrown. |
| 1637 | */ |
| 1638 | basic_string& |
| 1639 | insert(size_type __pos1, const basic_string& __str) |
| 1640 | { return this->replace(__pos1, size_type(0), |
| 1641 | __str._M_data(), __str.size()); } |
| 1642 | |
| 1643 | /** |
| 1644 | * @brief Insert a substring. |
| 1645 | * @param __pos1 Position in string to insert at. |
| 1646 | * @param __str The string to insert. |
| 1647 | * @param __pos2 Start of characters in str to insert. |
| 1648 | * @param __n Number of characters to insert. |
| 1649 | * @return Reference to this string. |
| 1650 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1651 | * @throw std::out_of_range If @a pos1 > size() or |
| 1652 | * @a __pos2 > @a str.size(). |
| 1653 | * |
| 1654 | * Starting at @a pos1, insert @a __n character of @a __str |
| 1655 | * beginning with @a __pos2. If adding characters causes the |
| 1656 | * length to exceed max_size(), length_error is thrown. If @a |
| 1657 | * __pos1 is beyond the end of this string or @a __pos2 is |
| 1658 | * beyond the end of @a __str, out_of_range is thrown. The |
| 1659 | * value of the string doesn't change if an error is thrown. |
| 1660 | */ |
| 1661 | basic_string& |
| 1662 | insert(size_type __pos1, const basic_string& __str, |
| 1663 | size_type __pos2, size_type __n = npos) |
| 1664 | { return this->replace(__pos1, size_type(0), __str._M_data() |
| 1665 | + __str._M_check(__pos2, "basic_string::insert"), |
| 1666 | __str._M_limit(__pos2, __n)); } |
| 1667 | |
| 1668 | /** |
| 1669 | * @brief Insert a C substring. |
| 1670 | * @param __pos Position in string to insert at. |
| 1671 | * @param __s The C string to insert. |
| 1672 | * @param __n The number of characters to insert. |
| 1673 | * @return Reference to this string. |
| 1674 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1675 | * @throw std::out_of_range If @a __pos is beyond the end of this |
| 1676 | * string. |
| 1677 | * |
| 1678 | * Inserts the first @a __n characters of @a __s starting at @a |
| 1679 | * __pos. If adding characters causes the length to exceed |
| 1680 | * max_size(), length_error is thrown. If @a __pos is beyond |
| 1681 | * end(), out_of_range is thrown. The value of the string |
| 1682 | * doesn't change if an error is thrown. |
| 1683 | */ |
| 1684 | basic_string& |
| 1685 | insert(size_type __pos, const _CharT* __s, size_type __n) |
| 1686 | { return this->replace(__pos, size_type(0), __s, __n); } |
| 1687 | |
| 1688 | /** |
| 1689 | * @brief Insert a C string. |
| 1690 | * @param __pos Position in string to insert at. |
| 1691 | * @param __s The C string to insert. |
| 1692 | * @return Reference to this string. |
| 1693 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1694 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 1695 | * string. |
| 1696 | * |
| 1697 | * Inserts the first @a n characters of @a __s starting at @a __pos. If |
| 1698 | * adding characters causes the length to exceed max_size(), |
| 1699 | * length_error is thrown. If @a __pos is beyond end(), out_of_range is |
| 1700 | * thrown. The value of the string doesn't change if an error is |
| 1701 | * thrown. |
| 1702 | */ |
| 1703 | basic_string& |
| 1704 | insert(size_type __pos, const _CharT* __s) |
| 1705 | { |
| 1706 | __glibcxx_requires_string(__s); |
| 1707 | return this->replace(__pos, size_type(0), __s, |
| 1708 | traits_type::length(__s)); |
| 1709 | } |
| 1710 | |
| 1711 | /** |
| 1712 | * @brief Insert multiple characters. |
| 1713 | * @param __pos Index in string to insert at. |
| 1714 | * @param __n Number of characters to insert |
| 1715 | * @param __c The character to insert. |
| 1716 | * @return Reference to this string. |
| 1717 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1718 | * @throw std::out_of_range If @a __pos is beyond the end of this |
| 1719 | * string. |
| 1720 | * |
| 1721 | * Inserts @a __n copies of character @a __c starting at index |
| 1722 | * @a __pos. If adding characters causes the length to exceed |
| 1723 | * max_size(), length_error is thrown. If @a __pos > length(), |
| 1724 | * out_of_range is thrown. The value of the string doesn't |
| 1725 | * change if an error is thrown. |
| 1726 | */ |
| 1727 | basic_string& |
| 1728 | insert(size_type __pos, size_type __n, _CharT __c) |
| 1729 | { return _M_replace_aux(_M_check(__pos, "basic_string::insert"), |
| 1730 | size_type(0), __n, __c); } |
| 1731 | |
| 1732 | /** |
| 1733 | * @brief Insert one character. |
| 1734 | * @param __p Iterator referencing position in string to insert at. |
| 1735 | * @param __c The character to insert. |
| 1736 | * @return Iterator referencing newly inserted char. |
| 1737 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1738 | * |
| 1739 | * Inserts character @a __c at position referenced by @a __p. |
| 1740 | * If adding character causes the length to exceed max_size(), |
| 1741 | * length_error is thrown. If @a __p is beyond end of string, |
| 1742 | * out_of_range is thrown. The value of the string doesn't |
| 1743 | * change if an error is thrown. |
| 1744 | */ |
| 1745 | iterator |
| 1746 | insert(__const_iterator __p, _CharT __c) |
| 1747 | { |
| 1748 | _GLIBCXX_DEBUG_PEDASSERT(__p >= begin() && __p <= end()); |
| 1749 | const size_type __pos = __p - begin(); |
| 1750 | _M_replace_aux(__pos, size_type(0), size_type(1), __c); |
| 1751 | return iterator(_M_data() + __pos); |
| 1752 | } |
| 1753 | |
| 1754 | #if __cplusplus201703L >= 201703L |
| 1755 | /** |
| 1756 | * @brief Insert a string_view. |
| 1757 | * @param __pos Position in string to insert at. |
| 1758 | * @param __svt The object convertible to string_view to insert. |
| 1759 | * @return Reference to this string. |
| 1760 | */ |
| 1761 | template<typename _Tp> |
| 1762 | _If_sv<_Tp, basic_string&> |
| 1763 | insert(size_type __pos, const _Tp& __svt) |
| 1764 | { |
| 1765 | __sv_type __sv = __svt; |
| 1766 | return this->insert(__pos, __sv.data(), __sv.size()); |
| 1767 | } |
| 1768 | |
| 1769 | /** |
| 1770 | * @brief Insert a string_view. |
| 1771 | * @param __pos1 Position in string to insert at. |
| 1772 | * @param __svt The object convertible to string_view to insert from. |
| 1773 | * @param __pos2 Start of characters in str to insert. |
| 1774 | * @param __n The number of characters to insert. |
| 1775 | * @return Reference to this string. |
| 1776 | */ |
| 1777 | template<typename _Tp> |
| 1778 | _If_sv<_Tp, basic_string&> |
| 1779 | insert(size_type __pos1, const _Tp& __svt, |
| 1780 | size_type __pos2, size_type __n = npos) |
| 1781 | { |
| 1782 | __sv_type __sv = __svt; |
| 1783 | return this->replace(__pos1, size_type(0), |
| 1784 | __sv.data() |
| 1785 | + std::__sv_check(__sv.size(), __pos2, "basic_string::insert"), |
| 1786 | std::__sv_limit(__sv.size(), __pos2, __n)); |
| 1787 | } |
| 1788 | #endif // C++17 |
| 1789 | |
| 1790 | /** |
| 1791 | * @brief Remove characters. |
| 1792 | * @param __pos Index of first character to remove (default 0). |
| 1793 | * @param __n Number of characters to remove (default remainder). |
| 1794 | * @return Reference to this string. |
| 1795 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 1796 | * string. |
| 1797 | * |
| 1798 | * Removes @a __n characters from this string starting at @a |
| 1799 | * __pos. The length of the string is reduced by @a __n. If |
| 1800 | * there are < @a __n characters to remove, the remainder of |
| 1801 | * the string is truncated. If @a __p is beyond end of string, |
| 1802 | * out_of_range is thrown. The value of the string doesn't |
| 1803 | * change if an error is thrown. |
| 1804 | */ |
| 1805 | basic_string& |
| 1806 | erase(size_type __pos = 0, size_type __n = npos) |
| 1807 | { |
| 1808 | _M_check(__pos, "basic_string::erase"); |
| 1809 | if (__n == npos) |
| 1810 | this->_M_set_length(__pos); |
| 1811 | else if (__n != 0) |
| 1812 | this->_M_erase(__pos, _M_limit(__pos, __n)); |
| 1813 | return *this; |
| 1814 | } |
| 1815 | |
| 1816 | /** |
| 1817 | * @brief Remove one character. |
| 1818 | * @param __position Iterator referencing the character to remove. |
| 1819 | * @return iterator referencing same location after removal. |
| 1820 | * |
| 1821 | * Removes the character at @a __position from this string. The value |
| 1822 | * of the string doesn't change if an error is thrown. |
| 1823 | */ |
| 1824 | iterator |
| 1825 | erase(__const_iterator __position) |
| 1826 | { |
| 1827 | _GLIBCXX_DEBUG_PEDASSERT(__position >= begin() |
| 1828 | && __position < end()); |
| 1829 | const size_type __pos = __position - begin(); |
| 1830 | this->_M_erase(__pos, size_type(1)); |
| 1831 | return iterator(_M_data() + __pos); |
| 1832 | } |
| 1833 | |
| 1834 | /** |
| 1835 | * @brief Remove a range of characters. |
| 1836 | * @param __first Iterator referencing the first character to remove. |
| 1837 | * @param __last Iterator referencing the end of the range. |
| 1838 | * @return Iterator referencing location of first after removal. |
| 1839 | * |
| 1840 | * Removes the characters in the range [first,last) from this string. |
| 1841 | * The value of the string doesn't change if an error is thrown. |
| 1842 | */ |
| 1843 | iterator |
| 1844 | erase(__const_iterator __first, __const_iterator __last) |
| 1845 | { |
| 1846 | _GLIBCXX_DEBUG_PEDASSERT(__first >= begin() && __first <= __last |
| 1847 | && __last <= end()); |
| 1848 | const size_type __pos = __first - begin(); |
| 1849 | if (__last == end()) |
| 1850 | this->_M_set_length(__pos); |
| 1851 | else |
| 1852 | this->_M_erase(__pos, __last - __first); |
| 1853 | return iterator(this->_M_data() + __pos); |
| 1854 | } |
| 1855 | |
| 1856 | #if __cplusplus201703L >= 201103L |
| 1857 | /** |
| 1858 | * @brief Remove the last character. |
| 1859 | * |
| 1860 | * The string must be non-empty. |
| 1861 | */ |
| 1862 | void |
| 1863 | pop_back() noexcept |
| 1864 | { |
| 1865 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 1865, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 1866 | _M_erase(size() - 1, 1); |
| 1867 | } |
| 1868 | #endif // C++11 |
| 1869 | |
| 1870 | /** |
| 1871 | * @brief Replace characters with value from another string. |
| 1872 | * @param __pos Index of first character to replace. |
| 1873 | * @param __n Number of characters to be replaced. |
| 1874 | * @param __str String to insert. |
| 1875 | * @return Reference to this string. |
| 1876 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 1877 | * string. |
| 1878 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1879 | * |
| 1880 | * Removes the characters in the range [__pos,__pos+__n) from |
| 1881 | * this string. In place, the value of @a __str is inserted. |
| 1882 | * If @a __pos is beyond end of string, out_of_range is thrown. |
| 1883 | * If the length of the result exceeds max_size(), length_error |
| 1884 | * is thrown. The value of the string doesn't change if an |
| 1885 | * error is thrown. |
| 1886 | */ |
| 1887 | basic_string& |
| 1888 | replace(size_type __pos, size_type __n, const basic_string& __str) |
| 1889 | { return this->replace(__pos, __n, __str._M_data(), __str.size()); } |
| 1890 | |
| 1891 | /** |
| 1892 | * @brief Replace characters with value from another string. |
| 1893 | * @param __pos1 Index of first character to replace. |
| 1894 | * @param __n1 Number of characters to be replaced. |
| 1895 | * @param __str String to insert. |
| 1896 | * @param __pos2 Index of first character of str to use. |
| 1897 | * @param __n2 Number of characters from str to use. |
| 1898 | * @return Reference to this string. |
| 1899 | * @throw std::out_of_range If @a __pos1 > size() or @a __pos2 > |
| 1900 | * __str.size(). |
| 1901 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1902 | * |
| 1903 | * Removes the characters in the range [__pos1,__pos1 + n) from this |
| 1904 | * string. In place, the value of @a __str is inserted. If @a __pos is |
| 1905 | * beyond end of string, out_of_range is thrown. If the length of the |
| 1906 | * result exceeds max_size(), length_error is thrown. The value of the |
| 1907 | * string doesn't change if an error is thrown. |
| 1908 | */ |
| 1909 | basic_string& |
| 1910 | replace(size_type __pos1, size_type __n1, const basic_string& __str, |
| 1911 | size_type __pos2, size_type __n2 = npos) |
| 1912 | { return this->replace(__pos1, __n1, __str._M_data() |
| 1913 | + __str._M_check(__pos2, "basic_string::replace"), |
| 1914 | __str._M_limit(__pos2, __n2)); } |
| 1915 | |
| 1916 | /** |
| 1917 | * @brief Replace characters with value of a C substring. |
| 1918 | * @param __pos Index of first character to replace. |
| 1919 | * @param __n1 Number of characters to be replaced. |
| 1920 | * @param __s C string to insert. |
| 1921 | * @param __n2 Number of characters from @a s to use. |
| 1922 | * @return Reference to this string. |
| 1923 | * @throw std::out_of_range If @a pos1 > size(). |
| 1924 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1925 | * |
| 1926 | * Removes the characters in the range [__pos,__pos + __n1) |
| 1927 | * from this string. In place, the first @a __n2 characters of |
| 1928 | * @a __s are inserted, or all of @a __s if @a __n2 is too large. If |
| 1929 | * @a __pos is beyond end of string, out_of_range is thrown. If |
| 1930 | * the length of result exceeds max_size(), length_error is |
| 1931 | * thrown. The value of the string doesn't change if an error |
| 1932 | * is thrown. |
| 1933 | */ |
| 1934 | basic_string& |
| 1935 | replace(size_type __pos, size_type __n1, const _CharT* __s, |
| 1936 | size_type __n2) |
| 1937 | { |
| 1938 | __glibcxx_requires_string_len(__s, __n2); |
| 1939 | return _M_replace(_M_check(__pos, "basic_string::replace"), |
| 1940 | _M_limit(__pos, __n1), __s, __n2); |
| 1941 | } |
| 1942 | |
| 1943 | /** |
| 1944 | * @brief Replace characters with value of a C string. |
| 1945 | * @param __pos Index of first character to replace. |
| 1946 | * @param __n1 Number of characters to be replaced. |
| 1947 | * @param __s C string to insert. |
| 1948 | * @return Reference to this string. |
| 1949 | * @throw std::out_of_range If @a pos > size(). |
| 1950 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1951 | * |
| 1952 | * Removes the characters in the range [__pos,__pos + __n1) |
| 1953 | * from this string. In place, the characters of @a __s are |
| 1954 | * inserted. If @a __pos is beyond end of string, out_of_range |
| 1955 | * is thrown. If the length of result exceeds max_size(), |
| 1956 | * length_error is thrown. The value of the string doesn't |
| 1957 | * change if an error is thrown. |
| 1958 | */ |
| 1959 | basic_string& |
| 1960 | replace(size_type __pos, size_type __n1, const _CharT* __s) |
| 1961 | { |
| 1962 | __glibcxx_requires_string(__s); |
| 1963 | return this->replace(__pos, __n1, __s, traits_type::length(__s)); |
| 1964 | } |
| 1965 | |
| 1966 | /** |
| 1967 | * @brief Replace characters with multiple characters. |
| 1968 | * @param __pos Index of first character to replace. |
| 1969 | * @param __n1 Number of characters to be replaced. |
| 1970 | * @param __n2 Number of characters to insert. |
| 1971 | * @param __c Character to insert. |
| 1972 | * @return Reference to this string. |
| 1973 | * @throw std::out_of_range If @a __pos > size(). |
| 1974 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1975 | * |
| 1976 | * Removes the characters in the range [pos,pos + n1) from this |
| 1977 | * string. In place, @a __n2 copies of @a __c are inserted. |
| 1978 | * If @a __pos is beyond end of string, out_of_range is thrown. |
| 1979 | * If the length of result exceeds max_size(), length_error is |
| 1980 | * thrown. The value of the string doesn't change if an error |
| 1981 | * is thrown. |
| 1982 | */ |
| 1983 | basic_string& |
| 1984 | replace(size_type __pos, size_type __n1, size_type __n2, _CharT __c) |
| 1985 | { return _M_replace_aux(_M_check(__pos, "basic_string::replace"), |
| 1986 | _M_limit(__pos, __n1), __n2, __c); } |
| 1987 | |
| 1988 | /** |
| 1989 | * @brief Replace range of characters with string. |
| 1990 | * @param __i1 Iterator referencing start of range to replace. |
| 1991 | * @param __i2 Iterator referencing end of range to replace. |
| 1992 | * @param __str String value to insert. |
| 1993 | * @return Reference to this string. |
| 1994 | * @throw std::length_error If new length exceeds @c max_size(). |
| 1995 | * |
| 1996 | * Removes the characters in the range [__i1,__i2). In place, |
| 1997 | * the value of @a __str is inserted. If the length of result |
| 1998 | * exceeds max_size(), length_error is thrown. The value of |
| 1999 | * the string doesn't change if an error is thrown. |
| 2000 | */ |
| 2001 | basic_string& |
| 2002 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2003 | const basic_string& __str) |
| 2004 | { return this->replace(__i1, __i2, __str._M_data(), __str.size()); } |
| 2005 | |
| 2006 | /** |
| 2007 | * @brief Replace range of characters with C substring. |
| 2008 | * @param __i1 Iterator referencing start of range to replace. |
| 2009 | * @param __i2 Iterator referencing end of range to replace. |
| 2010 | * @param __s C string value to insert. |
| 2011 | * @param __n Number of characters from s to insert. |
| 2012 | * @return Reference to this string. |
| 2013 | * @throw std::length_error If new length exceeds @c max_size(). |
| 2014 | * |
| 2015 | * Removes the characters in the range [__i1,__i2). In place, |
| 2016 | * the first @a __n characters of @a __s are inserted. If the |
| 2017 | * length of result exceeds max_size(), length_error is thrown. |
| 2018 | * The value of the string doesn't change if an error is |
| 2019 | * thrown. |
| 2020 | */ |
| 2021 | basic_string& |
| 2022 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2023 | const _CharT* __s, size_type __n) |
| 2024 | { |
| 2025 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2026 | && __i2 <= end()); |
| 2027 | return this->replace(__i1 - begin(), __i2 - __i1, __s, __n); |
| 2028 | } |
| 2029 | |
| 2030 | /** |
| 2031 | * @brief Replace range of characters with C string. |
| 2032 | * @param __i1 Iterator referencing start of range to replace. |
| 2033 | * @param __i2 Iterator referencing end of range to replace. |
| 2034 | * @param __s C string value to insert. |
| 2035 | * @return Reference to this string. |
| 2036 | * @throw std::length_error If new length exceeds @c max_size(). |
| 2037 | * |
| 2038 | * Removes the characters in the range [__i1,__i2). In place, |
| 2039 | * the characters of @a __s are inserted. If the length of |
| 2040 | * result exceeds max_size(), length_error is thrown. The |
| 2041 | * value of the string doesn't change if an error is thrown. |
| 2042 | */ |
| 2043 | basic_string& |
| 2044 | replace(__const_iterator __i1, __const_iterator __i2, const _CharT* __s) |
| 2045 | { |
| 2046 | __glibcxx_requires_string(__s); |
| 2047 | return this->replace(__i1, __i2, __s, traits_type::length(__s)); |
| 2048 | } |
| 2049 | |
| 2050 | /** |
| 2051 | * @brief Replace range of characters with multiple characters |
| 2052 | * @param __i1 Iterator referencing start of range to replace. |
| 2053 | * @param __i2 Iterator referencing end of range to replace. |
| 2054 | * @param __n Number of characters to insert. |
| 2055 | * @param __c Character to insert. |
| 2056 | * @return Reference to this string. |
| 2057 | * @throw std::length_error If new length exceeds @c max_size(). |
| 2058 | * |
| 2059 | * Removes the characters in the range [__i1,__i2). In place, |
| 2060 | * @a __n copies of @a __c are inserted. If the length of |
| 2061 | * result exceeds max_size(), length_error is thrown. The |
| 2062 | * value of the string doesn't change if an error is thrown. |
| 2063 | */ |
| 2064 | basic_string& |
| 2065 | replace(__const_iterator __i1, __const_iterator __i2, size_type __n, |
| 2066 | _CharT __c) |
| 2067 | { |
| 2068 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2069 | && __i2 <= end()); |
| 2070 | return _M_replace_aux(__i1 - begin(), __i2 - __i1, __n, __c); |
| 2071 | } |
| 2072 | |
| 2073 | /** |
| 2074 | * @brief Replace range of characters with range. |
| 2075 | * @param __i1 Iterator referencing start of range to replace. |
| 2076 | * @param __i2 Iterator referencing end of range to replace. |
| 2077 | * @param __k1 Iterator referencing start of range to insert. |
| 2078 | * @param __k2 Iterator referencing end of range to insert. |
| 2079 | * @return Reference to this string. |
| 2080 | * @throw std::length_error If new length exceeds @c max_size(). |
| 2081 | * |
| 2082 | * Removes the characters in the range [__i1,__i2). In place, |
| 2083 | * characters in the range [__k1,__k2) are inserted. If the |
| 2084 | * length of result exceeds max_size(), length_error is thrown. |
| 2085 | * The value of the string doesn't change if an error is |
| 2086 | * thrown. |
| 2087 | */ |
| 2088 | #if __cplusplus201703L >= 201103L |
| 2089 | template<class _InputIterator, |
| 2090 | typename = std::_RequireInputIter<_InputIterator>> |
| 2091 | basic_string& |
| 2092 | replace(const_iterator __i1, const_iterator __i2, |
| 2093 | _InputIterator __k1, _InputIterator __k2) |
| 2094 | { |
| 2095 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2096 | && __i2 <= end()); |
| 2097 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2098 | return this->_M_replace_dispatch(__i1, __i2, __k1, __k2, |
| 2099 | std::__false_type()); |
| 2100 | } |
| 2101 | #else |
| 2102 | template<class _InputIterator> |
| 2103 | #ifdef _GLIBCXX_DISAMBIGUATE_REPLACE_INST |
| 2104 | typename __enable_if_not_native_iterator<_InputIterator>::__type |
| 2105 | #else |
| 2106 | basic_string& |
| 2107 | #endif |
| 2108 | replace(iterator __i1, iterator __i2, |
| 2109 | _InputIterator __k1, _InputIterator __k2) |
| 2110 | { |
| 2111 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2112 | && __i2 <= end()); |
| 2113 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2114 | typedef typename std::__is_integer<_InputIterator>::__type _Integral; |
| 2115 | return _M_replace_dispatch(__i1, __i2, __k1, __k2, _Integral()); |
| 2116 | } |
| 2117 | #endif |
| 2118 | |
| 2119 | // Specializations for the common case of pointer and iterator: |
| 2120 | // useful to avoid the overhead of temporary buffering in _M_replace. |
| 2121 | basic_string& |
| 2122 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2123 | _CharT* __k1, _CharT* __k2) |
| 2124 | { |
| 2125 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2126 | && __i2 <= end()); |
| 2127 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2128 | return this->replace(__i1 - begin(), __i2 - __i1, |
| 2129 | __k1, __k2 - __k1); |
| 2130 | } |
| 2131 | |
| 2132 | basic_string& |
| 2133 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2134 | const _CharT* __k1, const _CharT* __k2) |
| 2135 | { |
| 2136 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2137 | && __i2 <= end()); |
| 2138 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2139 | return this->replace(__i1 - begin(), __i2 - __i1, |
| 2140 | __k1, __k2 - __k1); |
| 2141 | } |
| 2142 | |
| 2143 | basic_string& |
| 2144 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2145 | iterator __k1, iterator __k2) |
| 2146 | { |
| 2147 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2148 | && __i2 <= end()); |
| 2149 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2150 | return this->replace(__i1 - begin(), __i2 - __i1, |
| 2151 | __k1.base(), __k2 - __k1); |
| 2152 | } |
| 2153 | |
| 2154 | basic_string& |
| 2155 | replace(__const_iterator __i1, __const_iterator __i2, |
| 2156 | const_iterator __k1, const_iterator __k2) |
| 2157 | { |
| 2158 | _GLIBCXX_DEBUG_PEDASSERT(begin() <= __i1 && __i1 <= __i2 |
| 2159 | && __i2 <= end()); |
| 2160 | __glibcxx_requires_valid_range(__k1, __k2); |
| 2161 | return this->replace(__i1 - begin(), __i2 - __i1, |
| 2162 | __k1.base(), __k2 - __k1); |
| 2163 | } |
| 2164 | |
| 2165 | #if __cplusplus201703L >= 201103L |
| 2166 | /** |
| 2167 | * @brief Replace range of characters with initializer_list. |
| 2168 | * @param __i1 Iterator referencing start of range to replace. |
| 2169 | * @param __i2 Iterator referencing end of range to replace. |
| 2170 | * @param __l The initializer_list of characters to insert. |
| 2171 | * @return Reference to this string. |
| 2172 | * @throw std::length_error If new length exceeds @c max_size(). |
| 2173 | * |
| 2174 | * Removes the characters in the range [__i1,__i2). In place, |
| 2175 | * characters in the range [__k1,__k2) are inserted. If the |
| 2176 | * length of result exceeds max_size(), length_error is thrown. |
| 2177 | * The value of the string doesn't change if an error is |
| 2178 | * thrown. |
| 2179 | */ |
| 2180 | basic_string& replace(const_iterator __i1, const_iterator __i2, |
| 2181 | initializer_list<_CharT> __l) |
| 2182 | { return this->replace(__i1, __i2, __l.begin(), __l.size()); } |
| 2183 | #endif // C++11 |
| 2184 | |
| 2185 | #if __cplusplus201703L >= 201703L |
| 2186 | /** |
| 2187 | * @brief Replace range of characters with string_view. |
| 2188 | * @param __pos The position to replace at. |
| 2189 | * @param __n The number of characters to replace. |
| 2190 | * @param __svt The object convertible to string_view to insert. |
| 2191 | * @return Reference to this string. |
| 2192 | */ |
| 2193 | template<typename _Tp> |
| 2194 | _If_sv<_Tp, basic_string&> |
| 2195 | replace(size_type __pos, size_type __n, const _Tp& __svt) |
| 2196 | { |
| 2197 | __sv_type __sv = __svt; |
| 2198 | return this->replace(__pos, __n, __sv.data(), __sv.size()); |
| 2199 | } |
| 2200 | |
| 2201 | /** |
| 2202 | * @brief Replace range of characters with string_view. |
| 2203 | * @param __pos1 The position to replace at. |
| 2204 | * @param __n1 The number of characters to replace. |
| 2205 | * @param __svt The object convertible to string_view to insert from. |
| 2206 | * @param __pos2 The position in the string_view to insert from. |
| 2207 | * @param __n2 The number of characters to insert. |
| 2208 | * @return Reference to this string. |
| 2209 | */ |
| 2210 | template<typename _Tp> |
| 2211 | _If_sv<_Tp, basic_string&> |
| 2212 | replace(size_type __pos1, size_type __n1, const _Tp& __svt, |
| 2213 | size_type __pos2, size_type __n2 = npos) |
| 2214 | { |
| 2215 | __sv_type __sv = __svt; |
| 2216 | return this->replace(__pos1, __n1, |
| 2217 | __sv.data() |
| 2218 | + std::__sv_check(__sv.size(), __pos2, "basic_string::replace"), |
| 2219 | std::__sv_limit(__sv.size(), __pos2, __n2)); |
| 2220 | } |
| 2221 | |
| 2222 | /** |
| 2223 | * @brief Replace range of characters with string_view. |
| 2224 | * @param __i1 An iterator referencing the start position |
| 2225 | to replace at. |
| 2226 | * @param __i2 An iterator referencing the end position |
| 2227 | for the replace. |
| 2228 | * @param __svt The object convertible to string_view to insert from. |
| 2229 | * @return Reference to this string. |
| 2230 | */ |
| 2231 | template<typename _Tp> |
| 2232 | _If_sv<_Tp, basic_string&> |
| 2233 | replace(const_iterator __i1, const_iterator __i2, const _Tp& __svt) |
| 2234 | { |
| 2235 | __sv_type __sv = __svt; |
| 2236 | return this->replace(__i1 - begin(), __i2 - __i1, __sv); |
| 2237 | } |
| 2238 | #endif // C++17 |
| 2239 | |
| 2240 | private: |
| 2241 | template<class _Integer> |
| 2242 | basic_string& |
| 2243 | _M_replace_dispatch(const_iterator __i1, const_iterator __i2, |
| 2244 | _Integer __n, _Integer __val, __true_type) |
| 2245 | { return _M_replace_aux(__i1 - begin(), __i2 - __i1, __n, __val); } |
| 2246 | |
| 2247 | template<class _InputIterator> |
| 2248 | basic_string& |
| 2249 | _M_replace_dispatch(const_iterator __i1, const_iterator __i2, |
| 2250 | _InputIterator __k1, _InputIterator __k2, |
| 2251 | __false_type); |
| 2252 | |
| 2253 | basic_string& |
| 2254 | _M_replace_aux(size_type __pos1, size_type __n1, size_type __n2, |
| 2255 | _CharT __c); |
| 2256 | |
| 2257 | basic_string& |
| 2258 | _M_replace(size_type __pos, size_type __len1, const _CharT* __s, |
| 2259 | const size_type __len2); |
| 2260 | |
| 2261 | basic_string& |
| 2262 | _M_append(const _CharT* __s, size_type __n); |
| 2263 | |
| 2264 | public: |
| 2265 | |
| 2266 | /** |
| 2267 | * @brief Copy substring into C string. |
| 2268 | * @param __s C string to copy value into. |
| 2269 | * @param __n Number of characters to copy. |
| 2270 | * @param __pos Index of first character to copy. |
| 2271 | * @return Number of characters actually copied |
| 2272 | * @throw std::out_of_range If __pos > size(). |
| 2273 | * |
| 2274 | * Copies up to @a __n characters starting at @a __pos into the |
| 2275 | * C string @a __s. If @a __pos is %greater than size(), |
| 2276 | * out_of_range is thrown. |
| 2277 | */ |
| 2278 | size_type |
| 2279 | copy(_CharT* __s, size_type __n, size_type __pos = 0) const; |
| 2280 | |
| 2281 | /** |
| 2282 | * @brief Swap contents with another string. |
| 2283 | * @param __s String to swap with. |
| 2284 | * |
| 2285 | * Exchanges the contents of this string with that of @a __s in constant |
| 2286 | * time. |
| 2287 | */ |
| 2288 | void |
| 2289 | swap(basic_string& __s) _GLIBCXX_NOEXCEPTnoexcept; |
| 2290 | |
| 2291 | // String operations: |
| 2292 | /** |
| 2293 | * @brief Return const pointer to null-terminated contents. |
| 2294 | * |
| 2295 | * This is a handle to internal data. Do not modify or dire things may |
| 2296 | * happen. |
| 2297 | */ |
| 2298 | const _CharT* |
| 2299 | c_str() const _GLIBCXX_NOEXCEPTnoexcept |
| 2300 | { return _M_data(); } |
| 2301 | |
| 2302 | /** |
| 2303 | * @brief Return const pointer to contents. |
| 2304 | * |
| 2305 | * This is a pointer to internal data. It is undefined to modify |
| 2306 | * the contents through the returned pointer. To get a pointer that |
| 2307 | * allows modifying the contents use @c &str[0] instead, |
| 2308 | * (or in C++17 the non-const @c str.data() overload). |
| 2309 | */ |
| 2310 | const _CharT* |
| 2311 | data() const _GLIBCXX_NOEXCEPTnoexcept |
| 2312 | { return _M_data(); } |
| 2313 | |
| 2314 | #if __cplusplus201703L >= 201703L |
| 2315 | /** |
| 2316 | * @brief Return non-const pointer to contents. |
| 2317 | * |
| 2318 | * This is a pointer to the character sequence held by the string. |
| 2319 | * Modifying the characters in the sequence is allowed. |
| 2320 | */ |
| 2321 | _CharT* |
| 2322 | data() noexcept |
| 2323 | { return _M_data(); } |
| 2324 | #endif |
| 2325 | |
| 2326 | /** |
| 2327 | * @brief Return copy of allocator used to construct this string. |
| 2328 | */ |
| 2329 | allocator_type |
| 2330 | get_allocator() const _GLIBCXX_NOEXCEPTnoexcept |
| 2331 | { return _M_get_allocator(); } |
| 2332 | |
| 2333 | /** |
| 2334 | * @brief Find position of a C substring. |
| 2335 | * @param __s C string to locate. |
| 2336 | * @param __pos Index of character to search from. |
| 2337 | * @param __n Number of characters from @a s to search for. |
| 2338 | * @return Index of start of first occurrence. |
| 2339 | * |
| 2340 | * Starting from @a __pos, searches forward for the first @a |
| 2341 | * __n characters in @a __s within this string. If found, |
| 2342 | * returns the index where it begins. If not found, returns |
| 2343 | * npos. |
| 2344 | */ |
| 2345 | size_type |
| 2346 | find(const _CharT* __s, size_type __pos, size_type __n) const |
| 2347 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2348 | |
| 2349 | /** |
| 2350 | * @brief Find position of a string. |
| 2351 | * @param __str String to locate. |
| 2352 | * @param __pos Index of character to search from (default 0). |
| 2353 | * @return Index of start of first occurrence. |
| 2354 | * |
| 2355 | * Starting from @a __pos, searches forward for value of @a __str within |
| 2356 | * this string. If found, returns the index where it begins. If not |
| 2357 | * found, returns npos. |
| 2358 | */ |
| 2359 | size_type |
| 2360 | find(const basic_string& __str, size_type __pos = 0) const |
| 2361 | _GLIBCXX_NOEXCEPTnoexcept |
| 2362 | { return this->find(__str.data(), __pos, __str.size()); } |
| 2363 | |
| 2364 | #if __cplusplus201703L >= 201703L |
| 2365 | /** |
| 2366 | * @brief Find position of a string_view. |
| 2367 | * @param __svt The object convertible to string_view to locate. |
| 2368 | * @param __pos Index of character to search from (default 0). |
| 2369 | * @return Index of start of first occurrence. |
| 2370 | */ |
| 2371 | template<typename _Tp> |
| 2372 | _If_sv<_Tp, size_type> |
| 2373 | find(const _Tp& __svt, size_type __pos = 0) const |
| 2374 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2375 | { |
| 2376 | __sv_type __sv = __svt; |
| 2377 | return this->find(__sv.data(), __pos, __sv.size()); |
| 2378 | } |
| 2379 | #endif // C++17 |
| 2380 | |
| 2381 | /** |
| 2382 | * @brief Find position of a C string. |
| 2383 | * @param __s C string to locate. |
| 2384 | * @param __pos Index of character to search from (default 0). |
| 2385 | * @return Index of start of first occurrence. |
| 2386 | * |
| 2387 | * Starting from @a __pos, searches forward for the value of @a |
| 2388 | * __s within this string. If found, returns the index where |
| 2389 | * it begins. If not found, returns npos. |
| 2390 | */ |
| 2391 | size_type |
| 2392 | find(const _CharT* __s, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept |
| 2393 | { |
| 2394 | __glibcxx_requires_string(__s); |
| 2395 | return this->find(__s, __pos, traits_type::length(__s)); |
| 2396 | } |
| 2397 | |
| 2398 | /** |
| 2399 | * @brief Find position of a character. |
| 2400 | * @param __c Character to locate. |
| 2401 | * @param __pos Index of character to search from (default 0). |
| 2402 | * @return Index of first occurrence. |
| 2403 | * |
| 2404 | * Starting from @a __pos, searches forward for @a __c within |
| 2405 | * this string. If found, returns the index where it was |
| 2406 | * found. If not found, returns npos. |
| 2407 | */ |
| 2408 | size_type |
| 2409 | find(_CharT __c, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept; |
| 2410 | |
| 2411 | /** |
| 2412 | * @brief Find last position of a string. |
| 2413 | * @param __str String to locate. |
| 2414 | * @param __pos Index of character to search back from (default end). |
| 2415 | * @return Index of start of last occurrence. |
| 2416 | * |
| 2417 | * Starting from @a __pos, searches backward for value of @a |
| 2418 | * __str within this string. If found, returns the index where |
| 2419 | * it begins. If not found, returns npos. |
| 2420 | */ |
| 2421 | size_type |
| 2422 | rfind(const basic_string& __str, size_type __pos = npos) const |
| 2423 | _GLIBCXX_NOEXCEPTnoexcept |
| 2424 | { return this->rfind(__str.data(), __pos, __str.size()); } |
| 2425 | |
| 2426 | #if __cplusplus201703L >= 201703L |
| 2427 | /** |
| 2428 | * @brief Find last position of a string_view. |
| 2429 | * @param __svt The object convertible to string_view to locate. |
| 2430 | * @param __pos Index of character to search back from (default end). |
| 2431 | * @return Index of start of last occurrence. |
| 2432 | */ |
| 2433 | template<typename _Tp> |
| 2434 | _If_sv<_Tp, size_type> |
| 2435 | rfind(const _Tp& __svt, size_type __pos = npos) const |
| 2436 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2437 | { |
| 2438 | __sv_type __sv = __svt; |
| 2439 | return this->rfind(__sv.data(), __pos, __sv.size()); |
| 2440 | } |
| 2441 | #endif // C++17 |
| 2442 | |
| 2443 | /** |
| 2444 | * @brief Find last position of a C substring. |
| 2445 | * @param __s C string to locate. |
| 2446 | * @param __pos Index of character to search back from. |
| 2447 | * @param __n Number of characters from s to search for. |
| 2448 | * @return Index of start of last occurrence. |
| 2449 | * |
| 2450 | * Starting from @a __pos, searches backward for the first @a |
| 2451 | * __n characters in @a __s within this string. If found, |
| 2452 | * returns the index where it begins. If not found, returns |
| 2453 | * npos. |
| 2454 | */ |
| 2455 | size_type |
| 2456 | rfind(const _CharT* __s, size_type __pos, size_type __n) const |
| 2457 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2458 | |
| 2459 | /** |
| 2460 | * @brief Find last position of a C string. |
| 2461 | * @param __s C string to locate. |
| 2462 | * @param __pos Index of character to start search at (default end). |
| 2463 | * @return Index of start of last occurrence. |
| 2464 | * |
| 2465 | * Starting from @a __pos, searches backward for the value of |
| 2466 | * @a __s within this string. If found, returns the index |
| 2467 | * where it begins. If not found, returns npos. |
| 2468 | */ |
| 2469 | size_type |
| 2470 | rfind(const _CharT* __s, size_type __pos = npos) const |
| 2471 | { |
| 2472 | __glibcxx_requires_string(__s); |
| 2473 | return this->rfind(__s, __pos, traits_type::length(__s)); |
| 2474 | } |
| 2475 | |
| 2476 | /** |
| 2477 | * @brief Find last position of a character. |
| 2478 | * @param __c Character to locate. |
| 2479 | * @param __pos Index of character to search back from (default end). |
| 2480 | * @return Index of last occurrence. |
| 2481 | * |
| 2482 | * Starting from @a __pos, searches backward for @a __c within |
| 2483 | * this string. If found, returns the index where it was |
| 2484 | * found. If not found, returns npos. |
| 2485 | */ |
| 2486 | size_type |
| 2487 | rfind(_CharT __c, size_type __pos = npos) const _GLIBCXX_NOEXCEPTnoexcept; |
| 2488 | |
| 2489 | /** |
| 2490 | * @brief Find position of a character of string. |
| 2491 | * @param __str String containing characters to locate. |
| 2492 | * @param __pos Index of character to search from (default 0). |
| 2493 | * @return Index of first occurrence. |
| 2494 | * |
| 2495 | * Starting from @a __pos, searches forward for one of the |
| 2496 | * characters of @a __str within this string. If found, |
| 2497 | * returns the index where it was found. If not found, returns |
| 2498 | * npos. |
| 2499 | */ |
| 2500 | size_type |
| 2501 | find_first_of(const basic_string& __str, size_type __pos = 0) const |
| 2502 | _GLIBCXX_NOEXCEPTnoexcept |
| 2503 | { return this->find_first_of(__str.data(), __pos, __str.size()); } |
| 2504 | |
| 2505 | #if __cplusplus201703L >= 201703L |
| 2506 | /** |
| 2507 | * @brief Find position of a character of a string_view. |
| 2508 | * @param __svt An object convertible to string_view containing |
| 2509 | * characters to locate. |
| 2510 | * @param __pos Index of character to search from (default 0). |
| 2511 | * @return Index of first occurrence. |
| 2512 | */ |
| 2513 | template<typename _Tp> |
| 2514 | _If_sv<_Tp, size_type> |
| 2515 | find_first_of(const _Tp& __svt, size_type __pos = 0) const |
| 2516 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2517 | { |
| 2518 | __sv_type __sv = __svt; |
| 2519 | return this->find_first_of(__sv.data(), __pos, __sv.size()); |
| 2520 | } |
| 2521 | #endif // C++17 |
| 2522 | |
| 2523 | /** |
| 2524 | * @brief Find position of a character of C substring. |
| 2525 | * @param __s String containing characters to locate. |
| 2526 | * @param __pos Index of character to search from. |
| 2527 | * @param __n Number of characters from s to search for. |
| 2528 | * @return Index of first occurrence. |
| 2529 | * |
| 2530 | * Starting from @a __pos, searches forward for one of the |
| 2531 | * first @a __n characters of @a __s within this string. If |
| 2532 | * found, returns the index where it was found. If not found, |
| 2533 | * returns npos. |
| 2534 | */ |
| 2535 | size_type |
| 2536 | find_first_of(const _CharT* __s, size_type __pos, size_type __n) const |
| 2537 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2538 | |
| 2539 | /** |
| 2540 | * @brief Find position of a character of C string. |
| 2541 | * @param __s String containing characters to locate. |
| 2542 | * @param __pos Index of character to search from (default 0). |
| 2543 | * @return Index of first occurrence. |
| 2544 | * |
| 2545 | * Starting from @a __pos, searches forward for one of the |
| 2546 | * characters of @a __s within this string. If found, returns |
| 2547 | * the index where it was found. If not found, returns npos. |
| 2548 | */ |
| 2549 | size_type |
| 2550 | find_first_of(const _CharT* __s, size_type __pos = 0) const |
| 2551 | _GLIBCXX_NOEXCEPTnoexcept |
| 2552 | { |
| 2553 | __glibcxx_requires_string(__s); |
| 2554 | return this->find_first_of(__s, __pos, traits_type::length(__s)); |
| 2555 | } |
| 2556 | |
| 2557 | /** |
| 2558 | * @brief Find position of a character. |
| 2559 | * @param __c Character to locate. |
| 2560 | * @param __pos Index of character to search from (default 0). |
| 2561 | * @return Index of first occurrence. |
| 2562 | * |
| 2563 | * Starting from @a __pos, searches forward for the character |
| 2564 | * @a __c within this string. If found, returns the index |
| 2565 | * where it was found. If not found, returns npos. |
| 2566 | * |
| 2567 | * Note: equivalent to find(__c, __pos). |
| 2568 | */ |
| 2569 | size_type |
| 2570 | find_first_of(_CharT __c, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept |
| 2571 | { return this->find(__c, __pos); } |
| 2572 | |
| 2573 | /** |
| 2574 | * @brief Find last position of a character of string. |
| 2575 | * @param __str String containing characters to locate. |
| 2576 | * @param __pos Index of character to search back from (default end). |
| 2577 | * @return Index of last occurrence. |
| 2578 | * |
| 2579 | * Starting from @a __pos, searches backward for one of the |
| 2580 | * characters of @a __str within this string. If found, |
| 2581 | * returns the index where it was found. If not found, returns |
| 2582 | * npos. |
| 2583 | */ |
| 2584 | size_type |
| 2585 | find_last_of(const basic_string& __str, size_type __pos = npos) const |
| 2586 | _GLIBCXX_NOEXCEPTnoexcept |
| 2587 | { return this->find_last_of(__str.data(), __pos, __str.size()); } |
| 2588 | |
| 2589 | #if __cplusplus201703L >= 201703L |
| 2590 | /** |
| 2591 | * @brief Find last position of a character of string. |
| 2592 | * @param __svt An object convertible to string_view containing |
| 2593 | * characters to locate. |
| 2594 | * @param __pos Index of character to search back from (default end). |
| 2595 | * @return Index of last occurrence. |
| 2596 | */ |
| 2597 | template<typename _Tp> |
| 2598 | _If_sv<_Tp, size_type> |
| 2599 | find_last_of(const _Tp& __svt, size_type __pos = npos) const |
| 2600 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2601 | { |
| 2602 | __sv_type __sv = __svt; |
| 2603 | return this->find_last_of(__sv.data(), __pos, __sv.size()); |
| 2604 | } |
| 2605 | #endif // C++17 |
| 2606 | |
| 2607 | /** |
| 2608 | * @brief Find last position of a character of C substring. |
| 2609 | * @param __s C string containing characters to locate. |
| 2610 | * @param __pos Index of character to search back from. |
| 2611 | * @param __n Number of characters from s to search for. |
| 2612 | * @return Index of last occurrence. |
| 2613 | * |
| 2614 | * Starting from @a __pos, searches backward for one of the |
| 2615 | * first @a __n characters of @a __s within this string. If |
| 2616 | * found, returns the index where it was found. If not found, |
| 2617 | * returns npos. |
| 2618 | */ |
| 2619 | size_type |
| 2620 | find_last_of(const _CharT* __s, size_type __pos, size_type __n) const |
| 2621 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2622 | |
| 2623 | /** |
| 2624 | * @brief Find last position of a character of C string. |
| 2625 | * @param __s C string containing characters to locate. |
| 2626 | * @param __pos Index of character to search back from (default end). |
| 2627 | * @return Index of last occurrence. |
| 2628 | * |
| 2629 | * Starting from @a __pos, searches backward for one of the |
| 2630 | * characters of @a __s within this string. If found, returns |
| 2631 | * the index where it was found. If not found, returns npos. |
| 2632 | */ |
| 2633 | size_type |
| 2634 | find_last_of(const _CharT* __s, size_type __pos = npos) const |
| 2635 | _GLIBCXX_NOEXCEPTnoexcept |
| 2636 | { |
| 2637 | __glibcxx_requires_string(__s); |
| 2638 | return this->find_last_of(__s, __pos, traits_type::length(__s)); |
| 2639 | } |
| 2640 | |
| 2641 | /** |
| 2642 | * @brief Find last position of a character. |
| 2643 | * @param __c Character to locate. |
| 2644 | * @param __pos Index of character to search back from (default end). |
| 2645 | * @return Index of last occurrence. |
| 2646 | * |
| 2647 | * Starting from @a __pos, searches backward for @a __c within |
| 2648 | * this string. If found, returns the index where it was |
| 2649 | * found. If not found, returns npos. |
| 2650 | * |
| 2651 | * Note: equivalent to rfind(__c, __pos). |
| 2652 | */ |
| 2653 | size_type |
| 2654 | find_last_of(_CharT __c, size_type __pos = npos) const _GLIBCXX_NOEXCEPTnoexcept |
| 2655 | { return this->rfind(__c, __pos); } |
| 2656 | |
| 2657 | /** |
| 2658 | * @brief Find position of a character not in string. |
| 2659 | * @param __str String containing characters to avoid. |
| 2660 | * @param __pos Index of character to search from (default 0). |
| 2661 | * @return Index of first occurrence. |
| 2662 | * |
| 2663 | * Starting from @a __pos, searches forward for a character not contained |
| 2664 | * in @a __str within this string. If found, returns the index where it |
| 2665 | * was found. If not found, returns npos. |
| 2666 | */ |
| 2667 | size_type |
| 2668 | find_first_not_of(const basic_string& __str, size_type __pos = 0) const |
| 2669 | _GLIBCXX_NOEXCEPTnoexcept |
| 2670 | { return this->find_first_not_of(__str.data(), __pos, __str.size()); } |
| 2671 | |
| 2672 | #if __cplusplus201703L >= 201703L |
| 2673 | /** |
| 2674 | * @brief Find position of a character not in a string_view. |
| 2675 | * @param __svt A object convertible to string_view containing |
| 2676 | * characters to avoid. |
| 2677 | * @param __pos Index of character to search from (default 0). |
| 2678 | * @return Index of first occurrence. |
| 2679 | */ |
| 2680 | template<typename _Tp> |
| 2681 | _If_sv<_Tp, size_type> |
| 2682 | find_first_not_of(const _Tp& __svt, size_type __pos = 0) const |
| 2683 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2684 | { |
| 2685 | __sv_type __sv = __svt; |
| 2686 | return this->find_first_not_of(__sv.data(), __pos, __sv.size()); |
| 2687 | } |
| 2688 | #endif // C++17 |
| 2689 | |
| 2690 | /** |
| 2691 | * @brief Find position of a character not in C substring. |
| 2692 | * @param __s C string containing characters to avoid. |
| 2693 | * @param __pos Index of character to search from. |
| 2694 | * @param __n Number of characters from __s to consider. |
| 2695 | * @return Index of first occurrence. |
| 2696 | * |
| 2697 | * Starting from @a __pos, searches forward for a character not |
| 2698 | * contained in the first @a __n characters of @a __s within |
| 2699 | * this string. If found, returns the index where it was |
| 2700 | * found. If not found, returns npos. |
| 2701 | */ |
| 2702 | size_type |
| 2703 | find_first_not_of(const _CharT* __s, size_type __pos, |
| 2704 | size_type __n) const _GLIBCXX_NOEXCEPTnoexcept; |
| 2705 | |
| 2706 | /** |
| 2707 | * @brief Find position of a character not in C string. |
| 2708 | * @param __s C string containing characters to avoid. |
| 2709 | * @param __pos Index of character to search from (default 0). |
| 2710 | * @return Index of first occurrence. |
| 2711 | * |
| 2712 | * Starting from @a __pos, searches forward for a character not |
| 2713 | * contained in @a __s within this string. If found, returns |
| 2714 | * the index where it was found. If not found, returns npos. |
| 2715 | */ |
| 2716 | size_type |
| 2717 | find_first_not_of(const _CharT* __s, size_type __pos = 0) const |
| 2718 | _GLIBCXX_NOEXCEPTnoexcept |
| 2719 | { |
| 2720 | __glibcxx_requires_string(__s); |
| 2721 | return this->find_first_not_of(__s, __pos, traits_type::length(__s)); |
| 2722 | } |
| 2723 | |
| 2724 | /** |
| 2725 | * @brief Find position of a different character. |
| 2726 | * @param __c Character to avoid. |
| 2727 | * @param __pos Index of character to search from (default 0). |
| 2728 | * @return Index of first occurrence. |
| 2729 | * |
| 2730 | * Starting from @a __pos, searches forward for a character |
| 2731 | * other than @a __c within this string. If found, returns the |
| 2732 | * index where it was found. If not found, returns npos. |
| 2733 | */ |
| 2734 | size_type |
| 2735 | find_first_not_of(_CharT __c, size_type __pos = 0) const |
| 2736 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2737 | |
| 2738 | /** |
| 2739 | * @brief Find last position of a character not in string. |
| 2740 | * @param __str String containing characters to avoid. |
| 2741 | * @param __pos Index of character to search back from (default end). |
| 2742 | * @return Index of last occurrence. |
| 2743 | * |
| 2744 | * Starting from @a __pos, searches backward for a character |
| 2745 | * not contained in @a __str within this string. If found, |
| 2746 | * returns the index where it was found. If not found, returns |
| 2747 | * npos. |
| 2748 | */ |
| 2749 | size_type |
| 2750 | find_last_not_of(const basic_string& __str, size_type __pos = npos) const |
| 2751 | _GLIBCXX_NOEXCEPTnoexcept |
| 2752 | { return this->find_last_not_of(__str.data(), __pos, __str.size()); } |
| 2753 | |
| 2754 | #if __cplusplus201703L >= 201703L |
| 2755 | /** |
| 2756 | * @brief Find last position of a character not in a string_view. |
| 2757 | * @param __svt An object convertible to string_view containing |
| 2758 | * characters to avoid. |
| 2759 | * @param __pos Index of character to search back from (default end). |
| 2760 | * @return Index of last occurrence. |
| 2761 | */ |
| 2762 | template<typename _Tp> |
| 2763 | _If_sv<_Tp, size_type> |
| 2764 | find_last_not_of(const _Tp& __svt, size_type __pos = npos) const |
| 2765 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2766 | { |
| 2767 | __sv_type __sv = __svt; |
| 2768 | return this->find_last_not_of(__sv.data(), __pos, __sv.size()); |
| 2769 | } |
| 2770 | #endif // C++17 |
| 2771 | |
| 2772 | /** |
| 2773 | * @brief Find last position of a character not in C substring. |
| 2774 | * @param __s C string containing characters to avoid. |
| 2775 | * @param __pos Index of character to search back from. |
| 2776 | * @param __n Number of characters from s to consider. |
| 2777 | * @return Index of last occurrence. |
| 2778 | * |
| 2779 | * Starting from @a __pos, searches backward for a character not |
| 2780 | * contained in the first @a __n characters of @a __s within this string. |
| 2781 | * If found, returns the index where it was found. If not found, |
| 2782 | * returns npos. |
| 2783 | */ |
| 2784 | size_type |
| 2785 | find_last_not_of(const _CharT* __s, size_type __pos, |
| 2786 | size_type __n) const _GLIBCXX_NOEXCEPTnoexcept; |
| 2787 | /** |
| 2788 | * @brief Find last position of a character not in C string. |
| 2789 | * @param __s C string containing characters to avoid. |
| 2790 | * @param __pos Index of character to search back from (default end). |
| 2791 | * @return Index of last occurrence. |
| 2792 | * |
| 2793 | * Starting from @a __pos, searches backward for a character |
| 2794 | * not contained in @a __s within this string. If found, |
| 2795 | * returns the index where it was found. If not found, returns |
| 2796 | * npos. |
| 2797 | */ |
| 2798 | size_type |
| 2799 | find_last_not_of(const _CharT* __s, size_type __pos = npos) const |
| 2800 | _GLIBCXX_NOEXCEPTnoexcept |
| 2801 | { |
| 2802 | __glibcxx_requires_string(__s); |
| 2803 | return this->find_last_not_of(__s, __pos, traits_type::length(__s)); |
| 2804 | } |
| 2805 | |
| 2806 | /** |
| 2807 | * @brief Find last position of a different character. |
| 2808 | * @param __c Character to avoid. |
| 2809 | * @param __pos Index of character to search back from (default end). |
| 2810 | * @return Index of last occurrence. |
| 2811 | * |
| 2812 | * Starting from @a __pos, searches backward for a character other than |
| 2813 | * @a __c within this string. If found, returns the index where it was |
| 2814 | * found. If not found, returns npos. |
| 2815 | */ |
| 2816 | size_type |
| 2817 | find_last_not_of(_CharT __c, size_type __pos = npos) const |
| 2818 | _GLIBCXX_NOEXCEPTnoexcept; |
| 2819 | |
| 2820 | /** |
| 2821 | * @brief Get a substring. |
| 2822 | * @param __pos Index of first character (default 0). |
| 2823 | * @param __n Number of characters in substring (default remainder). |
| 2824 | * @return The new string. |
| 2825 | * @throw std::out_of_range If __pos > size(). |
| 2826 | * |
| 2827 | * Construct and return a new string using the @a __n |
| 2828 | * characters starting at @a __pos. If the string is too |
| 2829 | * short, use the remainder of the characters. If @a __pos is |
| 2830 | * beyond the end of the string, out_of_range is thrown. |
| 2831 | */ |
| 2832 | basic_string |
| 2833 | substr(size_type __pos = 0, size_type __n = npos) const |
| 2834 | { return basic_string(*this, |
| 2835 | _M_check(__pos, "basic_string::substr"), __n); } |
| 2836 | |
| 2837 | /** |
| 2838 | * @brief Compare to a string. |
| 2839 | * @param __str String to compare against. |
| 2840 | * @return Integer < 0, 0, or > 0. |
| 2841 | * |
| 2842 | * Returns an integer < 0 if this string is ordered before @a |
| 2843 | * __str, 0 if their values are equivalent, or > 0 if this |
| 2844 | * string is ordered after @a __str. Determines the effective |
| 2845 | * length rlen of the strings to compare as the smallest of |
| 2846 | * size() and str.size(). The function then compares the two |
| 2847 | * strings by calling traits::compare(data(), str.data(),rlen). |
| 2848 | * If the result of the comparison is nonzero returns it, |
| 2849 | * otherwise the shorter one is ordered first. |
| 2850 | */ |
| 2851 | int |
| 2852 | compare(const basic_string& __str) const |
| 2853 | { |
| 2854 | const size_type __size = this->size(); |
| 2855 | const size_type __osize = __str.size(); |
| 2856 | const size_type __len = std::min(__size, __osize); |
| 2857 | |
| 2858 | int __r = traits_type::compare(_M_data(), __str.data(), __len); |
| 2859 | if (!__r) |
| 2860 | __r = _S_compare(__size, __osize); |
| 2861 | return __r; |
| 2862 | } |
| 2863 | |
| 2864 | #if __cplusplus201703L >= 201703L |
| 2865 | /** |
| 2866 | * @brief Compare to a string_view. |
| 2867 | * @param __svt An object convertible to string_view to compare against. |
| 2868 | * @return Integer < 0, 0, or > 0. |
| 2869 | */ |
| 2870 | template<typename _Tp> |
| 2871 | _If_sv<_Tp, int> |
| 2872 | compare(const _Tp& __svt) const |
| 2873 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2874 | { |
| 2875 | __sv_type __sv = __svt; |
| 2876 | const size_type __size = this->size(); |
| 2877 | const size_type __osize = __sv.size(); |
| 2878 | const size_type __len = std::min(__size, __osize); |
| 2879 | |
| 2880 | int __r = traits_type::compare(_M_data(), __sv.data(), __len); |
| 2881 | if (!__r) |
| 2882 | __r = _S_compare(__size, __osize); |
| 2883 | return __r; |
| 2884 | } |
| 2885 | |
| 2886 | /** |
| 2887 | * @brief Compare to a string_view. |
| 2888 | * @param __pos A position in the string to start comparing from. |
| 2889 | * @param __n The number of characters to compare. |
| 2890 | * @param __svt An object convertible to string_view to compare |
| 2891 | * against. |
| 2892 | * @return Integer < 0, 0, or > 0. |
| 2893 | */ |
| 2894 | template<typename _Tp> |
| 2895 | _If_sv<_Tp, int> |
| 2896 | compare(size_type __pos, size_type __n, const _Tp& __svt) const |
| 2897 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2898 | { |
| 2899 | __sv_type __sv = __svt; |
| 2900 | return __sv_type(*this).substr(__pos, __n).compare(__sv); |
| 2901 | } |
| 2902 | |
| 2903 | /** |
| 2904 | * @brief Compare to a string_view. |
| 2905 | * @param __pos1 A position in the string to start comparing from. |
| 2906 | * @param __n1 The number of characters to compare. |
| 2907 | * @param __svt An object convertible to string_view to compare |
| 2908 | * against. |
| 2909 | * @param __pos2 A position in the string_view to start comparing from. |
| 2910 | * @param __n2 The number of characters to compare. |
| 2911 | * @return Integer < 0, 0, or > 0. |
| 2912 | */ |
| 2913 | template<typename _Tp> |
| 2914 | _If_sv<_Tp, int> |
| 2915 | compare(size_type __pos1, size_type __n1, const _Tp& __svt, |
| 2916 | size_type __pos2, size_type __n2 = npos) const |
| 2917 | noexcept(is_same<_Tp, __sv_type>::value) |
| 2918 | { |
| 2919 | __sv_type __sv = __svt; |
| 2920 | return __sv_type(*this) |
| 2921 | .substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2)); |
| 2922 | } |
| 2923 | #endif // C++17 |
| 2924 | |
| 2925 | /** |
| 2926 | * @brief Compare substring to a string. |
| 2927 | * @param __pos Index of first character of substring. |
| 2928 | * @param __n Number of characters in substring. |
| 2929 | * @param __str String to compare against. |
| 2930 | * @return Integer < 0, 0, or > 0. |
| 2931 | * |
| 2932 | * Form the substring of this string from the @a __n characters |
| 2933 | * starting at @a __pos. Returns an integer < 0 if the |
| 2934 | * substring is ordered before @a __str, 0 if their values are |
| 2935 | * equivalent, or > 0 if the substring is ordered after @a |
| 2936 | * __str. Determines the effective length rlen of the strings |
| 2937 | * to compare as the smallest of the length of the substring |
| 2938 | * and @a __str.size(). The function then compares the two |
| 2939 | * strings by calling |
| 2940 | * traits::compare(substring.data(),str.data(),rlen). If the |
| 2941 | * result of the comparison is nonzero returns it, otherwise |
| 2942 | * the shorter one is ordered first. |
| 2943 | */ |
| 2944 | int |
| 2945 | compare(size_type __pos, size_type __n, const basic_string& __str) const; |
| 2946 | |
| 2947 | /** |
| 2948 | * @brief Compare substring to a substring. |
| 2949 | * @param __pos1 Index of first character of substring. |
| 2950 | * @param __n1 Number of characters in substring. |
| 2951 | * @param __str String to compare against. |
| 2952 | * @param __pos2 Index of first character of substring of str. |
| 2953 | * @param __n2 Number of characters in substring of str. |
| 2954 | * @return Integer < 0, 0, or > 0. |
| 2955 | * |
| 2956 | * Form the substring of this string from the @a __n1 |
| 2957 | * characters starting at @a __pos1. Form the substring of @a |
| 2958 | * __str from the @a __n2 characters starting at @a __pos2. |
| 2959 | * Returns an integer < 0 if this substring is ordered before |
| 2960 | * the substring of @a __str, 0 if their values are equivalent, |
| 2961 | * or > 0 if this substring is ordered after the substring of |
| 2962 | * @a __str. Determines the effective length rlen of the |
| 2963 | * strings to compare as the smallest of the lengths of the |
| 2964 | * substrings. The function then compares the two strings by |
| 2965 | * calling |
| 2966 | * traits::compare(substring.data(),str.substr(pos2,n2).data(),rlen). |
| 2967 | * If the result of the comparison is nonzero returns it, |
| 2968 | * otherwise the shorter one is ordered first. |
| 2969 | */ |
| 2970 | int |
| 2971 | compare(size_type __pos1, size_type __n1, const basic_string& __str, |
| 2972 | size_type __pos2, size_type __n2 = npos) const; |
| 2973 | |
| 2974 | /** |
| 2975 | * @brief Compare to a C string. |
| 2976 | * @param __s C string to compare against. |
| 2977 | * @return Integer < 0, 0, or > 0. |
| 2978 | * |
| 2979 | * Returns an integer < 0 if this string is ordered before @a __s, 0 if |
| 2980 | * their values are equivalent, or > 0 if this string is ordered after |
| 2981 | * @a __s. Determines the effective length rlen of the strings to |
| 2982 | * compare as the smallest of size() and the length of a string |
| 2983 | * constructed from @a __s. The function then compares the two strings |
| 2984 | * by calling traits::compare(data(),s,rlen). If the result of the |
| 2985 | * comparison is nonzero returns it, otherwise the shorter one is |
| 2986 | * ordered first. |
| 2987 | */ |
| 2988 | int |
| 2989 | compare(const _CharT* __s) const _GLIBCXX_NOEXCEPTnoexcept; |
| 2990 | |
| 2991 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 2992 | // 5 String::compare specification questionable |
| 2993 | /** |
| 2994 | * @brief Compare substring to a C string. |
| 2995 | * @param __pos Index of first character of substring. |
| 2996 | * @param __n1 Number of characters in substring. |
| 2997 | * @param __s C string to compare against. |
| 2998 | * @return Integer < 0, 0, or > 0. |
| 2999 | * |
| 3000 | * Form the substring of this string from the @a __n1 |
| 3001 | * characters starting at @a pos. Returns an integer < 0 if |
| 3002 | * the substring is ordered before @a __s, 0 if their values |
| 3003 | * are equivalent, or > 0 if the substring is ordered after @a |
| 3004 | * __s. Determines the effective length rlen of the strings to |
| 3005 | * compare as the smallest of the length of the substring and |
| 3006 | * the length of a string constructed from @a __s. The |
| 3007 | * function then compares the two string by calling |
| 3008 | * traits::compare(substring.data(),__s,rlen). If the result of |
| 3009 | * the comparison is nonzero returns it, otherwise the shorter |
| 3010 | * one is ordered first. |
| 3011 | */ |
| 3012 | int |
| 3013 | compare(size_type __pos, size_type __n1, const _CharT* __s) const; |
| 3014 | |
| 3015 | /** |
| 3016 | * @brief Compare substring against a character %array. |
| 3017 | * @param __pos Index of first character of substring. |
| 3018 | * @param __n1 Number of characters in substring. |
| 3019 | * @param __s character %array to compare against. |
| 3020 | * @param __n2 Number of characters of s. |
| 3021 | * @return Integer < 0, 0, or > 0. |
| 3022 | * |
| 3023 | * Form the substring of this string from the @a __n1 |
| 3024 | * characters starting at @a __pos. Form a string from the |
| 3025 | * first @a __n2 characters of @a __s. Returns an integer < 0 |
| 3026 | * if this substring is ordered before the string from @a __s, |
| 3027 | * 0 if their values are equivalent, or > 0 if this substring |
| 3028 | * is ordered after the string from @a __s. Determines the |
| 3029 | * effective length rlen of the strings to compare as the |
| 3030 | * smallest of the length of the substring and @a __n2. The |
| 3031 | * function then compares the two strings by calling |
| 3032 | * traits::compare(substring.data(),s,rlen). If the result of |
| 3033 | * the comparison is nonzero returns it, otherwise the shorter |
| 3034 | * one is ordered first. |
| 3035 | * |
| 3036 | * NB: s must have at least n2 characters, '\\0' has |
| 3037 | * no special meaning. |
| 3038 | */ |
| 3039 | int |
| 3040 | compare(size_type __pos, size_type __n1, const _CharT* __s, |
| 3041 | size_type __n2) const; |
| 3042 | |
| 3043 | #if __cplusplus201703L > 201703L |
| 3044 | bool |
| 3045 | starts_with(basic_string_view<_CharT, _Traits> __x) const noexcept |
| 3046 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 3047 | |
| 3048 | bool |
| 3049 | starts_with(_CharT __x) const noexcept |
| 3050 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 3051 | |
| 3052 | bool |
| 3053 | starts_with(const _CharT* __x) const noexcept |
| 3054 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 3055 | |
| 3056 | bool |
| 3057 | ends_with(basic_string_view<_CharT, _Traits> __x) const noexcept |
| 3058 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 3059 | |
| 3060 | bool |
| 3061 | ends_with(_CharT __x) const noexcept |
| 3062 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 3063 | |
| 3064 | bool |
| 3065 | ends_with(const _CharT* __x) const noexcept |
| 3066 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 3067 | #endif // C++20 |
| 3068 | |
| 3069 | // Allow basic_stringbuf::__xfer_bufptrs to call _M_length: |
| 3070 | template<typename, typename, typename> friend class basic_stringbuf; |
| 3071 | }; |
| 3072 | _GLIBCXX_END_NAMESPACE_CXX11} |
| 3073 | #else // !_GLIBCXX_USE_CXX11_ABI |
| 3074 | // Reference-counted COW string implentation |
| 3075 | |
| 3076 | /** |
| 3077 | * @class basic_string basic_string.h <string> |
| 3078 | * @brief Managing sequences of characters and character-like objects. |
| 3079 | * |
| 3080 | * @ingroup strings |
| 3081 | * @ingroup sequences |
| 3082 | * |
| 3083 | * @tparam _CharT Type of character |
| 3084 | * @tparam _Traits Traits for character type, defaults to |
| 3085 | * char_traits<_CharT>. |
| 3086 | * @tparam _Alloc Allocator type, defaults to allocator<_CharT>. |
| 3087 | * |
| 3088 | * Meets the requirements of a <a href="tables.html#65">container</a>, a |
| 3089 | * <a href="tables.html#66">reversible container</a>, and a |
| 3090 | * <a href="tables.html#67">sequence</a>. Of the |
| 3091 | * <a href="tables.html#68">optional sequence requirements</a>, only |
| 3092 | * @c push_back, @c at, and @c %array access are supported. |
| 3093 | * |
| 3094 | * @doctodo |
| 3095 | * |
| 3096 | * |
| 3097 | * Documentation? What's that? |
| 3098 | * Nathan Myers <ncm@cantrip.org>. |
| 3099 | * |
| 3100 | * A string looks like this: |
| 3101 | * |
| 3102 | * @code |
| 3103 | * [_Rep] |
| 3104 | * _M_length |
| 3105 | * [basic_string<char_type>] _M_capacity |
| 3106 | * _M_dataplus _M_refcount |
| 3107 | * _M_p ----------------> unnamed array of char_type |
| 3108 | * @endcode |
| 3109 | * |
| 3110 | * Where the _M_p points to the first character in the string, and |
| 3111 | * you cast it to a pointer-to-_Rep and subtract 1 to get a |
| 3112 | * pointer to the header. |
| 3113 | * |
| 3114 | * This approach has the enormous advantage that a string object |
| 3115 | * requires only one allocation. All the ugliness is confined |
| 3116 | * within a single %pair of inline functions, which each compile to |
| 3117 | * a single @a add instruction: _Rep::_M_data(), and |
| 3118 | * string::_M_rep(); and the allocation function which gets a |
| 3119 | * block of raw bytes and with room enough and constructs a _Rep |
| 3120 | * object at the front. |
| 3121 | * |
| 3122 | * The reason you want _M_data pointing to the character %array and |
| 3123 | * not the _Rep is so that the debugger can see the string |
| 3124 | * contents. (Probably we should add a non-inline member to get |
| 3125 | * the _Rep for the debugger to use, so users can check the actual |
| 3126 | * string length.) |
| 3127 | * |
| 3128 | * Note that the _Rep object is a POD so that you can have a |
| 3129 | * static <em>empty string</em> _Rep object already @a constructed before |
| 3130 | * static constructors have run. The reference-count encoding is |
| 3131 | * chosen so that a 0 indicates one reference, so you never try to |
| 3132 | * destroy the empty-string _Rep object. |
| 3133 | * |
| 3134 | * All but the last paragraph is considered pretty conventional |
| 3135 | * for a C++ string implementation. |
| 3136 | */ |
| 3137 | // 21.3 Template class basic_string |
| 3138 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 3139 | class basic_string |
| 3140 | { |
| 3141 | typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template |
| 3142 | rebind<_CharT>::other _CharT_alloc_type; |
| 3143 | typedef __gnu_cxx::__alloc_traits<_CharT_alloc_type> _CharT_alloc_traits; |
| 3144 | |
| 3145 | // Types: |
| 3146 | public: |
| 3147 | typedef _Traits traits_type; |
| 3148 | typedef typename _Traits::char_type value_type; |
| 3149 | typedef _Alloc allocator_type; |
| 3150 | typedef typename _CharT_alloc_type::size_type size_type; |
| 3151 | typedef typename _CharT_alloc_type::difference_type difference_type; |
| 3152 | #if __cplusplus201703L < 201103L |
| 3153 | typedef typename _CharT_alloc_type::reference reference; |
| 3154 | typedef typename _CharT_alloc_type::const_reference const_reference; |
| 3155 | #else |
| 3156 | typedef value_type& reference; |
| 3157 | typedef const value_type& const_reference; |
| 3158 | #endif |
| 3159 | typedef typename _CharT_alloc_traits::pointer pointer; |
| 3160 | typedef typename _CharT_alloc_traits::const_pointer const_pointer; |
| 3161 | typedef __gnu_cxx::__normal_iterator<pointer, basic_string> iterator; |
| 3162 | typedef __gnu_cxx::__normal_iterator<const_pointer, basic_string> |
| 3163 | const_iterator; |
| 3164 | typedef std::reverse_iterator<const_iterator> const_reverse_iterator; |
| 3165 | typedef std::reverse_iterator<iterator> reverse_iterator; |
| 3166 | |
| 3167 | protected: |
| 3168 | // type used for positions in insert, erase etc. |
| 3169 | typedef iterator __const_iterator; |
| 3170 | |
| 3171 | private: |
| 3172 | // _Rep: string representation |
| 3173 | // Invariants: |
| 3174 | // 1. String really contains _M_length + 1 characters: due to 21.3.4 |
| 3175 | // must be kept null-terminated. |
| 3176 | // 2. _M_capacity >= _M_length |
| 3177 | // Allocated memory is always (_M_capacity + 1) * sizeof(_CharT). |
| 3178 | // 3. _M_refcount has three states: |
| 3179 | // -1: leaked, one reference, no ref-copies allowed, non-const. |
| 3180 | // 0: one reference, non-const. |
| 3181 | // n>0: n + 1 references, operations require a lock, const. |
| 3182 | // 4. All fields==0 is an empty string, given the extra storage |
| 3183 | // beyond-the-end for a null terminator; thus, the shared |
| 3184 | // empty string representation needs no constructor. |
| 3185 | |
| 3186 | struct _Rep_base |
| 3187 | { |
| 3188 | size_type _M_length; |
| 3189 | size_type _M_capacity; |
| 3190 | _Atomic_word _M_refcount; |
| 3191 | }; |
| 3192 | |
| 3193 | struct _Rep : _Rep_base |
| 3194 | { |
| 3195 | // Types: |
| 3196 | typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template |
| 3197 | rebind<char>::other _Raw_bytes_alloc; |
| 3198 | |
| 3199 | // (Public) Data members: |
| 3200 | |
| 3201 | // The maximum number of individual char_type elements of an |
| 3202 | // individual string is determined by _S_max_size. This is the |
| 3203 | // value that will be returned by max_size(). (Whereas npos |
| 3204 | // is the maximum number of bytes the allocator can allocate.) |
| 3205 | // If one was to divvy up the theoretical largest size string, |
| 3206 | // with a terminating character and m _CharT elements, it'd |
| 3207 | // look like this: |
| 3208 | // npos = sizeof(_Rep) + (m * sizeof(_CharT)) + sizeof(_CharT) |
| 3209 | // Solving for m: |
| 3210 | // m = ((npos - sizeof(_Rep))/sizeof(CharT)) - 1 |
| 3211 | // In addition, this implementation quarters this amount. |
| 3212 | static const size_type _S_max_size; |
| 3213 | static const _CharT _S_terminal; |
| 3214 | |
| 3215 | // The following storage is init'd to 0 by the linker, resulting |
| 3216 | // (carefully) in an empty string with one reference. |
| 3217 | static size_type _S_empty_rep_storage[]; |
| 3218 | |
| 3219 | static _Rep& |
| 3220 | _S_empty_rep() _GLIBCXX_NOEXCEPTnoexcept |
| 3221 | { |
| 3222 | // NB: Mild hack to avoid strict-aliasing warnings. Note that |
| 3223 | // _S_empty_rep_storage is never modified and the punning should |
| 3224 | // be reasonably safe in this case. |
| 3225 | void* __p = reinterpret_cast<void*>(&_S_empty_rep_storage); |
| 3226 | return *reinterpret_cast<_Rep*>(__p); |
| 3227 | } |
| 3228 | |
| 3229 | bool |
| 3230 | _M_is_leaked() const _GLIBCXX_NOEXCEPTnoexcept |
| 3231 | { |
| 3232 | #if defined(__GTHREADS1) |
| 3233 | // _M_refcount is mutated concurrently by _M_refcopy/_M_dispose, |
| 3234 | // so we need to use an atomic load. However, _M_is_leaked |
| 3235 | // predicate does not change concurrently (i.e. the string is either |
| 3236 | // leaked or not), so a relaxed load is enough. |
| 3237 | return __atomic_load_n(&this->_M_refcount, __ATOMIC_RELAXED0) < 0; |
| 3238 | #else |
| 3239 | return this->_M_refcount < 0; |
| 3240 | #endif |
| 3241 | } |
| 3242 | |
| 3243 | bool |
| 3244 | _M_is_shared() const _GLIBCXX_NOEXCEPTnoexcept |
| 3245 | { |
| 3246 | #if defined(__GTHREADS1) |
| 3247 | // _M_refcount is mutated concurrently by _M_refcopy/_M_dispose, |
| 3248 | // so we need to use an atomic load. Another thread can drop last |
| 3249 | // but one reference concurrently with this check, so we need this |
| 3250 | // load to be acquire to synchronize with release fetch_and_add in |
| 3251 | // _M_dispose. |
| 3252 | return __atomic_load_n(&this->_M_refcount, __ATOMIC_ACQUIRE2) > 0; |
| 3253 | #else |
| 3254 | return this->_M_refcount > 0; |
| 3255 | #endif |
| 3256 | } |
| 3257 | |
| 3258 | void |
| 3259 | _M_set_leaked() _GLIBCXX_NOEXCEPTnoexcept |
| 3260 | { this->_M_refcount = -1; } |
| 3261 | |
| 3262 | void |
| 3263 | _M_set_sharable() _GLIBCXX_NOEXCEPTnoexcept |
| 3264 | { this->_M_refcount = 0; } |
| 3265 | |
| 3266 | void |
| 3267 | _M_set_length_and_sharable(size_type __n) _GLIBCXX_NOEXCEPTnoexcept |
| 3268 | { |
| 3269 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3270 | if (__builtin_expect(this != &_S_empty_rep(), false)) |
| 3271 | #endif |
| 3272 | { |
| 3273 | this->_M_set_sharable(); // One reference. |
| 3274 | this->_M_length = __n; |
| 3275 | traits_type::assign(this->_M_refdata()[__n], _S_terminal); |
| 3276 | // grrr. (per 21.3.4) |
| 3277 | // You cannot leave those LWG people alone for a second. |
| 3278 | } |
| 3279 | } |
| 3280 | |
| 3281 | _CharT* |
| 3282 | _M_refdata() throw() |
| 3283 | { return reinterpret_cast<_CharT*>(this + 1); } |
| 3284 | |
| 3285 | _CharT* |
| 3286 | _M_grab(const _Alloc& __alloc1, const _Alloc& __alloc2) |
| 3287 | { |
| 3288 | return (!_M_is_leaked() && __alloc1 == __alloc2) |
| 3289 | ? _M_refcopy() : _M_clone(__alloc1); |
| 3290 | } |
| 3291 | |
| 3292 | // Create & Destroy |
| 3293 | static _Rep* |
| 3294 | _S_create(size_type, size_type, const _Alloc&); |
| 3295 | |
| 3296 | void |
| 3297 | _M_dispose(const _Alloc& __a) _GLIBCXX_NOEXCEPTnoexcept |
| 3298 | { |
| 3299 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3300 | if (__builtin_expect(this != &_S_empty_rep(), false)) |
| 3301 | #endif |
| 3302 | { |
| 3303 | // Be race-detector-friendly. For more info see bits/c++config. |
| 3304 | _GLIBCXX_SYNCHRONIZATION_HAPPENS_BEFORE(&this->_M_refcount); |
| 3305 | // Decrement of _M_refcount is acq_rel, because: |
| 3306 | // - all but last decrements need to release to synchronize with |
| 3307 | // the last decrement that will delete the object. |
| 3308 | // - the last decrement needs to acquire to synchronize with |
| 3309 | // all the previous decrements. |
| 3310 | // - last but one decrement needs to release to synchronize with |
| 3311 | // the acquire load in _M_is_shared that will conclude that |
| 3312 | // the object is not shared anymore. |
| 3313 | if (__gnu_cxx::__exchange_and_add_dispatch(&this->_M_refcount, |
| 3314 | -1) <= 0) |
| 3315 | { |
| 3316 | _GLIBCXX_SYNCHRONIZATION_HAPPENS_AFTER(&this->_M_refcount); |
| 3317 | _M_destroy(__a); |
| 3318 | } |
| 3319 | } |
| 3320 | } // XXX MT |
| 3321 | |
| 3322 | void |
| 3323 | _M_destroy(const _Alloc&) throw(); |
| 3324 | |
| 3325 | _CharT* |
| 3326 | _M_refcopy() throw() |
| 3327 | { |
| 3328 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3329 | if (__builtin_expect(this != &_S_empty_rep(), false)) |
| 3330 | #endif |
| 3331 | __gnu_cxx::__atomic_add_dispatch(&this->_M_refcount, 1); |
| 3332 | return _M_refdata(); |
| 3333 | } // XXX MT |
| 3334 | |
| 3335 | _CharT* |
| 3336 | _M_clone(const _Alloc&, size_type __res = 0); |
| 3337 | }; |
| 3338 | |
| 3339 | // Use empty-base optimization: http://www.cantrip.org/emptyopt.html |
| 3340 | struct _Alloc_hider : _Alloc |
| 3341 | { |
| 3342 | _Alloc_hider(_CharT* __dat, const _Alloc& __a) _GLIBCXX_NOEXCEPTnoexcept |
| 3343 | : _Alloc(__a), _M_p(__dat) { } |
| 3344 | |
| 3345 | _CharT* _M_p; // The actual data. |
| 3346 | }; |
| 3347 | |
| 3348 | public: |
| 3349 | // Data Members (public): |
| 3350 | // NB: This is an unsigned type, and thus represents the maximum |
| 3351 | // size that the allocator can hold. |
| 3352 | /// Value returned by various member functions when they fail. |
| 3353 | static const size_type npos = static_cast<size_type>(-1); |
| 3354 | |
| 3355 | private: |
| 3356 | // Data Members (private): |
| 3357 | mutable _Alloc_hider _M_dataplus; |
| 3358 | |
| 3359 | _CharT* |
| 3360 | _M_data() const _GLIBCXX_NOEXCEPTnoexcept |
| 3361 | { return _M_dataplus._M_p; } |
| 3362 | |
| 3363 | _CharT* |
| 3364 | _M_data(_CharT* __p) _GLIBCXX_NOEXCEPTnoexcept |
| 3365 | { return (_M_dataplus._M_p = __p); } |
| 3366 | |
| 3367 | _Rep* |
| 3368 | _M_rep() const _GLIBCXX_NOEXCEPTnoexcept |
| 3369 | { return &((reinterpret_cast<_Rep*> (_M_data()))[-1]); } |
| 3370 | |
| 3371 | // For the internal use we have functions similar to `begin'/`end' |
| 3372 | // but they do not call _M_leak. |
| 3373 | iterator |
| 3374 | _M_ibegin() const _GLIBCXX_NOEXCEPTnoexcept |
| 3375 | { return iterator(_M_data()); } |
| 3376 | |
| 3377 | iterator |
| 3378 | _M_iend() const _GLIBCXX_NOEXCEPTnoexcept |
| 3379 | { return iterator(_M_data() + this->size()); } |
| 3380 | |
| 3381 | void |
| 3382 | _M_leak() // for use in begin() & non-const op[] |
| 3383 | { |
| 3384 | if (!_M_rep()->_M_is_leaked()) |
| 3385 | _M_leak_hard(); |
| 3386 | } |
| 3387 | |
| 3388 | size_type |
| 3389 | _M_check(size_type __pos, const char* __s) const |
| 3390 | { |
| 3391 | if (__pos > this->size()) |
| 3392 | __throw_out_of_range_fmt(__N("%s: __pos (which is %zu) > "("%s: __pos (which is %zu) > " "this->size() (which is %zu)" ) |
| 3393 | "this->size() (which is %zu)")("%s: __pos (which is %zu) > " "this->size() (which is %zu)" ), |
| 3394 | __s, __pos, this->size()); |
| 3395 | return __pos; |
| 3396 | } |
| 3397 | |
| 3398 | void |
| 3399 | _M_check_length(size_type __n1, size_type __n2, const char* __s) const |
| 3400 | { |
| 3401 | if (this->max_size() - (this->size() - __n1) < __n2) |
| 3402 | __throw_length_error(__N(__s)(__s)); |
| 3403 | } |
| 3404 | |
| 3405 | // NB: _M_limit doesn't check for a bad __pos value. |
| 3406 | size_type |
| 3407 | _M_limit(size_type __pos, size_type __off) const _GLIBCXX_NOEXCEPTnoexcept |
| 3408 | { |
| 3409 | const bool __testoff = __off < this->size() - __pos; |
| 3410 | return __testoff ? __off : this->size() - __pos; |
| 3411 | } |
| 3412 | |
| 3413 | // True if _Rep and source do not overlap. |
| 3414 | bool |
| 3415 | _M_disjunct(const _CharT* __s) const _GLIBCXX_NOEXCEPTnoexcept |
| 3416 | { |
| 3417 | return (less<const _CharT*>()(__s, _M_data()) |
| 3418 | || less<const _CharT*>()(_M_data() + this->size(), __s)); |
| 3419 | } |
| 3420 | |
| 3421 | // When __n = 1 way faster than the general multichar |
| 3422 | // traits_type::copy/move/assign. |
| 3423 | static void |
| 3424 | _M_copy(_CharT* __d, const _CharT* __s, size_type __n) _GLIBCXX_NOEXCEPTnoexcept |
| 3425 | { |
| 3426 | if (__n == 1) |
| 3427 | traits_type::assign(*__d, *__s); |
| 3428 | else |
| 3429 | traits_type::copy(__d, __s, __n); |
| 3430 | } |
| 3431 | |
| 3432 | static void |
| 3433 | _M_move(_CharT* __d, const _CharT* __s, size_type __n) _GLIBCXX_NOEXCEPTnoexcept |
| 3434 | { |
| 3435 | if (__n == 1) |
| 3436 | traits_type::assign(*__d, *__s); |
| 3437 | else |
| 3438 | traits_type::move(__d, __s, __n); |
| 3439 | } |
| 3440 | |
| 3441 | static void |
| 3442 | _M_assign(_CharT* __d, size_type __n, _CharT __c) _GLIBCXX_NOEXCEPTnoexcept |
| 3443 | { |
| 3444 | if (__n == 1) |
| 3445 | traits_type::assign(*__d, __c); |
| 3446 | else |
| 3447 | traits_type::assign(__d, __n, __c); |
| 3448 | } |
| 3449 | |
| 3450 | // _S_copy_chars is a separate template to permit specialization |
| 3451 | // to optimize for the common case of pointers as iterators. |
| 3452 | template<class _Iterator> |
| 3453 | static void |
| 3454 | _S_copy_chars(_CharT* __p, _Iterator __k1, _Iterator __k2) |
| 3455 | { |
| 3456 | for (; __k1 != __k2; ++__k1, (void)++__p) |
| 3457 | traits_type::assign(*__p, *__k1); // These types are off. |
| 3458 | } |
| 3459 | |
| 3460 | static void |
| 3461 | _S_copy_chars(_CharT* __p, iterator __k1, iterator __k2) _GLIBCXX_NOEXCEPTnoexcept |
| 3462 | { _S_copy_chars(__p, __k1.base(), __k2.base()); } |
| 3463 | |
| 3464 | static void |
| 3465 | _S_copy_chars(_CharT* __p, const_iterator __k1, const_iterator __k2) |
| 3466 | _GLIBCXX_NOEXCEPTnoexcept |
| 3467 | { _S_copy_chars(__p, __k1.base(), __k2.base()); } |
| 3468 | |
| 3469 | static void |
| 3470 | _S_copy_chars(_CharT* __p, _CharT* __k1, _CharT* __k2) _GLIBCXX_NOEXCEPTnoexcept |
| 3471 | { _M_copy(__p, __k1, __k2 - __k1); } |
| 3472 | |
| 3473 | static void |
| 3474 | _S_copy_chars(_CharT* __p, const _CharT* __k1, const _CharT* __k2) |
| 3475 | _GLIBCXX_NOEXCEPTnoexcept |
| 3476 | { _M_copy(__p, __k1, __k2 - __k1); } |
| 3477 | |
| 3478 | static int |
| 3479 | _S_compare(size_type __n1, size_type __n2) _GLIBCXX_NOEXCEPTnoexcept |
| 3480 | { |
| 3481 | const difference_type __d = difference_type(__n1 - __n2); |
| 3482 | |
| 3483 | if (__d > __gnu_cxx::__numeric_traits<int>::__max) |
| 3484 | return __gnu_cxx::__numeric_traits<int>::__max; |
| 3485 | else if (__d < __gnu_cxx::__numeric_traits<int>::__min) |
| 3486 | return __gnu_cxx::__numeric_traits<int>::__min; |
| 3487 | else |
| 3488 | return int(__d); |
| 3489 | } |
| 3490 | |
| 3491 | void |
| 3492 | _M_mutate(size_type __pos, size_type __len1, size_type __len2); |
| 3493 | |
| 3494 | void |
| 3495 | _M_leak_hard(); |
| 3496 | |
| 3497 | static _Rep& |
| 3498 | _S_empty_rep() _GLIBCXX_NOEXCEPTnoexcept |
| 3499 | { return _Rep::_S_empty_rep(); } |
| 3500 | |
| 3501 | #if __cplusplus201703L >= 201703L |
| 3502 | // A helper type for avoiding boiler-plate. |
| 3503 | typedef basic_string_view<_CharT, _Traits> __sv_type; |
| 3504 | |
| 3505 | template<typename _Tp, typename _Res> |
| 3506 | using _If_sv = enable_if_t< |
| 3507 | __and_<is_convertible<const _Tp&, __sv_type>, |
| 3508 | __not_<is_convertible<const _Tp*, const basic_string*>>, |
| 3509 | __not_<is_convertible<const _Tp&, const _CharT*>>>::value, |
| 3510 | _Res>; |
| 3511 | |
| 3512 | // Allows an implicit conversion to __sv_type. |
| 3513 | static __sv_type |
| 3514 | _S_to_string_view(__sv_type __svt) noexcept |
| 3515 | { return __svt; } |
| 3516 | |
| 3517 | // Wraps a string_view by explicit conversion and thus |
| 3518 | // allows to add an internal constructor that does not |
| 3519 | // participate in overload resolution when a string_view |
| 3520 | // is provided. |
| 3521 | struct __sv_wrapper |
| 3522 | { |
| 3523 | explicit __sv_wrapper(__sv_type __sv) noexcept : _M_sv(__sv) { } |
| 3524 | __sv_type _M_sv; |
| 3525 | }; |
| 3526 | |
| 3527 | /** |
| 3528 | * @brief Only internally used: Construct string from a string view |
| 3529 | * wrapper. |
| 3530 | * @param __svw string view wrapper. |
| 3531 | * @param __a Allocator to use. |
| 3532 | */ |
| 3533 | explicit |
| 3534 | basic_string(__sv_wrapper __svw, const _Alloc& __a) |
| 3535 | : basic_string(__svw._M_sv.data(), __svw._M_sv.size(), __a) { } |
| 3536 | #endif |
| 3537 | |
| 3538 | public: |
| 3539 | // Construct/copy/destroy: |
| 3540 | // NB: We overload ctors in some cases instead of using default |
| 3541 | // arguments, per 17.4.4.4 para. 2 item 2. |
| 3542 | |
| 3543 | /** |
| 3544 | * @brief Default constructor creates an empty string. |
| 3545 | */ |
| 3546 | basic_string() |
| 3547 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3548 | _GLIBCXX_NOEXCEPTnoexcept |
| 3549 | : _M_dataplus(_S_empty_rep()._M_refdata(), _Alloc()) |
| 3550 | #else |
| 3551 | : _M_dataplus(_S_construct(size_type(), _CharT(), _Alloc()), _Alloc()) |
| 3552 | #endif |
| 3553 | { } |
| 3554 | |
| 3555 | /** |
| 3556 | * @brief Construct an empty string using allocator @a a. |
| 3557 | */ |
| 3558 | explicit |
| 3559 | basic_string(const _Alloc& __a); |
| 3560 | |
| 3561 | // NB: per LWG issue 42, semantics different from IS: |
| 3562 | /** |
| 3563 | * @brief Construct string with copy of value of @a str. |
| 3564 | * @param __str Source string. |
| 3565 | */ |
| 3566 | basic_string(const basic_string& __str); |
| 3567 | |
| 3568 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 3569 | // 2583. no way to supply an allocator for basic_string(str, pos) |
| 3570 | /** |
| 3571 | * @brief Construct string as copy of a substring. |
| 3572 | * @param __str Source string. |
| 3573 | * @param __pos Index of first character to copy from. |
| 3574 | * @param __a Allocator to use. |
| 3575 | */ |
| 3576 | basic_string(const basic_string& __str, size_type __pos, |
| 3577 | const _Alloc& __a = _Alloc()); |
| 3578 | |
| 3579 | /** |
| 3580 | * @brief Construct string as copy of a substring. |
| 3581 | * @param __str Source string. |
| 3582 | * @param __pos Index of first character to copy from. |
| 3583 | * @param __n Number of characters to copy. |
| 3584 | */ |
| 3585 | basic_string(const basic_string& __str, size_type __pos, |
| 3586 | size_type __n); |
| 3587 | /** |
| 3588 | * @brief Construct string as copy of a substring. |
| 3589 | * @param __str Source string. |
| 3590 | * @param __pos Index of first character to copy from. |
| 3591 | * @param __n Number of characters to copy. |
| 3592 | * @param __a Allocator to use. |
| 3593 | */ |
| 3594 | basic_string(const basic_string& __str, size_type __pos, |
| 3595 | size_type __n, const _Alloc& __a); |
| 3596 | |
| 3597 | /** |
| 3598 | * @brief Construct string initialized by a character %array. |
| 3599 | * @param __s Source character %array. |
| 3600 | * @param __n Number of characters to copy. |
| 3601 | * @param __a Allocator to use (default is default allocator). |
| 3602 | * |
| 3603 | * NB: @a __s must have at least @a __n characters, '\\0' |
| 3604 | * has no special meaning. |
| 3605 | */ |
| 3606 | basic_string(const _CharT* __s, size_type __n, |
| 3607 | const _Alloc& __a = _Alloc()); |
| 3608 | |
| 3609 | /** |
| 3610 | * @brief Construct string as copy of a C string. |
| 3611 | * @param __s Source C string. |
| 3612 | * @param __a Allocator to use (default is default allocator). |
| 3613 | */ |
| 3614 | #if __cpp_deduction_guides201703L && ! defined _GLIBCXX_DEFINING_STRING_INSTANTIATIONS |
| 3615 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 3616 | // 3076. basic_string CTAD ambiguity |
| 3617 | template<typename = _RequireAllocator<_Alloc>> |
| 3618 | #endif |
| 3619 | basic_string(const _CharT* __s, const _Alloc& __a = _Alloc()) |
| 3620 | : _M_dataplus(_S_construct(__s, __s ? __s + traits_type::length(__s) : |
| 3621 | __s + npos, __a), __a) |
| 3622 | { } |
| 3623 | |
| 3624 | /** |
| 3625 | * @brief Construct string as multiple characters. |
| 3626 | * @param __n Number of characters. |
| 3627 | * @param __c Character to use. |
| 3628 | * @param __a Allocator to use (default is default allocator). |
| 3629 | */ |
| 3630 | basic_string(size_type __n, _CharT __c, const _Alloc& __a = _Alloc()); |
| 3631 | |
| 3632 | #if __cplusplus201703L >= 201103L |
| 3633 | /** |
| 3634 | * @brief Move construct string. |
| 3635 | * @param __str Source string. |
| 3636 | * |
| 3637 | * The newly-created string contains the exact contents of @a __str. |
| 3638 | * @a __str is a valid, but unspecified string. |
| 3639 | **/ |
| 3640 | basic_string(basic_string&& __str) |
| 3641 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3642 | noexcept // FIXME C++11: should always be noexcept. |
| 3643 | #endif |
| 3644 | : _M_dataplus(std::move(__str._M_dataplus)) |
| 3645 | { |
| 3646 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3647 | __str._M_data(_S_empty_rep()._M_refdata()); |
| 3648 | #else |
| 3649 | __str._M_data(_S_construct(size_type(), _CharT(), get_allocator())); |
| 3650 | #endif |
| 3651 | } |
| 3652 | |
| 3653 | /** |
| 3654 | * @brief Construct string from an initializer %list. |
| 3655 | * @param __l std::initializer_list of characters. |
| 3656 | * @param __a Allocator to use (default is default allocator). |
| 3657 | */ |
| 3658 | basic_string(initializer_list<_CharT> __l, const _Alloc& __a = _Alloc()); |
| 3659 | |
| 3660 | basic_string(const basic_string& __str, const _Alloc& __a) |
| 3661 | : _M_dataplus(__str._M_rep()->_M_grab(__a, __str.get_allocator()), __a) |
| 3662 | { } |
| 3663 | |
| 3664 | basic_string(basic_string&& __str, const _Alloc& __a) |
| 3665 | : _M_dataplus(__str._M_data(), __a) |
| 3666 | { |
| 3667 | if (__a == __str.get_allocator()) |
| 3668 | { |
| 3669 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 3670 | __str._M_data(_S_empty_rep()._M_refdata()); |
| 3671 | #else |
| 3672 | __str._M_data(_S_construct(size_type(), _CharT(), __a)); |
| 3673 | #endif |
| 3674 | } |
| 3675 | else |
| 3676 | _M_dataplus._M_p = _S_construct(__str.begin(), __str.end(), __a); |
| 3677 | } |
| 3678 | #endif // C++11 |
| 3679 | |
| 3680 | /** |
| 3681 | * @brief Construct string as copy of a range. |
| 3682 | * @param __beg Start of range. |
| 3683 | * @param __end End of range. |
| 3684 | * @param __a Allocator to use (default is default allocator). |
| 3685 | */ |
| 3686 | template<class _InputIterator> |
| 3687 | basic_string(_InputIterator __beg, _InputIterator __end, |
| 3688 | const _Alloc& __a = _Alloc()); |
| 3689 | |
| 3690 | #if __cplusplus201703L >= 201703L |
| 3691 | /** |
| 3692 | * @brief Construct string from a substring of a string_view. |
| 3693 | * @param __t Source object convertible to string view. |
| 3694 | * @param __pos The index of the first character to copy from __t. |
| 3695 | * @param __n The number of characters to copy from __t. |
| 3696 | * @param __a Allocator to use. |
| 3697 | */ |
| 3698 | template<typename _Tp, typename = _If_sv<_Tp, void>> |
| 3699 | basic_string(const _Tp& __t, size_type __pos, size_type __n, |
| 3700 | const _Alloc& __a = _Alloc()) |
| 3701 | : basic_string(_S_to_string_view(__t).substr(__pos, __n), __a) { } |
| 3702 | |
| 3703 | /** |
| 3704 | * @brief Construct string from a string_view. |
| 3705 | * @param __t Source object convertible to string view. |
| 3706 | * @param __a Allocator to use (default is default allocator). |
| 3707 | */ |
| 3708 | template<typename _Tp, typename = _If_sv<_Tp, void>> |
| 3709 | explicit |
| 3710 | basic_string(const _Tp& __t, const _Alloc& __a = _Alloc()) |
| 3711 | : basic_string(__sv_wrapper(_S_to_string_view(__t)), __a) { } |
| 3712 | #endif // C++17 |
| 3713 | |
| 3714 | /** |
| 3715 | * @brief Destroy the string instance. |
| 3716 | */ |
| 3717 | ~basic_string() _GLIBCXX_NOEXCEPTnoexcept |
| 3718 | { _M_rep()->_M_dispose(this->get_allocator()); } |
| 3719 | |
| 3720 | /** |
| 3721 | * @brief Assign the value of @a str to this string. |
| 3722 | * @param __str Source string. |
| 3723 | */ |
| 3724 | basic_string& |
| 3725 | operator=(const basic_string& __str) |
| 3726 | { return this->assign(__str); } |
| 3727 | |
| 3728 | /** |
| 3729 | * @brief Copy contents of @a s into this string. |
| 3730 | * @param __s Source null-terminated string. |
| 3731 | */ |
| 3732 | basic_string& |
| 3733 | operator=(const _CharT* __s) |
| 3734 | { return this->assign(__s); } |
| 3735 | |
| 3736 | /** |
| 3737 | * @brief Set value to string of length 1. |
| 3738 | * @param __c Source character. |
| 3739 | * |
| 3740 | * Assigning to a character makes this string length 1 and |
| 3741 | * (*this)[0] == @a c. |
| 3742 | */ |
| 3743 | basic_string& |
| 3744 | operator=(_CharT __c) |
| 3745 | { |
| 3746 | this->assign(1, __c); |
| 3747 | return *this; |
| 3748 | } |
| 3749 | |
| 3750 | #if __cplusplus201703L >= 201103L |
| 3751 | /** |
| 3752 | * @brief Move assign the value of @a str to this string. |
| 3753 | * @param __str Source string. |
| 3754 | * |
| 3755 | * The contents of @a str are moved into this string (without copying). |
| 3756 | * @a str is a valid, but unspecified string. |
| 3757 | **/ |
| 3758 | basic_string& |
| 3759 | operator=(basic_string&& __str) |
| 3760 | _GLIBCXX_NOEXCEPT_IF(allocator_traits<_Alloc>::is_always_equal::value)noexcept(allocator_traits<_Alloc>::is_always_equal::value ) |
| 3761 | { |
| 3762 | // NB: DR 1204. |
| 3763 | this->swap(__str); |
| 3764 | return *this; |
| 3765 | } |
| 3766 | |
| 3767 | /** |
| 3768 | * @brief Set value to string constructed from initializer %list. |
| 3769 | * @param __l std::initializer_list. |
| 3770 | */ |
| 3771 | basic_string& |
| 3772 | operator=(initializer_list<_CharT> __l) |
| 3773 | { |
| 3774 | this->assign(__l.begin(), __l.size()); |
| 3775 | return *this; |
| 3776 | } |
| 3777 | #endif // C++11 |
| 3778 | |
| 3779 | #if __cplusplus201703L >= 201703L |
| 3780 | /** |
| 3781 | * @brief Set value to string constructed from a string_view. |
| 3782 | * @param __svt An object convertible to string_view. |
| 3783 | */ |
| 3784 | template<typename _Tp> |
| 3785 | _If_sv<_Tp, basic_string&> |
| 3786 | operator=(const _Tp& __svt) |
| 3787 | { return this->assign(__svt); } |
| 3788 | |
| 3789 | /** |
| 3790 | * @brief Convert to a string_view. |
| 3791 | * @return A string_view. |
| 3792 | */ |
| 3793 | operator __sv_type() const noexcept |
| 3794 | { return __sv_type(data(), size()); } |
| 3795 | #endif // C++17 |
| 3796 | |
| 3797 | // Iterators: |
| 3798 | /** |
| 3799 | * Returns a read/write iterator that points to the first character in |
| 3800 | * the %string. Unshares the string. |
| 3801 | */ |
| 3802 | iterator |
| 3803 | begin() // FIXME C++11: should be noexcept. |
| 3804 | { |
| 3805 | _M_leak(); |
| 3806 | return iterator(_M_data()); |
| 3807 | } |
| 3808 | |
| 3809 | /** |
| 3810 | * Returns a read-only (constant) iterator that points to the first |
| 3811 | * character in the %string. |
| 3812 | */ |
| 3813 | const_iterator |
| 3814 | begin() const _GLIBCXX_NOEXCEPTnoexcept |
| 3815 | { return const_iterator(_M_data()); } |
| 3816 | |
| 3817 | /** |
| 3818 | * Returns a read/write iterator that points one past the last |
| 3819 | * character in the %string. Unshares the string. |
| 3820 | */ |
| 3821 | iterator |
| 3822 | end() // FIXME C++11: should be noexcept. |
| 3823 | { |
| 3824 | _M_leak(); |
| 3825 | return iterator(_M_data() + this->size()); |
| 3826 | } |
| 3827 | |
| 3828 | /** |
| 3829 | * Returns a read-only (constant) iterator that points one past the |
| 3830 | * last character in the %string. |
| 3831 | */ |
| 3832 | const_iterator |
| 3833 | end() const _GLIBCXX_NOEXCEPTnoexcept |
| 3834 | { return const_iterator(_M_data() + this->size()); } |
| 3835 | |
| 3836 | /** |
| 3837 | * Returns a read/write reverse iterator that points to the last |
| 3838 | * character in the %string. Iteration is done in reverse element |
| 3839 | * order. Unshares the string. |
| 3840 | */ |
| 3841 | reverse_iterator |
| 3842 | rbegin() // FIXME C++11: should be noexcept. |
| 3843 | { return reverse_iterator(this->end()); } |
| 3844 | |
| 3845 | /** |
| 3846 | * Returns a read-only (constant) reverse iterator that points |
| 3847 | * to the last character in the %string. Iteration is done in |
| 3848 | * reverse element order. |
| 3849 | */ |
| 3850 | const_reverse_iterator |
| 3851 | rbegin() const _GLIBCXX_NOEXCEPTnoexcept |
| 3852 | { return const_reverse_iterator(this->end()); } |
| 3853 | |
| 3854 | /** |
| 3855 | * Returns a read/write reverse iterator that points to one before the |
| 3856 | * first character in the %string. Iteration is done in reverse |
| 3857 | * element order. Unshares the string. |
| 3858 | */ |
| 3859 | reverse_iterator |
| 3860 | rend() // FIXME C++11: should be noexcept. |
| 3861 | { return reverse_iterator(this->begin()); } |
| 3862 | |
| 3863 | /** |
| 3864 | * Returns a read-only (constant) reverse iterator that points |
| 3865 | * to one before the first character in the %string. Iteration |
| 3866 | * is done in reverse element order. |
| 3867 | */ |
| 3868 | const_reverse_iterator |
| 3869 | rend() const _GLIBCXX_NOEXCEPTnoexcept |
| 3870 | { return const_reverse_iterator(this->begin()); } |
| 3871 | |
| 3872 | #if __cplusplus201703L >= 201103L |
| 3873 | /** |
| 3874 | * Returns a read-only (constant) iterator that points to the first |
| 3875 | * character in the %string. |
| 3876 | */ |
| 3877 | const_iterator |
| 3878 | cbegin() const noexcept |
| 3879 | { return const_iterator(this->_M_data()); } |
| 3880 | |
| 3881 | /** |
| 3882 | * Returns a read-only (constant) iterator that points one past the |
| 3883 | * last character in the %string. |
| 3884 | */ |
| 3885 | const_iterator |
| 3886 | cend() const noexcept |
| 3887 | { return const_iterator(this->_M_data() + this->size()); } |
| 3888 | |
| 3889 | /** |
| 3890 | * Returns a read-only (constant) reverse iterator that points |
| 3891 | * to the last character in the %string. Iteration is done in |
| 3892 | * reverse element order. |
| 3893 | */ |
| 3894 | const_reverse_iterator |
| 3895 | crbegin() const noexcept |
| 3896 | { return const_reverse_iterator(this->end()); } |
| 3897 | |
| 3898 | /** |
| 3899 | * Returns a read-only (constant) reverse iterator that points |
| 3900 | * to one before the first character in the %string. Iteration |
| 3901 | * is done in reverse element order. |
| 3902 | */ |
| 3903 | const_reverse_iterator |
| 3904 | crend() const noexcept |
| 3905 | { return const_reverse_iterator(this->begin()); } |
| 3906 | #endif |
| 3907 | |
| 3908 | public: |
| 3909 | // Capacity: |
| 3910 | /// Returns the number of characters in the string, not including any |
| 3911 | /// null-termination. |
| 3912 | size_type |
| 3913 | size() const _GLIBCXX_NOEXCEPTnoexcept |
| 3914 | { return _M_rep()->_M_length; } |
| 3915 | |
| 3916 | /// Returns the number of characters in the string, not including any |
| 3917 | /// null-termination. |
| 3918 | size_type |
| 3919 | length() const _GLIBCXX_NOEXCEPTnoexcept |
| 3920 | { return _M_rep()->_M_length; } |
| 3921 | |
| 3922 | /// Returns the size() of the largest possible %string. |
| 3923 | size_type |
| 3924 | max_size() const _GLIBCXX_NOEXCEPTnoexcept |
| 3925 | { return _Rep::_S_max_size; } |
| 3926 | |
| 3927 | /** |
| 3928 | * @brief Resizes the %string to the specified number of characters. |
| 3929 | * @param __n Number of characters the %string should contain. |
| 3930 | * @param __c Character to fill any new elements. |
| 3931 | * |
| 3932 | * This function will %resize the %string to the specified |
| 3933 | * number of characters. If the number is smaller than the |
| 3934 | * %string's current size the %string is truncated, otherwise |
| 3935 | * the %string is extended and new elements are %set to @a __c. |
| 3936 | */ |
| 3937 | void |
| 3938 | resize(size_type __n, _CharT __c); |
| 3939 | |
| 3940 | /** |
| 3941 | * @brief Resizes the %string to the specified number of characters. |
| 3942 | * @param __n Number of characters the %string should contain. |
| 3943 | * |
| 3944 | * This function will resize the %string to the specified length. If |
| 3945 | * the new size is smaller than the %string's current size the %string |
| 3946 | * is truncated, otherwise the %string is extended and new characters |
| 3947 | * are default-constructed. For basic types such as char, this means |
| 3948 | * setting them to 0. |
| 3949 | */ |
| 3950 | void |
| 3951 | resize(size_type __n) |
| 3952 | { this->resize(__n, _CharT()); } |
| 3953 | |
| 3954 | #if __cplusplus201703L >= 201103L |
| 3955 | /// A non-binding request to reduce capacity() to size(). |
| 3956 | void |
| 3957 | shrink_to_fit() _GLIBCXX_NOEXCEPTnoexcept |
| 3958 | { |
| 3959 | #if __cpp_exceptions |
| 3960 | if (capacity() > size()) |
| 3961 | { |
| 3962 | try |
| 3963 | { reserve(0); } |
| 3964 | catch(...) |
| 3965 | { } |
| 3966 | } |
| 3967 | #endif |
| 3968 | } |
| 3969 | #endif |
| 3970 | |
| 3971 | /** |
| 3972 | * Returns the total number of characters that the %string can hold |
| 3973 | * before needing to allocate more memory. |
| 3974 | */ |
| 3975 | size_type |
| 3976 | capacity() const _GLIBCXX_NOEXCEPTnoexcept |
| 3977 | { return _M_rep()->_M_capacity; } |
| 3978 | |
| 3979 | /** |
| 3980 | * @brief Attempt to preallocate enough memory for specified number of |
| 3981 | * characters. |
| 3982 | * @param __res_arg Number of characters required. |
| 3983 | * @throw std::length_error If @a __res_arg exceeds @c max_size(). |
| 3984 | * |
| 3985 | * This function attempts to reserve enough memory for the |
| 3986 | * %string to hold the specified number of characters. If the |
| 3987 | * number requested is more than max_size(), length_error is |
| 3988 | * thrown. |
| 3989 | * |
| 3990 | * The advantage of this function is that if optimal code is a |
| 3991 | * necessity and the user can determine the string length that will be |
| 3992 | * required, the user can reserve the memory in %advance, and thus |
| 3993 | * prevent a possible reallocation of memory and copying of %string |
| 3994 | * data. |
| 3995 | */ |
| 3996 | void |
| 3997 | reserve(size_type __res_arg = 0); |
| 3998 | |
| 3999 | /** |
| 4000 | * Erases the string, making it empty. |
| 4001 | */ |
| 4002 | #if _GLIBCXX_FULLY_DYNAMIC_STRING0 == 0 |
| 4003 | void |
| 4004 | clear() _GLIBCXX_NOEXCEPTnoexcept |
| 4005 | { |
| 4006 | if (_M_rep()->_M_is_shared()) |
| 4007 | { |
| 4008 | _M_rep()->_M_dispose(this->get_allocator()); |
| 4009 | _M_data(_S_empty_rep()._M_refdata()); |
| 4010 | } |
| 4011 | else |
| 4012 | _M_rep()->_M_set_length_and_sharable(0); |
| 4013 | } |
| 4014 | #else |
| 4015 | // PR 56166: this should not throw. |
| 4016 | void |
| 4017 | clear() |
| 4018 | { _M_mutate(0, this->size(), 0); } |
| 4019 | #endif |
| 4020 | |
| 4021 | /** |
| 4022 | * Returns true if the %string is empty. Equivalent to |
| 4023 | * <code>*this == ""</code>. |
| 4024 | */ |
| 4025 | _GLIBCXX_NODISCARD[[__nodiscard__]] bool |
| 4026 | empty() const _GLIBCXX_NOEXCEPTnoexcept |
| 4027 | { return this->size() == 0; } |
| 4028 | |
| 4029 | // Element access: |
| 4030 | /** |
| 4031 | * @brief Subscript access to the data contained in the %string. |
| 4032 | * @param __pos The index of the character to access. |
| 4033 | * @return Read-only (constant) reference to the character. |
| 4034 | * |
| 4035 | * This operator allows for easy, array-style, data access. |
| 4036 | * Note that data access with this operator is unchecked and |
| 4037 | * out_of_range lookups are not defined. (For checked lookups |
| 4038 | * see at().) |
| 4039 | */ |
| 4040 | const_reference |
| 4041 | operator[] (size_type __pos) const _GLIBCXX_NOEXCEPTnoexcept |
| 4042 | { |
| 4043 | __glibcxx_assert(__pos <= size())do { if (! (__pos <= size())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4043, __PRETTY_FUNCTION__, "__pos <= size()"); } while ( false); |
| 4044 | return _M_data()[__pos]; |
| 4045 | } |
| 4046 | |
| 4047 | /** |
| 4048 | * @brief Subscript access to the data contained in the %string. |
| 4049 | * @param __pos The index of the character to access. |
| 4050 | * @return Read/write reference to the character. |
| 4051 | * |
| 4052 | * This operator allows for easy, array-style, data access. |
| 4053 | * Note that data access with this operator is unchecked and |
| 4054 | * out_of_range lookups are not defined. (For checked lookups |
| 4055 | * see at().) Unshares the string. |
| 4056 | */ |
| 4057 | reference |
| 4058 | operator[](size_type __pos) |
| 4059 | { |
| 4060 | // Allow pos == size() both in C++98 mode, as v3 extension, |
| 4061 | // and in C++11 mode. |
| 4062 | __glibcxx_assert(__pos <= size())do { if (! (__pos <= size())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4062, __PRETTY_FUNCTION__, "__pos <= size()"); } while ( false); |
| 4063 | // In pedantic mode be strict in C++98 mode. |
| 4064 | _GLIBCXX_DEBUG_PEDASSERT(__cplusplus >= 201103L || __pos < size()); |
| 4065 | _M_leak(); |
| 4066 | return _M_data()[__pos]; |
| 4067 | } |
| 4068 | |
| 4069 | /** |
| 4070 | * @brief Provides access to the data contained in the %string. |
| 4071 | * @param __n The index of the character to access. |
| 4072 | * @return Read-only (const) reference to the character. |
| 4073 | * @throw std::out_of_range If @a n is an invalid index. |
| 4074 | * |
| 4075 | * This function provides for safer data access. The parameter is |
| 4076 | * first checked that it is in the range of the string. The function |
| 4077 | * throws out_of_range if the check fails. |
| 4078 | */ |
| 4079 | const_reference |
| 4080 | at(size_type __n) const |
| 4081 | { |
| 4082 | if (__n >= this->size()) |
| 4083 | __throw_out_of_range_fmt(__N("basic_string::at: __n "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 4084 | "(which is %zu) >= this->size() "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 4085 | "(which is %zu)")("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)"), |
| 4086 | __n, this->size()); |
| 4087 | return _M_data()[__n]; |
| 4088 | } |
| 4089 | |
| 4090 | /** |
| 4091 | * @brief Provides access to the data contained in the %string. |
| 4092 | * @param __n The index of the character to access. |
| 4093 | * @return Read/write reference to the character. |
| 4094 | * @throw std::out_of_range If @a n is an invalid index. |
| 4095 | * |
| 4096 | * This function provides for safer data access. The parameter is |
| 4097 | * first checked that it is in the range of the string. The function |
| 4098 | * throws out_of_range if the check fails. Success results in |
| 4099 | * unsharing the string. |
| 4100 | */ |
| 4101 | reference |
| 4102 | at(size_type __n) |
| 4103 | { |
| 4104 | if (__n >= size()) |
| 4105 | __throw_out_of_range_fmt(__N("basic_string::at: __n "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 4106 | "(which is %zu) >= this->size() "("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)") |
| 4107 | "(which is %zu)")("basic_string::at: __n " "(which is %zu) >= this->size() " "(which is %zu)"), |
| 4108 | __n, this->size()); |
| 4109 | _M_leak(); |
| 4110 | return _M_data()[__n]; |
| 4111 | } |
| 4112 | |
| 4113 | #if __cplusplus201703L >= 201103L |
| 4114 | /** |
| 4115 | * Returns a read/write reference to the data at the first |
| 4116 | * element of the %string. |
| 4117 | */ |
| 4118 | reference |
| 4119 | front() |
| 4120 | { |
| 4121 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4121, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 4122 | return operator[](0); |
| 4123 | } |
| 4124 | |
| 4125 | /** |
| 4126 | * Returns a read-only (constant) reference to the data at the first |
| 4127 | * element of the %string. |
| 4128 | */ |
| 4129 | const_reference |
| 4130 | front() const noexcept |
| 4131 | { |
| 4132 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4132, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 4133 | return operator[](0); |
| 4134 | } |
| 4135 | |
| 4136 | /** |
| 4137 | * Returns a read/write reference to the data at the last |
| 4138 | * element of the %string. |
| 4139 | */ |
| 4140 | reference |
| 4141 | back() |
| 4142 | { |
| 4143 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4143, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 4144 | return operator[](this->size() - 1); |
| 4145 | } |
| 4146 | |
| 4147 | /** |
| 4148 | * Returns a read-only (constant) reference to the data at the |
| 4149 | * last element of the %string. |
| 4150 | */ |
| 4151 | const_reference |
| 4152 | back() const noexcept |
| 4153 | { |
| 4154 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4154, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 4155 | return operator[](this->size() - 1); |
| 4156 | } |
| 4157 | #endif |
| 4158 | |
| 4159 | // Modifiers: |
| 4160 | /** |
| 4161 | * @brief Append a string to this string. |
| 4162 | * @param __str The string to append. |
| 4163 | * @return Reference to this string. |
| 4164 | */ |
| 4165 | basic_string& |
| 4166 | operator+=(const basic_string& __str) |
| 4167 | { return this->append(__str); } |
| 4168 | |
| 4169 | /** |
| 4170 | * @brief Append a C string. |
| 4171 | * @param __s The C string to append. |
| 4172 | * @return Reference to this string. |
| 4173 | */ |
| 4174 | basic_string& |
| 4175 | operator+=(const _CharT* __s) |
| 4176 | { return this->append(__s); } |
| 4177 | |
| 4178 | /** |
| 4179 | * @brief Append a character. |
| 4180 | * @param __c The character to append. |
| 4181 | * @return Reference to this string. |
| 4182 | */ |
| 4183 | basic_string& |
| 4184 | operator+=(_CharT __c) |
| 4185 | { |
| 4186 | this->push_back(__c); |
| 4187 | return *this; |
| 4188 | } |
| 4189 | |
| 4190 | #if __cplusplus201703L >= 201103L |
| 4191 | /** |
| 4192 | * @brief Append an initializer_list of characters. |
| 4193 | * @param __l The initializer_list of characters to be appended. |
| 4194 | * @return Reference to this string. |
| 4195 | */ |
| 4196 | basic_string& |
| 4197 | operator+=(initializer_list<_CharT> __l) |
| 4198 | { return this->append(__l.begin(), __l.size()); } |
| 4199 | #endif // C++11 |
| 4200 | |
| 4201 | #if __cplusplus201703L >= 201703L |
| 4202 | /** |
| 4203 | * @brief Append a string_view. |
| 4204 | * @param __svt The object convertible to string_view to be appended. |
| 4205 | * @return Reference to this string. |
| 4206 | */ |
| 4207 | template<typename _Tp> |
| 4208 | _If_sv<_Tp, basic_string&> |
| 4209 | operator+=(const _Tp& __svt) |
| 4210 | { return this->append(__svt); } |
| 4211 | #endif // C++17 |
| 4212 | |
| 4213 | /** |
| 4214 | * @brief Append a string to this string. |
| 4215 | * @param __str The string to append. |
| 4216 | * @return Reference to this string. |
| 4217 | */ |
| 4218 | basic_string& |
| 4219 | append(const basic_string& __str); |
| 4220 | |
| 4221 | /** |
| 4222 | * @brief Append a substring. |
| 4223 | * @param __str The string to append. |
| 4224 | * @param __pos Index of the first character of str to append. |
| 4225 | * @param __n The number of characters to append. |
| 4226 | * @return Reference to this string. |
| 4227 | * @throw std::out_of_range if @a __pos is not a valid index. |
| 4228 | * |
| 4229 | * This function appends @a __n characters from @a __str |
| 4230 | * starting at @a __pos to this string. If @a __n is is larger |
| 4231 | * than the number of available characters in @a __str, the |
| 4232 | * remainder of @a __str is appended. |
| 4233 | */ |
| 4234 | basic_string& |
| 4235 | append(const basic_string& __str, size_type __pos, size_type __n = npos); |
| 4236 | |
| 4237 | /** |
| 4238 | * @brief Append a C substring. |
| 4239 | * @param __s The C string to append. |
| 4240 | * @param __n The number of characters to append. |
| 4241 | * @return Reference to this string. |
| 4242 | */ |
| 4243 | basic_string& |
| 4244 | append(const _CharT* __s, size_type __n); |
| 4245 | |
| 4246 | /** |
| 4247 | * @brief Append a C string. |
| 4248 | * @param __s The C string to append. |
| 4249 | * @return Reference to this string. |
| 4250 | */ |
| 4251 | basic_string& |
| 4252 | append(const _CharT* __s) |
| 4253 | { |
| 4254 | __glibcxx_requires_string(__s); |
| 4255 | return this->append(__s, traits_type::length(__s)); |
| 4256 | } |
| 4257 | |
| 4258 | /** |
| 4259 | * @brief Append multiple characters. |
| 4260 | * @param __n The number of characters to append. |
| 4261 | * @param __c The character to use. |
| 4262 | * @return Reference to this string. |
| 4263 | * |
| 4264 | * Appends __n copies of __c to this string. |
| 4265 | */ |
| 4266 | basic_string& |
| 4267 | append(size_type __n, _CharT __c); |
| 4268 | |
| 4269 | #if __cplusplus201703L >= 201103L |
| 4270 | /** |
| 4271 | * @brief Append an initializer_list of characters. |
| 4272 | * @param __l The initializer_list of characters to append. |
| 4273 | * @return Reference to this string. |
| 4274 | */ |
| 4275 | basic_string& |
| 4276 | append(initializer_list<_CharT> __l) |
| 4277 | { return this->append(__l.begin(), __l.size()); } |
| 4278 | #endif // C++11 |
| 4279 | |
| 4280 | /** |
| 4281 | * @brief Append a range of characters. |
| 4282 | * @param __first Iterator referencing the first character to append. |
| 4283 | * @param __last Iterator marking the end of the range. |
| 4284 | * @return Reference to this string. |
| 4285 | * |
| 4286 | * Appends characters in the range [__first,__last) to this string. |
| 4287 | */ |
| 4288 | template<class _InputIterator> |
| 4289 | basic_string& |
| 4290 | append(_InputIterator __first, _InputIterator __last) |
| 4291 | { return this->replace(_M_iend(), _M_iend(), __first, __last); } |
| 4292 | |
| 4293 | #if __cplusplus201703L >= 201703L |
| 4294 | /** |
| 4295 | * @brief Append a string_view. |
| 4296 | * @param __svt The object convertible to string_view to be appended. |
| 4297 | * @return Reference to this string. |
| 4298 | */ |
| 4299 | template<typename _Tp> |
| 4300 | _If_sv<_Tp, basic_string&> |
| 4301 | append(const _Tp& __svt) |
| 4302 | { |
| 4303 | __sv_type __sv = __svt; |
| 4304 | return this->append(__sv.data(), __sv.size()); |
| 4305 | } |
| 4306 | |
| 4307 | /** |
| 4308 | * @brief Append a range of characters from a string_view. |
| 4309 | * @param __svt The object convertible to string_view to be appended |
| 4310 | * from. |
| 4311 | * @param __pos The position in the string_view to append from. |
| 4312 | * @param __n The number of characters to append from the string_view. |
| 4313 | * @return Reference to this string. |
| 4314 | */ |
| 4315 | template<typename _Tp> |
| 4316 | _If_sv<_Tp, basic_string&> |
| 4317 | append(const _Tp& __svt, size_type __pos, size_type __n = npos) |
| 4318 | { |
| 4319 | __sv_type __sv = __svt; |
| 4320 | return append(__sv.data() |
| 4321 | + std::__sv_check(__sv.size(), __pos, "basic_string::append"), |
| 4322 | std::__sv_limit(__sv.size(), __pos, __n)); |
| 4323 | } |
| 4324 | #endif // C++17 |
| 4325 | |
| 4326 | /** |
| 4327 | * @brief Append a single character. |
| 4328 | * @param __c Character to append. |
| 4329 | */ |
| 4330 | void |
| 4331 | push_back(_CharT __c) |
| 4332 | { |
| 4333 | const size_type __len = 1 + this->size(); |
| 4334 | if (__len > this->capacity() || _M_rep()->_M_is_shared()) |
| 4335 | this->reserve(__len); |
| 4336 | traits_type::assign(_M_data()[this->size()], __c); |
| 4337 | _M_rep()->_M_set_length_and_sharable(__len); |
| 4338 | } |
| 4339 | |
| 4340 | /** |
| 4341 | * @brief Set value to contents of another string. |
| 4342 | * @param __str Source string to use. |
| 4343 | * @return Reference to this string. |
| 4344 | */ |
| 4345 | basic_string& |
| 4346 | assign(const basic_string& __str); |
| 4347 | |
| 4348 | #if __cplusplus201703L >= 201103L |
| 4349 | /** |
| 4350 | * @brief Set value to contents of another string. |
| 4351 | * @param __str Source string to use. |
| 4352 | * @return Reference to this string. |
| 4353 | * |
| 4354 | * This function sets this string to the exact contents of @a __str. |
| 4355 | * @a __str is a valid, but unspecified string. |
| 4356 | */ |
| 4357 | basic_string& |
| 4358 | assign(basic_string&& __str) |
| 4359 | noexcept(allocator_traits<_Alloc>::is_always_equal::value) |
| 4360 | { |
| 4361 | this->swap(__str); |
| 4362 | return *this; |
| 4363 | } |
| 4364 | #endif // C++11 |
| 4365 | |
| 4366 | /** |
| 4367 | * @brief Set value to a substring of a string. |
| 4368 | * @param __str The string to use. |
| 4369 | * @param __pos Index of the first character of str. |
| 4370 | * @param __n Number of characters to use. |
| 4371 | * @return Reference to this string. |
| 4372 | * @throw std::out_of_range if @a pos is not a valid index. |
| 4373 | * |
| 4374 | * This function sets this string to the substring of @a __str |
| 4375 | * consisting of @a __n characters at @a __pos. If @a __n is |
| 4376 | * is larger than the number of available characters in @a |
| 4377 | * __str, the remainder of @a __str is used. |
| 4378 | */ |
| 4379 | basic_string& |
| 4380 | assign(const basic_string& __str, size_type __pos, size_type __n = npos) |
| 4381 | { return this->assign(__str._M_data() |
| 4382 | + __str._M_check(__pos, "basic_string::assign"), |
| 4383 | __str._M_limit(__pos, __n)); } |
| 4384 | |
| 4385 | /** |
| 4386 | * @brief Set value to a C substring. |
| 4387 | * @param __s The C string to use. |
| 4388 | * @param __n Number of characters to use. |
| 4389 | * @return Reference to this string. |
| 4390 | * |
| 4391 | * This function sets the value of this string to the first @a __n |
| 4392 | * characters of @a __s. If @a __n is is larger than the number of |
| 4393 | * available characters in @a __s, the remainder of @a __s is used. |
| 4394 | */ |
| 4395 | basic_string& |
| 4396 | assign(const _CharT* __s, size_type __n); |
| 4397 | |
| 4398 | /** |
| 4399 | * @brief Set value to contents of a C string. |
| 4400 | * @param __s The C string to use. |
| 4401 | * @return Reference to this string. |
| 4402 | * |
| 4403 | * This function sets the value of this string to the value of @a __s. |
| 4404 | * The data is copied, so there is no dependence on @a __s once the |
| 4405 | * function returns. |
| 4406 | */ |
| 4407 | basic_string& |
| 4408 | assign(const _CharT* __s) |
| 4409 | { |
| 4410 | __glibcxx_requires_string(__s); |
| 4411 | return this->assign(__s, traits_type::length(__s)); |
| 4412 | } |
| 4413 | |
| 4414 | /** |
| 4415 | * @brief Set value to multiple characters. |
| 4416 | * @param __n Length of the resulting string. |
| 4417 | * @param __c The character to use. |
| 4418 | * @return Reference to this string. |
| 4419 | * |
| 4420 | * This function sets the value of this string to @a __n copies of |
| 4421 | * character @a __c. |
| 4422 | */ |
| 4423 | basic_string& |
| 4424 | assign(size_type __n, _CharT __c) |
| 4425 | { return _M_replace_aux(size_type(0), this->size(), __n, __c); } |
| 4426 | |
| 4427 | /** |
| 4428 | * @brief Set value to a range of characters. |
| 4429 | * @param __first Iterator referencing the first character to append. |
| 4430 | * @param __last Iterator marking the end of the range. |
| 4431 | * @return Reference to this string. |
| 4432 | * |
| 4433 | * Sets value of string to characters in the range [__first,__last). |
| 4434 | */ |
| 4435 | template<class _InputIterator> |
| 4436 | basic_string& |
| 4437 | assign(_InputIterator __first, _InputIterator __last) |
| 4438 | { return this->replace(_M_ibegin(), _M_iend(), __first, __last); } |
| 4439 | |
| 4440 | #if __cplusplus201703L >= 201103L |
| 4441 | /** |
| 4442 | * @brief Set value to an initializer_list of characters. |
| 4443 | * @param __l The initializer_list of characters to assign. |
| 4444 | * @return Reference to this string. |
| 4445 | */ |
| 4446 | basic_string& |
| 4447 | assign(initializer_list<_CharT> __l) |
| 4448 | { return this->assign(__l.begin(), __l.size()); } |
| 4449 | #endif // C++11 |
| 4450 | |
| 4451 | #if __cplusplus201703L >= 201703L |
| 4452 | /** |
| 4453 | * @brief Set value from a string_view. |
| 4454 | * @param __svt The source object convertible to string_view. |
| 4455 | * @return Reference to this string. |
| 4456 | */ |
| 4457 | template<typename _Tp> |
| 4458 | _If_sv<_Tp, basic_string&> |
| 4459 | assign(const _Tp& __svt) |
| 4460 | { |
| 4461 | __sv_type __sv = __svt; |
| 4462 | return this->assign(__sv.data(), __sv.size()); |
| 4463 | } |
| 4464 | |
| 4465 | /** |
| 4466 | * @brief Set value from a range of characters in a string_view. |
| 4467 | * @param __svt The source object convertible to string_view. |
| 4468 | * @param __pos The position in the string_view to assign from. |
| 4469 | * @param __n The number of characters to assign. |
| 4470 | * @return Reference to this string. |
| 4471 | */ |
| 4472 | template<typename _Tp> |
| 4473 | _If_sv<_Tp, basic_string&> |
| 4474 | assign(const _Tp& __svt, size_type __pos, size_type __n = npos) |
| 4475 | { |
| 4476 | __sv_type __sv = __svt; |
| 4477 | return assign(__sv.data() |
| 4478 | + std::__sv_check(__sv.size(), __pos, "basic_string::assign"), |
| 4479 | std::__sv_limit(__sv.size(), __pos, __n)); |
| 4480 | } |
| 4481 | #endif // C++17 |
| 4482 | |
| 4483 | /** |
| 4484 | * @brief Insert multiple characters. |
| 4485 | * @param __p Iterator referencing location in string to insert at. |
| 4486 | * @param __n Number of characters to insert |
| 4487 | * @param __c The character to insert. |
| 4488 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4489 | * |
| 4490 | * Inserts @a __n copies of character @a __c starting at the |
| 4491 | * position referenced by iterator @a __p. If adding |
| 4492 | * characters causes the length to exceed max_size(), |
| 4493 | * length_error is thrown. The value of the string doesn't |
| 4494 | * change if an error is thrown. |
| 4495 | */ |
| 4496 | void |
| 4497 | insert(iterator __p, size_type __n, _CharT __c) |
| 4498 | { this->replace(__p, __p, __n, __c); } |
| 4499 | |
| 4500 | /** |
| 4501 | * @brief Insert a range of characters. |
| 4502 | * @param __p Iterator referencing location in string to insert at. |
| 4503 | * @param __beg Start of range. |
| 4504 | * @param __end End of range. |
| 4505 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4506 | * |
| 4507 | * Inserts characters in range [__beg,__end). If adding |
| 4508 | * characters causes the length to exceed max_size(), |
| 4509 | * length_error is thrown. The value of the string doesn't |
| 4510 | * change if an error is thrown. |
| 4511 | */ |
| 4512 | template<class _InputIterator> |
| 4513 | void |
| 4514 | insert(iterator __p, _InputIterator __beg, _InputIterator __end) |
| 4515 | { this->replace(__p, __p, __beg, __end); } |
| 4516 | |
| 4517 | #if __cplusplus201703L >= 201103L |
| 4518 | /** |
| 4519 | * @brief Insert an initializer_list of characters. |
| 4520 | * @param __p Iterator referencing location in string to insert at. |
| 4521 | * @param __l The initializer_list of characters to insert. |
| 4522 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4523 | */ |
| 4524 | void |
| 4525 | insert(iterator __p, initializer_list<_CharT> __l) |
| 4526 | { |
| 4527 | _GLIBCXX_DEBUG_PEDASSERT(__p >= _M_ibegin() && __p <= _M_iend()); |
| 4528 | this->insert(__p - _M_ibegin(), __l.begin(), __l.size()); |
| 4529 | } |
| 4530 | #endif // C++11 |
| 4531 | |
| 4532 | /** |
| 4533 | * @brief Insert value of a string. |
| 4534 | * @param __pos1 Position in string to insert at. |
| 4535 | * @param __str The string to insert. |
| 4536 | * @return Reference to this string. |
| 4537 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4538 | * |
| 4539 | * Inserts value of @a __str starting at @a __pos1. If adding |
| 4540 | * characters causes the length to exceed max_size(), |
| 4541 | * length_error is thrown. The value of the string doesn't |
| 4542 | * change if an error is thrown. |
| 4543 | */ |
| 4544 | basic_string& |
| 4545 | insert(size_type __pos1, const basic_string& __str) |
| 4546 | { return this->insert(__pos1, __str, size_type(0), __str.size()); } |
| 4547 | |
| 4548 | /** |
| 4549 | * @brief Insert a substring. |
| 4550 | * @param __pos1 Position in string to insert at. |
| 4551 | * @param __str The string to insert. |
| 4552 | * @param __pos2 Start of characters in str to insert. |
| 4553 | * @param __n Number of characters to insert. |
| 4554 | * @return Reference to this string. |
| 4555 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4556 | * @throw std::out_of_range If @a pos1 > size() or |
| 4557 | * @a __pos2 > @a str.size(). |
| 4558 | * |
| 4559 | * Starting at @a pos1, insert @a __n character of @a __str |
| 4560 | * beginning with @a __pos2. If adding characters causes the |
| 4561 | * length to exceed max_size(), length_error is thrown. If @a |
| 4562 | * __pos1 is beyond the end of this string or @a __pos2 is |
| 4563 | * beyond the end of @a __str, out_of_range is thrown. The |
| 4564 | * value of the string doesn't change if an error is thrown. |
| 4565 | */ |
| 4566 | basic_string& |
| 4567 | insert(size_type __pos1, const basic_string& __str, |
| 4568 | size_type __pos2, size_type __n = npos) |
| 4569 | { return this->insert(__pos1, __str._M_data() |
| 4570 | + __str._M_check(__pos2, "basic_string::insert"), |
| 4571 | __str._M_limit(__pos2, __n)); } |
| 4572 | |
| 4573 | /** |
| 4574 | * @brief Insert a C substring. |
| 4575 | * @param __pos Position in string to insert at. |
| 4576 | * @param __s The C string to insert. |
| 4577 | * @param __n The number of characters to insert. |
| 4578 | * @return Reference to this string. |
| 4579 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4580 | * @throw std::out_of_range If @a __pos is beyond the end of this |
| 4581 | * string. |
| 4582 | * |
| 4583 | * Inserts the first @a __n characters of @a __s starting at @a |
| 4584 | * __pos. If adding characters causes the length to exceed |
| 4585 | * max_size(), length_error is thrown. If @a __pos is beyond |
| 4586 | * end(), out_of_range is thrown. The value of the string |
| 4587 | * doesn't change if an error is thrown. |
| 4588 | */ |
| 4589 | basic_string& |
| 4590 | insert(size_type __pos, const _CharT* __s, size_type __n); |
| 4591 | |
| 4592 | /** |
| 4593 | * @brief Insert a C string. |
| 4594 | * @param __pos Position in string to insert at. |
| 4595 | * @param __s The C string to insert. |
| 4596 | * @return Reference to this string. |
| 4597 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4598 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 4599 | * string. |
| 4600 | * |
| 4601 | * Inserts the first @a n characters of @a __s starting at @a __pos. If |
| 4602 | * adding characters causes the length to exceed max_size(), |
| 4603 | * length_error is thrown. If @a __pos is beyond end(), out_of_range is |
| 4604 | * thrown. The value of the string doesn't change if an error is |
| 4605 | * thrown. |
| 4606 | */ |
| 4607 | basic_string& |
| 4608 | insert(size_type __pos, const _CharT* __s) |
| 4609 | { |
| 4610 | __glibcxx_requires_string(__s); |
| 4611 | return this->insert(__pos, __s, traits_type::length(__s)); |
| 4612 | } |
| 4613 | |
| 4614 | /** |
| 4615 | * @brief Insert multiple characters. |
| 4616 | * @param __pos Index in string to insert at. |
| 4617 | * @param __n Number of characters to insert |
| 4618 | * @param __c The character to insert. |
| 4619 | * @return Reference to this string. |
| 4620 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4621 | * @throw std::out_of_range If @a __pos is beyond the end of this |
| 4622 | * string. |
| 4623 | * |
| 4624 | * Inserts @a __n copies of character @a __c starting at index |
| 4625 | * @a __pos. If adding characters causes the length to exceed |
| 4626 | * max_size(), length_error is thrown. If @a __pos > length(), |
| 4627 | * out_of_range is thrown. The value of the string doesn't |
| 4628 | * change if an error is thrown. |
| 4629 | */ |
| 4630 | basic_string& |
| 4631 | insert(size_type __pos, size_type __n, _CharT __c) |
| 4632 | { return _M_replace_aux(_M_check(__pos, "basic_string::insert"), |
| 4633 | size_type(0), __n, __c); } |
| 4634 | |
| 4635 | /** |
| 4636 | * @brief Insert one character. |
| 4637 | * @param __p Iterator referencing position in string to insert at. |
| 4638 | * @param __c The character to insert. |
| 4639 | * @return Iterator referencing newly inserted char. |
| 4640 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4641 | * |
| 4642 | * Inserts character @a __c at position referenced by @a __p. |
| 4643 | * If adding character causes the length to exceed max_size(), |
| 4644 | * length_error is thrown. If @a __p is beyond end of string, |
| 4645 | * out_of_range is thrown. The value of the string doesn't |
| 4646 | * change if an error is thrown. |
| 4647 | */ |
| 4648 | iterator |
| 4649 | insert(iterator __p, _CharT __c) |
| 4650 | { |
| 4651 | _GLIBCXX_DEBUG_PEDASSERT(__p >= _M_ibegin() && __p <= _M_iend()); |
| 4652 | const size_type __pos = __p - _M_ibegin(); |
| 4653 | _M_replace_aux(__pos, size_type(0), size_type(1), __c); |
| 4654 | _M_rep()->_M_set_leaked(); |
| 4655 | return iterator(_M_data() + __pos); |
| 4656 | } |
| 4657 | |
| 4658 | #if __cplusplus201703L >= 201703L |
| 4659 | /** |
| 4660 | * @brief Insert a string_view. |
| 4661 | * @param __pos Position in string to insert at. |
| 4662 | * @param __svt The object convertible to string_view to insert. |
| 4663 | * @return Reference to this string. |
| 4664 | */ |
| 4665 | template<typename _Tp> |
| 4666 | _If_sv<_Tp, basic_string&> |
| 4667 | insert(size_type __pos, const _Tp& __svt) |
| 4668 | { |
| 4669 | __sv_type __sv = __svt; |
| 4670 | return this->insert(__pos, __sv.data(), __sv.size()); |
| 4671 | } |
| 4672 | |
| 4673 | /** |
| 4674 | * @brief Insert a string_view. |
| 4675 | * @param __pos Position in string to insert at. |
| 4676 | * @param __svt The object convertible to string_view to insert from. |
| 4677 | * @param __pos Position in string_view to insert |
| 4678 | * from. |
| 4679 | * @param __n The number of characters to insert. |
| 4680 | * @return Reference to this string. |
| 4681 | */ |
| 4682 | template<typename _Tp> |
| 4683 | _If_sv<_Tp, basic_string&> |
| 4684 | insert(size_type __pos1, const _Tp& __svt, |
| 4685 | size_type __pos2, size_type __n = npos) |
| 4686 | { |
| 4687 | __sv_type __sv = __svt; |
| 4688 | return this->replace(__pos1, size_type(0), __sv.data() |
| 4689 | + std::__sv_check(__sv.size(), __pos2, "basic_string::insert"), |
| 4690 | std::__sv_limit(__sv.size(), __pos2, __n)); |
| 4691 | } |
| 4692 | #endif // C++17 |
| 4693 | |
| 4694 | /** |
| 4695 | * @brief Remove characters. |
| 4696 | * @param __pos Index of first character to remove (default 0). |
| 4697 | * @param __n Number of characters to remove (default remainder). |
| 4698 | * @return Reference to this string. |
| 4699 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 4700 | * string. |
| 4701 | * |
| 4702 | * Removes @a __n characters from this string starting at @a |
| 4703 | * __pos. The length of the string is reduced by @a __n. If |
| 4704 | * there are < @a __n characters to remove, the remainder of |
| 4705 | * the string is truncated. If @a __p is beyond end of string, |
| 4706 | * out_of_range is thrown. The value of the string doesn't |
| 4707 | * change if an error is thrown. |
| 4708 | */ |
| 4709 | basic_string& |
| 4710 | erase(size_type __pos = 0, size_type __n = npos) |
| 4711 | { |
| 4712 | _M_mutate(_M_check(__pos, "basic_string::erase"), |
| 4713 | _M_limit(__pos, __n), size_type(0)); |
| 4714 | return *this; |
| 4715 | } |
| 4716 | |
| 4717 | /** |
| 4718 | * @brief Remove one character. |
| 4719 | * @param __position Iterator referencing the character to remove. |
| 4720 | * @return iterator referencing same location after removal. |
| 4721 | * |
| 4722 | * Removes the character at @a __position from this string. The value |
| 4723 | * of the string doesn't change if an error is thrown. |
| 4724 | */ |
| 4725 | iterator |
| 4726 | erase(iterator __position) |
| 4727 | { |
| 4728 | _GLIBCXX_DEBUG_PEDASSERT(__position >= _M_ibegin() |
| 4729 | && __position < _M_iend()); |
| 4730 | const size_type __pos = __position - _M_ibegin(); |
| 4731 | _M_mutate(__pos, size_type(1), size_type(0)); |
| 4732 | _M_rep()->_M_set_leaked(); |
| 4733 | return iterator(_M_data() + __pos); |
| 4734 | } |
| 4735 | |
| 4736 | /** |
| 4737 | * @brief Remove a range of characters. |
| 4738 | * @param __first Iterator referencing the first character to remove. |
| 4739 | * @param __last Iterator referencing the end of the range. |
| 4740 | * @return Iterator referencing location of first after removal. |
| 4741 | * |
| 4742 | * Removes the characters in the range [first,last) from this string. |
| 4743 | * The value of the string doesn't change if an error is thrown. |
| 4744 | */ |
| 4745 | iterator |
| 4746 | erase(iterator __first, iterator __last); |
| 4747 | |
| 4748 | #if __cplusplus201703L >= 201103L |
| 4749 | /** |
| 4750 | * @brief Remove the last character. |
| 4751 | * |
| 4752 | * The string must be non-empty. |
| 4753 | */ |
| 4754 | void |
| 4755 | pop_back() // FIXME C++11: should be noexcept. |
| 4756 | { |
| 4757 | __glibcxx_assert(!empty())do { if (! (!empty())) std::__replacement_assert("/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/basic_string.h" , 4757, __PRETTY_FUNCTION__, "!empty()"); } while (false); |
| 4758 | erase(size() - 1, 1); |
| 4759 | } |
| 4760 | #endif // C++11 |
| 4761 | |
| 4762 | /** |
| 4763 | * @brief Replace characters with value from another string. |
| 4764 | * @param __pos Index of first character to replace. |
| 4765 | * @param __n Number of characters to be replaced. |
| 4766 | * @param __str String to insert. |
| 4767 | * @return Reference to this string. |
| 4768 | * @throw std::out_of_range If @a pos is beyond the end of this |
| 4769 | * string. |
| 4770 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4771 | * |
| 4772 | * Removes the characters in the range [__pos,__pos+__n) from |
| 4773 | * this string. In place, the value of @a __str is inserted. |
| 4774 | * If @a __pos is beyond end of string, out_of_range is thrown. |
| 4775 | * If the length of the result exceeds max_size(), length_error |
| 4776 | * is thrown. The value of the string doesn't change if an |
| 4777 | * error is thrown. |
| 4778 | */ |
| 4779 | basic_string& |
| 4780 | replace(size_type __pos, size_type __n, const basic_string& __str) |
| 4781 | { return this->replace(__pos, __n, __str._M_data(), __str.size()); } |
| 4782 | |
| 4783 | /** |
| 4784 | * @brief Replace characters with value from another string. |
| 4785 | * @param __pos1 Index of first character to replace. |
| 4786 | * @param __n1 Number of characters to be replaced. |
| 4787 | * @param __str String to insert. |
| 4788 | * @param __pos2 Index of first character of str to use. |
| 4789 | * @param __n2 Number of characters from str to use. |
| 4790 | * @return Reference to this string. |
| 4791 | * @throw std::out_of_range If @a __pos1 > size() or @a __pos2 > |
| 4792 | * __str.size(). |
| 4793 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4794 | * |
| 4795 | * Removes the characters in the range [__pos1,__pos1 + n) from this |
| 4796 | * string. In place, the value of @a __str is inserted. If @a __pos is |
| 4797 | * beyond end of string, out_of_range is thrown. If the length of the |
| 4798 | * result exceeds max_size(), length_error is thrown. The value of the |
| 4799 | * string doesn't change if an error is thrown. |
| 4800 | */ |
| 4801 | basic_string& |
| 4802 | replace(size_type __pos1, size_type __n1, const basic_string& __str, |
| 4803 | size_type __pos2, size_type __n2 = npos) |
| 4804 | { return this->replace(__pos1, __n1, __str._M_data() |
| 4805 | + __str._M_check(__pos2, "basic_string::replace"), |
| 4806 | __str._M_limit(__pos2, __n2)); } |
| 4807 | |
| 4808 | /** |
| 4809 | * @brief Replace characters with value of a C substring. |
| 4810 | * @param __pos Index of first character to replace. |
| 4811 | * @param __n1 Number of characters to be replaced. |
| 4812 | * @param __s C string to insert. |
| 4813 | * @param __n2 Number of characters from @a s to use. |
| 4814 | * @return Reference to this string. |
| 4815 | * @throw std::out_of_range If @a pos1 > size(). |
| 4816 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4817 | * |
| 4818 | * Removes the characters in the range [__pos,__pos + __n1) |
| 4819 | * from this string. In place, the first @a __n2 characters of |
| 4820 | * @a __s are inserted, or all of @a __s if @a __n2 is too large. If |
| 4821 | * @a __pos is beyond end of string, out_of_range is thrown. If |
| 4822 | * the length of result exceeds max_size(), length_error is |
| 4823 | * thrown. The value of the string doesn't change if an error |
| 4824 | * is thrown. |
| 4825 | */ |
| 4826 | basic_string& |
| 4827 | replace(size_type __pos, size_type __n1, const _CharT* __s, |
| 4828 | size_type __n2); |
| 4829 | |
| 4830 | /** |
| 4831 | * @brief Replace characters with value of a C string. |
| 4832 | * @param __pos Index of first character to replace. |
| 4833 | * @param __n1 Number of characters to be replaced. |
| 4834 | * @param __s C string to insert. |
| 4835 | * @return Reference to this string. |
| 4836 | * @throw std::out_of_range If @a pos > size(). |
| 4837 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4838 | * |
| 4839 | * Removes the characters in the range [__pos,__pos + __n1) |
| 4840 | * from this string. In place, the characters of @a __s are |
| 4841 | * inserted. If @a __pos is beyond end of string, out_of_range |
| 4842 | * is thrown. If the length of result exceeds max_size(), |
| 4843 | * length_error is thrown. The value of the string doesn't |
| 4844 | * change if an error is thrown. |
| 4845 | */ |
| 4846 | basic_string& |
| 4847 | replace(size_type __pos, size_type __n1, const _CharT* __s) |
| 4848 | { |
| 4849 | __glibcxx_requires_string(__s); |
| 4850 | return this->replace(__pos, __n1, __s, traits_type::length(__s)); |
| 4851 | } |
| 4852 | |
| 4853 | /** |
| 4854 | * @brief Replace characters with multiple characters. |
| 4855 | * @param __pos Index of first character to replace. |
| 4856 | * @param __n1 Number of characters to be replaced. |
| 4857 | * @param __n2 Number of characters to insert. |
| 4858 | * @param __c Character to insert. |
| 4859 | * @return Reference to this string. |
| 4860 | * @throw std::out_of_range If @a __pos > size(). |
| 4861 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4862 | * |
| 4863 | * Removes the characters in the range [pos,pos + n1) from this |
| 4864 | * string. In place, @a __n2 copies of @a __c are inserted. |
| 4865 | * If @a __pos is beyond end of string, out_of_range is thrown. |
| 4866 | * If the length of result exceeds max_size(), length_error is |
| 4867 | * thrown. The value of the string doesn't change if an error |
| 4868 | * is thrown. |
| 4869 | */ |
| 4870 | basic_string& |
| 4871 | replace(size_type __pos, size_type __n1, size_type __n2, _CharT __c) |
| 4872 | { return _M_replace_aux(_M_check(__pos, "basic_string::replace"), |
| 4873 | _M_limit(__pos, __n1), __n2, __c); } |
| 4874 | |
| 4875 | /** |
| 4876 | * @brief Replace range of characters with string. |
| 4877 | * @param __i1 Iterator referencing start of range to replace. |
| 4878 | * @param __i2 Iterator referencing end of range to replace. |
| 4879 | * @param __str String value to insert. |
| 4880 | * @return Reference to this string. |
| 4881 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4882 | * |
| 4883 | * Removes the characters in the range [__i1,__i2). In place, |
| 4884 | * the value of @a __str is inserted. If the length of result |
| 4885 | * exceeds max_size(), length_error is thrown. The value of |
| 4886 | * the string doesn't change if an error is thrown. |
| 4887 | */ |
| 4888 | basic_string& |
| 4889 | replace(iterator __i1, iterator __i2, const basic_string& __str) |
| 4890 | { return this->replace(__i1, __i2, __str._M_data(), __str.size()); } |
| 4891 | |
| 4892 | /** |
| 4893 | * @brief Replace range of characters with C substring. |
| 4894 | * @param __i1 Iterator referencing start of range to replace. |
| 4895 | * @param __i2 Iterator referencing end of range to replace. |
| 4896 | * @param __s C string value to insert. |
| 4897 | * @param __n Number of characters from s to insert. |
| 4898 | * @return Reference to this string. |
| 4899 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4900 | * |
| 4901 | * Removes the characters in the range [__i1,__i2). In place, |
| 4902 | * the first @a __n characters of @a __s are inserted. If the |
| 4903 | * length of result exceeds max_size(), length_error is thrown. |
| 4904 | * The value of the string doesn't change if an error is |
| 4905 | * thrown. |
| 4906 | */ |
| 4907 | basic_string& |
| 4908 | replace(iterator __i1, iterator __i2, const _CharT* __s, size_type __n) |
| 4909 | { |
| 4910 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 4911 | && __i2 <= _M_iend()); |
| 4912 | return this->replace(__i1 - _M_ibegin(), __i2 - __i1, __s, __n); |
| 4913 | } |
| 4914 | |
| 4915 | /** |
| 4916 | * @brief Replace range of characters with C string. |
| 4917 | * @param __i1 Iterator referencing start of range to replace. |
| 4918 | * @param __i2 Iterator referencing end of range to replace. |
| 4919 | * @param __s C string value to insert. |
| 4920 | * @return Reference to this string. |
| 4921 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4922 | * |
| 4923 | * Removes the characters in the range [__i1,__i2). In place, |
| 4924 | * the characters of @a __s are inserted. If the length of |
| 4925 | * result exceeds max_size(), length_error is thrown. The |
| 4926 | * value of the string doesn't change if an error is thrown. |
| 4927 | */ |
| 4928 | basic_string& |
| 4929 | replace(iterator __i1, iterator __i2, const _CharT* __s) |
| 4930 | { |
| 4931 | __glibcxx_requires_string(__s); |
| 4932 | return this->replace(__i1, __i2, __s, traits_type::length(__s)); |
| 4933 | } |
| 4934 | |
| 4935 | /** |
| 4936 | * @brief Replace range of characters with multiple characters |
| 4937 | * @param __i1 Iterator referencing start of range to replace. |
| 4938 | * @param __i2 Iterator referencing end of range to replace. |
| 4939 | * @param __n Number of characters to insert. |
| 4940 | * @param __c Character to insert. |
| 4941 | * @return Reference to this string. |
| 4942 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4943 | * |
| 4944 | * Removes the characters in the range [__i1,__i2). In place, |
| 4945 | * @a __n copies of @a __c are inserted. If the length of |
| 4946 | * result exceeds max_size(), length_error is thrown. The |
| 4947 | * value of the string doesn't change if an error is thrown. |
| 4948 | */ |
| 4949 | basic_string& |
| 4950 | replace(iterator __i1, iterator __i2, size_type __n, _CharT __c) |
| 4951 | { |
| 4952 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 4953 | && __i2 <= _M_iend()); |
| 4954 | return _M_replace_aux(__i1 - _M_ibegin(), __i2 - __i1, __n, __c); |
| 4955 | } |
| 4956 | |
| 4957 | /** |
| 4958 | * @brief Replace range of characters with range. |
| 4959 | * @param __i1 Iterator referencing start of range to replace. |
| 4960 | * @param __i2 Iterator referencing end of range to replace. |
| 4961 | * @param __k1 Iterator referencing start of range to insert. |
| 4962 | * @param __k2 Iterator referencing end of range to insert. |
| 4963 | * @return Reference to this string. |
| 4964 | * @throw std::length_error If new length exceeds @c max_size(). |
| 4965 | * |
| 4966 | * Removes the characters in the range [__i1,__i2). In place, |
| 4967 | * characters in the range [__k1,__k2) are inserted. If the |
| 4968 | * length of result exceeds max_size(), length_error is thrown. |
| 4969 | * The value of the string doesn't change if an error is |
| 4970 | * thrown. |
| 4971 | */ |
| 4972 | template<class _InputIterator> |
| 4973 | basic_string& |
| 4974 | replace(iterator __i1, iterator __i2, |
| 4975 | _InputIterator __k1, _InputIterator __k2) |
| 4976 | { |
| 4977 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 4978 | && __i2 <= _M_iend()); |
| 4979 | __glibcxx_requires_valid_range(__k1, __k2); |
| 4980 | typedef typename std::__is_integer<_InputIterator>::__type _Integral; |
| 4981 | return _M_replace_dispatch(__i1, __i2, __k1, __k2, _Integral()); |
| 4982 | } |
| 4983 | |
| 4984 | // Specializations for the common case of pointer and iterator: |
| 4985 | // useful to avoid the overhead of temporary buffering in _M_replace. |
| 4986 | basic_string& |
| 4987 | replace(iterator __i1, iterator __i2, _CharT* __k1, _CharT* __k2) |
| 4988 | { |
| 4989 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 4990 | && __i2 <= _M_iend()); |
| 4991 | __glibcxx_requires_valid_range(__k1, __k2); |
| 4992 | return this->replace(__i1 - _M_ibegin(), __i2 - __i1, |
| 4993 | __k1, __k2 - __k1); |
| 4994 | } |
| 4995 | |
| 4996 | basic_string& |
| 4997 | replace(iterator __i1, iterator __i2, |
| 4998 | const _CharT* __k1, const _CharT* __k2) |
| 4999 | { |
| 5000 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 5001 | && __i2 <= _M_iend()); |
| 5002 | __glibcxx_requires_valid_range(__k1, __k2); |
| 5003 | return this->replace(__i1 - _M_ibegin(), __i2 - __i1, |
| 5004 | __k1, __k2 - __k1); |
| 5005 | } |
| 5006 | |
| 5007 | basic_string& |
| 5008 | replace(iterator __i1, iterator __i2, iterator __k1, iterator __k2) |
| 5009 | { |
| 5010 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 5011 | && __i2 <= _M_iend()); |
| 5012 | __glibcxx_requires_valid_range(__k1, __k2); |
| 5013 | return this->replace(__i1 - _M_ibegin(), __i2 - __i1, |
| 5014 | __k1.base(), __k2 - __k1); |
| 5015 | } |
| 5016 | |
| 5017 | basic_string& |
| 5018 | replace(iterator __i1, iterator __i2, |
| 5019 | const_iterator __k1, const_iterator __k2) |
| 5020 | { |
| 5021 | _GLIBCXX_DEBUG_PEDASSERT(_M_ibegin() <= __i1 && __i1 <= __i2 |
| 5022 | && __i2 <= _M_iend()); |
| 5023 | __glibcxx_requires_valid_range(__k1, __k2); |
| 5024 | return this->replace(__i1 - _M_ibegin(), __i2 - __i1, |
| 5025 | __k1.base(), __k2 - __k1); |
| 5026 | } |
| 5027 | |
| 5028 | #if __cplusplus201703L >= 201103L |
| 5029 | /** |
| 5030 | * @brief Replace range of characters with initializer_list. |
| 5031 | * @param __i1 Iterator referencing start of range to replace. |
| 5032 | * @param __i2 Iterator referencing end of range to replace. |
| 5033 | * @param __l The initializer_list of characters to insert. |
| 5034 | * @return Reference to this string. |
| 5035 | * @throw std::length_error If new length exceeds @c max_size(). |
| 5036 | * |
| 5037 | * Removes the characters in the range [__i1,__i2). In place, |
| 5038 | * characters in the range [__k1,__k2) are inserted. If the |
| 5039 | * length of result exceeds max_size(), length_error is thrown. |
| 5040 | * The value of the string doesn't change if an error is |
| 5041 | * thrown. |
| 5042 | */ |
| 5043 | basic_string& replace(iterator __i1, iterator __i2, |
| 5044 | initializer_list<_CharT> __l) |
| 5045 | { return this->replace(__i1, __i2, __l.begin(), __l.end()); } |
| 5046 | #endif // C++11 |
| 5047 | |
| 5048 | #if __cplusplus201703L >= 201703L |
| 5049 | /** |
| 5050 | * @brief Replace range of characters with string_view. |
| 5051 | * @param __pos The position to replace at. |
| 5052 | * @param __n The number of characters to replace. |
| 5053 | * @param __svt The object convertible to string_view to insert. |
| 5054 | * @return Reference to this string. |
| 5055 | */ |
| 5056 | template<typename _Tp> |
| 5057 | _If_sv<_Tp, basic_string&> |
| 5058 | replace(size_type __pos, size_type __n, const _Tp& __svt) |
| 5059 | { |
| 5060 | __sv_type __sv = __svt; |
| 5061 | return this->replace(__pos, __n, __sv.data(), __sv.size()); |
| 5062 | } |
| 5063 | |
| 5064 | /** |
| 5065 | * @brief Replace range of characters with string_view. |
| 5066 | * @param __pos1 The position to replace at. |
| 5067 | * @param __n1 The number of characters to replace. |
| 5068 | * @param __svt The object convertible to string_view to insert from. |
| 5069 | * @param __pos2 The position in the string_view to insert from. |
| 5070 | * @param __n2 The number of characters to insert. |
| 5071 | * @return Reference to this string. |
| 5072 | */ |
| 5073 | template<typename _Tp> |
| 5074 | _If_sv<_Tp, basic_string&> |
| 5075 | replace(size_type __pos1, size_type __n1, const _Tp& __svt, |
| 5076 | size_type __pos2, size_type __n2 = npos) |
| 5077 | { |
| 5078 | __sv_type __sv = __svt; |
| 5079 | return this->replace(__pos1, __n1, |
| 5080 | __sv.data() |
| 5081 | + std::__sv_check(__sv.size(), __pos2, "basic_string::replace"), |
| 5082 | std::__sv_limit(__sv.size(), __pos2, __n2)); |
| 5083 | } |
| 5084 | |
| 5085 | /** |
| 5086 | * @brief Replace range of characters with string_view. |
| 5087 | * @param __i1 An iterator referencing the start position |
| 5088 | to replace at. |
| 5089 | * @param __i2 An iterator referencing the end position |
| 5090 | for the replace. |
| 5091 | * @param __svt The object convertible to string_view to insert from. |
| 5092 | * @return Reference to this string. |
| 5093 | */ |
| 5094 | template<typename _Tp> |
| 5095 | _If_sv<_Tp, basic_string&> |
| 5096 | replace(const_iterator __i1, const_iterator __i2, const _Tp& __svt) |
| 5097 | { |
| 5098 | __sv_type __sv = __svt; |
| 5099 | return this->replace(__i1 - begin(), __i2 - __i1, __sv); |
| 5100 | } |
| 5101 | #endif // C++17 |
| 5102 | |
| 5103 | private: |
| 5104 | template<class _Integer> |
| 5105 | basic_string& |
| 5106 | _M_replace_dispatch(iterator __i1, iterator __i2, _Integer __n, |
| 5107 | _Integer __val, __true_type) |
| 5108 | { return _M_replace_aux(__i1 - _M_ibegin(), __i2 - __i1, __n, __val); } |
| 5109 | |
| 5110 | template<class _InputIterator> |
| 5111 | basic_string& |
| 5112 | _M_replace_dispatch(iterator __i1, iterator __i2, _InputIterator __k1, |
| 5113 | _InputIterator __k2, __false_type); |
| 5114 | |
| 5115 | basic_string& |
| 5116 | _M_replace_aux(size_type __pos1, size_type __n1, size_type __n2, |
| 5117 | _CharT __c); |
| 5118 | |
| 5119 | basic_string& |
| 5120 | _M_replace_safe(size_type __pos1, size_type __n1, const _CharT* __s, |
| 5121 | size_type __n2); |
| 5122 | |
| 5123 | // _S_construct_aux is used to implement the 21.3.1 para 15 which |
| 5124 | // requires special behaviour if _InIter is an integral type |
| 5125 | template<class _InIterator> |
| 5126 | static _CharT* |
| 5127 | _S_construct_aux(_InIterator __beg, _InIterator __end, |
| 5128 | const _Alloc& __a, __false_type) |
| 5129 | { |
| 5130 | typedef typename iterator_traits<_InIterator>::iterator_category _Tag; |
| 5131 | return _S_construct(__beg, __end, __a, _Tag()); |
| 5132 | } |
| 5133 | |
| 5134 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 5135 | // 438. Ambiguity in the "do the right thing" clause |
| 5136 | template<class _Integer> |
| 5137 | static _CharT* |
| 5138 | _S_construct_aux(_Integer __beg, _Integer __end, |
| 5139 | const _Alloc& __a, __true_type) |
| 5140 | { return _S_construct_aux_2(static_cast<size_type>(__beg), |
| 5141 | __end, __a); } |
| 5142 | |
| 5143 | static _CharT* |
| 5144 | _S_construct_aux_2(size_type __req, _CharT __c, const _Alloc& __a) |
| 5145 | { return _S_construct(__req, __c, __a); } |
| 5146 | |
| 5147 | template<class _InIterator> |
| 5148 | static _CharT* |
| 5149 | _S_construct(_InIterator __beg, _InIterator __end, const _Alloc& __a) |
| 5150 | { |
| 5151 | typedef typename std::__is_integer<_InIterator>::__type _Integral; |
| 5152 | return _S_construct_aux(__beg, __end, __a, _Integral()); |
| 5153 | } |
| 5154 | |
| 5155 | // For Input Iterators, used in istreambuf_iterators, etc. |
| 5156 | template<class _InIterator> |
| 5157 | static _CharT* |
| 5158 | _S_construct(_InIterator __beg, _InIterator __end, const _Alloc& __a, |
| 5159 | input_iterator_tag); |
| 5160 | |
| 5161 | // For forward_iterators up to random_access_iterators, used for |
| 5162 | // string::iterator, _CharT*, etc. |
| 5163 | template<class _FwdIterator> |
| 5164 | static _CharT* |
| 5165 | _S_construct(_FwdIterator __beg, _FwdIterator __end, const _Alloc& __a, |
| 5166 | forward_iterator_tag); |
| 5167 | |
| 5168 | static _CharT* |
| 5169 | _S_construct(size_type __req, _CharT __c, const _Alloc& __a); |
| 5170 | |
| 5171 | public: |
| 5172 | |
| 5173 | /** |
| 5174 | * @brief Copy substring into C string. |
| 5175 | * @param __s C string to copy value into. |
| 5176 | * @param __n Number of characters to copy. |
| 5177 | * @param __pos Index of first character to copy. |
| 5178 | * @return Number of characters actually copied |
| 5179 | * @throw std::out_of_range If __pos > size(). |
| 5180 | * |
| 5181 | * Copies up to @a __n characters starting at @a __pos into the |
| 5182 | * C string @a __s. If @a __pos is %greater than size(), |
| 5183 | * out_of_range is thrown. |
| 5184 | */ |
| 5185 | size_type |
| 5186 | copy(_CharT* __s, size_type __n, size_type __pos = 0) const; |
| 5187 | |
| 5188 | /** |
| 5189 | * @brief Swap contents with another string. |
| 5190 | * @param __s String to swap with. |
| 5191 | * |
| 5192 | * Exchanges the contents of this string with that of @a __s in constant |
| 5193 | * time. |
| 5194 | */ |
| 5195 | void |
| 5196 | swap(basic_string& __s) |
| 5197 | _GLIBCXX_NOEXCEPT_IF(allocator_traits<_Alloc>::is_always_equal::value)noexcept(allocator_traits<_Alloc>::is_always_equal::value ); |
| 5198 | |
| 5199 | // String operations: |
| 5200 | /** |
| 5201 | * @brief Return const pointer to null-terminated contents. |
| 5202 | * |
| 5203 | * This is a handle to internal data. Do not modify or dire things may |
| 5204 | * happen. |
| 5205 | */ |
| 5206 | const _CharT* |
| 5207 | c_str() const _GLIBCXX_NOEXCEPTnoexcept |
| 5208 | { return _M_data(); } |
| 5209 | |
| 5210 | /** |
| 5211 | * @brief Return const pointer to contents. |
| 5212 | * |
| 5213 | * This is a pointer to internal data. It is undefined to modify |
| 5214 | * the contents through the returned pointer. To get a pointer that |
| 5215 | * allows modifying the contents use @c &str[0] instead, |
| 5216 | * (or in C++17 the non-const @c str.data() overload). |
| 5217 | */ |
| 5218 | const _CharT* |
| 5219 | data() const _GLIBCXX_NOEXCEPTnoexcept |
| 5220 | { return _M_data(); } |
| 5221 | |
| 5222 | #if __cplusplus201703L >= 201703L |
| 5223 | /** |
| 5224 | * @brief Return non-const pointer to contents. |
| 5225 | * |
| 5226 | * This is a pointer to the character sequence held by the string. |
| 5227 | * Modifying the characters in the sequence is allowed. |
| 5228 | */ |
| 5229 | _CharT* |
| 5230 | data() noexcept |
| 5231 | { |
| 5232 | _M_leak(); |
| 5233 | return _M_data(); |
| 5234 | } |
| 5235 | #endif |
| 5236 | |
| 5237 | /** |
| 5238 | * @brief Return copy of allocator used to construct this string. |
| 5239 | */ |
| 5240 | allocator_type |
| 5241 | get_allocator() const _GLIBCXX_NOEXCEPTnoexcept |
| 5242 | { return _M_dataplus; } |
| 5243 | |
| 5244 | /** |
| 5245 | * @brief Find position of a C substring. |
| 5246 | * @param __s C string to locate. |
| 5247 | * @param __pos Index of character to search from. |
| 5248 | * @param __n Number of characters from @a s to search for. |
| 5249 | * @return Index of start of first occurrence. |
| 5250 | * |
| 5251 | * Starting from @a __pos, searches forward for the first @a |
| 5252 | * __n characters in @a __s within this string. If found, |
| 5253 | * returns the index where it begins. If not found, returns |
| 5254 | * npos. |
| 5255 | */ |
| 5256 | size_type |
| 5257 | find(const _CharT* __s, size_type __pos, size_type __n) const |
| 5258 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5259 | |
| 5260 | /** |
| 5261 | * @brief Find position of a string. |
| 5262 | * @param __str String to locate. |
| 5263 | * @param __pos Index of character to search from (default 0). |
| 5264 | * @return Index of start of first occurrence. |
| 5265 | * |
| 5266 | * Starting from @a __pos, searches forward for value of @a __str within |
| 5267 | * this string. If found, returns the index where it begins. If not |
| 5268 | * found, returns npos. |
| 5269 | */ |
| 5270 | size_type |
| 5271 | find(const basic_string& __str, size_type __pos = 0) const |
| 5272 | _GLIBCXX_NOEXCEPTnoexcept |
| 5273 | { return this->find(__str.data(), __pos, __str.size()); } |
| 5274 | |
| 5275 | /** |
| 5276 | * @brief Find position of a C string. |
| 5277 | * @param __s C string to locate. |
| 5278 | * @param __pos Index of character to search from (default 0). |
| 5279 | * @return Index of start of first occurrence. |
| 5280 | * |
| 5281 | * Starting from @a __pos, searches forward for the value of @a |
| 5282 | * __s within this string. If found, returns the index where |
| 5283 | * it begins. If not found, returns npos. |
| 5284 | */ |
| 5285 | size_type |
| 5286 | find(const _CharT* __s, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept |
| 5287 | { |
| 5288 | __glibcxx_requires_string(__s); |
| 5289 | return this->find(__s, __pos, traits_type::length(__s)); |
| 5290 | } |
| 5291 | |
| 5292 | /** |
| 5293 | * @brief Find position of a character. |
| 5294 | * @param __c Character to locate. |
| 5295 | * @param __pos Index of character to search from (default 0). |
| 5296 | * @return Index of first occurrence. |
| 5297 | * |
| 5298 | * Starting from @a __pos, searches forward for @a __c within |
| 5299 | * this string. If found, returns the index where it was |
| 5300 | * found. If not found, returns npos. |
| 5301 | */ |
| 5302 | size_type |
| 5303 | find(_CharT __c, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept; |
| 5304 | |
| 5305 | #if __cplusplus201703L >= 201703L |
| 5306 | /** |
| 5307 | * @brief Find position of a string_view. |
| 5308 | * @param __svt The object convertible to string_view to locate. |
| 5309 | * @param __pos Index of character to search from (default 0). |
| 5310 | * @return Index of start of first occurrence. |
| 5311 | */ |
| 5312 | template<typename _Tp> |
| 5313 | _If_sv<_Tp, size_type> |
| 5314 | find(const _Tp& __svt, size_type __pos = 0) const |
| 5315 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5316 | { |
| 5317 | __sv_type __sv = __svt; |
| 5318 | return this->find(__sv.data(), __pos, __sv.size()); |
| 5319 | } |
| 5320 | #endif // C++17 |
| 5321 | |
| 5322 | /** |
| 5323 | * @brief Find last position of a string. |
| 5324 | * @param __str String to locate. |
| 5325 | * @param __pos Index of character to search back from (default end). |
| 5326 | * @return Index of start of last occurrence. |
| 5327 | * |
| 5328 | * Starting from @a __pos, searches backward for value of @a |
| 5329 | * __str within this string. If found, returns the index where |
| 5330 | * it begins. If not found, returns npos. |
| 5331 | */ |
| 5332 | size_type |
| 5333 | rfind(const basic_string& __str, size_type __pos = npos) const |
| 5334 | _GLIBCXX_NOEXCEPTnoexcept |
| 5335 | { return this->rfind(__str.data(), __pos, __str.size()); } |
| 5336 | |
| 5337 | /** |
| 5338 | * @brief Find last position of a C substring. |
| 5339 | * @param __s C string to locate. |
| 5340 | * @param __pos Index of character to search back from. |
| 5341 | * @param __n Number of characters from s to search for. |
| 5342 | * @return Index of start of last occurrence. |
| 5343 | * |
| 5344 | * Starting from @a __pos, searches backward for the first @a |
| 5345 | * __n characters in @a __s within this string. If found, |
| 5346 | * returns the index where it begins. If not found, returns |
| 5347 | * npos. |
| 5348 | */ |
| 5349 | size_type |
| 5350 | rfind(const _CharT* __s, size_type __pos, size_type __n) const |
| 5351 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5352 | |
| 5353 | /** |
| 5354 | * @brief Find last position of a C string. |
| 5355 | * @param __s C string to locate. |
| 5356 | * @param __pos Index of character to start search at (default end). |
| 5357 | * @return Index of start of last occurrence. |
| 5358 | * |
| 5359 | * Starting from @a __pos, searches backward for the value of |
| 5360 | * @a __s within this string. If found, returns the index |
| 5361 | * where it begins. If not found, returns npos. |
| 5362 | */ |
| 5363 | size_type |
| 5364 | rfind(const _CharT* __s, size_type __pos = npos) const _GLIBCXX_NOEXCEPTnoexcept |
| 5365 | { |
| 5366 | __glibcxx_requires_string(__s); |
| 5367 | return this->rfind(__s, __pos, traits_type::length(__s)); |
| 5368 | } |
| 5369 | |
| 5370 | /** |
| 5371 | * @brief Find last position of a character. |
| 5372 | * @param __c Character to locate. |
| 5373 | * @param __pos Index of character to search back from (default end). |
| 5374 | * @return Index of last occurrence. |
| 5375 | * |
| 5376 | * Starting from @a __pos, searches backward for @a __c within |
| 5377 | * this string. If found, returns the index where it was |
| 5378 | * found. If not found, returns npos. |
| 5379 | */ |
| 5380 | size_type |
| 5381 | rfind(_CharT __c, size_type __pos = npos) const _GLIBCXX_NOEXCEPTnoexcept; |
| 5382 | |
| 5383 | #if __cplusplus201703L >= 201703L |
| 5384 | /** |
| 5385 | * @brief Find last position of a string_view. |
| 5386 | * @param __svt The object convertible to string_view to locate. |
| 5387 | * @param __pos Index of character to search back from (default end). |
| 5388 | * @return Index of start of last occurrence. |
| 5389 | */ |
| 5390 | template<typename _Tp> |
| 5391 | _If_sv<_Tp, size_type> |
| 5392 | rfind(const _Tp& __svt, size_type __pos = npos) const |
| 5393 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5394 | { |
| 5395 | __sv_type __sv = __svt; |
| 5396 | return this->rfind(__sv.data(), __pos, __sv.size()); |
| 5397 | } |
| 5398 | #endif // C++17 |
| 5399 | |
| 5400 | /** |
| 5401 | * @brief Find position of a character of string. |
| 5402 | * @param __str String containing characters to locate. |
| 5403 | * @param __pos Index of character to search from (default 0). |
| 5404 | * @return Index of first occurrence. |
| 5405 | * |
| 5406 | * Starting from @a __pos, searches forward for one of the |
| 5407 | * characters of @a __str within this string. If found, |
| 5408 | * returns the index where it was found. If not found, returns |
| 5409 | * npos. |
| 5410 | */ |
| 5411 | size_type |
| 5412 | find_first_of(const basic_string& __str, size_type __pos = 0) const |
| 5413 | _GLIBCXX_NOEXCEPTnoexcept |
| 5414 | { return this->find_first_of(__str.data(), __pos, __str.size()); } |
| 5415 | |
| 5416 | /** |
| 5417 | * @brief Find position of a character of C substring. |
| 5418 | * @param __s String containing characters to locate. |
| 5419 | * @param __pos Index of character to search from. |
| 5420 | * @param __n Number of characters from s to search for. |
| 5421 | * @return Index of first occurrence. |
| 5422 | * |
| 5423 | * Starting from @a __pos, searches forward for one of the |
| 5424 | * first @a __n characters of @a __s within this string. If |
| 5425 | * found, returns the index where it was found. If not found, |
| 5426 | * returns npos. |
| 5427 | */ |
| 5428 | size_type |
| 5429 | find_first_of(const _CharT* __s, size_type __pos, size_type __n) const |
| 5430 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5431 | |
| 5432 | /** |
| 5433 | * @brief Find position of a character of C string. |
| 5434 | * @param __s String containing characters to locate. |
| 5435 | * @param __pos Index of character to search from (default 0). |
| 5436 | * @return Index of first occurrence. |
| 5437 | * |
| 5438 | * Starting from @a __pos, searches forward for one of the |
| 5439 | * characters of @a __s within this string. If found, returns |
| 5440 | * the index where it was found. If not found, returns npos. |
| 5441 | */ |
| 5442 | size_type |
| 5443 | find_first_of(const _CharT* __s, size_type __pos = 0) const |
| 5444 | _GLIBCXX_NOEXCEPTnoexcept |
| 5445 | { |
| 5446 | __glibcxx_requires_string(__s); |
| 5447 | return this->find_first_of(__s, __pos, traits_type::length(__s)); |
| 5448 | } |
| 5449 | |
| 5450 | /** |
| 5451 | * @brief Find position of a character. |
| 5452 | * @param __c Character to locate. |
| 5453 | * @param __pos Index of character to search from (default 0). |
| 5454 | * @return Index of first occurrence. |
| 5455 | * |
| 5456 | * Starting from @a __pos, searches forward for the character |
| 5457 | * @a __c within this string. If found, returns the index |
| 5458 | * where it was found. If not found, returns npos. |
| 5459 | * |
| 5460 | * Note: equivalent to find(__c, __pos). |
| 5461 | */ |
| 5462 | size_type |
| 5463 | find_first_of(_CharT __c, size_type __pos = 0) const _GLIBCXX_NOEXCEPTnoexcept |
| 5464 | { return this->find(__c, __pos); } |
| 5465 | |
| 5466 | #if __cplusplus201703L >= 201703L |
| 5467 | /** |
| 5468 | * @brief Find position of a character of a string_view. |
| 5469 | * @param __svt An object convertible to string_view containing |
| 5470 | * characters to locate. |
| 5471 | * @param __pos Index of character to search from (default 0). |
| 5472 | * @return Index of first occurrence. |
| 5473 | */ |
| 5474 | template<typename _Tp> |
| 5475 | _If_sv<_Tp, size_type> |
| 5476 | find_first_of(const _Tp& __svt, size_type __pos = 0) const |
| 5477 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5478 | { |
| 5479 | __sv_type __sv = __svt; |
| 5480 | return this->find_first_of(__sv.data(), __pos, __sv.size()); |
| 5481 | } |
| 5482 | #endif // C++17 |
| 5483 | |
| 5484 | /** |
| 5485 | * @brief Find last position of a character of string. |
| 5486 | * @param __str String containing characters to locate. |
| 5487 | * @param __pos Index of character to search back from (default end). |
| 5488 | * @return Index of last occurrence. |
| 5489 | * |
| 5490 | * Starting from @a __pos, searches backward for one of the |
| 5491 | * characters of @a __str within this string. If found, |
| 5492 | * returns the index where it was found. If not found, returns |
| 5493 | * npos. |
| 5494 | */ |
| 5495 | size_type |
| 5496 | find_last_of(const basic_string& __str, size_type __pos = npos) const |
| 5497 | _GLIBCXX_NOEXCEPTnoexcept |
| 5498 | { return this->find_last_of(__str.data(), __pos, __str.size()); } |
| 5499 | |
| 5500 | /** |
| 5501 | * @brief Find last position of a character of C substring. |
| 5502 | * @param __s C string containing characters to locate. |
| 5503 | * @param __pos Index of character to search back from. |
| 5504 | * @param __n Number of characters from s to search for. |
| 5505 | * @return Index of last occurrence. |
| 5506 | * |
| 5507 | * Starting from @a __pos, searches backward for one of the |
| 5508 | * first @a __n characters of @a __s within this string. If |
| 5509 | * found, returns the index where it was found. If not found, |
| 5510 | * returns npos. |
| 5511 | */ |
| 5512 | size_type |
| 5513 | find_last_of(const _CharT* __s, size_type __pos, size_type __n) const |
| 5514 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5515 | |
| 5516 | /** |
| 5517 | * @brief Find last position of a character of C string. |
| 5518 | * @param __s C string containing characters to locate. |
| 5519 | * @param __pos Index of character to search back from (default end). |
| 5520 | * @return Index of last occurrence. |
| 5521 | * |
| 5522 | * Starting from @a __pos, searches backward for one of the |
| 5523 | * characters of @a __s within this string. If found, returns |
| 5524 | * the index where it was found. If not found, returns npos. |
| 5525 | */ |
| 5526 | size_type |
| 5527 | find_last_of(const _CharT* __s, size_type __pos = npos) const |
| 5528 | _GLIBCXX_NOEXCEPTnoexcept |
| 5529 | { |
| 5530 | __glibcxx_requires_string(__s); |
| 5531 | return this->find_last_of(__s, __pos, traits_type::length(__s)); |
| 5532 | } |
| 5533 | |
| 5534 | /** |
| 5535 | * @brief Find last position of a character. |
| 5536 | * @param __c Character to locate. |
| 5537 | * @param __pos Index of character to search back from (default end). |
| 5538 | * @return Index of last occurrence. |
| 5539 | * |
| 5540 | * Starting from @a __pos, searches backward for @a __c within |
| 5541 | * this string. If found, returns the index where it was |
| 5542 | * found. If not found, returns npos. |
| 5543 | * |
| 5544 | * Note: equivalent to rfind(__c, __pos). |
| 5545 | */ |
| 5546 | size_type |
| 5547 | find_last_of(_CharT __c, size_type __pos = npos) const _GLIBCXX_NOEXCEPTnoexcept |
| 5548 | { return this->rfind(__c, __pos); } |
| 5549 | |
| 5550 | #if __cplusplus201703L >= 201703L |
| 5551 | /** |
| 5552 | * @brief Find last position of a character of string. |
| 5553 | * @param __svt An object convertible to string_view containing |
| 5554 | * characters to locate. |
| 5555 | * @param __pos Index of character to search back from (default end). |
| 5556 | * @return Index of last occurrence. |
| 5557 | */ |
| 5558 | template<typename _Tp> |
| 5559 | _If_sv<_Tp, size_type> |
| 5560 | find_last_of(const _Tp& __svt, size_type __pos = npos) const |
| 5561 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5562 | { |
| 5563 | __sv_type __sv = __svt; |
| 5564 | return this->find_last_of(__sv.data(), __pos, __sv.size()); |
| 5565 | } |
| 5566 | #endif // C++17 |
| 5567 | |
| 5568 | /** |
| 5569 | * @brief Find position of a character not in string. |
| 5570 | * @param __str String containing characters to avoid. |
| 5571 | * @param __pos Index of character to search from (default 0). |
| 5572 | * @return Index of first occurrence. |
| 5573 | * |
| 5574 | * Starting from @a __pos, searches forward for a character not contained |
| 5575 | * in @a __str within this string. If found, returns the index where it |
| 5576 | * was found. If not found, returns npos. |
| 5577 | */ |
| 5578 | size_type |
| 5579 | find_first_not_of(const basic_string& __str, size_type __pos = 0) const |
| 5580 | _GLIBCXX_NOEXCEPTnoexcept |
| 5581 | { return this->find_first_not_of(__str.data(), __pos, __str.size()); } |
| 5582 | |
| 5583 | /** |
| 5584 | * @brief Find position of a character not in C substring. |
| 5585 | * @param __s C string containing characters to avoid. |
| 5586 | * @param __pos Index of character to search from. |
| 5587 | * @param __n Number of characters from __s to consider. |
| 5588 | * @return Index of first occurrence. |
| 5589 | * |
| 5590 | * Starting from @a __pos, searches forward for a character not |
| 5591 | * contained in the first @a __n characters of @a __s within |
| 5592 | * this string. If found, returns the index where it was |
| 5593 | * found. If not found, returns npos. |
| 5594 | */ |
| 5595 | size_type |
| 5596 | find_first_not_of(const _CharT* __s, size_type __pos, |
| 5597 | size_type __n) const _GLIBCXX_NOEXCEPTnoexcept; |
| 5598 | |
| 5599 | /** |
| 5600 | * @brief Find position of a character not in C string. |
| 5601 | * @param __s C string containing characters to avoid. |
| 5602 | * @param __pos Index of character to search from (default 0). |
| 5603 | * @return Index of first occurrence. |
| 5604 | * |
| 5605 | * Starting from @a __pos, searches forward for a character not |
| 5606 | * contained in @a __s within this string. If found, returns |
| 5607 | * the index where it was found. If not found, returns npos. |
| 5608 | */ |
| 5609 | size_type |
| 5610 | find_first_not_of(const _CharT* __s, size_type __pos = 0) const |
| 5611 | _GLIBCXX_NOEXCEPTnoexcept |
| 5612 | { |
| 5613 | __glibcxx_requires_string(__s); |
| 5614 | return this->find_first_not_of(__s, __pos, traits_type::length(__s)); |
| 5615 | } |
| 5616 | |
| 5617 | /** |
| 5618 | * @brief Find position of a different character. |
| 5619 | * @param __c Character to avoid. |
| 5620 | * @param __pos Index of character to search from (default 0). |
| 5621 | * @return Index of first occurrence. |
| 5622 | * |
| 5623 | * Starting from @a __pos, searches forward for a character |
| 5624 | * other than @a __c within this string. If found, returns the |
| 5625 | * index where it was found. If not found, returns npos. |
| 5626 | */ |
| 5627 | size_type |
| 5628 | find_first_not_of(_CharT __c, size_type __pos = 0) const |
| 5629 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5630 | |
| 5631 | #if __cplusplus201703L >= 201703L |
| 5632 | /** |
| 5633 | * @brief Find position of a character not in a string_view. |
| 5634 | * @param __svt An object convertible to string_view containing |
| 5635 | * characters to avoid. |
| 5636 | * @param __pos Index of character to search from (default 0). |
| 5637 | * @return Index of first occurrence. |
| 5638 | */ |
| 5639 | template<typename _Tp> |
| 5640 | _If_sv<_Tp, size_type> |
| 5641 | find_first_not_of(const _Tp& __svt, size_type __pos = 0) const |
| 5642 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5643 | { |
| 5644 | __sv_type __sv = __svt; |
| 5645 | return this->find_first_not_of(__sv.data(), __pos, __sv.size()); |
| 5646 | } |
| 5647 | #endif // C++17 |
| 5648 | |
| 5649 | /** |
| 5650 | * @brief Find last position of a character not in string. |
| 5651 | * @param __str String containing characters to avoid. |
| 5652 | * @param __pos Index of character to search back from (default end). |
| 5653 | * @return Index of last occurrence. |
| 5654 | * |
| 5655 | * Starting from @a __pos, searches backward for a character |
| 5656 | * not contained in @a __str within this string. If found, |
| 5657 | * returns the index where it was found. If not found, returns |
| 5658 | * npos. |
| 5659 | */ |
| 5660 | size_type |
| 5661 | find_last_not_of(const basic_string& __str, size_type __pos = npos) const |
| 5662 | _GLIBCXX_NOEXCEPTnoexcept |
| 5663 | { return this->find_last_not_of(__str.data(), __pos, __str.size()); } |
| 5664 | |
| 5665 | /** |
| 5666 | * @brief Find last position of a character not in C substring. |
| 5667 | * @param __s C string containing characters to avoid. |
| 5668 | * @param __pos Index of character to search back from. |
| 5669 | * @param __n Number of characters from s to consider. |
| 5670 | * @return Index of last occurrence. |
| 5671 | * |
| 5672 | * Starting from @a __pos, searches backward for a character not |
| 5673 | * contained in the first @a __n characters of @a __s within this string. |
| 5674 | * If found, returns the index where it was found. If not found, |
| 5675 | * returns npos. |
| 5676 | */ |
| 5677 | size_type |
| 5678 | find_last_not_of(const _CharT* __s, size_type __pos, |
| 5679 | size_type __n) const _GLIBCXX_NOEXCEPTnoexcept; |
| 5680 | /** |
| 5681 | * @brief Find last position of a character not in C string. |
| 5682 | * @param __s C string containing characters to avoid. |
| 5683 | * @param __pos Index of character to search back from (default end). |
| 5684 | * @return Index of last occurrence. |
| 5685 | * |
| 5686 | * Starting from @a __pos, searches backward for a character |
| 5687 | * not contained in @a __s within this string. If found, |
| 5688 | * returns the index where it was found. If not found, returns |
| 5689 | * npos. |
| 5690 | */ |
| 5691 | size_type |
| 5692 | find_last_not_of(const _CharT* __s, size_type __pos = npos) const |
| 5693 | _GLIBCXX_NOEXCEPTnoexcept |
| 5694 | { |
| 5695 | __glibcxx_requires_string(__s); |
| 5696 | return this->find_last_not_of(__s, __pos, traits_type::length(__s)); |
| 5697 | } |
| 5698 | |
| 5699 | /** |
| 5700 | * @brief Find last position of a different character. |
| 5701 | * @param __c Character to avoid. |
| 5702 | * @param __pos Index of character to search back from (default end). |
| 5703 | * @return Index of last occurrence. |
| 5704 | * |
| 5705 | * Starting from @a __pos, searches backward for a character other than |
| 5706 | * @a __c within this string. If found, returns the index where it was |
| 5707 | * found. If not found, returns npos. |
| 5708 | */ |
| 5709 | size_type |
| 5710 | find_last_not_of(_CharT __c, size_type __pos = npos) const |
| 5711 | _GLIBCXX_NOEXCEPTnoexcept; |
| 5712 | |
| 5713 | #if __cplusplus201703L >= 201703L |
| 5714 | /** |
| 5715 | * @brief Find last position of a character not in a string_view. |
| 5716 | * @param __svt An object convertible to string_view containing |
| 5717 | * characters to avoid. |
| 5718 | * @param __pos Index of character to search back from (default end). |
| 5719 | * @return Index of last occurrence. |
| 5720 | */ |
| 5721 | template<typename _Tp> |
| 5722 | _If_sv<_Tp, size_type> |
| 5723 | find_last_not_of(const _Tp& __svt, size_type __pos = npos) const |
| 5724 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5725 | { |
| 5726 | __sv_type __sv = __svt; |
| 5727 | return this->find_last_not_of(__sv.data(), __pos, __sv.size()); |
| 5728 | } |
| 5729 | #endif // C++17 |
| 5730 | |
| 5731 | /** |
| 5732 | * @brief Get a substring. |
| 5733 | * @param __pos Index of first character (default 0). |
| 5734 | * @param __n Number of characters in substring (default remainder). |
| 5735 | * @return The new string. |
| 5736 | * @throw std::out_of_range If __pos > size(). |
| 5737 | * |
| 5738 | * Construct and return a new string using the @a __n |
| 5739 | * characters starting at @a __pos. If the string is too |
| 5740 | * short, use the remainder of the characters. If @a __pos is |
| 5741 | * beyond the end of the string, out_of_range is thrown. |
| 5742 | */ |
| 5743 | basic_string |
| 5744 | substr(size_type __pos = 0, size_type __n = npos) const |
| 5745 | { return basic_string(*this, |
| 5746 | _M_check(__pos, "basic_string::substr"), __n); } |
| 5747 | |
| 5748 | /** |
| 5749 | * @brief Compare to a string. |
| 5750 | * @param __str String to compare against. |
| 5751 | * @return Integer < 0, 0, or > 0. |
| 5752 | * |
| 5753 | * Returns an integer < 0 if this string is ordered before @a |
| 5754 | * __str, 0 if their values are equivalent, or > 0 if this |
| 5755 | * string is ordered after @a __str. Determines the effective |
| 5756 | * length rlen of the strings to compare as the smallest of |
| 5757 | * size() and str.size(). The function then compares the two |
| 5758 | * strings by calling traits::compare(data(), str.data(),rlen). |
| 5759 | * If the result of the comparison is nonzero returns it, |
| 5760 | * otherwise the shorter one is ordered first. |
| 5761 | */ |
| 5762 | int |
| 5763 | compare(const basic_string& __str) const |
| 5764 | { |
| 5765 | const size_type __size = this->size(); |
| 5766 | const size_type __osize = __str.size(); |
| 5767 | const size_type __len = std::min(__size, __osize); |
| 5768 | |
| 5769 | int __r = traits_type::compare(_M_data(), __str.data(), __len); |
| 5770 | if (!__r) |
| 5771 | __r = _S_compare(__size, __osize); |
| 5772 | return __r; |
| 5773 | } |
| 5774 | |
| 5775 | #if __cplusplus201703L >= 201703L |
| 5776 | /** |
| 5777 | * @brief Compare to a string_view. |
| 5778 | * @param __svt An object convertible to string_view to compare against. |
| 5779 | * @return Integer < 0, 0, or > 0. |
| 5780 | */ |
| 5781 | template<typename _Tp> |
| 5782 | _If_sv<_Tp, int> |
| 5783 | compare(const _Tp& __svt) const |
| 5784 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5785 | { |
| 5786 | __sv_type __sv = __svt; |
| 5787 | const size_type __size = this->size(); |
| 5788 | const size_type __osize = __sv.size(); |
| 5789 | const size_type __len = std::min(__size, __osize); |
| 5790 | |
| 5791 | int __r = traits_type::compare(_M_data(), __sv.data(), __len); |
| 5792 | if (!__r) |
| 5793 | __r = _S_compare(__size, __osize); |
| 5794 | return __r; |
| 5795 | } |
| 5796 | |
| 5797 | /** |
| 5798 | * @brief Compare to a string_view. |
| 5799 | * @param __pos A position in the string to start comparing from. |
| 5800 | * @param __n The number of characters to compare. |
| 5801 | * @param __svt An object convertible to string_view to compare |
| 5802 | * against. |
| 5803 | * @return Integer < 0, 0, or > 0. |
| 5804 | */ |
| 5805 | template<typename _Tp> |
| 5806 | _If_sv<_Tp, int> |
| 5807 | compare(size_type __pos, size_type __n, const _Tp& __svt) const |
| 5808 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5809 | { |
| 5810 | __sv_type __sv = __svt; |
| 5811 | return __sv_type(*this).substr(__pos, __n).compare(__sv); |
| 5812 | } |
| 5813 | |
| 5814 | /** |
| 5815 | * @brief Compare to a string_view. |
| 5816 | * @param __pos1 A position in the string to start comparing from. |
| 5817 | * @param __n1 The number of characters to compare. |
| 5818 | * @param __svt An object convertible to string_view to compare |
| 5819 | * against. |
| 5820 | * @param __pos2 A position in the string_view to start comparing from. |
| 5821 | * @param __n2 The number of characters to compare. |
| 5822 | * @return Integer < 0, 0, or > 0. |
| 5823 | */ |
| 5824 | template<typename _Tp> |
| 5825 | _If_sv<_Tp, int> |
| 5826 | compare(size_type __pos1, size_type __n1, const _Tp& __svt, |
| 5827 | size_type __pos2, size_type __n2 = npos) const |
| 5828 | noexcept(is_same<_Tp, __sv_type>::value) |
| 5829 | { |
| 5830 | __sv_type __sv = __svt; |
| 5831 | return __sv_type(*this) |
| 5832 | .substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2)); |
| 5833 | } |
| 5834 | #endif // C++17 |
| 5835 | |
| 5836 | /** |
| 5837 | * @brief Compare substring to a string. |
| 5838 | * @param __pos Index of first character of substring. |
| 5839 | * @param __n Number of characters in substring. |
| 5840 | * @param __str String to compare against. |
| 5841 | * @return Integer < 0, 0, or > 0. |
| 5842 | * |
| 5843 | * Form the substring of this string from the @a __n characters |
| 5844 | * starting at @a __pos. Returns an integer < 0 if the |
| 5845 | * substring is ordered before @a __str, 0 if their values are |
| 5846 | * equivalent, or > 0 if the substring is ordered after @a |
| 5847 | * __str. Determines the effective length rlen of the strings |
| 5848 | * to compare as the smallest of the length of the substring |
| 5849 | * and @a __str.size(). The function then compares the two |
| 5850 | * strings by calling |
| 5851 | * traits::compare(substring.data(),str.data(),rlen). If the |
| 5852 | * result of the comparison is nonzero returns it, otherwise |
| 5853 | * the shorter one is ordered first. |
| 5854 | */ |
| 5855 | int |
| 5856 | compare(size_type __pos, size_type __n, const basic_string& __str) const; |
| 5857 | |
| 5858 | /** |
| 5859 | * @brief Compare substring to a substring. |
| 5860 | * @param __pos1 Index of first character of substring. |
| 5861 | * @param __n1 Number of characters in substring. |
| 5862 | * @param __str String to compare against. |
| 5863 | * @param __pos2 Index of first character of substring of str. |
| 5864 | * @param __n2 Number of characters in substring of str. |
| 5865 | * @return Integer < 0, 0, or > 0. |
| 5866 | * |
| 5867 | * Form the substring of this string from the @a __n1 |
| 5868 | * characters starting at @a __pos1. Form the substring of @a |
| 5869 | * __str from the @a __n2 characters starting at @a __pos2. |
| 5870 | * Returns an integer < 0 if this substring is ordered before |
| 5871 | * the substring of @a __str, 0 if their values are equivalent, |
| 5872 | * or > 0 if this substring is ordered after the substring of |
| 5873 | * @a __str. Determines the effective length rlen of the |
| 5874 | * strings to compare as the smallest of the lengths of the |
| 5875 | * substrings. The function then compares the two strings by |
| 5876 | * calling |
| 5877 | * traits::compare(substring.data(),str.substr(pos2,n2).data(),rlen). |
| 5878 | * If the result of the comparison is nonzero returns it, |
| 5879 | * otherwise the shorter one is ordered first. |
| 5880 | */ |
| 5881 | int |
| 5882 | compare(size_type __pos1, size_type __n1, const basic_string& __str, |
| 5883 | size_type __pos2, size_type __n2 = npos) const; |
| 5884 | |
| 5885 | /** |
| 5886 | * @brief Compare to a C string. |
| 5887 | * @param __s C string to compare against. |
| 5888 | * @return Integer < 0, 0, or > 0. |
| 5889 | * |
| 5890 | * Returns an integer < 0 if this string is ordered before @a __s, 0 if |
| 5891 | * their values are equivalent, or > 0 if this string is ordered after |
| 5892 | * @a __s. Determines the effective length rlen of the strings to |
| 5893 | * compare as the smallest of size() and the length of a string |
| 5894 | * constructed from @a __s. The function then compares the two strings |
| 5895 | * by calling traits::compare(data(),s,rlen). If the result of the |
| 5896 | * comparison is nonzero returns it, otherwise the shorter one is |
| 5897 | * ordered first. |
| 5898 | */ |
| 5899 | int |
| 5900 | compare(const _CharT* __s) const _GLIBCXX_NOEXCEPTnoexcept; |
| 5901 | |
| 5902 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 5903 | // 5 String::compare specification questionable |
| 5904 | /** |
| 5905 | * @brief Compare substring to a C string. |
| 5906 | * @param __pos Index of first character of substring. |
| 5907 | * @param __n1 Number of characters in substring. |
| 5908 | * @param __s C string to compare against. |
| 5909 | * @return Integer < 0, 0, or > 0. |
| 5910 | * |
| 5911 | * Form the substring of this string from the @a __n1 |
| 5912 | * characters starting at @a pos. Returns an integer < 0 if |
| 5913 | * the substring is ordered before @a __s, 0 if their values |
| 5914 | * are equivalent, or > 0 if the substring is ordered after @a |
| 5915 | * __s. Determines the effective length rlen of the strings to |
| 5916 | * compare as the smallest of the length of the substring and |
| 5917 | * the length of a string constructed from @a __s. The |
| 5918 | * function then compares the two string by calling |
| 5919 | * traits::compare(substring.data(),__s,rlen). If the result of |
| 5920 | * the comparison is nonzero returns it, otherwise the shorter |
| 5921 | * one is ordered first. |
| 5922 | */ |
| 5923 | int |
| 5924 | compare(size_type __pos, size_type __n1, const _CharT* __s) const; |
| 5925 | |
| 5926 | /** |
| 5927 | * @brief Compare substring against a character %array. |
| 5928 | * @param __pos Index of first character of substring. |
| 5929 | * @param __n1 Number of characters in substring. |
| 5930 | * @param __s character %array to compare against. |
| 5931 | * @param __n2 Number of characters of s. |
| 5932 | * @return Integer < 0, 0, or > 0. |
| 5933 | * |
| 5934 | * Form the substring of this string from the @a __n1 |
| 5935 | * characters starting at @a __pos. Form a string from the |
| 5936 | * first @a __n2 characters of @a __s. Returns an integer < 0 |
| 5937 | * if this substring is ordered before the string from @a __s, |
| 5938 | * 0 if their values are equivalent, or > 0 if this substring |
| 5939 | * is ordered after the string from @a __s. Determines the |
| 5940 | * effective length rlen of the strings to compare as the |
| 5941 | * smallest of the length of the substring and @a __n2. The |
| 5942 | * function then compares the two strings by calling |
| 5943 | * traits::compare(substring.data(),s,rlen). If the result of |
| 5944 | * the comparison is nonzero returns it, otherwise the shorter |
| 5945 | * one is ordered first. |
| 5946 | * |
| 5947 | * NB: s must have at least n2 characters, '\\0' has |
| 5948 | * no special meaning. |
| 5949 | */ |
| 5950 | int |
| 5951 | compare(size_type __pos, size_type __n1, const _CharT* __s, |
| 5952 | size_type __n2) const; |
| 5953 | |
| 5954 | #if __cplusplus201703L > 201703L |
| 5955 | bool |
| 5956 | starts_with(basic_string_view<_CharT, _Traits> __x) const noexcept |
| 5957 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 5958 | |
| 5959 | bool |
| 5960 | starts_with(_CharT __x) const noexcept |
| 5961 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 5962 | |
| 5963 | bool |
| 5964 | starts_with(const _CharT* __x) const noexcept |
| 5965 | { return __sv_type(this->data(), this->size()).starts_with(__x); } |
| 5966 | |
| 5967 | bool |
| 5968 | ends_with(basic_string_view<_CharT, _Traits> __x) const noexcept |
| 5969 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 5970 | |
| 5971 | bool |
| 5972 | ends_with(_CharT __x) const noexcept |
| 5973 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 5974 | |
| 5975 | bool |
| 5976 | ends_with(const _CharT* __x) const noexcept |
| 5977 | { return __sv_type(this->data(), this->size()).ends_with(__x); } |
| 5978 | #endif // C++20 |
| 5979 | |
| 5980 | # ifdef _GLIBCXX_TM_TS_INTERNAL |
| 5981 | friend void |
| 5982 | ::_txnal_cow_string_C1_for_exceptions(void* that, const char* s, |
| 5983 | void* exc); |
| 5984 | friend const char* |
| 5985 | ::_txnal_cow_string_c_str(const void *that); |
| 5986 | friend void |
| 5987 | ::_txnal_cow_string_D1(void *that); |
| 5988 | friend void |
| 5989 | ::_txnal_cow_string_D1_commit(void *that); |
| 5990 | # endif |
| 5991 | }; |
| 5992 | #endif // !_GLIBCXX_USE_CXX11_ABI |
| 5993 | |
| 5994 | #if __cpp_deduction_guides201703L >= 201606 |
| 5995 | _GLIBCXX_BEGIN_NAMESPACE_CXX11namespace __cxx11 { |
| 5996 | template<typename _InputIterator, typename _CharT |
| 5997 | = typename iterator_traits<_InputIterator>::value_type, |
| 5998 | typename _Allocator = allocator<_CharT>, |
| 5999 | typename = _RequireInputIter<_InputIterator>, |
| 6000 | typename = _RequireAllocator<_Allocator>> |
| 6001 | basic_string(_InputIterator, _InputIterator, _Allocator = _Allocator()) |
| 6002 | -> basic_string<_CharT, char_traits<_CharT>, _Allocator>; |
| 6003 | |
| 6004 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 6005 | // 3075. basic_string needs deduction guides from basic_string_view |
| 6006 | template<typename _CharT, typename _Traits, |
| 6007 | typename _Allocator = allocator<_CharT>, |
| 6008 | typename = _RequireAllocator<_Allocator>> |
| 6009 | basic_string(basic_string_view<_CharT, _Traits>, const _Allocator& = _Allocator()) |
| 6010 | -> basic_string<_CharT, _Traits, _Allocator>; |
| 6011 | |
| 6012 | template<typename _CharT, typename _Traits, |
| 6013 | typename _Allocator = allocator<_CharT>, |
| 6014 | typename = _RequireAllocator<_Allocator>> |
| 6015 | basic_string(basic_string_view<_CharT, _Traits>, |
| 6016 | typename basic_string<_CharT, _Traits, _Allocator>::size_type, |
| 6017 | typename basic_string<_CharT, _Traits, _Allocator>::size_type, |
| 6018 | const _Allocator& = _Allocator()) |
| 6019 | -> basic_string<_CharT, _Traits, _Allocator>; |
| 6020 | _GLIBCXX_END_NAMESPACE_CXX11} |
| 6021 | #endif |
| 6022 | |
| 6023 | // operator+ |
| 6024 | /** |
| 6025 | * @brief Concatenate two strings. |
| 6026 | * @param __lhs First string. |
| 6027 | * @param __rhs Last string. |
| 6028 | * @return New string with value of @a __lhs followed by @a __rhs. |
| 6029 | */ |
| 6030 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6031 | basic_string<_CharT, _Traits, _Alloc> |
| 6032 | operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6033 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6034 | { |
| 6035 | basic_string<_CharT, _Traits, _Alloc> __str(__lhs); |
| 6036 | __str.append(__rhs); |
| 6037 | return __str; |
| 6038 | } |
| 6039 | |
| 6040 | /** |
| 6041 | * @brief Concatenate C string and string. |
| 6042 | * @param __lhs First string. |
| 6043 | * @param __rhs Last string. |
| 6044 | * @return New string with value of @a __lhs followed by @a __rhs. |
| 6045 | */ |
| 6046 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6047 | basic_string<_CharT,_Traits,_Alloc> |
| 6048 | operator+(const _CharT* __lhs, |
| 6049 | const basic_string<_CharT,_Traits,_Alloc>& __rhs); |
| 6050 | |
| 6051 | /** |
| 6052 | * @brief Concatenate character and string. |
| 6053 | * @param __lhs First string. |
| 6054 | * @param __rhs Last string. |
| 6055 | * @return New string with @a __lhs followed by @a __rhs. |
| 6056 | */ |
| 6057 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6058 | basic_string<_CharT,_Traits,_Alloc> |
| 6059 | operator+(_CharT __lhs, const basic_string<_CharT,_Traits,_Alloc>& __rhs); |
| 6060 | |
| 6061 | /** |
| 6062 | * @brief Concatenate string and C string. |
| 6063 | * @param __lhs First string. |
| 6064 | * @param __rhs Last string. |
| 6065 | * @return New string with @a __lhs followed by @a __rhs. |
| 6066 | */ |
| 6067 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6068 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6069 | operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6070 | const _CharT* __rhs) |
| 6071 | { |
| 6072 | basic_string<_CharT, _Traits, _Alloc> __str(__lhs); |
| 6073 | __str.append(__rhs); |
| 6074 | return __str; |
| 6075 | } |
| 6076 | |
| 6077 | /** |
| 6078 | * @brief Concatenate string and character. |
| 6079 | * @param __lhs First string. |
| 6080 | * @param __rhs Last string. |
| 6081 | * @return New string with @a __lhs followed by @a __rhs. |
| 6082 | */ |
| 6083 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6084 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6085 | operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs, _CharT __rhs) |
| 6086 | { |
| 6087 | typedef basic_string<_CharT, _Traits, _Alloc> __string_type; |
| 6088 | typedef typename __string_type::size_type __size_type; |
| 6089 | __string_type __str(__lhs); |
| 6090 | __str.append(__size_type(1), __rhs); |
| 6091 | return __str; |
| 6092 | } |
| 6093 | |
| 6094 | #if __cplusplus201703L >= 201103L |
| 6095 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6096 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6097 | operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs, |
| 6098 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6099 | { return std::move(__lhs.append(__rhs)); } |
| 6100 | |
| 6101 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6102 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6103 | operator+(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6104 | basic_string<_CharT, _Traits, _Alloc>&& __rhs) |
| 6105 | { return std::move(__rhs.insert(0, __lhs)); } |
| 6106 | |
| 6107 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6108 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6109 | operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs, |
| 6110 | basic_string<_CharT, _Traits, _Alloc>&& __rhs) |
| 6111 | { |
| 6112 | #if _GLIBCXX_USE_CXX11_ABI1 |
| 6113 | using _Alloc_traits = allocator_traits<_Alloc>; |
| 6114 | bool __use_rhs = false; |
| 6115 | if _GLIBCXX17_CONSTEXPRconstexpr (typename _Alloc_traits::is_always_equal{}) |
| 6116 | __use_rhs = true; |
| 6117 | else if (__lhs.get_allocator() == __rhs.get_allocator()) |
| 6118 | __use_rhs = true; |
| 6119 | if (__use_rhs) |
| 6120 | #endif |
| 6121 | { |
| 6122 | const auto __size = __lhs.size() + __rhs.size(); |
| 6123 | if (__size > __lhs.capacity() && __size <= __rhs.capacity()) |
| 6124 | return std::move(__rhs.insert(0, __lhs)); |
| 6125 | } |
| 6126 | return std::move(__lhs.append(__rhs)); |
| 6127 | } |
| 6128 | |
| 6129 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6130 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6131 | operator+(const _CharT* __lhs, |
| 6132 | basic_string<_CharT, _Traits, _Alloc>&& __rhs) |
| 6133 | { return std::move(__rhs.insert(0, __lhs)); } |
| 6134 | |
| 6135 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6136 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6137 | operator+(_CharT __lhs, |
| 6138 | basic_string<_CharT, _Traits, _Alloc>&& __rhs) |
| 6139 | { return std::move(__rhs.insert(0, 1, __lhs)); } |
| 6140 | |
| 6141 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6142 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6143 | operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs, |
| 6144 | const _CharT* __rhs) |
| 6145 | { return std::move(__lhs.append(__rhs)); } |
| 6146 | |
| 6147 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6148 | inline basic_string<_CharT, _Traits, _Alloc> |
| 6149 | operator+(basic_string<_CharT, _Traits, _Alloc>&& __lhs, |
| 6150 | _CharT __rhs) |
| 6151 | { return std::move(__lhs.append(1, __rhs)); } |
| 6152 | #endif |
| 6153 | |
| 6154 | // operator == |
| 6155 | /** |
| 6156 | * @brief Test equivalence of two strings. |
| 6157 | * @param __lhs First string. |
| 6158 | * @param __rhs Second string. |
| 6159 | * @return True if @a __lhs.compare(@a __rhs) == 0. False otherwise. |
| 6160 | */ |
| 6161 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6162 | inline bool |
| 6163 | operator==(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6164 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6165 | _GLIBCXX_NOEXCEPTnoexcept |
| 6166 | { return __lhs.compare(__rhs) == 0; } |
| 6167 | |
| 6168 | template<typename _CharT> |
| 6169 | inline |
| 6170 | typename __gnu_cxx::__enable_if<__is_char<_CharT>::__value, bool>::__type |
| 6171 | operator==(const basic_string<_CharT>& __lhs, |
| 6172 | const basic_string<_CharT>& __rhs) _GLIBCXX_NOEXCEPTnoexcept |
| 6173 | { return (__lhs.size() == __rhs.size() |
| 6174 | && !std::char_traits<_CharT>::compare(__lhs.data(), __rhs.data(), |
| 6175 | __lhs.size())); } |
| 6176 | |
| 6177 | /** |
| 6178 | * @brief Test equivalence of string and C string. |
| 6179 | * @param __lhs String. |
| 6180 | * @param __rhs C string. |
| 6181 | * @return True if @a __lhs.compare(@a __rhs) == 0. False otherwise. |
| 6182 | */ |
| 6183 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6184 | inline bool |
| 6185 | operator==(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6186 | const _CharT* __rhs) |
| 6187 | { return __lhs.compare(__rhs) == 0; } |
| 6188 | |
| 6189 | #if __cpp_lib_three_way_comparison |
| 6190 | /** |
| 6191 | * @brief Three-way comparison of a string and a C string. |
| 6192 | * @param __lhs A string. |
| 6193 | * @param __rhs A null-terminated string. |
| 6194 | * @return A value indicating whether `__lhs` is less than, equal to, |
| 6195 | * greater than, or incomparable with `__rhs`. |
| 6196 | */ |
| 6197 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6198 | inline auto |
| 6199 | operator<=>(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6200 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) noexcept |
| 6201 | -> decltype(__detail::__char_traits_cmp_cat<_Traits>(0)) |
| 6202 | { return __detail::__char_traits_cmp_cat<_Traits>(__lhs.compare(__rhs)); } |
| 6203 | |
| 6204 | /** |
| 6205 | * @brief Three-way comparison of a string and a C string. |
| 6206 | * @param __lhs A string. |
| 6207 | * @param __rhs A null-terminated string. |
| 6208 | * @return A value indicating whether `__lhs` is less than, equal to, |
| 6209 | * greater than, or incomparable with `__rhs`. |
| 6210 | */ |
| 6211 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6212 | inline auto |
| 6213 | operator<=>(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6214 | const _CharT* __rhs) noexcept |
| 6215 | -> decltype(__detail::__char_traits_cmp_cat<_Traits>(0)) |
| 6216 | { return __detail::__char_traits_cmp_cat<_Traits>(__lhs.compare(__rhs)); } |
| 6217 | #else |
| 6218 | /** |
| 6219 | * @brief Test equivalence of C string and string. |
| 6220 | * @param __lhs C string. |
| 6221 | * @param __rhs String. |
| 6222 | * @return True if @a __rhs.compare(@a __lhs) == 0. False otherwise. |
| 6223 | */ |
| 6224 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6225 | inline bool |
| 6226 | operator==(const _CharT* __lhs, |
| 6227 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6228 | { return __rhs.compare(__lhs) == 0; } |
| 6229 | |
| 6230 | // operator != |
| 6231 | /** |
| 6232 | * @brief Test difference of two strings. |
| 6233 | * @param __lhs First string. |
| 6234 | * @param __rhs Second string. |
| 6235 | * @return True if @a __lhs.compare(@a __rhs) != 0. False otherwise. |
| 6236 | */ |
| 6237 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6238 | inline bool |
| 6239 | operator!=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6240 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6241 | _GLIBCXX_NOEXCEPTnoexcept |
| 6242 | { return !(__lhs == __rhs); } |
| 6243 | |
| 6244 | /** |
| 6245 | * @brief Test difference of C string and string. |
| 6246 | * @param __lhs C string. |
| 6247 | * @param __rhs String. |
| 6248 | * @return True if @a __rhs.compare(@a __lhs) != 0. False otherwise. |
| 6249 | */ |
| 6250 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6251 | inline bool |
| 6252 | operator!=(const _CharT* __lhs, |
| 6253 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6254 | { return !(__lhs == __rhs); } |
| 6255 | |
| 6256 | /** |
| 6257 | * @brief Test difference of string and C string. |
| 6258 | * @param __lhs String. |
| 6259 | * @param __rhs C string. |
| 6260 | * @return True if @a __lhs.compare(@a __rhs) != 0. False otherwise. |
| 6261 | */ |
| 6262 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6263 | inline bool |
| 6264 | operator!=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6265 | const _CharT* __rhs) |
| 6266 | { return !(__lhs == __rhs); } |
| 6267 | |
| 6268 | // operator < |
| 6269 | /** |
| 6270 | * @brief Test if string precedes string. |
| 6271 | * @param __lhs First string. |
| 6272 | * @param __rhs Second string. |
| 6273 | * @return True if @a __lhs precedes @a __rhs. False otherwise. |
| 6274 | */ |
| 6275 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6276 | inline bool |
| 6277 | operator<(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6278 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6279 | _GLIBCXX_NOEXCEPTnoexcept |
| 6280 | { return __lhs.compare(__rhs) < 0; } |
| 6281 | |
| 6282 | /** |
| 6283 | * @brief Test if string precedes C string. |
| 6284 | * @param __lhs String. |
| 6285 | * @param __rhs C string. |
| 6286 | * @return True if @a __lhs precedes @a __rhs. False otherwise. |
| 6287 | */ |
| 6288 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6289 | inline bool |
| 6290 | operator<(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6291 | const _CharT* __rhs) |
| 6292 | { return __lhs.compare(__rhs) < 0; } |
| 6293 | |
| 6294 | /** |
| 6295 | * @brief Test if C string precedes string. |
| 6296 | * @param __lhs C string. |
| 6297 | * @param __rhs String. |
| 6298 | * @return True if @a __lhs precedes @a __rhs. False otherwise. |
| 6299 | */ |
| 6300 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6301 | inline bool |
| 6302 | operator<(const _CharT* __lhs, |
| 6303 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6304 | { return __rhs.compare(__lhs) > 0; } |
| 6305 | |
| 6306 | // operator > |
| 6307 | /** |
| 6308 | * @brief Test if string follows string. |
| 6309 | * @param __lhs First string. |
| 6310 | * @param __rhs Second string. |
| 6311 | * @return True if @a __lhs follows @a __rhs. False otherwise. |
| 6312 | */ |
| 6313 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6314 | inline bool |
| 6315 | operator>(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6316 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6317 | _GLIBCXX_NOEXCEPTnoexcept |
| 6318 | { return __lhs.compare(__rhs) > 0; } |
| 6319 | |
| 6320 | /** |
| 6321 | * @brief Test if string follows C string. |
| 6322 | * @param __lhs String. |
| 6323 | * @param __rhs C string. |
| 6324 | * @return True if @a __lhs follows @a __rhs. False otherwise. |
| 6325 | */ |
| 6326 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6327 | inline bool |
| 6328 | operator>(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6329 | const _CharT* __rhs) |
| 6330 | { return __lhs.compare(__rhs) > 0; } |
| 6331 | |
| 6332 | /** |
| 6333 | * @brief Test if C string follows string. |
| 6334 | * @param __lhs C string. |
| 6335 | * @param __rhs String. |
| 6336 | * @return True if @a __lhs follows @a __rhs. False otherwise. |
| 6337 | */ |
| 6338 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6339 | inline bool |
| 6340 | operator>(const _CharT* __lhs, |
| 6341 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6342 | { return __rhs.compare(__lhs) < 0; } |
| 6343 | |
| 6344 | // operator <= |
| 6345 | /** |
| 6346 | * @brief Test if string doesn't follow string. |
| 6347 | * @param __lhs First string. |
| 6348 | * @param __rhs Second string. |
| 6349 | * @return True if @a __lhs doesn't follow @a __rhs. False otherwise. |
| 6350 | */ |
| 6351 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6352 | inline bool |
| 6353 | operator<=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6354 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6355 | _GLIBCXX_NOEXCEPTnoexcept |
| 6356 | { return __lhs.compare(__rhs) <= 0; } |
| 6357 | |
| 6358 | /** |
| 6359 | * @brief Test if string doesn't follow C string. |
| 6360 | * @param __lhs String. |
| 6361 | * @param __rhs C string. |
| 6362 | * @return True if @a __lhs doesn't follow @a __rhs. False otherwise. |
| 6363 | */ |
| 6364 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6365 | inline bool |
| 6366 | operator<=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6367 | const _CharT* __rhs) |
| 6368 | { return __lhs.compare(__rhs) <= 0; } |
| 6369 | |
| 6370 | /** |
| 6371 | * @brief Test if C string doesn't follow string. |
| 6372 | * @param __lhs C string. |
| 6373 | * @param __rhs String. |
| 6374 | * @return True if @a __lhs doesn't follow @a __rhs. False otherwise. |
| 6375 | */ |
| 6376 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6377 | inline bool |
| 6378 | operator<=(const _CharT* __lhs, |
| 6379 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6380 | { return __rhs.compare(__lhs) >= 0; } |
| 6381 | |
| 6382 | // operator >= |
| 6383 | /** |
| 6384 | * @brief Test if string doesn't precede string. |
| 6385 | * @param __lhs First string. |
| 6386 | * @param __rhs Second string. |
| 6387 | * @return True if @a __lhs doesn't precede @a __rhs. False otherwise. |
| 6388 | */ |
| 6389 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6390 | inline bool |
| 6391 | operator>=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6392 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6393 | _GLIBCXX_NOEXCEPTnoexcept |
| 6394 | { return __lhs.compare(__rhs) >= 0; } |
| 6395 | |
| 6396 | /** |
| 6397 | * @brief Test if string doesn't precede C string. |
| 6398 | * @param __lhs String. |
| 6399 | * @param __rhs C string. |
| 6400 | * @return True if @a __lhs doesn't precede @a __rhs. False otherwise. |
| 6401 | */ |
| 6402 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6403 | inline bool |
| 6404 | operator>=(const basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6405 | const _CharT* __rhs) |
| 6406 | { return __lhs.compare(__rhs) >= 0; } |
| 6407 | |
| 6408 | /** |
| 6409 | * @brief Test if C string doesn't precede string. |
| 6410 | * @param __lhs C string. |
| 6411 | * @param __rhs String. |
| 6412 | * @return True if @a __lhs doesn't precede @a __rhs. False otherwise. |
| 6413 | */ |
| 6414 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6415 | inline bool |
| 6416 | operator>=(const _CharT* __lhs, |
| 6417 | const basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6418 | { return __rhs.compare(__lhs) <= 0; } |
| 6419 | #endif // three-way comparison |
| 6420 | |
| 6421 | /** |
| 6422 | * @brief Swap contents of two strings. |
| 6423 | * @param __lhs First string. |
| 6424 | * @param __rhs Second string. |
| 6425 | * |
| 6426 | * Exchanges the contents of @a __lhs and @a __rhs in constant time. |
| 6427 | */ |
| 6428 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6429 | inline void |
| 6430 | swap(basic_string<_CharT, _Traits, _Alloc>& __lhs, |
| 6431 | basic_string<_CharT, _Traits, _Alloc>& __rhs) |
| 6432 | _GLIBCXX_NOEXCEPT_IF(noexcept(__lhs.swap(__rhs)))noexcept(noexcept(__lhs.swap(__rhs))) |
| 6433 | { __lhs.swap(__rhs); } |
| 6434 | |
| 6435 | |
| 6436 | /** |
| 6437 | * @brief Read stream into a string. |
| 6438 | * @param __is Input stream. |
| 6439 | * @param __str Buffer to store into. |
| 6440 | * @return Reference to the input stream. |
| 6441 | * |
| 6442 | * Stores characters from @a __is into @a __str until whitespace is |
| 6443 | * found, the end of the stream is encountered, or str.max_size() |
| 6444 | * is reached. If is.width() is non-zero, that is the limit on the |
| 6445 | * number of characters stored into @a __str. Any previous |
| 6446 | * contents of @a __str are erased. |
| 6447 | */ |
| 6448 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6449 | basic_istream<_CharT, _Traits>& |
| 6450 | operator>>(basic_istream<_CharT, _Traits>& __is, |
| 6451 | basic_string<_CharT, _Traits, _Alloc>& __str); |
| 6452 | |
| 6453 | template<> |
| 6454 | basic_istream<char>& |
| 6455 | operator>>(basic_istream<char>& __is, basic_string<char>& __str); |
| 6456 | |
| 6457 | /** |
| 6458 | * @brief Write string to a stream. |
| 6459 | * @param __os Output stream. |
| 6460 | * @param __str String to write out. |
| 6461 | * @return Reference to the output stream. |
| 6462 | * |
| 6463 | * Output characters of @a __str into os following the same rules as for |
| 6464 | * writing a C string. |
| 6465 | */ |
| 6466 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6467 | inline basic_ostream<_CharT, _Traits>& |
| 6468 | operator<<(basic_ostream<_CharT, _Traits>& __os, |
| 6469 | const basic_string<_CharT, _Traits, _Alloc>& __str) |
| 6470 | { |
| 6471 | // _GLIBCXX_RESOLVE_LIB_DEFECTS |
| 6472 | // 586. string inserter not a formatted function |
| 6473 | return __ostream_insert(__os, __str.data(), __str.size()); |
| 6474 | } |
| 6475 | |
| 6476 | /** |
| 6477 | * @brief Read a line from stream into a string. |
| 6478 | * @param __is Input stream. |
| 6479 | * @param __str Buffer to store into. |
| 6480 | * @param __delim Character marking end of line. |
| 6481 | * @return Reference to the input stream. |
| 6482 | * |
| 6483 | * Stores characters from @a __is into @a __str until @a __delim is |
| 6484 | * found, the end of the stream is encountered, or str.max_size() |
| 6485 | * is reached. Any previous contents of @a __str are erased. If |
| 6486 | * @a __delim is encountered, it is extracted but not stored into |
| 6487 | * @a __str. |
| 6488 | */ |
| 6489 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6490 | basic_istream<_CharT, _Traits>& |
| 6491 | getline(basic_istream<_CharT, _Traits>& __is, |
| 6492 | basic_string<_CharT, _Traits, _Alloc>& __str, _CharT __delim); |
| 6493 | |
| 6494 | /** |
| 6495 | * @brief Read a line from stream into a string. |
| 6496 | * @param __is Input stream. |
| 6497 | * @param __str Buffer to store into. |
| 6498 | * @return Reference to the input stream. |
| 6499 | * |
| 6500 | * Stores characters from is into @a __str until '\n' is |
| 6501 | * found, the end of the stream is encountered, or str.max_size() |
| 6502 | * is reached. Any previous contents of @a __str are erased. If |
| 6503 | * end of line is encountered, it is extracted but not stored into |
| 6504 | * @a __str. |
| 6505 | */ |
| 6506 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6507 | inline basic_istream<_CharT, _Traits>& |
| 6508 | getline(basic_istream<_CharT, _Traits>& __is, |
| 6509 | basic_string<_CharT, _Traits, _Alloc>& __str) |
| 6510 | { return std::getline(__is, __str, __is.widen('\n')); } |
| 6511 | |
| 6512 | #if __cplusplus201703L >= 201103L |
| 6513 | /// Read a line from an rvalue stream into a string. |
| 6514 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6515 | inline basic_istream<_CharT, _Traits>& |
| 6516 | getline(basic_istream<_CharT, _Traits>&& __is, |
| 6517 | basic_string<_CharT, _Traits, _Alloc>& __str, _CharT __delim) |
| 6518 | { return std::getline(__is, __str, __delim); } |
| 6519 | |
| 6520 | /// Read a line from an rvalue stream into a string. |
| 6521 | template<typename _CharT, typename _Traits, typename _Alloc> |
| 6522 | inline basic_istream<_CharT, _Traits>& |
| 6523 | getline(basic_istream<_CharT, _Traits>&& __is, |
| 6524 | basic_string<_CharT, _Traits, _Alloc>& __str) |
| 6525 | { return std::getline(__is, __str); } |
| 6526 | #endif |
| 6527 | |
| 6528 | template<> |
| 6529 | basic_istream<char>& |
| 6530 | getline(basic_istream<char>& __in, basic_string<char>& __str, |
| 6531 | char __delim); |
| 6532 | |
| 6533 | #ifdef _GLIBCXX_USE_WCHAR_T1 |
| 6534 | template<> |
| 6535 | basic_istream<wchar_t>& |
| 6536 | getline(basic_istream<wchar_t>& __in, basic_string<wchar_t>& __str, |
| 6537 | wchar_t __delim); |
| 6538 | #endif |
| 6539 | |
| 6540 | _GLIBCXX_END_NAMESPACE_VERSION |
| 6541 | } // namespace |
| 6542 | |
| 6543 | #if __cplusplus201703L >= 201103L |
| 6544 | |
| 6545 | #include <ext/string_conversions.h> |
| 6546 | #include <bits/charconv.h> |
| 6547 | |
| 6548 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 6549 | { |
| 6550 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 6551 | _GLIBCXX_BEGIN_NAMESPACE_CXX11namespace __cxx11 { |
| 6552 | |
| 6553 | #if _GLIBCXX_USE_C99_STDLIB1 |
| 6554 | // 21.4 Numeric Conversions [string.conversions]. |
| 6555 | inline int |
| 6556 | stoi(const string& __str, size_t* __idx = 0, int __base = 10) |
| 6557 | { return __gnu_cxx::__stoa<long, int>(&std::strtol, "stoi", __str.c_str(), |
| 6558 | __idx, __base); } |
| 6559 | |
| 6560 | inline long |
| 6561 | stol(const string& __str, size_t* __idx = 0, int __base = 10) |
| 6562 | { return __gnu_cxx::__stoa(&std::strtol, "stol", __str.c_str(), |
| 6563 | __idx, __base); } |
| 6564 | |
| 6565 | inline unsigned long |
| 6566 | stoul(const string& __str, size_t* __idx = 0, int __base = 10) |
| 6567 | { return __gnu_cxx::__stoa(&std::strtoul, "stoul", __str.c_str(), |
| 6568 | __idx, __base); } |
| 6569 | |
| 6570 | inline long long |
| 6571 | stoll(const string& __str, size_t* __idx = 0, int __base = 10) |
| 6572 | { return __gnu_cxx::__stoa(&std::strtoll, "stoll", __str.c_str(), |
| 6573 | __idx, __base); } |
| 6574 | |
| 6575 | inline unsigned long long |
| 6576 | stoull(const string& __str, size_t* __idx = 0, int __base = 10) |
| 6577 | { return __gnu_cxx::__stoa(&std::strtoull, "stoull", __str.c_str(), |
| 6578 | __idx, __base); } |
| 6579 | |
| 6580 | // NB: strtof vs strtod. |
| 6581 | inline float |
| 6582 | stof(const string& __str, size_t* __idx = 0) |
| 6583 | { return __gnu_cxx::__stoa(&std::strtof, "stof", __str.c_str(), __idx); } |
| 6584 | |
| 6585 | inline double |
| 6586 | stod(const string& __str, size_t* __idx = 0) |
| 6587 | { return __gnu_cxx::__stoa(&std::strtod, "stod", __str.c_str(), __idx); } |
| 6588 | |
| 6589 | inline long double |
| 6590 | stold(const string& __str, size_t* __idx = 0) |
| 6591 | { return __gnu_cxx::__stoa(&std::strtold, "stold", __str.c_str(), __idx); } |
| 6592 | #endif // _GLIBCXX_USE_C99_STDLIB |
| 6593 | |
| 6594 | // DR 1261. Insufficent overloads for to_string / to_wstring |
| 6595 | |
| 6596 | inline string |
| 6597 | to_string(int __val) |
| 6598 | { |
| 6599 | const bool __neg = __val < 0; |
| 6600 | const unsigned __uval = __neg ? (unsigned)~__val + 1u : __val; |
| 6601 | const auto __len = __detail::__to_chars_len(__uval); |
| 6602 | string __str(__neg + __len, '-'); |
| 6603 | __detail::__to_chars_10_impl(&__str[__neg], __len, __uval); |
| 6604 | return __str; |
| 6605 | } |
| 6606 | |
| 6607 | inline string |
| 6608 | to_string(unsigned __val) |
| 6609 | { |
| 6610 | string __str(__detail::__to_chars_len(__val), '\0'); |
| 6611 | __detail::__to_chars_10_impl(&__str[0], __str.size(), __val); |
| 6612 | return __str; |
| 6613 | } |
| 6614 | |
| 6615 | inline string |
| 6616 | to_string(long __val) |
| 6617 | { |
| 6618 | const bool __neg = __val < 0; |
| 6619 | const unsigned long __uval = __neg ? (unsigned long)~__val + 1ul : __val; |
| 6620 | const auto __len = __detail::__to_chars_len(__uval); |
| 6621 | string __str(__neg + __len, '-'); |
| 6622 | __detail::__to_chars_10_impl(&__str[__neg], __len, __uval); |
| 6623 | return __str; |
| 6624 | } |
| 6625 | |
| 6626 | inline string |
| 6627 | to_string(unsigned long __val) |
| 6628 | { |
| 6629 | string __str(__detail::__to_chars_len(__val), '\0'); |
| 6630 | __detail::__to_chars_10_impl(&__str[0], __str.size(), __val); |
| 6631 | return __str; |
| 6632 | } |
| 6633 | |
| 6634 | inline string |
| 6635 | to_string(long long __val) |
| 6636 | { |
| 6637 | const bool __neg = __val < 0; |
| 6638 | const unsigned long long __uval |
| 6639 | = __neg ? (unsigned long long)~__val + 1ull : __val; |
| 6640 | const auto __len = __detail::__to_chars_len(__uval); |
| 6641 | string __str(__neg + __len, '-'); |
| 6642 | __detail::__to_chars_10_impl(&__str[__neg], __len, __uval); |
| 6643 | return __str; |
| 6644 | } |
| 6645 | |
| 6646 | inline string |
| 6647 | to_string(unsigned long long __val) |
| 6648 | { |
| 6649 | string __str(__detail::__to_chars_len(__val), '\0'); |
| 6650 | __detail::__to_chars_10_impl(&__str[0], __str.size(), __val); |
| 6651 | return __str; |
| 6652 | } |
| 6653 | |
| 6654 | #if _GLIBCXX_USE_C99_STDIO1 |
| 6655 | // NB: (v)snprintf vs sprintf. |
| 6656 | |
| 6657 | inline string |
| 6658 | to_string(float __val) |
| 6659 | { |
| 6660 | const int __n = |
| 6661 | __gnu_cxx::__numeric_traits<float>::__max_exponent10 + 20; |
| 6662 | return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n, |
| 6663 | "%f", __val); |
| 6664 | } |
| 6665 | |
| 6666 | inline string |
| 6667 | to_string(double __val) |
| 6668 | { |
| 6669 | const int __n = |
| 6670 | __gnu_cxx::__numeric_traits<double>::__max_exponent10 + 20; |
| 6671 | return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n, |
| 6672 | "%f", __val); |
| 6673 | } |
| 6674 | |
| 6675 | inline string |
| 6676 | to_string(long double __val) |
| 6677 | { |
| 6678 | const int __n = |
| 6679 | __gnu_cxx::__numeric_traits<long double>::__max_exponent10 + 20; |
| 6680 | return __gnu_cxx::__to_xstring<string>(&std::vsnprintf, __n, |
| 6681 | "%Lf", __val); |
| 6682 | } |
| 6683 | #endif // _GLIBCXX_USE_C99_STDIO |
| 6684 | |
| 6685 | #if defined(_GLIBCXX_USE_WCHAR_T1) && _GLIBCXX_USE_C99_WCHAR1 |
| 6686 | inline int |
| 6687 | stoi(const wstring& __str, size_t* __idx = 0, int __base = 10) |
| 6688 | { return __gnu_cxx::__stoa<long, int>(&std::wcstol, "stoi", __str.c_str(), |
| 6689 | __idx, __base); } |
| 6690 | |
| 6691 | inline long |
| 6692 | stol(const wstring& __str, size_t* __idx = 0, int __base = 10) |
| 6693 | { return __gnu_cxx::__stoa(&std::wcstol, "stol", __str.c_str(), |
| 6694 | __idx, __base); } |
| 6695 | |
| 6696 | inline unsigned long |
| 6697 | stoul(const wstring& __str, size_t* __idx = 0, int __base = 10) |
| 6698 | { return __gnu_cxx::__stoa(&std::wcstoul, "stoul", __str.c_str(), |
| 6699 | __idx, __base); } |
| 6700 | |
| 6701 | inline long long |
| 6702 | stoll(const wstring& __str, size_t* __idx = 0, int __base = 10) |
| 6703 | { return __gnu_cxx::__stoa(&std::wcstoll, "stoll", __str.c_str(), |
| 6704 | __idx, __base); } |
| 6705 | |
| 6706 | inline unsigned long long |
| 6707 | stoull(const wstring& __str, size_t* __idx = 0, int __base = 10) |
| 6708 | { return __gnu_cxx::__stoa(&std::wcstoull, "stoull", __str.c_str(), |
| 6709 | __idx, __base); } |
| 6710 | |
| 6711 | // NB: wcstof vs wcstod. |
| 6712 | inline float |
| 6713 | stof(const wstring& __str, size_t* __idx = 0) |
| 6714 | { return __gnu_cxx::__stoa(&std::wcstof, "stof", __str.c_str(), __idx); } |
| 6715 | |
| 6716 | inline double |
| 6717 | stod(const wstring& __str, size_t* __idx = 0) |
| 6718 | { return __gnu_cxx::__stoa(&std::wcstod, "stod", __str.c_str(), __idx); } |
| 6719 | |
| 6720 | inline long double |
| 6721 | stold(const wstring& __str, size_t* __idx = 0) |
| 6722 | { return __gnu_cxx::__stoa(&std::wcstold, "stold", __str.c_str(), __idx); } |
| 6723 | |
| 6724 | #ifndef _GLIBCXX_HAVE_BROKEN_VSWPRINTF |
| 6725 | // DR 1261. |
| 6726 | inline wstring |
| 6727 | to_wstring(int __val) |
| 6728 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, 4 * sizeof(int), |
| 6729 | L"%d", __val); } |
| 6730 | |
| 6731 | inline wstring |
| 6732 | to_wstring(unsigned __val) |
| 6733 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, |
| 6734 | 4 * sizeof(unsigned), |
| 6735 | L"%u", __val); } |
| 6736 | |
| 6737 | inline wstring |
| 6738 | to_wstring(long __val) |
| 6739 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, 4 * sizeof(long), |
| 6740 | L"%ld", __val); } |
| 6741 | |
| 6742 | inline wstring |
| 6743 | to_wstring(unsigned long __val) |
| 6744 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, |
| 6745 | 4 * sizeof(unsigned long), |
| 6746 | L"%lu", __val); } |
| 6747 | |
| 6748 | inline wstring |
| 6749 | to_wstring(long long __val) |
| 6750 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, |
| 6751 | 4 * sizeof(long long), |
| 6752 | L"%lld", __val); } |
| 6753 | |
| 6754 | inline wstring |
| 6755 | to_wstring(unsigned long long __val) |
| 6756 | { return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, |
| 6757 | 4 * sizeof(unsigned long long), |
| 6758 | L"%llu", __val); } |
| 6759 | |
| 6760 | inline wstring |
| 6761 | to_wstring(float __val) |
| 6762 | { |
| 6763 | const int __n = |
| 6764 | __gnu_cxx::__numeric_traits<float>::__max_exponent10 + 20; |
| 6765 | return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n, |
| 6766 | L"%f", __val); |
| 6767 | } |
| 6768 | |
| 6769 | inline wstring |
| 6770 | to_wstring(double __val) |
| 6771 | { |
| 6772 | const int __n = |
| 6773 | __gnu_cxx::__numeric_traits<double>::__max_exponent10 + 20; |
| 6774 | return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n, |
| 6775 | L"%f", __val); |
| 6776 | } |
| 6777 | |
| 6778 | inline wstring |
| 6779 | to_wstring(long double __val) |
| 6780 | { |
| 6781 | const int __n = |
| 6782 | __gnu_cxx::__numeric_traits<long double>::__max_exponent10 + 20; |
| 6783 | return __gnu_cxx::__to_xstring<wstring>(&std::vswprintf, __n, |
| 6784 | L"%Lf", __val); |
| 6785 | } |
| 6786 | #endif // _GLIBCXX_HAVE_BROKEN_VSWPRINTF |
| 6787 | #endif // _GLIBCXX_USE_WCHAR_T && _GLIBCXX_USE_C99_WCHAR |
| 6788 | |
| 6789 | _GLIBCXX_END_NAMESPACE_CXX11} |
| 6790 | _GLIBCXX_END_NAMESPACE_VERSION |
| 6791 | } // namespace |
| 6792 | |
| 6793 | #endif /* C++11 */ |
| 6794 | |
| 6795 | #if __cplusplus201703L >= 201103L |
| 6796 | |
| 6797 | #include <bits/functional_hash.h> |
| 6798 | |
| 6799 | namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default"))) |
| 6800 | { |
| 6801 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
| 6802 | |
| 6803 | // DR 1182. |
| 6804 | |
| 6805 | #ifndef _GLIBCXX_COMPATIBILITY_CXX0X |
| 6806 | /// std::hash specialization for string. |
| 6807 | template<> |
| 6808 | struct hash<string> |
| 6809 | : public __hash_base<size_t, string> |
| 6810 | { |
| 6811 | size_t |
| 6812 | operator()(const string& __s) const noexcept |
| 6813 | { return std::_Hash_impl::hash(__s.data(), __s.length()); } |
| 6814 | }; |
| 6815 | |
| 6816 | template<> |
| 6817 | struct __is_fast_hash<hash<string>> : std::false_type |
| 6818 | { }; |
| 6819 | |
| 6820 | #ifdef _GLIBCXX_USE_WCHAR_T1 |
| 6821 | /// std::hash specialization for wstring. |
| 6822 | template<> |
| 6823 | struct hash<wstring> |
| 6824 | : public __hash_base<size_t, wstring> |
| 6825 | { |
| 6826 | size_t |
| 6827 | operator()(const wstring& __s) const noexcept |
| 6828 | { return std::_Hash_impl::hash(__s.data(), |
| 6829 | __s.length() * sizeof(wchar_t)); } |
| 6830 | }; |
| 6831 | |
| 6832 | template<> |
| 6833 | struct __is_fast_hash<hash<wstring>> : std::false_type |
| 6834 | { }; |
| 6835 | #endif |
| 6836 | #endif /* _GLIBCXX_COMPATIBILITY_CXX0X */ |
| 6837 | |
| 6838 | #ifdef _GLIBCXX_USE_CHAR8_T |
| 6839 | /// std::hash specialization for u8string. |
| 6840 | template<> |
| 6841 | struct hash<u8string> |
| 6842 | : public __hash_base<size_t, u8string> |
| 6843 | { |
| 6844 | size_t |
| 6845 | operator()(const u8string& __s) const noexcept |
| 6846 | { return std::_Hash_impl::hash(__s.data(), |
| 6847 | __s.length() * sizeof(char8_t)); } |
| 6848 | }; |
| 6849 | |
| 6850 | template<> |
| 6851 | struct __is_fast_hash<hash<u8string>> : std::false_type |
| 6852 | { }; |
| 6853 | #endif |
| 6854 | |
| 6855 | /// std::hash specialization for u16string. |
| 6856 | template<> |
| 6857 | struct hash<u16string> |
| 6858 | : public __hash_base<size_t, u16string> |
| 6859 | { |
| 6860 | size_t |
| 6861 | operator()(const u16string& __s) const noexcept |
| 6862 | { return std::_Hash_impl::hash(__s.data(), |
| 6863 | __s.length() * sizeof(char16_t)); } |
| 6864 | }; |
| 6865 | |
| 6866 | template<> |
| 6867 | struct __is_fast_hash<hash<u16string>> : std::false_type |
| 6868 | { }; |
| 6869 | |
| 6870 | /// std::hash specialization for u32string. |
| 6871 | template<> |
| 6872 | struct hash<u32string> |
| 6873 | : public __hash_base<size_t, u32string> |
| 6874 | { |
| 6875 | size_t |
| 6876 | operator()(const u32string& __s) const noexcept |
| 6877 | { return std::_Hash_impl::hash(__s.data(), |
| 6878 | __s.length() * sizeof(char32_t)); } |
| 6879 | }; |
| 6880 | |
| 6881 | template<> |
| 6882 | struct __is_fast_hash<hash<u32string>> : std::false_type |
| 6883 | { }; |
| 6884 | |
| 6885 | #if __cplusplus201703L >= 201402L |
| 6886 | |
| 6887 | #define __cpp_lib_string_udls201304 201304 |
| 6888 | |
| 6889 | inline namespace literals |
| 6890 | { |
| 6891 | inline namespace string_literals |
| 6892 | { |
| 6893 | #pragma GCC diagnostic push |
| 6894 | #pragma GCC diagnostic ignored "-Wliteral-suffix" |
| 6895 | _GLIBCXX_DEFAULT_ABI_TAG__attribute ((__abi_tag__ ("cxx11"))) |
| 6896 | inline basic_string<char> |
| 6897 | operator""s(const char* __str, size_t __len) |
| 6898 | { return basic_string<char>{__str, __len}; } |
| 6899 | |
| 6900 | #ifdef _GLIBCXX_USE_WCHAR_T1 |
| 6901 | _GLIBCXX_DEFAULT_ABI_TAG__attribute ((__abi_tag__ ("cxx11"))) |
| 6902 | inline basic_string<wchar_t> |
| 6903 | operator""s(const wchar_t* __str, size_t __len) |
| 6904 | { return basic_string<wchar_t>{__str, __len}; } |
| 6905 | #endif |
| 6906 | |
| 6907 | #ifdef _GLIBCXX_USE_CHAR8_T |
| 6908 | _GLIBCXX_DEFAULT_ABI_TAG__attribute ((__abi_tag__ ("cxx11"))) |
| 6909 | inline basic_string<char8_t> |
| 6910 | operator""s(const char8_t* __str, size_t __len) |
| 6911 | { return basic_string<char8_t>{__str, __len}; } |
| 6912 | #endif |
| 6913 | |
| 6914 | _GLIBCXX_DEFAULT_ABI_TAG__attribute ((__abi_tag__ ("cxx11"))) |
| 6915 | inline basic_string<char16_t> |
| 6916 | operator""s(const char16_t* __str, size_t __len) |
| 6917 | { return basic_string<char16_t>{__str, __len}; } |
| 6918 | |
| 6919 | _GLIBCXX_DEFAULT_ABI_TAG__attribute ((__abi_tag__ ("cxx11"))) |
| 6920 | inline basic_string<char32_t> |
| 6921 | operator""s(const char32_t* __str, size_t __len) |
| 6922 | { return basic_string<char32_t>{__str, __len}; } |
| 6923 | |
| 6924 | #pragma GCC diagnostic pop |
| 6925 | } // inline namespace string_literals |
| 6926 | } // inline namespace literals |
| 6927 | |
| 6928 | #if __cplusplus201703L >= 201703L |
| 6929 | namespace __detail::__variant |
| 6930 | { |
| 6931 | template<typename> struct _Never_valueless_alt; // see <variant> |
| 6932 | |
| 6933 | // Provide the strong exception-safety guarantee when emplacing a |
| 6934 | // basic_string into a variant, but only if moving the string cannot throw. |
| 6935 | template<typename _Tp, typename _Traits, typename _Alloc> |
| 6936 | struct _Never_valueless_alt<std::basic_string<_Tp, _Traits, _Alloc>> |
| 6937 | : __and_< |
| 6938 | is_nothrow_move_constructible<std::basic_string<_Tp, _Traits, _Alloc>>, |
| 6939 | is_nothrow_move_assignable<std::basic_string<_Tp, _Traits, _Alloc>> |
| 6940 | >::type |
| 6941 | { }; |
| 6942 | } // namespace __detail::__variant |
| 6943 | #endif // C++17 |
| 6944 | #endif // C++14 |
| 6945 | |
| 6946 | _GLIBCXX_END_NAMESPACE_VERSION |
| 6947 | } // namespace std |
| 6948 | |
| 6949 | #endif // C++11 |
| 6950 | |
| 6951 | #endif /* _BASIC_STRING_H */ |
| 1 | //===- llvm/Transforms/Utils/LoopUtils.h - Loop utilities -------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines some loop transformation utilities. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H |
| 14 | #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H |
| 15 | |
| 16 | #include "llvm/Analysis/IVDescriptors.h" |
| 17 | #include "llvm/Analysis/LoopAccessAnalysis.h" |
| 18 | #include "llvm/Transforms/Utils/ValueMapper.h" |
| 19 | |
| 20 | namespace llvm { |
| 21 | |
| 22 | template <typename T> class DomTreeNodeBase; |
| 23 | using DomTreeNode = DomTreeNodeBase<BasicBlock>; |
| 24 | class AssumptionCache; |
| 25 | class StringRef; |
| 26 | class AnalysisUsage; |
| 27 | class TargetTransformInfo; |
| 28 | class AAResults; |
| 29 | class BasicBlock; |
| 30 | class ICFLoopSafetyInfo; |
| 31 | class IRBuilderBase; |
| 32 | class Loop; |
| 33 | class LoopInfo; |
| 34 | class MemoryAccess; |
| 35 | class MemorySSA; |
| 36 | class MemorySSAUpdater; |
| 37 | class OptimizationRemarkEmitter; |
| 38 | class PredIteratorCache; |
| 39 | class ScalarEvolution; |
| 40 | class SCEV; |
| 41 | class SCEVExpander; |
| 42 | class TargetLibraryInfo; |
| 43 | class LPPassManager; |
| 44 | class Instruction; |
| 45 | struct RuntimeCheckingPtrGroup; |
| 46 | typedef std::pair<const RuntimeCheckingPtrGroup *, |
| 47 | const RuntimeCheckingPtrGroup *> |
| 48 | RuntimePointerCheck; |
| 49 | |
| 50 | template <typename T, unsigned N> class SmallSetVector; |
| 51 | template <typename T, unsigned N> class SmallPriorityWorklist; |
| 52 | |
| 53 | BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, |
| 54 | MemorySSAUpdater *MSSAU, bool PreserveLCSSA); |
| 55 | |
| 56 | /// Ensure that all exit blocks of the loop are dedicated exits. |
| 57 | /// |
| 58 | /// For any loop exit block with non-loop predecessors, we split the loop |
| 59 | /// predecessors to use a dedicated loop exit block. We update the dominator |
| 60 | /// tree and loop info if provided, and will preserve LCSSA if requested. |
| 61 | bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, |
| 62 | MemorySSAUpdater *MSSAU, bool PreserveLCSSA); |
| 63 | |
| 64 | /// Ensures LCSSA form for every instruction from the Worklist in the scope of |
| 65 | /// innermost containing loop. |
| 66 | /// |
| 67 | /// For the given instruction which have uses outside of the loop, an LCSSA PHI |
| 68 | /// node is inserted and the uses outside the loop are rewritten to use this |
| 69 | /// node. |
| 70 | /// |
| 71 | /// LoopInfo and DominatorTree are required and, since the routine makes no |
| 72 | /// changes to CFG, preserved. |
| 73 | /// |
| 74 | /// Returns true if any modifications are made. |
| 75 | /// |
| 76 | /// This function may introduce unused PHI nodes. If \p PHIsToRemove is not |
| 77 | /// nullptr, those are added to it (before removing, the caller has to check if |
| 78 | /// they still do not have any uses). Otherwise the PHIs are directly removed. |
| 79 | bool formLCSSAForInstructions( |
| 80 | SmallVectorImpl<Instruction *> &Worklist, const DominatorTree &DT, |
| 81 | const LoopInfo &LI, IRBuilderBase &Builder, |
| 82 | SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr); |
| 83 | |
| 84 | /// Put loop into LCSSA form. |
| 85 | /// |
| 86 | /// Looks at all instructions in the loop which have uses outside of the |
| 87 | /// current loop. For each, an LCSSA PHI node is inserted and the uses outside |
| 88 | /// the loop are rewritten to use this node. Sub-loops must be in LCSSA form |
| 89 | /// already. |
| 90 | /// |
| 91 | /// LoopInfo and DominatorTree are required and preserved. ScalarEvolution is |
| 92 | /// preserved. |
| 93 | /// |
| 94 | /// Returns true if any modifications are made to the loop. |
| 95 | bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI); |
| 96 | |
| 97 | /// Put a loop nest into LCSSA form. |
| 98 | /// |
| 99 | /// This recursively forms LCSSA for a loop nest. |
| 100 | /// |
| 101 | /// LoopInfo and DominatorTree are required and preserved. ScalarEvolution is |
| 102 | /// preserved. |
| 103 | /// |
| 104 | /// Returns true if any modifications are made to the loop. |
| 105 | bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI); |
| 106 | |
| 107 | /// Flags controlling how much is checked when sinking or hoisting |
| 108 | /// instructions. The number of memory access in the loop (and whether there |
| 109 | /// are too many) is determined in the constructors when using MemorySSA. |
| 110 | class SinkAndHoistLICMFlags { |
| 111 | public: |
| 112 | // Explicitly set limits. |
| 113 | SinkAndHoistLICMFlags(unsigned LicmMssaOptCap, |
| 114 | unsigned LicmMssaNoAccForPromotionCap, bool IsSink, |
| 115 | Loop &L, MemorySSA &MSSA); |
| 116 | // Use default limits. |
| 117 | SinkAndHoistLICMFlags(bool IsSink, Loop &L, MemorySSA &MSSA); |
| 118 | |
| 119 | void setIsSink(bool B) { IsSink = B; } |
| 120 | bool getIsSink() { return IsSink; } |
| 121 | bool tooManyMemoryAccesses() { return NoOfMemAccTooLarge; } |
| 122 | bool tooManyClobberingCalls() { return LicmMssaOptCounter >= LicmMssaOptCap; } |
| 123 | void incrementClobberingCalls() { ++LicmMssaOptCounter; } |
| 124 | |
| 125 | protected: |
| 126 | bool NoOfMemAccTooLarge = false; |
| 127 | unsigned LicmMssaOptCounter = 0; |
| 128 | unsigned LicmMssaOptCap; |
| 129 | unsigned LicmMssaNoAccForPromotionCap; |
| 130 | bool IsSink; |
| 131 | }; |
| 132 | |
| 133 | /// Walk the specified region of the CFG (defined by all blocks |
| 134 | /// dominated by the specified block, and that are in the current loop) in |
| 135 | /// reverse depth first order w.r.t the DominatorTree. This allows us to visit |
| 136 | /// uses before definitions, allowing us to sink a loop body in one pass without |
| 137 | /// iteration. Takes DomTreeNode, AAResults, LoopInfo, DominatorTree, |
| 138 | /// TargetLibraryInfo, Loop, AliasSet information for all |
| 139 | /// instructions of the loop and loop safety information as |
| 140 | /// arguments. Diagnostics is emitted via \p ORE. It returns changed status. |
| 141 | /// \p CurLoop is a loop to do sinking on. \p OutermostLoop is used only when |
| 142 | /// this function is called by \p sinkRegionForLoopNest. |
| 143 | bool sinkRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, |
| 144 | TargetLibraryInfo *, TargetTransformInfo *, Loop *CurLoop, |
| 145 | MemorySSAUpdater &, ICFLoopSafetyInfo *, |
| 146 | SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, |
| 147 | Loop *OutermostLoop = nullptr); |
| 148 | |
| 149 | /// Call sinkRegion on loops contained within the specified loop |
| 150 | /// in order from innermost to outermost. |
| 151 | bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *, |
| 152 | DominatorTree *, TargetLibraryInfo *, |
| 153 | TargetTransformInfo *, Loop *, MemorySSAUpdater &, |
| 154 | ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, |
| 155 | OptimizationRemarkEmitter *); |
| 156 | |
| 157 | /// Walk the specified region of the CFG (defined by all blocks |
| 158 | /// dominated by the specified block, and that are in the current loop) in depth |
| 159 | /// first order w.r.t the DominatorTree. This allows us to visit definitions |
| 160 | /// before uses, allowing us to hoist a loop body in one pass without iteration. |
| 161 | /// Takes DomTreeNode, AAResults, LoopInfo, DominatorTree, |
| 162 | /// TargetLibraryInfo, Loop, AliasSet information for all |
| 163 | /// instructions of the loop and loop safety information as arguments. |
| 164 | /// Diagnostics is emitted via \p ORE. It returns changed status. |
| 165 | /// \p AllowSpeculation is whether values should be hoisted even if they are not |
| 166 | /// guaranteed to execute in the loop, but are safe to speculatively execute. |
| 167 | bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, |
| 168 | AssumptionCache *, TargetLibraryInfo *, Loop *, |
| 169 | MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *, |
| 170 | SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool, |
| 171 | bool AllowSpeculation); |
| 172 | |
| 173 | /// Return true if the induction variable \p IV in a Loop whose latch is |
| 174 | /// \p LatchBlock would become dead if the exit test \p Cond were removed. |
| 175 | /// Conservatively returns false if analysis is insufficient. |
| 176 | bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond); |
| 177 | |
| 178 | /// This function deletes dead loops. The caller of this function needs to |
| 179 | /// guarantee that the loop is infact dead. |
| 180 | /// The function requires a bunch or prerequisites to be present: |
| 181 | /// - The loop needs to be in LCSSA form |
| 182 | /// - The loop needs to have a Preheader |
| 183 | /// - A unique dedicated exit block must exist |
| 184 | /// |
| 185 | /// This also updates the relevant analysis information in \p DT, \p SE, \p LI |
| 186 | /// and \p MSSA if pointers to those are provided. |
| 187 | /// It also updates the loop PM if an updater struct is provided. |
| 188 | |
| 189 | void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE, |
| 190 | LoopInfo *LI, MemorySSA *MSSA = nullptr); |
| 191 | |
| 192 | /// Remove the backedge of the specified loop. Handles loop nests and general |
| 193 | /// loop structures subject to the precondition that the loop has no parent |
| 194 | /// loop and has a single latch block. Preserves all listed analyses. |
| 195 | void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE, |
| 196 | LoopInfo &LI, MemorySSA *MSSA); |
| 197 | |
| 198 | /// Try to promote memory values to scalars by sinking stores out of |
| 199 | /// the loop and moving loads to before the loop. We do this by looping over |
| 200 | /// the stores in the loop, looking for stores to Must pointers which are |
| 201 | /// loop invariant. It takes a set of must-alias values, Loop exit blocks |
| 202 | /// vector, loop exit blocks insertion point vector, PredIteratorCache, |
| 203 | /// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions |
| 204 | /// of the loop and loop safety information as arguments. |
| 205 | /// Diagnostics is emitted via \p ORE. It returns changed status. |
| 206 | /// \p AllowSpeculation is whether values should be hoisted even if they are not |
| 207 | /// guaranteed to execute in the loop, but are safe to speculatively execute. |
| 208 | bool promoteLoopAccessesToScalars( |
| 209 | const SmallSetVector<Value *, 8> &, SmallVectorImpl<BasicBlock *> &, |
| 210 | SmallVectorImpl<Instruction *> &, SmallVectorImpl<MemoryAccess *> &, |
| 211 | PredIteratorCache &, LoopInfo *, DominatorTree *, AssumptionCache *AC, |
| 212 | const TargetLibraryInfo *, TargetTransformInfo *, Loop *, |
| 213 | MemorySSAUpdater &, ICFLoopSafetyInfo *, OptimizationRemarkEmitter *, |
| 214 | bool AllowSpeculation, bool HasReadsOutsideSet); |
| 215 | |
| 216 | /// Does a BFS from a given node to all of its children inside a given loop. |
| 217 | /// The returned vector of nodes includes the starting point. |
| 218 | SmallVector<DomTreeNode *, 16> collectChildrenInLoop(DomTreeNode *N, |
| 219 | const Loop *CurLoop); |
| 220 | |
| 221 | /// Returns the instructions that use values defined in the loop. |
| 222 | SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L); |
| 223 | |
| 224 | /// Find a combination of metadata ("llvm.loop.vectorize.width" and |
| 225 | /// "llvm.loop.vectorize.scalable.enable") for a loop and use it to construct a |
| 226 | /// ElementCount. If the metadata "llvm.loop.vectorize.width" cannot be found |
| 227 | /// then std::nullopt is returned. |
| 228 | std::optional<ElementCount> |
| 229 | getOptionalElementCountLoopAttribute(const Loop *TheLoop); |
| 230 | |
| 231 | /// Create a new loop identifier for a loop created from a loop transformation. |
| 232 | /// |
| 233 | /// @param OrigLoopID The loop ID of the loop before the transformation. |
| 234 | /// @param FollowupAttrs List of attribute names that contain attributes to be |
| 235 | /// added to the new loop ID. |
| 236 | /// @param InheritOptionsAttrsPrefix Selects which attributes should be inherited |
| 237 | /// from the original loop. The following values |
| 238 | /// are considered: |
| 239 | /// nullptr : Inherit all attributes from @p OrigLoopID. |
| 240 | /// "" : Do not inherit any attribute from @p OrigLoopID; only use |
| 241 | /// those specified by a followup attribute. |
| 242 | /// "<prefix>": Inherit all attributes except those which start with |
| 243 | /// <prefix>; commonly used to remove metadata for the |
| 244 | /// applied transformation. |
| 245 | /// @param AlwaysNew If true, do not try to reuse OrigLoopID and never return |
| 246 | /// std::nullopt. |
| 247 | /// |
| 248 | /// @return The loop ID for the after-transformation loop. The following values |
| 249 | /// can be returned: |
| 250 | /// std::nullopt : No followup attribute was found; it is up to the |
| 251 | /// transformation to choose attributes that make sense. |
| 252 | /// @p OrigLoopID: The original identifier can be reused. |
| 253 | /// nullptr : The new loop has no attributes. |
| 254 | /// MDNode* : A new unique loop identifier. |
| 255 | std::optional<MDNode *> |
| 256 | makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef<StringRef> FollowupAttrs, |
| 257 | const char *InheritOptionsAttrsPrefix = "", |
| 258 | bool AlwaysNew = false); |
| 259 | |
| 260 | /// Look for the loop attribute that disables all transformation heuristic. |
| 261 | bool hasDisableAllTransformsHint(const Loop *L); |
| 262 | |
| 263 | /// Look for the loop attribute that disables the LICM transformation heuristics. |
| 264 | bool hasDisableLICMTransformsHint(const Loop *L); |
| 265 | |
| 266 | /// The mode sets how eager a transformation should be applied. |
| 267 | enum TransformationMode { |
| 268 | /// The pass can use heuristics to determine whether a transformation should |
| 269 | /// be applied. |
| 270 | TM_Unspecified, |
| 271 | |
| 272 | /// The transformation should be applied without considering a cost model. |
| 273 | TM_Enable, |
| 274 | |
| 275 | /// The transformation should not be applied. |
| 276 | TM_Disable, |
| 277 | |
| 278 | /// Force is a flag and should not be used alone. |
| 279 | TM_Force = 0x04, |
| 280 | |
| 281 | /// The transformation was directed by the user, e.g. by a #pragma in |
| 282 | /// the source code. If the transformation could not be applied, a |
| 283 | /// warning should be emitted. |
| 284 | TM_ForcedByUser = TM_Enable | TM_Force, |
| 285 | |
| 286 | /// The transformation must not be applied. For instance, `#pragma clang loop |
| 287 | /// unroll(disable)` explicitly forbids any unrolling to take place. Unlike |
| 288 | /// general loop metadata, it must not be dropped. Most passes should not |
| 289 | /// behave differently under TM_Disable and TM_SuppressedByUser. |
| 290 | TM_SuppressedByUser = TM_Disable | TM_Force |
| 291 | }; |
| 292 | |
| 293 | /// @{ |
| 294 | /// Get the mode for LLVM's supported loop transformations. |
| 295 | TransformationMode hasUnrollTransformation(const Loop *L); |
| 296 | TransformationMode hasUnrollAndJamTransformation(const Loop *L); |
| 297 | TransformationMode hasVectorizeTransformation(const Loop *L); |
| 298 | TransformationMode hasDistributeTransformation(const Loop *L); |
| 299 | TransformationMode hasLICMVersioningTransformation(const Loop *L); |
| 300 | /// @} |
| 301 | |
| 302 | /// Set input string into loop metadata by keeping other values intact. |
| 303 | /// If the string is already in loop metadata update value if it is |
| 304 | /// different. |
| 305 | void addStringMetadataToLoop(Loop *TheLoop, const char *MDString, |
| 306 | unsigned V = 0); |
| 307 | |
| 308 | /// Returns a loop's estimated trip count based on branch weight metadata. |
| 309 | /// In addition if \p EstimatedLoopInvocationWeight is not null it is |
| 310 | /// initialized with weight of loop's latch leading to the exit. |
| 311 | /// Returns 0 when the count is estimated to be 0, or std::nullopt when a |
| 312 | /// meaningful estimate can not be made. |
| 313 | std::optional<unsigned> |
| 314 | getLoopEstimatedTripCount(Loop *L, |
| 315 | unsigned *EstimatedLoopInvocationWeight = nullptr); |
| 316 | |
| 317 | /// Set a loop's branch weight metadata to reflect that loop has \p |
| 318 | /// EstimatedTripCount iterations and \p EstimatedLoopInvocationWeight exits |
| 319 | /// through latch. Returns true if metadata is successfully updated, false |
| 320 | /// otherwise. Note that loop must have a latch block which controls loop exit |
| 321 | /// in order to succeed. |
| 322 | bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount, |
| 323 | unsigned EstimatedLoopInvocationWeight); |
| 324 | |
| 325 | /// Check inner loop (L) backedge count is known to be invariant on all |
| 326 | /// iterations of its outer loop. If the loop has no parent, this is trivially |
| 327 | /// true. |
| 328 | bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE); |
| 329 | |
| 330 | /// Helper to consistently add the set of standard passes to a loop pass's \c |
| 331 | /// AnalysisUsage. |
| 332 | /// |
| 333 | /// All loop passes should call this as part of implementing their \c |
| 334 | /// getAnalysisUsage. |
| 335 | void getLoopAnalysisUsage(AnalysisUsage &AU); |
| 336 | |
| 337 | /// Returns true if is legal to hoist or sink this instruction disregarding the |
| 338 | /// possible introduction of faults. Reasoning about potential faulting |
| 339 | /// instructions is the responsibility of the caller since it is challenging to |
| 340 | /// do efficiently from within this routine. |
| 341 | /// \p TargetExecutesOncePerLoop is true only when it is guaranteed that the |
| 342 | /// target executes at most once per execution of the loop body. This is used |
| 343 | /// to assess the legality of duplicating atomic loads. Generally, this is |
| 344 | /// true when moving out of loop and not true when moving into loops. |
| 345 | /// If \p ORE is set use it to emit optimization remarks. |
| 346 | bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT, |
| 347 | Loop *CurLoop, MemorySSAUpdater &MSSAU, |
| 348 | bool TargetExecutesOncePerLoop, |
| 349 | SinkAndHoistLICMFlags &LICMFlags, |
| 350 | OptimizationRemarkEmitter *ORE = nullptr); |
| 351 | |
| 352 | /// Returns the min/max intrinsic used when expanding a min/max reduction. |
| 353 | Intrinsic::ID getMinMaxReductionIntrinsicOp(RecurKind RK); |
| 354 | |
| 355 | /// Returns the comparison predicate used when expanding a min/max reduction. |
| 356 | CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK); |
| 357 | |
| 358 | /// See RecurrenceDescriptor::isSelectCmpPattern for a description of the |
| 359 | /// pattern we are trying to match. In this pattern we are only ever selecting |
| 360 | /// between two values: 1) an initial PHI start value, and 2) a loop invariant |
| 361 | /// value. This function uses \p LoopExitInst to determine 2), which we then use |
| 362 | /// to select between \p Left and \p Right. Any lane value in \p Left that |
| 363 | /// matches 2) will be merged into \p Right. |
| 364 | Value *createSelectCmpOp(IRBuilderBase &Builder, Value *StartVal, RecurKind RK, |
| 365 | Value *Left, Value *Right); |
| 366 | |
| 367 | /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind. |
| 368 | /// The Builder's fast-math-flags must be set to propagate the expected values. |
| 369 | Value *createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, |
| 370 | Value *Right); |
| 371 | |
| 372 | /// Generates an ordered vector reduction using extracts to reduce the value. |
| 373 | Value *getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src, |
| 374 | unsigned Op, RecurKind MinMaxKind = RecurKind::None); |
| 375 | |
| 376 | /// Generates a vector reduction using shufflevectors to reduce the value. |
| 377 | /// Fast-math-flags are propagated using the IRBuilder's setting. |
| 378 | Value *getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op, |
| 379 | RecurKind MinMaxKind = RecurKind::None); |
| 380 | |
| 381 | /// Create a target reduction of the given vector. The reduction operation |
| 382 | /// is described by the \p Opcode parameter. min/max reductions require |
| 383 | /// additional information supplied in \p RdxKind. |
| 384 | /// The target is queried to determine if intrinsics or shuffle sequences are |
| 385 | /// required to implement the reduction. |
| 386 | /// Fast-math-flags are propagated using the IRBuilder's setting. |
| 387 | Value *createSimpleTargetReduction(IRBuilderBase &B, |
| 388 | const TargetTransformInfo *TTI, Value *Src, |
| 389 | RecurKind RdxKind); |
| 390 | |
| 391 | /// Create a target reduction of the given vector \p Src for a reduction of the |
| 392 | /// kind RecurKind::SelectICmp or RecurKind::SelectFCmp. The reduction operation |
| 393 | /// is described by \p Desc. |
| 394 | Value *createSelectCmpTargetReduction(IRBuilderBase &B, |
| 395 | const TargetTransformInfo *TTI, |
| 396 | Value *Src, |
| 397 | const RecurrenceDescriptor &Desc, |
| 398 | PHINode *OrigPhi); |
| 399 | |
| 400 | /// Create a generic target reduction using a recurrence descriptor \p Desc |
| 401 | /// The target is queried to determine if intrinsics or shuffle sequences are |
| 402 | /// required to implement the reduction. |
| 403 | /// Fast-math-flags are propagated using the RecurrenceDescriptor. |
| 404 | Value *createTargetReduction(IRBuilderBase &B, const TargetTransformInfo *TTI, |
| 405 | const RecurrenceDescriptor &Desc, Value *Src, |
| 406 | PHINode *OrigPhi = nullptr); |
| 407 | |
| 408 | /// Create an ordered reduction intrinsic using the given recurrence |
| 409 | /// descriptor \p Desc. |
| 410 | Value *createOrderedReduction(IRBuilderBase &B, |
| 411 | const RecurrenceDescriptor &Desc, Value *Src, |
| 412 | Value *Start); |
| 413 | |
| 414 | /// Get the intersection (logical and) of all of the potential IR flags |
| 415 | /// of each scalar operation (VL) that will be converted into a vector (I). |
| 416 | /// If OpValue is non-null, we only consider operations similar to OpValue |
| 417 | /// when intersecting. |
| 418 | /// Flag set: NSW, NUW (if IncludeWrapFlags is true), exact, and all of |
| 419 | /// fast-math. |
| 420 | void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr, |
| 421 | bool IncludeWrapFlags = true); |
| 422 | |
| 423 | /// Returns true if we can prove that \p S is defined and always negative in |
| 424 | /// loop \p L. |
| 425 | bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE); |
| 426 | |
| 427 | /// Returns true if we can prove that \p S is defined and always non-negative in |
| 428 | /// loop \p L. |
| 429 | bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L, |
| 430 | ScalarEvolution &SE); |
| 431 | /// Returns true if we can prove that \p S is defined and always positive in |
| 432 | /// loop \p L. |
| 433 | bool isKnownPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE); |
| 434 | |
| 435 | /// Returns true if we can prove that \p S is defined and always non-positive in |
| 436 | /// loop \p L. |
| 437 | bool isKnownNonPositiveInLoop(const SCEV *S, const Loop *L, |
| 438 | ScalarEvolution &SE); |
| 439 | |
| 440 | /// Returns true if \p S is defined and never is equal to signed/unsigned max. |
| 441 | bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, |
| 442 | bool Signed); |
| 443 | |
| 444 | /// Returns true if \p S is defined and never is equal to signed/unsigned min. |
| 445 | bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, |
| 446 | bool Signed); |
| 447 | |
| 448 | enum ReplaceExitVal { |
| 449 | NeverRepl, |
| 450 | OnlyCheapRepl, |
| 451 | NoHardUse, |
| 452 | UnusedIndVarInLoop, |
| 453 | AlwaysRepl |
| 454 | }; |
| 455 | |
| 456 | /// If the final value of any expressions that are recurrent in the loop can |
| 457 | /// be computed, substitute the exit values from the loop into any instructions |
| 458 | /// outside of the loop that use the final values of the current expressions. |
| 459 | /// Return the number of loop exit values that have been replaced, and the |
| 460 | /// corresponding phi node will be added to DeadInsts. |
| 461 | int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI, |
| 462 | ScalarEvolution *SE, const TargetTransformInfo *TTI, |
| 463 | SCEVExpander &Rewriter, DominatorTree *DT, |
| 464 | ReplaceExitVal ReplaceExitValue, |
| 465 | SmallVector<WeakTrackingVH, 16> &DeadInsts); |
| 466 | |
| 467 | /// Set weights for \p UnrolledLoop and \p RemainderLoop based on weights for |
| 468 | /// \p OrigLoop and the following distribution of \p OrigLoop iteration among \p |
| 469 | /// UnrolledLoop and \p RemainderLoop. \p UnrolledLoop receives weights that |
| 470 | /// reflect TC/UF iterations, and \p RemainderLoop receives weights that reflect |
| 471 | /// the remaining TC%UF iterations. |
| 472 | /// |
| 473 | /// Note that \p OrigLoop may be equal to either \p UnrolledLoop or \p |
| 474 | /// RemainderLoop in which case weights for \p OrigLoop are updated accordingly. |
| 475 | /// Note also behavior is undefined if \p UnrolledLoop and \p RemainderLoop are |
| 476 | /// equal. \p UF must be greater than zero. |
| 477 | /// If \p OrigLoop has no profile info associated nothing happens. |
| 478 | /// |
| 479 | /// This utility may be useful for such optimizations as unroller and |
| 480 | /// vectorizer as it's typical transformation for them. |
| 481 | void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop, |
| 482 | Loop *RemainderLoop, uint64_t UF); |
| 483 | |
| 484 | /// Utility that implements appending of loops onto a worklist given a range. |
| 485 | /// We want to process loops in postorder, but the worklist is a LIFO data |
| 486 | /// structure, so we append to it in *reverse* postorder. |
| 487 | /// For trees, a preorder traversal is a viable reverse postorder, so we |
| 488 | /// actually append using a preorder walk algorithm. |
| 489 | template <typename RangeT> |
| 490 | void appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist<Loop *, 4> &); |
| 491 | /// Utility that implements appending of loops onto a worklist given a range. |
| 492 | /// It has the same behavior as appendLoopsToWorklist, but assumes the range of |
| 493 | /// loops has already been reversed, so it processes loops in the given order. |
| 494 | template <typename RangeT> |
| 495 | void appendReversedLoopsToWorklist(RangeT &&, |
| 496 | SmallPriorityWorklist<Loop *, 4> &); |
| 497 | |
| 498 | /// Utility that implements appending of loops onto a worklist given LoopInfo. |
| 499 | /// Calls the templated utility taking a Range of loops, handing it the Loops |
| 500 | /// in LoopInfo, iterated in reverse. This is because the loops are stored in |
| 501 | /// RPO w.r.t. the control flow graph in LoopInfo. For the purpose of unrolling, |
| 502 | /// loop deletion, and LICM, we largely want to work forward across the CFG so |
| 503 | /// that we visit defs before uses and can propagate simplifications from one |
| 504 | /// loop nest into the next. Calls appendReversedLoopsToWorklist with the |
| 505 | /// already reversed loops in LI. |
| 506 | /// FIXME: Consider changing the order in LoopInfo. |
| 507 | void appendLoopsToWorklist(LoopInfo &, SmallPriorityWorklist<Loop *, 4> &); |
| 508 | |
| 509 | /// Recursively clone the specified loop and all of its children, |
| 510 | /// mapping the blocks with the specified map. |
| 511 | Loop *cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, |
| 512 | LoopInfo *LI, LPPassManager *LPM); |
| 513 | |
| 514 | /// Add code that checks at runtime if the accessed arrays in \p PointerChecks |
| 515 | /// overlap. Returns the final comparator value or NULL if no check is needed. |
| 516 | Value * |
| 517 | addRuntimeChecks(Instruction *Loc, Loop *TheLoop, |
| 518 | const SmallVectorImpl<RuntimePointerCheck> &PointerChecks, |
| 519 | SCEVExpander &Expander); |
| 520 | |
| 521 | Value *addDiffRuntimeChecks( |
| 522 | Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander, |
| 523 | function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC); |
| 524 | |
| 525 | /// Struct to hold information about a partially invariant condition. |
| 526 | struct IVConditionInfo { |
| 527 | /// Instructions that need to be duplicated and checked for the unswitching |
| 528 | /// condition. |
| 529 | SmallVector<Instruction *> InstToDuplicate; |
| 530 | |
| 531 | /// Constant to indicate for which value the condition is invariant. |
| 532 | Constant *KnownValue = nullptr; |
| 533 | |
| 534 | /// True if the partially invariant path is no-op (=does not have any |
| 535 | /// side-effects and no loop value is used outside the loop). |
| 536 | bool PathIsNoop = true; |
| 537 | |
| 538 | /// If the partially invariant path reaches a single exit block, ExitForPath |
| 539 | /// is set to that block. Otherwise it is nullptr. |
| 540 | BasicBlock *ExitForPath = nullptr; |
| 541 | }; |
| 542 | |
| 543 | /// Check if the loop header has a conditional branch that is not |
| 544 | /// loop-invariant, because it involves load instructions. If all paths from |
| 545 | /// either the true or false successor to the header or loop exists do not |
| 546 | /// modify the memory feeding the condition, perform 'partial unswitching'. That |
| 547 | /// is, duplicate the instructions feeding the condition in the pre-header. Then |
| 548 | /// unswitch on the duplicated condition. The condition is now known in the |
| 549 | /// unswitched version for the 'invariant' path through the original loop. |
| 550 | /// |
| 551 | /// If the branch condition of the header is partially invariant, return a pair |
| 552 | /// containing the instructions to duplicate and a boolean Constant to update |
| 553 | /// the condition in the loops created for the true or false successors. |
| 554 | std::optional<IVConditionInfo> hasPartialIVCondition(const Loop &L, |
| 555 | unsigned MSSAThreshold, |
| 556 | const MemorySSA &MSSA, |
| 557 | AAResults &AA); |
| 558 | |
| 559 | } // end namespace llvm |
| 560 | |
| 561 | #endif // LLVM_TRANSFORMS_UTILS_LOOPUTILS_H |