| File: | build/source/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp |
| Warning: | line 590, column 11 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===--- ExpandLargeFpConvert.cpp - Expand large fp convert----------------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | ||||
| 10 | // This pass expands ‘fptoui .. to’, ‘fptosi .. to’, ‘uitofp .. to’, | |||
| 11 | // ‘sitofp .. to’ instructions with a bitwidth above a threshold into | |||
| 12 | // auto-generated functions. This is useful for targets like x86_64 that cannot | |||
| 13 | // lower fp convertions with more than 128 bits. | |||
| 14 | // | |||
| 15 | //===----------------------------------------------------------------------===// | |||
| 16 | ||||
| 17 | #include "llvm/ADT/SmallVector.h" | |||
| 18 | #include "llvm/ADT/StringExtras.h" | |||
| 19 | #include "llvm/Analysis/GlobalsModRef.h" | |||
| 20 | #include "llvm/CodeGen/Passes.h" | |||
| 21 | #include "llvm/CodeGen/TargetLowering.h" | |||
| 22 | #include "llvm/CodeGen/TargetPassConfig.h" | |||
| 23 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
| 24 | #include "llvm/IR/IRBuilder.h" | |||
| 25 | #include "llvm/IR/InstIterator.h" | |||
| 26 | #include "llvm/IR/PassManager.h" | |||
| 27 | #include "llvm/InitializePasses.h" | |||
| 28 | #include "llvm/Pass.h" | |||
| 29 | #include "llvm/Support/CommandLine.h" | |||
| 30 | #include "llvm/Target/TargetMachine.h" | |||
| 31 | ||||
| 32 | using namespace llvm; | |||
| 33 | ||||
| 34 | static cl::opt<unsigned> | |||
| 35 | ExpandFpConvertBits("expand-fp-convert-bits", cl::Hidden, | |||
| 36 | cl::init(llvm::IntegerType::MAX_INT_BITS), | |||
| 37 | cl::desc("fp convert instructions on integers with " | |||
| 38 | "more than <N> bits are expanded.")); | |||
| 39 | ||||
| 40 | /// Generate code to convert a fp number to integer, replacing FPToS(U)I with | |||
| 41 | /// the generated code. This currently generates code similarly to compiler-rt's | |||
| 42 | /// implementations. | |||
| 43 | /// | |||
| 44 | /// An example IR generated from compiler-rt/fixsfdi.c looks like below: | |||
| 45 | /// define dso_local i64 @foo(float noundef %a) local_unnamed_addr #0 { | |||
| 46 | /// entry: | |||
| 47 | /// %0 = bitcast float %a to i32 | |||
| 48 | /// %conv.i = zext i32 %0 to i64 | |||
| 49 | /// %tobool.not = icmp sgt i32 %0, -1 | |||
| 50 | /// %conv = select i1 %tobool.not, i64 1, i64 -1 | |||
| 51 | /// %and = lshr i64 %conv.i, 23 | |||
| 52 | /// %shr = and i64 %and, 255 | |||
| 53 | /// %and2 = and i64 %conv.i, 8388607 | |||
| 54 | /// %or = or i64 %and2, 8388608 | |||
| 55 | /// %cmp = icmp ult i64 %shr, 127 | |||
| 56 | /// br i1 %cmp, label %cleanup, label %if.end | |||
| 57 | /// | |||
| 58 | /// if.end: ; preds = %entry | |||
| 59 | /// %sub = add nuw nsw i64 %shr, 4294967169 | |||
| 60 | /// %conv5 = and i64 %sub, 4294967232 | |||
| 61 | /// %cmp6.not = icmp eq i64 %conv5, 0 | |||
| 62 | /// br i1 %cmp6.not, label %if.end12, label %if.then8 | |||
| 63 | /// | |||
| 64 | /// if.then8: ; preds = %if.end | |||
| 65 | /// %cond11 = select i1 %tobool.not, i64 9223372036854775807, i64 -9223372036854775808 | |||
| 66 | /// br label %cleanup | |||
| 67 | /// | |||
| 68 | /// if.end12: ; preds = %if.end | |||
| 69 | /// %cmp13 = icmp ult i64 %shr, 150 | |||
| 70 | /// br i1 %cmp13, label %if.then15, label %if.else | |||
| 71 | /// | |||
| 72 | /// if.then15: ; preds = %if.end12 | |||
| 73 | /// %sub16 = sub nuw nsw i64 150, %shr | |||
| 74 | /// %shr17 = lshr i64 %or, %sub16 | |||
| 75 | /// %mul = mul nsw i64 %shr17, %conv | |||
| 76 | /// br label %cleanup | |||
| 77 | /// | |||
| 78 | /// if.else: ; preds = %if.end12 | |||
| 79 | /// %sub18 = add nsw i64 %shr, -150 | |||
| 80 | /// %shl = shl i64 %or, %sub18 | |||
| 81 | /// %mul19 = mul nsw i64 %shl, %conv | |||
| 82 | /// br label %cleanup | |||
| 83 | /// | |||
| 84 | /// cleanup: ; preds = %entry, %if.else, %if.then15, %if.then8 | |||
| 85 | /// %retval.0 = phi i64 [ %cond11, %if.then8 ], [ %mul, %if.then15 ], [ %mul19, %if.else ], [ 0, %entry ] | |||
| 86 | /// ret i64 %retval.0 | |||
| 87 | /// } | |||
| 88 | /// | |||
| 89 | /// Replace fp to integer with generated code. | |||
| 90 | static void expandFPToI(Instruction *FPToI) { | |||
| 91 | IRBuilder<> Builder(FPToI); | |||
| 92 | auto *FloatVal = FPToI->getOperand(0); | |||
| 93 | IntegerType *IntTy = cast<IntegerType>(FPToI->getType()); | |||
| 94 | ||||
| 95 | unsigned BitWidth = FPToI->getType()->getIntegerBitWidth(); | |||
| 96 | unsigned FPMantissaWidth = FloatVal->getType()->getFPMantissaWidth() - 1; | |||
| 97 | ||||
| 98 | // FIXME: fp16's range is covered by i32. So `fptoi half` can convert | |||
| 99 | // to i32 first following a sext/zext to target integer type. | |||
| 100 | Value *A1 = nullptr; | |||
| 101 | if (FloatVal->getType()->isHalfTy()) { | |||
| 102 | if (FPToI->getOpcode() == Instruction::FPToUI) { | |||
| 103 | Value *A0 = Builder.CreateFPToUI(FloatVal, Builder.getIntNTy(32)); | |||
| 104 | A1 = Builder.CreateZExt(A0, IntTy); | |||
| 105 | } else { // FPToSI | |||
| 106 | Value *A0 = Builder.CreateFPToSI(FloatVal, Builder.getIntNTy(32)); | |||
| 107 | A1 = Builder.CreateSExt(A0, IntTy); | |||
| 108 | } | |||
| 109 | FPToI->replaceAllUsesWith(A1); | |||
| 110 | FPToI->dropAllReferences(); | |||
| 111 | FPToI->eraseFromParent(); | |||
| 112 | return; | |||
| 113 | } | |||
| 114 | ||||
| 115 | // fp80 conversion is implemented by fpext to fp128 first then do the | |||
| 116 | // conversion. | |||
| 117 | FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth; | |||
| 118 | unsigned FloatWidth = PowerOf2Ceil(FPMantissaWidth); | |||
| 119 | unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1; | |||
| 120 | unsigned ExponentBias = (1 << (ExponentWidth - 1)) - 1; | |||
| 121 | Value *ImplicitBit = Builder.CreateShl( | |||
| 122 | Builder.getIntN(BitWidth, 1), Builder.getIntN(BitWidth, FPMantissaWidth)); | |||
| 123 | Value *SignificandMask = | |||
| 124 | Builder.CreateSub(ImplicitBit, Builder.getIntN(BitWidth, 1)); | |||
| 125 | Value *NegOne = Builder.CreateSExt( | |||
| 126 | ConstantInt::getSigned(Builder.getInt32Ty(), -1), IntTy); | |||
| 127 | Value *NegInf = | |||
| 128 | Builder.CreateShl(ConstantInt::getSigned(IntTy, 1), | |||
| 129 | ConstantInt::getSigned(IntTy, BitWidth - 1)); | |||
| 130 | ||||
| 131 | BasicBlock *Entry = Builder.GetInsertBlock(); | |||
| 132 | Function *F = Entry->getParent(); | |||
| 133 | Entry->setName(Twine(Entry->getName(), "fp-to-i-entry")); | |||
| 134 | BasicBlock *End = | |||
| 135 | Entry->splitBasicBlock(Builder.GetInsertPoint(), "fp-to-i-cleanup"); | |||
| 136 | BasicBlock *IfEnd = | |||
| 137 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-end", F, End); | |||
| 138 | BasicBlock *IfThen5 = | |||
| 139 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-then5", F, End); | |||
| 140 | BasicBlock *IfEnd9 = | |||
| 141 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-end9", F, End); | |||
| 142 | BasicBlock *IfThen12 = | |||
| 143 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-then12", F, End); | |||
| 144 | BasicBlock *IfElse = | |||
| 145 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-else", F, End); | |||
| 146 | ||||
| 147 | Entry->getTerminator()->eraseFromParent(); | |||
| 148 | ||||
| 149 | // entry: | |||
| 150 | Builder.SetInsertPoint(Entry); | |||
| 151 | Value *FloatVal0 = FloatVal; | |||
| 152 | // fp80 conversion is implemented by fpext to fp128 first then do the | |||
| 153 | // conversion. | |||
| 154 | if (FloatVal->getType()->isX86_FP80Ty()) | |||
| 155 | FloatVal0 = | |||
| 156 | Builder.CreateFPExt(FloatVal, Type::getFP128Ty(Builder.getContext())); | |||
| 157 | Value *ARep0 = | |||
| 158 | Builder.CreateBitCast(FloatVal0, Builder.getIntNTy(FloatWidth)); | |||
| 159 | Value *ARep = Builder.CreateZExt(ARep0, FPToI->getType()); | |||
| 160 | Value *PosOrNeg = Builder.CreateICmpSGT( | |||
| 161 | ARep0, ConstantInt::getSigned(Builder.getIntNTy(FloatWidth), -1)); | |||
| 162 | Value *Sign = Builder.CreateSelect(PosOrNeg, ConstantInt::getSigned(IntTy, 1), | |||
| 163 | ConstantInt::getSigned(IntTy, -1)); | |||
| 164 | Value *And = | |||
| 165 | Builder.CreateLShr(ARep, Builder.getIntN(BitWidth, FPMantissaWidth)); | |||
| 166 | Value *And2 = Builder.CreateAnd( | |||
| 167 | And, Builder.getIntN(BitWidth, (1 << ExponentWidth) - 1)); | |||
| 168 | Value *Abs = Builder.CreateAnd(ARep, SignificandMask); | |||
| 169 | Value *Or = Builder.CreateOr(Abs, ImplicitBit); | |||
| 170 | Value *Cmp = | |||
| 171 | Builder.CreateICmpULT(And2, Builder.getIntN(BitWidth, ExponentBias)); | |||
| 172 | Builder.CreateCondBr(Cmp, End, IfEnd); | |||
| 173 | ||||
| 174 | // if.end: | |||
| 175 | Builder.SetInsertPoint(IfEnd); | |||
| 176 | Value *Add1 = Builder.CreateAdd( | |||
| 177 | And2, ConstantInt::getSigned(IntTy, -int64_t(ExponentBias + BitWidth))); | |||
| 178 | Value *Cmp3 = | |||
| 179 | Builder.CreateICmpULT(Add1, ConstantInt::getSigned(IntTy, -BitWidth)); | |||
| 180 | Builder.CreateCondBr(Cmp3, IfThen5, IfEnd9); | |||
| 181 | ||||
| 182 | // if.then5: | |||
| 183 | Builder.SetInsertPoint(IfThen5); | |||
| 184 | Value *PosInf = Builder.CreateXor(NegOne, NegInf); | |||
| 185 | Value *Cond8 = Builder.CreateSelect(PosOrNeg, PosInf, NegInf); | |||
| 186 | Builder.CreateBr(End); | |||
| 187 | ||||
| 188 | // if.end9: | |||
| 189 | Builder.SetInsertPoint(IfEnd9); | |||
| 190 | Value *Cmp10 = Builder.CreateICmpULT( | |||
| 191 | And2, Builder.getIntN(BitWidth, ExponentBias + FPMantissaWidth)); | |||
| 192 | Builder.CreateCondBr(Cmp10, IfThen12, IfElse); | |||
| 193 | ||||
| 194 | // if.then12: | |||
| 195 | Builder.SetInsertPoint(IfThen12); | |||
| 196 | Value *Sub13 = Builder.CreateSub( | |||
| 197 | Builder.getIntN(BitWidth, ExponentBias + FPMantissaWidth), And2); | |||
| 198 | Value *Shr14 = Builder.CreateLShr(Or, Sub13); | |||
| 199 | Value *Mul = Builder.CreateMul(Shr14, Sign); | |||
| 200 | Builder.CreateBr(End); | |||
| 201 | ||||
| 202 | // if.else: | |||
| 203 | Builder.SetInsertPoint(IfElse); | |||
| 204 | Value *Sub15 = Builder.CreateAdd( | |||
| 205 | And2, | |||
| 206 | ConstantInt::getSigned(IntTy, -(ExponentBias + FPMantissaWidth))); | |||
| 207 | Value *Shl = Builder.CreateShl(Or, Sub15); | |||
| 208 | Value *Mul16 = Builder.CreateMul(Shl, Sign); | |||
| 209 | Builder.CreateBr(End); | |||
| 210 | ||||
| 211 | // cleanup: | |||
| 212 | Builder.SetInsertPoint(End, End->begin()); | |||
| 213 | PHINode *Retval0 = Builder.CreatePHI(FPToI->getType(), 4); | |||
| 214 | ||||
| 215 | Retval0->addIncoming(Cond8, IfThen5); | |||
| 216 | Retval0->addIncoming(Mul, IfThen12); | |||
| 217 | Retval0->addIncoming(Mul16, IfElse); | |||
| 218 | Retval0->addIncoming(Builder.getIntN(BitWidth, 0), Entry); | |||
| 219 | ||||
| 220 | FPToI->replaceAllUsesWith(Retval0); | |||
| 221 | FPToI->dropAllReferences(); | |||
| 222 | FPToI->eraseFromParent(); | |||
| 223 | } | |||
| 224 | ||||
| 225 | /// Generate code to convert a fp number to integer, replacing S(U)IToFP with | |||
| 226 | /// the generated code. This currently generates code similarly to compiler-rt's | |||
| 227 | /// implementations. This implementation has an implicit assumption that integer | |||
| 228 | /// width is larger than fp. | |||
| 229 | /// | |||
| 230 | /// An example IR generated from compiler-rt/floatdisf.c looks like below: | |||
| 231 | /// define dso_local float @__floatdisf(i64 noundef %a) local_unnamed_addr #0 { | |||
| 232 | /// entry: | |||
| 233 | /// %cmp = icmp eq i64 %a, 0 | |||
| 234 | /// br i1 %cmp, label %return, label %if.end | |||
| 235 | /// | |||
| 236 | /// if.end: ; preds = %entry | |||
| 237 | /// %shr = ashr i64 %a, 63 | |||
| 238 | /// %xor = xor i64 %shr, %a | |||
| 239 | /// %sub = sub nsw i64 %xor, %shr | |||
| 240 | /// %0 = tail call i64 @llvm.ctlz.i64(i64 %sub, i1 true), !range !5 | |||
| 241 | /// %cast = trunc i64 %0 to i32 | |||
| 242 | /// %sub1 = sub nuw nsw i32 64, %cast | |||
| 243 | /// %sub2 = xor i32 %cast, 63 | |||
| 244 | /// %cmp3 = icmp ult i32 %cast, 40 | |||
| 245 | /// br i1 %cmp3, label %if.then4, label %if.else | |||
| 246 | /// | |||
| 247 | /// if.then4: ; preds = %if.end | |||
| 248 | /// switch i32 %sub1, label %sw.default [ | |||
| 249 | /// i32 25, label %sw.bb | |||
| 250 | /// i32 26, label %sw.epilog | |||
| 251 | /// ] | |||
| 252 | /// | |||
| 253 | /// sw.bb: ; preds = %if.then4 | |||
| 254 | /// %shl = shl i64 %sub, 1 | |||
| 255 | /// br label %sw.epilog | |||
| 256 | /// | |||
| 257 | /// sw.default: ; preds = %if.then4 | |||
| 258 | /// %sub5 = sub nsw i64 38, %0 | |||
| 259 | /// %sh_prom = and i64 %sub5, 4294967295 | |||
| 260 | /// %shr6 = lshr i64 %sub, %sh_prom | |||
| 261 | /// %shr9 = lshr i64 274877906943, %0 | |||
| 262 | /// %and = and i64 %shr9, %sub | |||
| 263 | /// %cmp10 = icmp ne i64 %and, 0 | |||
| 264 | /// %conv11 = zext i1 %cmp10 to i64 | |||
| 265 | /// %or = or i64 %shr6, %conv11 | |||
| 266 | /// br label %sw.epilog | |||
| 267 | /// | |||
| 268 | /// sw.epilog: ; preds = %sw.default, %if.then4, %sw.bb | |||
| 269 | /// %a.addr.0 = phi i64 [ %or, %sw.default ], [ %sub, %if.then4 ], [ %shl, %sw.bb ] | |||
| 270 | /// %1 = lshr i64 %a.addr.0, 2 | |||
| 271 | /// %2 = and i64 %1, 1 | |||
| 272 | /// %or16 = or i64 %2, %a.addr.0 | |||
| 273 | /// %inc = add nsw i64 %or16, 1 | |||
| 274 | /// %3 = and i64 %inc, 67108864 | |||
| 275 | /// %tobool.not = icmp eq i64 %3, 0 | |||
| 276 | /// %spec.select.v = select i1 %tobool.not, i64 2, i64 3 | |||
| 277 | /// %spec.select = ashr i64 %inc, %spec.select.v | |||
| 278 | /// %spec.select56 = select i1 %tobool.not, i32 %sub2, i32 %sub1 | |||
| 279 | /// br label %if.end26 | |||
| 280 | /// | |||
| 281 | /// if.else: ; preds = %if.end | |||
| 282 | /// %sub23 = add nuw nsw i64 %0, 4294967256 | |||
| 283 | /// %sh_prom24 = and i64 %sub23, 4294967295 | |||
| 284 | /// %shl25 = shl i64 %sub, %sh_prom24 | |||
| 285 | /// br label %if.end26 | |||
| 286 | /// | |||
| 287 | /// if.end26: ; preds = %sw.epilog, %if.else | |||
| 288 | /// %a.addr.1 = phi i64 [ %shl25, %if.else ], [ %spec.select, %sw.epilog ] | |||
| 289 | /// %e.0 = phi i32 [ %sub2, %if.else ], [ %spec.select56, %sw.epilog ] | |||
| 290 | /// %conv27 = trunc i64 %shr to i32 | |||
| 291 | /// %and28 = and i32 %conv27, -2147483648 | |||
| 292 | /// %add = shl nuw nsw i32 %e.0, 23 | |||
| 293 | /// %shl29 = add nuw nsw i32 %add, 1065353216 | |||
| 294 | /// %conv31 = trunc i64 %a.addr.1 to i32 | |||
| 295 | /// %and32 = and i32 %conv31, 8388607 | |||
| 296 | /// %or30 = or i32 %and32, %and28 | |||
| 297 | /// %or33 = or i32 %or30, %shl29 | |||
| 298 | /// %4 = bitcast i32 %or33 to float | |||
| 299 | /// br label %return | |||
| 300 | /// | |||
| 301 | /// return: ; preds = %entry, %if.end26 | |||
| 302 | /// %retval.0 = phi float [ %4, %if.end26 ], [ 0.000000e+00, %entry ] | |||
| 303 | /// ret float %retval.0 | |||
| 304 | /// } | |||
| 305 | /// | |||
| 306 | /// Replace integer to fp with generated code. | |||
| 307 | static void expandIToFP(Instruction *IToFP) { | |||
| 308 | IRBuilder<> Builder(IToFP); | |||
| 309 | auto *IntVal = IToFP->getOperand(0); | |||
| 310 | IntegerType *IntTy = cast<IntegerType>(IntVal->getType()); | |||
| 311 | ||||
| 312 | unsigned BitWidth = IntVal->getType()->getIntegerBitWidth(); | |||
| 313 | unsigned FPMantissaWidth = IToFP->getType()->getFPMantissaWidth() - 1; | |||
| 314 | // fp80 conversion is implemented by conversion tp fp128 first following | |||
| 315 | // a fptrunc to fp80. | |||
| 316 | FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth; | |||
| 317 | // FIXME: As there is no related builtins added in compliler-rt, | |||
| 318 | // here currently utilized the fp32 <-> fp16 lib calls to implement. | |||
| 319 | FPMantissaWidth = FPMantissaWidth == 10 ? 23 : FPMantissaWidth; | |||
| 320 | unsigned FloatWidth = PowerOf2Ceil(FPMantissaWidth); | |||
| 321 | bool IsSigned = IToFP->getOpcode() == Instruction::SIToFP; | |||
| 322 | ||||
| 323 | assert(BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() "(static_cast <bool> (BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() " "assumes integer width is larger than fp." ) ? void (0) : __assert_fail ("BitWidth > FloatWidth && \"Unexpected conversion. expandIToFP() \" \"assumes integer width is larger than fp.\"" , "llvm/lib/CodeGen/ExpandLargeFpConvert.cpp", 324, __extension__ __PRETTY_FUNCTION__)) | |||
| 324 | "assumes integer width is larger than fp.")(static_cast <bool> (BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() " "assumes integer width is larger than fp." ) ? void (0) : __assert_fail ("BitWidth > FloatWidth && \"Unexpected conversion. expandIToFP() \" \"assumes integer width is larger than fp.\"" , "llvm/lib/CodeGen/ExpandLargeFpConvert.cpp", 324, __extension__ __PRETTY_FUNCTION__)); | |||
| 325 | ||||
| 326 | Value *Temp1 = | |||
| 327 | Builder.CreateShl(Builder.getIntN(BitWidth, 1), | |||
| 328 | Builder.getIntN(BitWidth, FPMantissaWidth + 3)); | |||
| 329 | ||||
| 330 | BasicBlock *Entry = Builder.GetInsertBlock(); | |||
| 331 | Function *F = Entry->getParent(); | |||
| 332 | Entry->setName(Twine(Entry->getName(), "itofp-entry")); | |||
| 333 | BasicBlock *End = | |||
| 334 | Entry->splitBasicBlock(Builder.GetInsertPoint(), "itofp-return"); | |||
| 335 | BasicBlock *IfEnd = | |||
| 336 | BasicBlock::Create(Builder.getContext(), "itofp-if-end", F, End); | |||
| 337 | BasicBlock *IfThen4 = | |||
| 338 | BasicBlock::Create(Builder.getContext(), "itofp-if-then4", F, End); | |||
| 339 | BasicBlock *SwBB = | |||
| 340 | BasicBlock::Create(Builder.getContext(), "itofp-sw-bb", F, End); | |||
| 341 | BasicBlock *SwDefault = | |||
| 342 | BasicBlock::Create(Builder.getContext(), "itofp-sw-default", F, End); | |||
| 343 | BasicBlock *SwEpilog = | |||
| 344 | BasicBlock::Create(Builder.getContext(), "itofp-sw-epilog", F, End); | |||
| 345 | BasicBlock *IfThen20 = | |||
| 346 | BasicBlock::Create(Builder.getContext(), "itofp-if-then20", F, End); | |||
| 347 | BasicBlock *IfElse = | |||
| 348 | BasicBlock::Create(Builder.getContext(), "itofp-if-else", F, End); | |||
| 349 | BasicBlock *IfEnd26 = | |||
| 350 | BasicBlock::Create(Builder.getContext(), "itofp-if-end26", F, End); | |||
| 351 | ||||
| 352 | Entry->getTerminator()->eraseFromParent(); | |||
| 353 | ||||
| 354 | Function *CTLZ = | |||
| 355 | Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, IntTy); | |||
| 356 | ConstantInt *True = Builder.getTrue(); | |||
| 357 | ||||
| 358 | // entry: | |||
| 359 | Builder.SetInsertPoint(Entry); | |||
| 360 | Value *Cmp = Builder.CreateICmpEQ(IntVal, ConstantInt::getSigned(IntTy, 0)); | |||
| 361 | Builder.CreateCondBr(Cmp, End, IfEnd); | |||
| 362 | ||||
| 363 | // if.end: | |||
| 364 | Builder.SetInsertPoint(IfEnd); | |||
| 365 | Value *Shr = | |||
| 366 | Builder.CreateAShr(IntVal, Builder.getIntN(BitWidth, BitWidth - 1)); | |||
| 367 | Value *Xor = Builder.CreateXor(Shr, IntVal); | |||
| 368 | Value *Sub = Builder.CreateSub(Xor, Shr); | |||
| 369 | Value *Call = Builder.CreateCall(CTLZ, {IsSigned ? Sub : IntVal, True}); | |||
| 370 | Value *Cast = Builder.CreateTrunc(Call, Builder.getInt32Ty()); | |||
| 371 | int BitWidthNew = FloatWidth == 128 ? BitWidth : 32; | |||
| 372 | Value *Sub1 = Builder.CreateSub(Builder.getIntN(BitWidthNew, BitWidth), | |||
| 373 | FloatWidth == 128 ? Call : Cast); | |||
| 374 | Value *Sub2 = Builder.CreateSub(Builder.getIntN(BitWidthNew, BitWidth - 1), | |||
| 375 | FloatWidth == 128 ? Call : Cast); | |||
| 376 | Value *Cmp3 = Builder.CreateICmpSGT( | |||
| 377 | Sub2, Builder.getIntN(BitWidthNew, FPMantissaWidth + 1)); | |||
| 378 | Builder.CreateCondBr(Cmp3, IfThen4, IfElse); | |||
| 379 | ||||
| 380 | // if.then4: | |||
| 381 | Builder.SetInsertPoint(IfThen4); | |||
| 382 | llvm::SwitchInst *SI = Builder.CreateSwitch(Sub1, SwDefault); | |||
| 383 | SI->addCase(Builder.getIntN(BitWidthNew, FPMantissaWidth + 2), SwBB); | |||
| 384 | SI->addCase(Builder.getIntN(BitWidthNew, FPMantissaWidth + 3), SwEpilog); | |||
| 385 | ||||
| 386 | // sw.bb: | |||
| 387 | Builder.SetInsertPoint(SwBB); | |||
| 388 | Value *Shl = | |||
| 389 | Builder.CreateShl(IsSigned ? Sub : IntVal, Builder.getIntN(BitWidth, 1)); | |||
| 390 | Builder.CreateBr(SwEpilog); | |||
| 391 | ||||
| 392 | // sw.default: | |||
| 393 | Builder.SetInsertPoint(SwDefault); | |||
| 394 | Value *Sub5 = Builder.CreateSub( | |||
| 395 | Builder.getIntN(BitWidthNew, BitWidth - FPMantissaWidth - 3), | |||
| 396 | FloatWidth == 128 ? Call : Cast); | |||
| 397 | Value *ShProm = Builder.CreateZExt(Sub5, IntTy); | |||
| 398 | Value *Shr6 = Builder.CreateLShr(IsSigned ? Sub : IntVal, | |||
| 399 | FloatWidth == 128 ? Sub5 : ShProm); | |||
| 400 | Value *Sub8 = | |||
| 401 | Builder.CreateAdd(FloatWidth == 128 ? Call : Cast, | |||
| 402 | Builder.getIntN(BitWidthNew, FPMantissaWidth + 3)); | |||
| 403 | Value *ShProm9 = Builder.CreateZExt(Sub8, IntTy); | |||
| 404 | Value *Shr9 = Builder.CreateLShr(ConstantInt::getSigned(IntTy, -1), | |||
| 405 | FloatWidth == 128 ? Sub8 : ShProm9); | |||
| 406 | Value *And = Builder.CreateAnd(Shr9, IsSigned ? Sub : IntVal); | |||
| 407 | Value *Cmp10 = Builder.CreateICmpNE(And, Builder.getIntN(BitWidth, 0)); | |||
| 408 | Value *Conv11 = Builder.CreateZExt(Cmp10, IntTy); | |||
| 409 | Value *Or = Builder.CreateOr(Shr6, Conv11); | |||
| 410 | Builder.CreateBr(SwEpilog); | |||
| 411 | ||||
| 412 | // sw.epilog: | |||
| 413 | Builder.SetInsertPoint(SwEpilog); | |||
| 414 | PHINode *AAddr0 = Builder.CreatePHI(IntTy, 3); | |||
| 415 | AAddr0->addIncoming(Or, SwDefault); | |||
| 416 | AAddr0->addIncoming(IsSigned ? Sub : IntVal, IfThen4); | |||
| 417 | AAddr0->addIncoming(Shl, SwBB); | |||
| 418 | Value *A0 = Builder.CreateTrunc(AAddr0, Builder.getInt32Ty()); | |||
| 419 | Value *A1 = Builder.CreateLShr(A0, Builder.getIntN(32, 2)); | |||
| 420 | Value *A2 = Builder.CreateAnd(A1, Builder.getIntN(32, 1)); | |||
| 421 | Value *Conv16 = Builder.CreateZExt(A2, IntTy); | |||
| 422 | Value *Or17 = Builder.CreateOr(AAddr0, Conv16); | |||
| 423 | Value *Inc = Builder.CreateAdd(Or17, Builder.getIntN(BitWidth, 1)); | |||
| 424 | Value *Shr18 = nullptr; | |||
| 425 | if (IsSigned) | |||
| 426 | Shr18 = Builder.CreateAShr(Inc, Builder.getIntN(BitWidth, 2)); | |||
| 427 | else | |||
| 428 | Shr18 = Builder.CreateLShr(Inc, Builder.getIntN(BitWidth, 2)); | |||
| 429 | Value *A3 = Builder.CreateAnd(Inc, Temp1, "a3"); | |||
| 430 | Value *PosOrNeg = Builder.CreateICmpEQ(A3, Builder.getIntN(BitWidth, 0)); | |||
| 431 | Value *ExtractT60 = Builder.CreateTrunc(Shr18, Builder.getIntNTy(FloatWidth)); | |||
| 432 | Value *Extract63 = Builder.CreateLShr(Shr18, Builder.getIntN(BitWidth, 32)); | |||
| 433 | Value *ExtractT64 = nullptr; | |||
| 434 | if (FloatWidth > 80) | |||
| 435 | ExtractT64 = Builder.CreateTrunc(Sub2, Builder.getInt64Ty()); | |||
| 436 | else | |||
| 437 | ExtractT64 = Builder.CreateTrunc(Extract63, Builder.getInt32Ty()); | |||
| 438 | Builder.CreateCondBr(PosOrNeg, IfEnd26, IfThen20); | |||
| 439 | ||||
| 440 | // if.then20 | |||
| 441 | Builder.SetInsertPoint(IfThen20); | |||
| 442 | Value *Shr21 = nullptr; | |||
| 443 | if (IsSigned) | |||
| 444 | Shr21 = Builder.CreateAShr(Inc, Builder.getIntN(BitWidth, 3)); | |||
| 445 | else | |||
| 446 | Shr21 = Builder.CreateLShr(Inc, Builder.getIntN(BitWidth, 3)); | |||
| 447 | Value *ExtractT = Builder.CreateTrunc(Shr21, Builder.getIntNTy(FloatWidth)); | |||
| 448 | Value *Extract = Builder.CreateLShr(Shr21, Builder.getIntN(BitWidth, 32)); | |||
| 449 | Value *ExtractT62 = nullptr; | |||
| 450 | if (FloatWidth > 80) | |||
| 451 | ExtractT62 = Builder.CreateTrunc(Sub1, Builder.getIntNTy(64)); | |||
| 452 | else | |||
| 453 | ExtractT62 = Builder.CreateTrunc(Extract, Builder.getIntNTy(32)); | |||
| 454 | Builder.CreateBr(IfEnd26); | |||
| 455 | ||||
| 456 | // if.else: | |||
| 457 | Builder.SetInsertPoint(IfElse); | |||
| 458 | Value *Sub24 = Builder.CreateAdd( | |||
| 459 | FloatWidth == 128 ? Call : Cast, | |||
| 460 | ConstantInt::getSigned(Builder.getIntNTy(BitWidthNew), | |||
| 461 | -(BitWidth - FPMantissaWidth - 1))); | |||
| 462 | Value *ShProm25 = Builder.CreateZExt(Sub24, IntTy); | |||
| 463 | Value *Shl26 = Builder.CreateShl(IsSigned ? Sub : IntVal, | |||
| 464 | FloatWidth == 128 ? Sub24 : ShProm25); | |||
| 465 | Value *ExtractT61 = Builder.CreateTrunc(Shl26, Builder.getIntNTy(FloatWidth)); | |||
| 466 | Value *Extract65 = Builder.CreateLShr(Shl26, Builder.getIntN(BitWidth, 32)); | |||
| 467 | Value *ExtractT66 = nullptr; | |||
| 468 | if (FloatWidth > 80) | |||
| 469 | ExtractT66 = Builder.CreateTrunc(Sub2, Builder.getIntNTy(64)); | |||
| 470 | else | |||
| 471 | ExtractT66 = Builder.CreateTrunc(Extract65, Builder.getInt32Ty()); | |||
| 472 | Builder.CreateBr(IfEnd26); | |||
| 473 | ||||
| 474 | // if.end26: | |||
| 475 | Builder.SetInsertPoint(IfEnd26); | |||
| 476 | PHINode *AAddr1Off0 = Builder.CreatePHI(Builder.getIntNTy(FloatWidth), 3); | |||
| 477 | AAddr1Off0->addIncoming(ExtractT, IfThen20); | |||
| 478 | AAddr1Off0->addIncoming(ExtractT60, SwEpilog); | |||
| 479 | AAddr1Off0->addIncoming(ExtractT61, IfElse); | |||
| 480 | PHINode *AAddr1Off32 = nullptr; | |||
| 481 | if (FloatWidth > 32) { | |||
| 482 | AAddr1Off32 = | |||
| 483 | Builder.CreatePHI(Builder.getIntNTy(FloatWidth > 80 ? 64 : 32), 3); | |||
| 484 | AAddr1Off32->addIncoming(ExtractT62, IfThen20); | |||
| 485 | AAddr1Off32->addIncoming(ExtractT64, SwEpilog); | |||
| 486 | AAddr1Off32->addIncoming(ExtractT66, IfElse); | |||
| 487 | } | |||
| 488 | PHINode *E0 = nullptr; | |||
| 489 | if (FloatWidth <= 80) { | |||
| 490 | E0 = Builder.CreatePHI(Builder.getIntNTy(BitWidthNew), 3); | |||
| 491 | E0->addIncoming(Sub1, IfThen20); | |||
| 492 | E0->addIncoming(Sub2, SwEpilog); | |||
| 493 | E0->addIncoming(Sub2, IfElse); | |||
| 494 | } | |||
| 495 | Value *And29 = nullptr; | |||
| 496 | if (FloatWidth > 80) { | |||
| 497 | Value *Temp2 = Builder.CreateShl(Builder.getIntN(BitWidth, 1), | |||
| 498 | Builder.getIntN(BitWidth, 63)); | |||
| 499 | And29 = Builder.CreateAnd(Shr, Temp2, "and29"); | |||
| 500 | } else { | |||
| 501 | Value *Conv28 = Builder.CreateTrunc(Shr, Builder.getIntNTy(32)); | |||
| 502 | And29 = Builder.CreateAnd( | |||
| 503 | Conv28, ConstantInt::getSigned(Builder.getIntNTy(32), 0x80000000)); | |||
| 504 | } | |||
| 505 | unsigned TempMod = FPMantissaWidth % 32; | |||
| 506 | Value *And34 = nullptr; | |||
| 507 | Value *Shl30 = nullptr; | |||
| 508 | if (FloatWidth > 80) { | |||
| 509 | TempMod += 32; | |||
| 510 | Value *Add = Builder.CreateShl(AAddr1Off32, Builder.getIntN(64, TempMod)); | |||
| 511 | Shl30 = Builder.CreateAdd( | |||
| 512 | Add, | |||
| 513 | Builder.getIntN(64, ((1ull << (62ull - TempMod)) - 1ull) << TempMod)); | |||
| 514 | And34 = Builder.CreateZExt(Shl30, Builder.getIntNTy(128)); | |||
| 515 | } else { | |||
| 516 | Value *Add = Builder.CreateShl(E0, Builder.getIntN(32, TempMod)); | |||
| 517 | Shl30 = Builder.CreateAdd( | |||
| 518 | Add, Builder.getIntN(32, ((1 << (30 - TempMod)) - 1) << TempMod)); | |||
| 519 | And34 = Builder.CreateAnd(FloatWidth > 32 ? AAddr1Off32 : AAddr1Off0, | |||
| 520 | Builder.getIntN(32, (1 << TempMod) - 1)); | |||
| 521 | } | |||
| 522 | Value *Or35 = nullptr; | |||
| 523 | if (FloatWidth > 80) { | |||
| 524 | Value *And29Trunc = Builder.CreateTrunc(And29, Builder.getIntNTy(128)); | |||
| 525 | Value *Or31 = Builder.CreateOr(And29Trunc, And34); | |||
| 526 | Value *Or34 = Builder.CreateShl(Or31, Builder.getIntN(128, 64)); | |||
| 527 | Value *Temp3 = Builder.CreateShl(Builder.getIntN(128, 1), | |||
| 528 | Builder.getIntN(128, FPMantissaWidth)); | |||
| 529 | Value *Temp4 = Builder.CreateSub(Temp3, Builder.getIntN(128, 1)); | |||
| 530 | Value *A6 = Builder.CreateAnd(AAddr1Off0, Temp4); | |||
| 531 | Or35 = Builder.CreateOr(Or34, A6); | |||
| 532 | } else { | |||
| 533 | Value *Or31 = Builder.CreateOr(And34, And29); | |||
| 534 | Or35 = Builder.CreateOr(IsSigned ? Or31 : And34, Shl30); | |||
| 535 | } | |||
| 536 | Value *A4 = nullptr; | |||
| 537 | if (IToFP->getType()->isDoubleTy()) { | |||
| 538 | Value *ZExt1 = Builder.CreateZExt(Or35, Builder.getIntNTy(FloatWidth)); | |||
| 539 | Value *Shl1 = Builder.CreateShl(ZExt1, Builder.getIntN(FloatWidth, 32)); | |||
| 540 | Value *And1 = | |||
| 541 | Builder.CreateAnd(AAddr1Off0, Builder.getIntN(FloatWidth, 0xFFFFFFFF)); | |||
| 542 | Value *Or1 = Builder.CreateOr(Shl1, And1); | |||
| 543 | A4 = Builder.CreateBitCast(Or1, IToFP->getType()); | |||
| 544 | } else if (IToFP->getType()->isX86_FP80Ty()) { | |||
| 545 | Value *A40 = | |||
| 546 | Builder.CreateBitCast(Or35, Type::getFP128Ty(Builder.getContext())); | |||
| 547 | A4 = Builder.CreateFPTrunc(A40, IToFP->getType()); | |||
| 548 | } else if (IToFP->getType()->isHalfTy()) { | |||
| 549 | // Deal with "half" situation. This is a workaround since we don't have | |||
| 550 | // floattihf.c currently as referring. | |||
| 551 | Value *A40 = | |||
| 552 | Builder.CreateBitCast(Or35, Type::getFloatTy(Builder.getContext())); | |||
| 553 | A4 = Builder.CreateFPTrunc(A40, IToFP->getType()); | |||
| 554 | } else // float type | |||
| 555 | A4 = Builder.CreateBitCast(Or35, IToFP->getType()); | |||
| 556 | Builder.CreateBr(End); | |||
| 557 | ||||
| 558 | // return: | |||
| 559 | Builder.SetInsertPoint(End, End->begin()); | |||
| 560 | PHINode *Retval0 = Builder.CreatePHI(IToFP->getType(), 2); | |||
| 561 | Retval0->addIncoming(A4, IfEnd26); | |||
| 562 | Retval0->addIncoming(ConstantFP::getZero(IToFP->getType(), false), Entry); | |||
| 563 | ||||
| 564 | IToFP->replaceAllUsesWith(Retval0); | |||
| 565 | IToFP->dropAllReferences(); | |||
| 566 | IToFP->eraseFromParent(); | |||
| 567 | } | |||
| 568 | ||||
| 569 | static bool runImpl(Function &F, const TargetLowering &TLI) { | |||
| 570 | SmallVector<Instruction *, 4> Replace; | |||
| 571 | bool Modified = false; | |||
| 572 | ||||
| 573 | unsigned MaxLegalFpConvertBitWidth = | |||
| 574 | TLI.getMaxLargeFPConvertBitWidthSupported(); | |||
| 575 | if (ExpandFpConvertBits != llvm::IntegerType::MAX_INT_BITS) | |||
| 576 | MaxLegalFpConvertBitWidth = ExpandFpConvertBits; | |||
| 577 | ||||
| 578 | if (MaxLegalFpConvertBitWidth >= llvm::IntegerType::MAX_INT_BITS) | |||
| 579 | return false; | |||
| 580 | ||||
| 581 | for (auto &I : instructions(F)) { | |||
| 582 | switch (I.getOpcode()) { | |||
| 583 | case Instruction::FPToUI: | |||
| 584 | case Instruction::FPToSI: { | |||
| 585 | // TODO: This pass doesn't handle vectors. | |||
| 586 | if (I.getOperand(0)->getType()->isVectorTy()) | |||
| 587 | continue; | |||
| 588 | ||||
| 589 | auto *IntTy = dyn_cast<IntegerType>(I.getType()); | |||
| 590 | if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) | |||
| ||||
| 591 | continue; | |||
| 592 | ||||
| 593 | Replace.push_back(&I); | |||
| 594 | Modified = true; | |||
| 595 | break; | |||
| 596 | } | |||
| 597 | case Instruction::UIToFP: | |||
| 598 | case Instruction::SIToFP: { | |||
| 599 | // TODO: This pass doesn't handle vectors. | |||
| 600 | if (I.getOperand(0)->getType()->isVectorTy()) | |||
| 601 | continue; | |||
| 602 | ||||
| 603 | auto *IntTy = dyn_cast<IntegerType>(I.getOperand(0)->getType()); | |||
| 604 | if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) | |||
| 605 | continue; | |||
| 606 | ||||
| 607 | Replace.push_back(&I); | |||
| 608 | Modified = true; | |||
| 609 | break; | |||
| 610 | } | |||
| 611 | default: | |||
| 612 | break; | |||
| 613 | } | |||
| 614 | } | |||
| 615 | ||||
| 616 | if (Replace.empty()) | |||
| 617 | return false; | |||
| 618 | ||||
| 619 | while (!Replace.empty()) { | |||
| 620 | Instruction *I = Replace.pop_back_val(); | |||
| 621 | if (I->getOpcode() == Instruction::FPToUI || | |||
| 622 | I->getOpcode() == Instruction::FPToSI) { | |||
| 623 | expandFPToI(I); | |||
| 624 | } else { | |||
| 625 | expandIToFP(I); | |||
| 626 | } | |||
| 627 | } | |||
| 628 | ||||
| 629 | return Modified; | |||
| 630 | } | |||
| 631 | ||||
| 632 | namespace { | |||
| 633 | class ExpandLargeFpConvertLegacyPass : public FunctionPass { | |||
| 634 | public: | |||
| 635 | static char ID; | |||
| 636 | ||||
| 637 | ExpandLargeFpConvertLegacyPass() : FunctionPass(ID) { | |||
| 638 | initializeExpandLargeFpConvertLegacyPassPass( | |||
| 639 | *PassRegistry::getPassRegistry()); | |||
| 640 | } | |||
| 641 | ||||
| 642 | bool runOnFunction(Function &F) override { | |||
| 643 | auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); | |||
| 644 | auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); | |||
| 645 | return runImpl(F, *TLI); | |||
| ||||
| 646 | } | |||
| 647 | ||||
| 648 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
| 649 | AU.addRequired<TargetPassConfig>(); | |||
| 650 | AU.addPreserved<AAResultsWrapperPass>(); | |||
| 651 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||
| 652 | } | |||
| 653 | }; | |||
| 654 | } // namespace | |||
| 655 | ||||
| 656 | char ExpandLargeFpConvertLegacyPass::ID = 0; | |||
| 657 | INITIALIZE_PASS_BEGIN(ExpandLargeFpConvertLegacyPass, "expand-large-fp-convert",static void *initializeExpandLargeFpConvertLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 658 | "Expand large fp convert", false, false)static void *initializeExpandLargeFpConvertLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 659 | INITIALIZE_PASS_END(ExpandLargeFpConvertLegacyPass, "expand-large-fp-convert",PassInfo *PI = new PassInfo( "Expand large fp convert", "expand-large-fp-convert" , &ExpandLargeFpConvertLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<ExpandLargeFpConvertLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeExpandLargeFpConvertLegacyPassPassFlag ; void llvm::initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeExpandLargeFpConvertLegacyPassPassFlag , initializeExpandLargeFpConvertLegacyPassPassOnce, std::ref( Registry)); } | |||
| 660 | "Expand large fp convert", false, false)PassInfo *PI = new PassInfo( "Expand large fp convert", "expand-large-fp-convert" , &ExpandLargeFpConvertLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<ExpandLargeFpConvertLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeExpandLargeFpConvertLegacyPassPassFlag ; void llvm::initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeExpandLargeFpConvertLegacyPassPassFlag , initializeExpandLargeFpConvertLegacyPassPassOnce, std::ref( Registry)); } | |||
| 661 | ||||
| 662 | FunctionPass *llvm::createExpandLargeFpConvertPass() { | |||
| 663 | return new ExpandLargeFpConvertLegacyPass(); | |||
| 664 | } |