File: | build/source/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp |
Warning: | line 590, column 11 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===--- ExpandLargeFpConvert.cpp - Expand large fp convert----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | ||||
10 | // This pass expands ‘fptoui .. to’, ‘fptosi .. to’, ‘uitofp .. to’, | |||
11 | // ‘sitofp .. to’ instructions with a bitwidth above a threshold into | |||
12 | // auto-generated functions. This is useful for targets like x86_64 that cannot | |||
13 | // lower fp convertions with more than 128 bits. | |||
14 | // | |||
15 | //===----------------------------------------------------------------------===// | |||
16 | ||||
17 | #include "llvm/ADT/SmallVector.h" | |||
18 | #include "llvm/ADT/StringExtras.h" | |||
19 | #include "llvm/Analysis/GlobalsModRef.h" | |||
20 | #include "llvm/CodeGen/Passes.h" | |||
21 | #include "llvm/CodeGen/TargetLowering.h" | |||
22 | #include "llvm/CodeGen/TargetPassConfig.h" | |||
23 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
24 | #include "llvm/IR/IRBuilder.h" | |||
25 | #include "llvm/IR/InstIterator.h" | |||
26 | #include "llvm/IR/PassManager.h" | |||
27 | #include "llvm/InitializePasses.h" | |||
28 | #include "llvm/Pass.h" | |||
29 | #include "llvm/Support/CommandLine.h" | |||
30 | #include "llvm/Target/TargetMachine.h" | |||
31 | ||||
32 | using namespace llvm; | |||
33 | ||||
34 | static cl::opt<unsigned> | |||
35 | ExpandFpConvertBits("expand-fp-convert-bits", cl::Hidden, | |||
36 | cl::init(llvm::IntegerType::MAX_INT_BITS), | |||
37 | cl::desc("fp convert instructions on integers with " | |||
38 | "more than <N> bits are expanded.")); | |||
39 | ||||
40 | /// Generate code to convert a fp number to integer, replacing FPToS(U)I with | |||
41 | /// the generated code. This currently generates code similarly to compiler-rt's | |||
42 | /// implementations. | |||
43 | /// | |||
44 | /// An example IR generated from compiler-rt/fixsfdi.c looks like below: | |||
45 | /// define dso_local i64 @foo(float noundef %a) local_unnamed_addr #0 { | |||
46 | /// entry: | |||
47 | /// %0 = bitcast float %a to i32 | |||
48 | /// %conv.i = zext i32 %0 to i64 | |||
49 | /// %tobool.not = icmp sgt i32 %0, -1 | |||
50 | /// %conv = select i1 %tobool.not, i64 1, i64 -1 | |||
51 | /// %and = lshr i64 %conv.i, 23 | |||
52 | /// %shr = and i64 %and, 255 | |||
53 | /// %and2 = and i64 %conv.i, 8388607 | |||
54 | /// %or = or i64 %and2, 8388608 | |||
55 | /// %cmp = icmp ult i64 %shr, 127 | |||
56 | /// br i1 %cmp, label %cleanup, label %if.end | |||
57 | /// | |||
58 | /// if.end: ; preds = %entry | |||
59 | /// %sub = add nuw nsw i64 %shr, 4294967169 | |||
60 | /// %conv5 = and i64 %sub, 4294967232 | |||
61 | /// %cmp6.not = icmp eq i64 %conv5, 0 | |||
62 | /// br i1 %cmp6.not, label %if.end12, label %if.then8 | |||
63 | /// | |||
64 | /// if.then8: ; preds = %if.end | |||
65 | /// %cond11 = select i1 %tobool.not, i64 9223372036854775807, i64 -9223372036854775808 | |||
66 | /// br label %cleanup | |||
67 | /// | |||
68 | /// if.end12: ; preds = %if.end | |||
69 | /// %cmp13 = icmp ult i64 %shr, 150 | |||
70 | /// br i1 %cmp13, label %if.then15, label %if.else | |||
71 | /// | |||
72 | /// if.then15: ; preds = %if.end12 | |||
73 | /// %sub16 = sub nuw nsw i64 150, %shr | |||
74 | /// %shr17 = lshr i64 %or, %sub16 | |||
75 | /// %mul = mul nsw i64 %shr17, %conv | |||
76 | /// br label %cleanup | |||
77 | /// | |||
78 | /// if.else: ; preds = %if.end12 | |||
79 | /// %sub18 = add nsw i64 %shr, -150 | |||
80 | /// %shl = shl i64 %or, %sub18 | |||
81 | /// %mul19 = mul nsw i64 %shl, %conv | |||
82 | /// br label %cleanup | |||
83 | /// | |||
84 | /// cleanup: ; preds = %entry, %if.else, %if.then15, %if.then8 | |||
85 | /// %retval.0 = phi i64 [ %cond11, %if.then8 ], [ %mul, %if.then15 ], [ %mul19, %if.else ], [ 0, %entry ] | |||
86 | /// ret i64 %retval.0 | |||
87 | /// } | |||
88 | /// | |||
89 | /// Replace fp to integer with generated code. | |||
90 | static void expandFPToI(Instruction *FPToI) { | |||
91 | IRBuilder<> Builder(FPToI); | |||
92 | auto *FloatVal = FPToI->getOperand(0); | |||
93 | IntegerType *IntTy = cast<IntegerType>(FPToI->getType()); | |||
94 | ||||
95 | unsigned BitWidth = FPToI->getType()->getIntegerBitWidth(); | |||
96 | unsigned FPMantissaWidth = FloatVal->getType()->getFPMantissaWidth() - 1; | |||
97 | ||||
98 | // FIXME: fp16's range is covered by i32. So `fptoi half` can convert | |||
99 | // to i32 first following a sext/zext to target integer type. | |||
100 | Value *A1 = nullptr; | |||
101 | if (FloatVal->getType()->isHalfTy()) { | |||
102 | if (FPToI->getOpcode() == Instruction::FPToUI) { | |||
103 | Value *A0 = Builder.CreateFPToUI(FloatVal, Builder.getIntNTy(32)); | |||
104 | A1 = Builder.CreateZExt(A0, IntTy); | |||
105 | } else { // FPToSI | |||
106 | Value *A0 = Builder.CreateFPToSI(FloatVal, Builder.getIntNTy(32)); | |||
107 | A1 = Builder.CreateSExt(A0, IntTy); | |||
108 | } | |||
109 | FPToI->replaceAllUsesWith(A1); | |||
110 | FPToI->dropAllReferences(); | |||
111 | FPToI->eraseFromParent(); | |||
112 | return; | |||
113 | } | |||
114 | ||||
115 | // fp80 conversion is implemented by fpext to fp128 first then do the | |||
116 | // conversion. | |||
117 | FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth; | |||
118 | unsigned FloatWidth = PowerOf2Ceil(FPMantissaWidth); | |||
119 | unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1; | |||
120 | unsigned ExponentBias = (1 << (ExponentWidth - 1)) - 1; | |||
121 | Value *ImplicitBit = Builder.CreateShl( | |||
122 | Builder.getIntN(BitWidth, 1), Builder.getIntN(BitWidth, FPMantissaWidth)); | |||
123 | Value *SignificandMask = | |||
124 | Builder.CreateSub(ImplicitBit, Builder.getIntN(BitWidth, 1)); | |||
125 | Value *NegOne = Builder.CreateSExt( | |||
126 | ConstantInt::getSigned(Builder.getInt32Ty(), -1), IntTy); | |||
127 | Value *NegInf = | |||
128 | Builder.CreateShl(ConstantInt::getSigned(IntTy, 1), | |||
129 | ConstantInt::getSigned(IntTy, BitWidth - 1)); | |||
130 | ||||
131 | BasicBlock *Entry = Builder.GetInsertBlock(); | |||
132 | Function *F = Entry->getParent(); | |||
133 | Entry->setName(Twine(Entry->getName(), "fp-to-i-entry")); | |||
134 | BasicBlock *End = | |||
135 | Entry->splitBasicBlock(Builder.GetInsertPoint(), "fp-to-i-cleanup"); | |||
136 | BasicBlock *IfEnd = | |||
137 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-end", F, End); | |||
138 | BasicBlock *IfThen5 = | |||
139 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-then5", F, End); | |||
140 | BasicBlock *IfEnd9 = | |||
141 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-end9", F, End); | |||
142 | BasicBlock *IfThen12 = | |||
143 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-then12", F, End); | |||
144 | BasicBlock *IfElse = | |||
145 | BasicBlock::Create(Builder.getContext(), "fp-to-i-if-else", F, End); | |||
146 | ||||
147 | Entry->getTerminator()->eraseFromParent(); | |||
148 | ||||
149 | // entry: | |||
150 | Builder.SetInsertPoint(Entry); | |||
151 | Value *FloatVal0 = FloatVal; | |||
152 | // fp80 conversion is implemented by fpext to fp128 first then do the | |||
153 | // conversion. | |||
154 | if (FloatVal->getType()->isX86_FP80Ty()) | |||
155 | FloatVal0 = | |||
156 | Builder.CreateFPExt(FloatVal, Type::getFP128Ty(Builder.getContext())); | |||
157 | Value *ARep0 = | |||
158 | Builder.CreateBitCast(FloatVal0, Builder.getIntNTy(FloatWidth)); | |||
159 | Value *ARep = Builder.CreateZExt(ARep0, FPToI->getType()); | |||
160 | Value *PosOrNeg = Builder.CreateICmpSGT( | |||
161 | ARep0, ConstantInt::getSigned(Builder.getIntNTy(FloatWidth), -1)); | |||
162 | Value *Sign = Builder.CreateSelect(PosOrNeg, ConstantInt::getSigned(IntTy, 1), | |||
163 | ConstantInt::getSigned(IntTy, -1)); | |||
164 | Value *And = | |||
165 | Builder.CreateLShr(ARep, Builder.getIntN(BitWidth, FPMantissaWidth)); | |||
166 | Value *And2 = Builder.CreateAnd( | |||
167 | And, Builder.getIntN(BitWidth, (1 << ExponentWidth) - 1)); | |||
168 | Value *Abs = Builder.CreateAnd(ARep, SignificandMask); | |||
169 | Value *Or = Builder.CreateOr(Abs, ImplicitBit); | |||
170 | Value *Cmp = | |||
171 | Builder.CreateICmpULT(And2, Builder.getIntN(BitWidth, ExponentBias)); | |||
172 | Builder.CreateCondBr(Cmp, End, IfEnd); | |||
173 | ||||
174 | // if.end: | |||
175 | Builder.SetInsertPoint(IfEnd); | |||
176 | Value *Add1 = Builder.CreateAdd( | |||
177 | And2, ConstantInt::getSigned(IntTy, -int64_t(ExponentBias + BitWidth))); | |||
178 | Value *Cmp3 = | |||
179 | Builder.CreateICmpULT(Add1, ConstantInt::getSigned(IntTy, -BitWidth)); | |||
180 | Builder.CreateCondBr(Cmp3, IfThen5, IfEnd9); | |||
181 | ||||
182 | // if.then5: | |||
183 | Builder.SetInsertPoint(IfThen5); | |||
184 | Value *PosInf = Builder.CreateXor(NegOne, NegInf); | |||
185 | Value *Cond8 = Builder.CreateSelect(PosOrNeg, PosInf, NegInf); | |||
186 | Builder.CreateBr(End); | |||
187 | ||||
188 | // if.end9: | |||
189 | Builder.SetInsertPoint(IfEnd9); | |||
190 | Value *Cmp10 = Builder.CreateICmpULT( | |||
191 | And2, Builder.getIntN(BitWidth, ExponentBias + FPMantissaWidth)); | |||
192 | Builder.CreateCondBr(Cmp10, IfThen12, IfElse); | |||
193 | ||||
194 | // if.then12: | |||
195 | Builder.SetInsertPoint(IfThen12); | |||
196 | Value *Sub13 = Builder.CreateSub( | |||
197 | Builder.getIntN(BitWidth, ExponentBias + FPMantissaWidth), And2); | |||
198 | Value *Shr14 = Builder.CreateLShr(Or, Sub13); | |||
199 | Value *Mul = Builder.CreateMul(Shr14, Sign); | |||
200 | Builder.CreateBr(End); | |||
201 | ||||
202 | // if.else: | |||
203 | Builder.SetInsertPoint(IfElse); | |||
204 | Value *Sub15 = Builder.CreateAdd( | |||
205 | And2, | |||
206 | ConstantInt::getSigned(IntTy, -(ExponentBias + FPMantissaWidth))); | |||
207 | Value *Shl = Builder.CreateShl(Or, Sub15); | |||
208 | Value *Mul16 = Builder.CreateMul(Shl, Sign); | |||
209 | Builder.CreateBr(End); | |||
210 | ||||
211 | // cleanup: | |||
212 | Builder.SetInsertPoint(End, End->begin()); | |||
213 | PHINode *Retval0 = Builder.CreatePHI(FPToI->getType(), 4); | |||
214 | ||||
215 | Retval0->addIncoming(Cond8, IfThen5); | |||
216 | Retval0->addIncoming(Mul, IfThen12); | |||
217 | Retval0->addIncoming(Mul16, IfElse); | |||
218 | Retval0->addIncoming(Builder.getIntN(BitWidth, 0), Entry); | |||
219 | ||||
220 | FPToI->replaceAllUsesWith(Retval0); | |||
221 | FPToI->dropAllReferences(); | |||
222 | FPToI->eraseFromParent(); | |||
223 | } | |||
224 | ||||
225 | /// Generate code to convert a fp number to integer, replacing S(U)IToFP with | |||
226 | /// the generated code. This currently generates code similarly to compiler-rt's | |||
227 | /// implementations. This implementation has an implicit assumption that integer | |||
228 | /// width is larger than fp. | |||
229 | /// | |||
230 | /// An example IR generated from compiler-rt/floatdisf.c looks like below: | |||
231 | /// define dso_local float @__floatdisf(i64 noundef %a) local_unnamed_addr #0 { | |||
232 | /// entry: | |||
233 | /// %cmp = icmp eq i64 %a, 0 | |||
234 | /// br i1 %cmp, label %return, label %if.end | |||
235 | /// | |||
236 | /// if.end: ; preds = %entry | |||
237 | /// %shr = ashr i64 %a, 63 | |||
238 | /// %xor = xor i64 %shr, %a | |||
239 | /// %sub = sub nsw i64 %xor, %shr | |||
240 | /// %0 = tail call i64 @llvm.ctlz.i64(i64 %sub, i1 true), !range !5 | |||
241 | /// %cast = trunc i64 %0 to i32 | |||
242 | /// %sub1 = sub nuw nsw i32 64, %cast | |||
243 | /// %sub2 = xor i32 %cast, 63 | |||
244 | /// %cmp3 = icmp ult i32 %cast, 40 | |||
245 | /// br i1 %cmp3, label %if.then4, label %if.else | |||
246 | /// | |||
247 | /// if.then4: ; preds = %if.end | |||
248 | /// switch i32 %sub1, label %sw.default [ | |||
249 | /// i32 25, label %sw.bb | |||
250 | /// i32 26, label %sw.epilog | |||
251 | /// ] | |||
252 | /// | |||
253 | /// sw.bb: ; preds = %if.then4 | |||
254 | /// %shl = shl i64 %sub, 1 | |||
255 | /// br label %sw.epilog | |||
256 | /// | |||
257 | /// sw.default: ; preds = %if.then4 | |||
258 | /// %sub5 = sub nsw i64 38, %0 | |||
259 | /// %sh_prom = and i64 %sub5, 4294967295 | |||
260 | /// %shr6 = lshr i64 %sub, %sh_prom | |||
261 | /// %shr9 = lshr i64 274877906943, %0 | |||
262 | /// %and = and i64 %shr9, %sub | |||
263 | /// %cmp10 = icmp ne i64 %and, 0 | |||
264 | /// %conv11 = zext i1 %cmp10 to i64 | |||
265 | /// %or = or i64 %shr6, %conv11 | |||
266 | /// br label %sw.epilog | |||
267 | /// | |||
268 | /// sw.epilog: ; preds = %sw.default, %if.then4, %sw.bb | |||
269 | /// %a.addr.0 = phi i64 [ %or, %sw.default ], [ %sub, %if.then4 ], [ %shl, %sw.bb ] | |||
270 | /// %1 = lshr i64 %a.addr.0, 2 | |||
271 | /// %2 = and i64 %1, 1 | |||
272 | /// %or16 = or i64 %2, %a.addr.0 | |||
273 | /// %inc = add nsw i64 %or16, 1 | |||
274 | /// %3 = and i64 %inc, 67108864 | |||
275 | /// %tobool.not = icmp eq i64 %3, 0 | |||
276 | /// %spec.select.v = select i1 %tobool.not, i64 2, i64 3 | |||
277 | /// %spec.select = ashr i64 %inc, %spec.select.v | |||
278 | /// %spec.select56 = select i1 %tobool.not, i32 %sub2, i32 %sub1 | |||
279 | /// br label %if.end26 | |||
280 | /// | |||
281 | /// if.else: ; preds = %if.end | |||
282 | /// %sub23 = add nuw nsw i64 %0, 4294967256 | |||
283 | /// %sh_prom24 = and i64 %sub23, 4294967295 | |||
284 | /// %shl25 = shl i64 %sub, %sh_prom24 | |||
285 | /// br label %if.end26 | |||
286 | /// | |||
287 | /// if.end26: ; preds = %sw.epilog, %if.else | |||
288 | /// %a.addr.1 = phi i64 [ %shl25, %if.else ], [ %spec.select, %sw.epilog ] | |||
289 | /// %e.0 = phi i32 [ %sub2, %if.else ], [ %spec.select56, %sw.epilog ] | |||
290 | /// %conv27 = trunc i64 %shr to i32 | |||
291 | /// %and28 = and i32 %conv27, -2147483648 | |||
292 | /// %add = shl nuw nsw i32 %e.0, 23 | |||
293 | /// %shl29 = add nuw nsw i32 %add, 1065353216 | |||
294 | /// %conv31 = trunc i64 %a.addr.1 to i32 | |||
295 | /// %and32 = and i32 %conv31, 8388607 | |||
296 | /// %or30 = or i32 %and32, %and28 | |||
297 | /// %or33 = or i32 %or30, %shl29 | |||
298 | /// %4 = bitcast i32 %or33 to float | |||
299 | /// br label %return | |||
300 | /// | |||
301 | /// return: ; preds = %entry, %if.end26 | |||
302 | /// %retval.0 = phi float [ %4, %if.end26 ], [ 0.000000e+00, %entry ] | |||
303 | /// ret float %retval.0 | |||
304 | /// } | |||
305 | /// | |||
306 | /// Replace integer to fp with generated code. | |||
307 | static void expandIToFP(Instruction *IToFP) { | |||
308 | IRBuilder<> Builder(IToFP); | |||
309 | auto *IntVal = IToFP->getOperand(0); | |||
310 | IntegerType *IntTy = cast<IntegerType>(IntVal->getType()); | |||
311 | ||||
312 | unsigned BitWidth = IntVal->getType()->getIntegerBitWidth(); | |||
313 | unsigned FPMantissaWidth = IToFP->getType()->getFPMantissaWidth() - 1; | |||
314 | // fp80 conversion is implemented by conversion tp fp128 first following | |||
315 | // a fptrunc to fp80. | |||
316 | FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth; | |||
317 | // FIXME: As there is no related builtins added in compliler-rt, | |||
318 | // here currently utilized the fp32 <-> fp16 lib calls to implement. | |||
319 | FPMantissaWidth = FPMantissaWidth == 10 ? 23 : FPMantissaWidth; | |||
320 | unsigned FloatWidth = PowerOf2Ceil(FPMantissaWidth); | |||
321 | bool IsSigned = IToFP->getOpcode() == Instruction::SIToFP; | |||
322 | ||||
323 | assert(BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() "(static_cast <bool> (BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() " "assumes integer width is larger than fp." ) ? void (0) : __assert_fail ("BitWidth > FloatWidth && \"Unexpected conversion. expandIToFP() \" \"assumes integer width is larger than fp.\"" , "llvm/lib/CodeGen/ExpandLargeFpConvert.cpp", 324, __extension__ __PRETTY_FUNCTION__)) | |||
324 | "assumes integer width is larger than fp.")(static_cast <bool> (BitWidth > FloatWidth && "Unexpected conversion. expandIToFP() " "assumes integer width is larger than fp." ) ? void (0) : __assert_fail ("BitWidth > FloatWidth && \"Unexpected conversion. expandIToFP() \" \"assumes integer width is larger than fp.\"" , "llvm/lib/CodeGen/ExpandLargeFpConvert.cpp", 324, __extension__ __PRETTY_FUNCTION__)); | |||
325 | ||||
326 | Value *Temp1 = | |||
327 | Builder.CreateShl(Builder.getIntN(BitWidth, 1), | |||
328 | Builder.getIntN(BitWidth, FPMantissaWidth + 3)); | |||
329 | ||||
330 | BasicBlock *Entry = Builder.GetInsertBlock(); | |||
331 | Function *F = Entry->getParent(); | |||
332 | Entry->setName(Twine(Entry->getName(), "itofp-entry")); | |||
333 | BasicBlock *End = | |||
334 | Entry->splitBasicBlock(Builder.GetInsertPoint(), "itofp-return"); | |||
335 | BasicBlock *IfEnd = | |||
336 | BasicBlock::Create(Builder.getContext(), "itofp-if-end", F, End); | |||
337 | BasicBlock *IfThen4 = | |||
338 | BasicBlock::Create(Builder.getContext(), "itofp-if-then4", F, End); | |||
339 | BasicBlock *SwBB = | |||
340 | BasicBlock::Create(Builder.getContext(), "itofp-sw-bb", F, End); | |||
341 | BasicBlock *SwDefault = | |||
342 | BasicBlock::Create(Builder.getContext(), "itofp-sw-default", F, End); | |||
343 | BasicBlock *SwEpilog = | |||
344 | BasicBlock::Create(Builder.getContext(), "itofp-sw-epilog", F, End); | |||
345 | BasicBlock *IfThen20 = | |||
346 | BasicBlock::Create(Builder.getContext(), "itofp-if-then20", F, End); | |||
347 | BasicBlock *IfElse = | |||
348 | BasicBlock::Create(Builder.getContext(), "itofp-if-else", F, End); | |||
349 | BasicBlock *IfEnd26 = | |||
350 | BasicBlock::Create(Builder.getContext(), "itofp-if-end26", F, End); | |||
351 | ||||
352 | Entry->getTerminator()->eraseFromParent(); | |||
353 | ||||
354 | Function *CTLZ = | |||
355 | Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, IntTy); | |||
356 | ConstantInt *True = Builder.getTrue(); | |||
357 | ||||
358 | // entry: | |||
359 | Builder.SetInsertPoint(Entry); | |||
360 | Value *Cmp = Builder.CreateICmpEQ(IntVal, ConstantInt::getSigned(IntTy, 0)); | |||
361 | Builder.CreateCondBr(Cmp, End, IfEnd); | |||
362 | ||||
363 | // if.end: | |||
364 | Builder.SetInsertPoint(IfEnd); | |||
365 | Value *Shr = | |||
366 | Builder.CreateAShr(IntVal, Builder.getIntN(BitWidth, BitWidth - 1)); | |||
367 | Value *Xor = Builder.CreateXor(Shr, IntVal); | |||
368 | Value *Sub = Builder.CreateSub(Xor, Shr); | |||
369 | Value *Call = Builder.CreateCall(CTLZ, {IsSigned ? Sub : IntVal, True}); | |||
370 | Value *Cast = Builder.CreateTrunc(Call, Builder.getInt32Ty()); | |||
371 | int BitWidthNew = FloatWidth == 128 ? BitWidth : 32; | |||
372 | Value *Sub1 = Builder.CreateSub(Builder.getIntN(BitWidthNew, BitWidth), | |||
373 | FloatWidth == 128 ? Call : Cast); | |||
374 | Value *Sub2 = Builder.CreateSub(Builder.getIntN(BitWidthNew, BitWidth - 1), | |||
375 | FloatWidth == 128 ? Call : Cast); | |||
376 | Value *Cmp3 = Builder.CreateICmpSGT( | |||
377 | Sub2, Builder.getIntN(BitWidthNew, FPMantissaWidth + 1)); | |||
378 | Builder.CreateCondBr(Cmp3, IfThen4, IfElse); | |||
379 | ||||
380 | // if.then4: | |||
381 | Builder.SetInsertPoint(IfThen4); | |||
382 | llvm::SwitchInst *SI = Builder.CreateSwitch(Sub1, SwDefault); | |||
383 | SI->addCase(Builder.getIntN(BitWidthNew, FPMantissaWidth + 2), SwBB); | |||
384 | SI->addCase(Builder.getIntN(BitWidthNew, FPMantissaWidth + 3), SwEpilog); | |||
385 | ||||
386 | // sw.bb: | |||
387 | Builder.SetInsertPoint(SwBB); | |||
388 | Value *Shl = | |||
389 | Builder.CreateShl(IsSigned ? Sub : IntVal, Builder.getIntN(BitWidth, 1)); | |||
390 | Builder.CreateBr(SwEpilog); | |||
391 | ||||
392 | // sw.default: | |||
393 | Builder.SetInsertPoint(SwDefault); | |||
394 | Value *Sub5 = Builder.CreateSub( | |||
395 | Builder.getIntN(BitWidthNew, BitWidth - FPMantissaWidth - 3), | |||
396 | FloatWidth == 128 ? Call : Cast); | |||
397 | Value *ShProm = Builder.CreateZExt(Sub5, IntTy); | |||
398 | Value *Shr6 = Builder.CreateLShr(IsSigned ? Sub : IntVal, | |||
399 | FloatWidth == 128 ? Sub5 : ShProm); | |||
400 | Value *Sub8 = | |||
401 | Builder.CreateAdd(FloatWidth == 128 ? Call : Cast, | |||
402 | Builder.getIntN(BitWidthNew, FPMantissaWidth + 3)); | |||
403 | Value *ShProm9 = Builder.CreateZExt(Sub8, IntTy); | |||
404 | Value *Shr9 = Builder.CreateLShr(ConstantInt::getSigned(IntTy, -1), | |||
405 | FloatWidth == 128 ? Sub8 : ShProm9); | |||
406 | Value *And = Builder.CreateAnd(Shr9, IsSigned ? Sub : IntVal); | |||
407 | Value *Cmp10 = Builder.CreateICmpNE(And, Builder.getIntN(BitWidth, 0)); | |||
408 | Value *Conv11 = Builder.CreateZExt(Cmp10, IntTy); | |||
409 | Value *Or = Builder.CreateOr(Shr6, Conv11); | |||
410 | Builder.CreateBr(SwEpilog); | |||
411 | ||||
412 | // sw.epilog: | |||
413 | Builder.SetInsertPoint(SwEpilog); | |||
414 | PHINode *AAddr0 = Builder.CreatePHI(IntTy, 3); | |||
415 | AAddr0->addIncoming(Or, SwDefault); | |||
416 | AAddr0->addIncoming(IsSigned ? Sub : IntVal, IfThen4); | |||
417 | AAddr0->addIncoming(Shl, SwBB); | |||
418 | Value *A0 = Builder.CreateTrunc(AAddr0, Builder.getInt32Ty()); | |||
419 | Value *A1 = Builder.CreateLShr(A0, Builder.getIntN(32, 2)); | |||
420 | Value *A2 = Builder.CreateAnd(A1, Builder.getIntN(32, 1)); | |||
421 | Value *Conv16 = Builder.CreateZExt(A2, IntTy); | |||
422 | Value *Or17 = Builder.CreateOr(AAddr0, Conv16); | |||
423 | Value *Inc = Builder.CreateAdd(Or17, Builder.getIntN(BitWidth, 1)); | |||
424 | Value *Shr18 = nullptr; | |||
425 | if (IsSigned) | |||
426 | Shr18 = Builder.CreateAShr(Inc, Builder.getIntN(BitWidth, 2)); | |||
427 | else | |||
428 | Shr18 = Builder.CreateLShr(Inc, Builder.getIntN(BitWidth, 2)); | |||
429 | Value *A3 = Builder.CreateAnd(Inc, Temp1, "a3"); | |||
430 | Value *PosOrNeg = Builder.CreateICmpEQ(A3, Builder.getIntN(BitWidth, 0)); | |||
431 | Value *ExtractT60 = Builder.CreateTrunc(Shr18, Builder.getIntNTy(FloatWidth)); | |||
432 | Value *Extract63 = Builder.CreateLShr(Shr18, Builder.getIntN(BitWidth, 32)); | |||
433 | Value *ExtractT64 = nullptr; | |||
434 | if (FloatWidth > 80) | |||
435 | ExtractT64 = Builder.CreateTrunc(Sub2, Builder.getInt64Ty()); | |||
436 | else | |||
437 | ExtractT64 = Builder.CreateTrunc(Extract63, Builder.getInt32Ty()); | |||
438 | Builder.CreateCondBr(PosOrNeg, IfEnd26, IfThen20); | |||
439 | ||||
440 | // if.then20 | |||
441 | Builder.SetInsertPoint(IfThen20); | |||
442 | Value *Shr21 = nullptr; | |||
443 | if (IsSigned) | |||
444 | Shr21 = Builder.CreateAShr(Inc, Builder.getIntN(BitWidth, 3)); | |||
445 | else | |||
446 | Shr21 = Builder.CreateLShr(Inc, Builder.getIntN(BitWidth, 3)); | |||
447 | Value *ExtractT = Builder.CreateTrunc(Shr21, Builder.getIntNTy(FloatWidth)); | |||
448 | Value *Extract = Builder.CreateLShr(Shr21, Builder.getIntN(BitWidth, 32)); | |||
449 | Value *ExtractT62 = nullptr; | |||
450 | if (FloatWidth > 80) | |||
451 | ExtractT62 = Builder.CreateTrunc(Sub1, Builder.getIntNTy(64)); | |||
452 | else | |||
453 | ExtractT62 = Builder.CreateTrunc(Extract, Builder.getIntNTy(32)); | |||
454 | Builder.CreateBr(IfEnd26); | |||
455 | ||||
456 | // if.else: | |||
457 | Builder.SetInsertPoint(IfElse); | |||
458 | Value *Sub24 = Builder.CreateAdd( | |||
459 | FloatWidth == 128 ? Call : Cast, | |||
460 | ConstantInt::getSigned(Builder.getIntNTy(BitWidthNew), | |||
461 | -(BitWidth - FPMantissaWidth - 1))); | |||
462 | Value *ShProm25 = Builder.CreateZExt(Sub24, IntTy); | |||
463 | Value *Shl26 = Builder.CreateShl(IsSigned ? Sub : IntVal, | |||
464 | FloatWidth == 128 ? Sub24 : ShProm25); | |||
465 | Value *ExtractT61 = Builder.CreateTrunc(Shl26, Builder.getIntNTy(FloatWidth)); | |||
466 | Value *Extract65 = Builder.CreateLShr(Shl26, Builder.getIntN(BitWidth, 32)); | |||
467 | Value *ExtractT66 = nullptr; | |||
468 | if (FloatWidth > 80) | |||
469 | ExtractT66 = Builder.CreateTrunc(Sub2, Builder.getIntNTy(64)); | |||
470 | else | |||
471 | ExtractT66 = Builder.CreateTrunc(Extract65, Builder.getInt32Ty()); | |||
472 | Builder.CreateBr(IfEnd26); | |||
473 | ||||
474 | // if.end26: | |||
475 | Builder.SetInsertPoint(IfEnd26); | |||
476 | PHINode *AAddr1Off0 = Builder.CreatePHI(Builder.getIntNTy(FloatWidth), 3); | |||
477 | AAddr1Off0->addIncoming(ExtractT, IfThen20); | |||
478 | AAddr1Off0->addIncoming(ExtractT60, SwEpilog); | |||
479 | AAddr1Off0->addIncoming(ExtractT61, IfElse); | |||
480 | PHINode *AAddr1Off32 = nullptr; | |||
481 | if (FloatWidth > 32) { | |||
482 | AAddr1Off32 = | |||
483 | Builder.CreatePHI(Builder.getIntNTy(FloatWidth > 80 ? 64 : 32), 3); | |||
484 | AAddr1Off32->addIncoming(ExtractT62, IfThen20); | |||
485 | AAddr1Off32->addIncoming(ExtractT64, SwEpilog); | |||
486 | AAddr1Off32->addIncoming(ExtractT66, IfElse); | |||
487 | } | |||
488 | PHINode *E0 = nullptr; | |||
489 | if (FloatWidth <= 80) { | |||
490 | E0 = Builder.CreatePHI(Builder.getIntNTy(BitWidthNew), 3); | |||
491 | E0->addIncoming(Sub1, IfThen20); | |||
492 | E0->addIncoming(Sub2, SwEpilog); | |||
493 | E0->addIncoming(Sub2, IfElse); | |||
494 | } | |||
495 | Value *And29 = nullptr; | |||
496 | if (FloatWidth > 80) { | |||
497 | Value *Temp2 = Builder.CreateShl(Builder.getIntN(BitWidth, 1), | |||
498 | Builder.getIntN(BitWidth, 63)); | |||
499 | And29 = Builder.CreateAnd(Shr, Temp2, "and29"); | |||
500 | } else { | |||
501 | Value *Conv28 = Builder.CreateTrunc(Shr, Builder.getIntNTy(32)); | |||
502 | And29 = Builder.CreateAnd( | |||
503 | Conv28, ConstantInt::getSigned(Builder.getIntNTy(32), 0x80000000)); | |||
504 | } | |||
505 | unsigned TempMod = FPMantissaWidth % 32; | |||
506 | Value *And34 = nullptr; | |||
507 | Value *Shl30 = nullptr; | |||
508 | if (FloatWidth > 80) { | |||
509 | TempMod += 32; | |||
510 | Value *Add = Builder.CreateShl(AAddr1Off32, Builder.getIntN(64, TempMod)); | |||
511 | Shl30 = Builder.CreateAdd( | |||
512 | Add, | |||
513 | Builder.getIntN(64, ((1ull << (62ull - TempMod)) - 1ull) << TempMod)); | |||
514 | And34 = Builder.CreateZExt(Shl30, Builder.getIntNTy(128)); | |||
515 | } else { | |||
516 | Value *Add = Builder.CreateShl(E0, Builder.getIntN(32, TempMod)); | |||
517 | Shl30 = Builder.CreateAdd( | |||
518 | Add, Builder.getIntN(32, ((1 << (30 - TempMod)) - 1) << TempMod)); | |||
519 | And34 = Builder.CreateAnd(FloatWidth > 32 ? AAddr1Off32 : AAddr1Off0, | |||
520 | Builder.getIntN(32, (1 << TempMod) - 1)); | |||
521 | } | |||
522 | Value *Or35 = nullptr; | |||
523 | if (FloatWidth > 80) { | |||
524 | Value *And29Trunc = Builder.CreateTrunc(And29, Builder.getIntNTy(128)); | |||
525 | Value *Or31 = Builder.CreateOr(And29Trunc, And34); | |||
526 | Value *Or34 = Builder.CreateShl(Or31, Builder.getIntN(128, 64)); | |||
527 | Value *Temp3 = Builder.CreateShl(Builder.getIntN(128, 1), | |||
528 | Builder.getIntN(128, FPMantissaWidth)); | |||
529 | Value *Temp4 = Builder.CreateSub(Temp3, Builder.getIntN(128, 1)); | |||
530 | Value *A6 = Builder.CreateAnd(AAddr1Off0, Temp4); | |||
531 | Or35 = Builder.CreateOr(Or34, A6); | |||
532 | } else { | |||
533 | Value *Or31 = Builder.CreateOr(And34, And29); | |||
534 | Or35 = Builder.CreateOr(IsSigned ? Or31 : And34, Shl30); | |||
535 | } | |||
536 | Value *A4 = nullptr; | |||
537 | if (IToFP->getType()->isDoubleTy()) { | |||
538 | Value *ZExt1 = Builder.CreateZExt(Or35, Builder.getIntNTy(FloatWidth)); | |||
539 | Value *Shl1 = Builder.CreateShl(ZExt1, Builder.getIntN(FloatWidth, 32)); | |||
540 | Value *And1 = | |||
541 | Builder.CreateAnd(AAddr1Off0, Builder.getIntN(FloatWidth, 0xFFFFFFFF)); | |||
542 | Value *Or1 = Builder.CreateOr(Shl1, And1); | |||
543 | A4 = Builder.CreateBitCast(Or1, IToFP->getType()); | |||
544 | } else if (IToFP->getType()->isX86_FP80Ty()) { | |||
545 | Value *A40 = | |||
546 | Builder.CreateBitCast(Or35, Type::getFP128Ty(Builder.getContext())); | |||
547 | A4 = Builder.CreateFPTrunc(A40, IToFP->getType()); | |||
548 | } else if (IToFP->getType()->isHalfTy()) { | |||
549 | // Deal with "half" situation. This is a workaround since we don't have | |||
550 | // floattihf.c currently as referring. | |||
551 | Value *A40 = | |||
552 | Builder.CreateBitCast(Or35, Type::getFloatTy(Builder.getContext())); | |||
553 | A4 = Builder.CreateFPTrunc(A40, IToFP->getType()); | |||
554 | } else // float type | |||
555 | A4 = Builder.CreateBitCast(Or35, IToFP->getType()); | |||
556 | Builder.CreateBr(End); | |||
557 | ||||
558 | // return: | |||
559 | Builder.SetInsertPoint(End, End->begin()); | |||
560 | PHINode *Retval0 = Builder.CreatePHI(IToFP->getType(), 2); | |||
561 | Retval0->addIncoming(A4, IfEnd26); | |||
562 | Retval0->addIncoming(ConstantFP::getZero(IToFP->getType(), false), Entry); | |||
563 | ||||
564 | IToFP->replaceAllUsesWith(Retval0); | |||
565 | IToFP->dropAllReferences(); | |||
566 | IToFP->eraseFromParent(); | |||
567 | } | |||
568 | ||||
569 | static bool runImpl(Function &F, const TargetLowering &TLI) { | |||
570 | SmallVector<Instruction *, 4> Replace; | |||
571 | bool Modified = false; | |||
572 | ||||
573 | unsigned MaxLegalFpConvertBitWidth = | |||
574 | TLI.getMaxLargeFPConvertBitWidthSupported(); | |||
575 | if (ExpandFpConvertBits != llvm::IntegerType::MAX_INT_BITS) | |||
576 | MaxLegalFpConvertBitWidth = ExpandFpConvertBits; | |||
577 | ||||
578 | if (MaxLegalFpConvertBitWidth >= llvm::IntegerType::MAX_INT_BITS) | |||
579 | return false; | |||
580 | ||||
581 | for (auto &I : instructions(F)) { | |||
582 | switch (I.getOpcode()) { | |||
583 | case Instruction::FPToUI: | |||
584 | case Instruction::FPToSI: { | |||
585 | // TODO: This pass doesn't handle vectors. | |||
586 | if (I.getOperand(0)->getType()->isVectorTy()) | |||
587 | continue; | |||
588 | ||||
589 | auto *IntTy = dyn_cast<IntegerType>(I.getType()); | |||
590 | if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) | |||
| ||||
591 | continue; | |||
592 | ||||
593 | Replace.push_back(&I); | |||
594 | Modified = true; | |||
595 | break; | |||
596 | } | |||
597 | case Instruction::UIToFP: | |||
598 | case Instruction::SIToFP: { | |||
599 | // TODO: This pass doesn't handle vectors. | |||
600 | if (I.getOperand(0)->getType()->isVectorTy()) | |||
601 | continue; | |||
602 | ||||
603 | auto *IntTy = dyn_cast<IntegerType>(I.getOperand(0)->getType()); | |||
604 | if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) | |||
605 | continue; | |||
606 | ||||
607 | Replace.push_back(&I); | |||
608 | Modified = true; | |||
609 | break; | |||
610 | } | |||
611 | default: | |||
612 | break; | |||
613 | } | |||
614 | } | |||
615 | ||||
616 | if (Replace.empty()) | |||
617 | return false; | |||
618 | ||||
619 | while (!Replace.empty()) { | |||
620 | Instruction *I = Replace.pop_back_val(); | |||
621 | if (I->getOpcode() == Instruction::FPToUI || | |||
622 | I->getOpcode() == Instruction::FPToSI) { | |||
623 | expandFPToI(I); | |||
624 | } else { | |||
625 | expandIToFP(I); | |||
626 | } | |||
627 | } | |||
628 | ||||
629 | return Modified; | |||
630 | } | |||
631 | ||||
632 | namespace { | |||
633 | class ExpandLargeFpConvertLegacyPass : public FunctionPass { | |||
634 | public: | |||
635 | static char ID; | |||
636 | ||||
637 | ExpandLargeFpConvertLegacyPass() : FunctionPass(ID) { | |||
638 | initializeExpandLargeFpConvertLegacyPassPass( | |||
639 | *PassRegistry::getPassRegistry()); | |||
640 | } | |||
641 | ||||
642 | bool runOnFunction(Function &F) override { | |||
643 | auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); | |||
644 | auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); | |||
645 | return runImpl(F, *TLI); | |||
| ||||
646 | } | |||
647 | ||||
648 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
649 | AU.addRequired<TargetPassConfig>(); | |||
650 | AU.addPreserved<AAResultsWrapperPass>(); | |||
651 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||
652 | } | |||
653 | }; | |||
654 | } // namespace | |||
655 | ||||
656 | char ExpandLargeFpConvertLegacyPass::ID = 0; | |||
657 | INITIALIZE_PASS_BEGIN(ExpandLargeFpConvertLegacyPass, "expand-large-fp-convert",static void *initializeExpandLargeFpConvertLegacyPassPassOnce (PassRegistry &Registry) { | |||
658 | "Expand large fp convert", false, false)static void *initializeExpandLargeFpConvertLegacyPassPassOnce (PassRegistry &Registry) { | |||
659 | INITIALIZE_PASS_END(ExpandLargeFpConvertLegacyPass, "expand-large-fp-convert",PassInfo *PI = new PassInfo( "Expand large fp convert", "expand-large-fp-convert" , &ExpandLargeFpConvertLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<ExpandLargeFpConvertLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeExpandLargeFpConvertLegacyPassPassFlag ; void llvm::initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeExpandLargeFpConvertLegacyPassPassFlag , initializeExpandLargeFpConvertLegacyPassPassOnce, std::ref( Registry)); } | |||
660 | "Expand large fp convert", false, false)PassInfo *PI = new PassInfo( "Expand large fp convert", "expand-large-fp-convert" , &ExpandLargeFpConvertLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<ExpandLargeFpConvertLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeExpandLargeFpConvertLegacyPassPassFlag ; void llvm::initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeExpandLargeFpConvertLegacyPassPassFlag , initializeExpandLargeFpConvertLegacyPassPassOnce, std::ref( Registry)); } | |||
661 | ||||
662 | FunctionPass *llvm::createExpandLargeFpConvertPass() { | |||
663 | return new ExpandLargeFpConvertLegacyPass(); | |||
664 | } |