/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp

1

//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This file implements the auto-upgrade helper functions.

10

// This is where deprecated IR intrinsics and other IR features are updated to

11

// current specifications.

12

//

13

//===----------------------------------------------------------------------===//

14

15

#include "llvm/IR/AutoUpgrade.h"

16

#include "llvm/ADT/StringSwitch.h"

17

#include "llvm/IR/Constants.h"

18

#include "llvm/IR/DIBuilder.h"

19

#include "llvm/IR/DebugInfo.h"

20

#include "llvm/IR/DiagnosticInfo.h"

21

#include "llvm/IR/Function.h"

22

#include "llvm/IR/IRBuilder.h"

23

#include "llvm/IR/InstVisitor.h"

24

#include "llvm/IR/Instruction.h"

25

#include "llvm/IR/IntrinsicInst.h"

26

#include "llvm/IR/Intrinsics.h"

27

#include "llvm/IR/IntrinsicsAArch64.h"

28

#include "llvm/IR/IntrinsicsARM.h"

29

#include "llvm/IR/IntrinsicsX86.h"

30

#include "llvm/IR/LLVMContext.h"

31

#include "llvm/IR/Module.h"

32

#include "llvm/IR/Verifier.h"

33

#include "llvm/Support/ErrorHandling.h"

34

#include "llvm/Support/Regex.h"

35

#include <cstring>

36

using namespace llvm;

37

38

static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }

39

40

// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have

41

// changed their type from v4f32 to v2i64.

42

static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,

43

Function *&NewFn) {

44

// Check whether this is an old version of the function, which received

45

// v4f32 arguments.

46

Type *Arg0Type = F->getFunctionType()->getParamType(0);

47

if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))

48

return false;

49

50

// Yes, it's old, replace it with new version.

51

rename(F);

52

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

53

return true;

54

}

55

56

// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask

57

// arguments have changed their type from i32 to i8.

58

static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,

59

Function *&NewFn) {

60

// Check that the last argument is an i32.

61

Type *LastArgType = F->getFunctionType()->getParamType(

62

F->getFunctionType()->getNumParams() - 1);

63

if (!LastArgType->isIntegerTy(32))

64

return false;

65

66

// Move this function aside and map down.

67

rename(F);

68

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

69

return true;

70

}

71

72

// Upgrade the declaration of fp compare intrinsics that change return type

73

// from scalar to vXi1 mask.

74

static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,

75

Function *&NewFn) {

76

// Check if the return type is a vector.

77

if (F->getReturnType()->isVectorTy())

78

return false;

79

80

rename(F);

81

NewFn = Intrinsic::getDeclaration(F->getParent(), IID);

82

return true;

83

}

84

85

static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {

86

// All of the intrinsics matches below should be marked with which llvm

87

// version started autoupgrading them. At some point in the future we would

88

// like to use this information to remove upgrade code for some older

89

// intrinsics. It is currently undecided how we will determine that future

90

// point.

91

if (Name == "addcarryx.u32" || // Added in 8.0

92

Name == "addcarryx.u64" || // Added in 8.0

93

Name == "addcarry.u32" || // Added in 8.0

94

Name == "addcarry.u64" || // Added in 8.0

95

Name == "subborrow.u32" || // Added in 8.0

96

Name == "subborrow.u64" || // Added in 8.0

97

Name.startswith("sse2.padds.") || // Added in 8.0

98

Name.startswith("sse2.psubs.") || // Added in 8.0

99

Name.startswith("sse2.paddus.") || // Added in 8.0

100

Name.startswith("sse2.psubus.") || // Added in 8.0

101

Name.startswith("avx2.padds.") || // Added in 8.0

102

Name.startswith("avx2.psubs.") || // Added in 8.0

103

Name.startswith("avx2.paddus.") || // Added in 8.0

104

Name.startswith("avx2.psubus.") || // Added in 8.0

105

Name.startswith("avx512.padds.") || // Added in 8.0

106

Name.startswith("avx512.psubs.") || // Added in 8.0

107

Name.startswith("avx512.mask.padds.") || // Added in 8.0

108

Name.startswith("avx512.mask.psubs.") || // Added in 8.0

109

Name.startswith("avx512.mask.paddus.") || // Added in 8.0

110

Name.startswith("avx512.mask.psubus.") || // Added in 8.0

111

Name=="ssse3.pabs.b.128" || // Added in 6.0

112

Name=="ssse3.pabs.w.128" || // Added in 6.0

113

Name=="ssse3.pabs.d.128" || // Added in 6.0

114

Name.startswith("fma4.vfmadd.s") || // Added in 7.0

115

Name.startswith("fma.vfmadd.") || // Added in 7.0

116

Name.startswith("fma.vfmsub.") || // Added in 7.0

117

Name.startswith("fma.vfmsubadd.") || // Added in 7.0

118

Name.startswith("fma.vfnmadd.") || // Added in 7.0

119

Name.startswith("fma.vfnmsub.") || // Added in 7.0

120

Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0

121

Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0

122

Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0

123

Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0

124

Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0

125

Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0

126

Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0

127

Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0

128

Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0

129

Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0

130

Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0

131

Name.startswith("avx512.mask.shuf.i") || // Added in 6.0

132

Name.startswith("avx512.mask.shuf.f") || // Added in 6.0

133

Name.startswith("avx512.kunpck") || //added in 6.0

134

Name.startswith("avx2.pabs.") || // Added in 6.0

135

Name.startswith("avx512.mask.pabs.") || // Added in 6.0

136

Name.startswith("avx512.broadcastm") || // Added in 6.0

137

Name == "sse.sqrt.ss" || // Added in 7.0

138

Name == "sse2.sqrt.sd" || // Added in 7.0

139

Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0

140

Name.startswith("avx.sqrt.p") || // Added in 7.0

141

Name.startswith("sse2.sqrt.p") || // Added in 7.0

142

Name.startswith("sse.sqrt.p") || // Added in 7.0

143

Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0

144

Name.startswith("sse2.pcmpeq.") || // Added in 3.1

145

Name.startswith("sse2.pcmpgt.") || // Added in 3.1

146

Name.startswith("avx2.pcmpeq.") || // Added in 3.1

147

Name.startswith("avx2.pcmpgt.") || // Added in 3.1

148

Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9

149

Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9

150

Name.startswith("avx.vperm2f128.") || // Added in 6.0

151

Name == "avx2.vperm2i128" || // Added in 6.0

152

Name == "sse.add.ss" || // Added in 4.0

153

Name == "sse2.add.sd" || // Added in 4.0

154

Name == "sse.sub.ss" || // Added in 4.0

155

Name == "sse2.sub.sd" || // Added in 4.0

156

Name == "sse.mul.ss" || // Added in 4.0

157

Name == "sse2.mul.sd" || // Added in 4.0

158

Name == "sse.div.ss" || // Added in 4.0

159

Name == "sse2.div.sd" || // Added in 4.0

160

Name == "sse41.pmaxsb" || // Added in 3.9

161

Name == "sse2.pmaxs.w" || // Added in 3.9

162

Name == "sse41.pmaxsd" || // Added in 3.9

163

Name == "sse2.pmaxu.b" || // Added in 3.9

164

Name == "sse41.pmaxuw" || // Added in 3.9

165

Name == "sse41.pmaxud" || // Added in 3.9

166

Name == "sse41.pminsb" || // Added in 3.9

167

Name == "sse2.pmins.w" || // Added in 3.9

168

Name == "sse41.pminsd" || // Added in 3.9

169

Name == "sse2.pminu.b" || // Added in 3.9

170

Name == "sse41.pminuw" || // Added in 3.9

171

Name == "sse41.pminud" || // Added in 3.9

172

Name == "avx512.kand.w" || // Added in 7.0

173

Name == "avx512.kandn.w" || // Added in 7.0

174

Name == "avx512.knot.w" || // Added in 7.0

175

Name == "avx512.kor.w" || // Added in 7.0

176

Name == "avx512.kxor.w" || // Added in 7.0

177

Name == "avx512.kxnor.w" || // Added in 7.0

178

Name == "avx512.kortestc.w" || // Added in 7.0

179

Name == "avx512.kortestz.w" || // Added in 7.0

180

Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0

181

Name.startswith("avx2.pmax") || // Added in 3.9

182

Name.startswith("avx2.pmin") || // Added in 3.9

183

Name.startswith("avx512.mask.pmax") || // Added in 4.0

184

Name.startswith("avx512.mask.pmin") || // Added in 4.0

185

Name.startswith("avx2.vbroadcast") || // Added in 3.8

186

Name.startswith("avx2.pbroadcast") || // Added in 3.8

187

Name.startswith("avx.vpermil.") || // Added in 3.1

188

Name.startswith("sse2.pshuf") || // Added in 3.9

189

Name.startswith("avx512.pbroadcast") || // Added in 3.9

190

Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9

191

Name.startswith("avx512.mask.movddup") || // Added in 3.9

192

Name.startswith("avx512.mask.movshdup") || // Added in 3.9

193

Name.startswith("avx512.mask.movsldup") || // Added in 3.9

194

Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9

195

Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9

196

Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9

197

Name.startswith("avx512.mask.shuf.p") || // Added in 4.0

198

Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9

199

Name.startswith("avx512.mask.perm.df.") || // Added in 3.9

200

Name.startswith("avx512.mask.perm.di.") || // Added in 3.9

201

Name.startswith("avx512.mask.punpckl") || // Added in 3.9

202

Name.startswith("avx512.mask.punpckh") || // Added in 3.9

203

Name.startswith("avx512.mask.unpckl.") || // Added in 3.9

204

Name.startswith("avx512.mask.unpckh.") || // Added in 3.9

205

Name.startswith("avx512.mask.pand.") || // Added in 3.9

206

Name.startswith("avx512.mask.pandn.") || // Added in 3.9

207

Name.startswith("avx512.mask.por.") || // Added in 3.9

208

Name.startswith("avx512.mask.pxor.") || // Added in 3.9

209

Name.startswith("avx512.mask.and.") || // Added in 3.9

210

Name.startswith("avx512.mask.andn.") || // Added in 3.9

211

Name.startswith("avx512.mask.or.") || // Added in 3.9

212

Name.startswith("avx512.mask.xor.") || // Added in 3.9

213

Name.startswith("avx512.mask.padd.") || // Added in 4.0

214

Name.startswith("avx512.mask.psub.") || // Added in 4.0

215

Name.startswith("avx512.mask.pmull.") || // Added in 4.0

216

Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0

217

Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0

218

Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0

219

Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0

220

Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0

221

Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0

222

Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0

223

Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0

224

Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0

225

Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0

226

Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0

227

Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0

228

Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0

229

Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0

230

Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0

231

Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0

232

Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0

233

Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0

234

Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0

235

Name == "avx512.cvtusi2sd" || // Added in 7.0

236

Name.startswith("avx512.mask.permvar.") || // Added in 7.0

237

Name == "sse2.pmulu.dq" || // Added in 7.0

238

Name == "sse41.pmuldq" || // Added in 7.0

239

Name == "avx2.pmulu.dq" || // Added in 7.0

240

Name == "avx2.pmul.dq" || // Added in 7.0

241

Name == "avx512.pmulu.dq.512" || // Added in 7.0

242

Name == "avx512.pmul.dq.512" || // Added in 7.0

243

Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0

244

Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0

245

Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0

246

Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0

247

Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0

248

Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0

249

Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0

250

Name.startswith("avx512.mask.packsswb.") || // Added in 5.0

251

Name.startswith("avx512.mask.packssdw.") || // Added in 5.0

252

Name.startswith("avx512.mask.packuswb.") || // Added in 5.0

253

Name.startswith("avx512.mask.packusdw.") || // Added in 5.0

254

Name.startswith("avx512.mask.cmp.b") || // Added in 5.0

255

Name.startswith("avx512.mask.cmp.d") || // Added in 5.0

256

Name.startswith("avx512.mask.cmp.q") || // Added in 5.0

257

Name.startswith("avx512.mask.cmp.w") || // Added in 5.0

258

Name.startswith("avx512.cmp.p") || // Added in 12.0

259

Name.startswith("avx512.mask.ucmp.") || // Added in 5.0

260

Name.startswith("avx512.cvtb2mask.") || // Added in 7.0

261

Name.startswith("avx512.cvtw2mask.") || // Added in 7.0

262

Name.startswith("avx512.cvtd2mask.") || // Added in 7.0

263

Name.startswith("avx512.cvtq2mask.") || // Added in 7.0

264

Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0

265

Name.startswith("avx512.mask.psll.d") || // Added in 4.0

266

Name.startswith("avx512.mask.psll.q") || // Added in 4.0

267

Name.startswith("avx512.mask.psll.w") || // Added in 4.0

268

Name.startswith("avx512.mask.psra.d") || // Added in 4.0

269

Name.startswith("avx512.mask.psra.q") || // Added in 4.0

270

Name.startswith("avx512.mask.psra.w") || // Added in 4.0

271

Name.startswith("avx512.mask.psrl.d") || // Added in 4.0

272

Name.startswith("avx512.mask.psrl.q") || // Added in 4.0

273

Name.startswith("avx512.mask.psrl.w") || // Added in 4.0

274

Name.startswith("avx512.mask.pslli") || // Added in 4.0

275

Name.startswith("avx512.mask.psrai") || // Added in 4.0

276

Name.startswith("avx512.mask.psrli") || // Added in 4.0

277

Name.startswith("avx512.mask.psllv") || // Added in 4.0

278

Name.startswith("avx512.mask.psrav") || // Added in 4.0

279

Name.startswith("avx512.mask.psrlv") || // Added in 4.0

280

Name.startswith("sse41.pmovsx") || // Added in 3.8

281

Name.startswith("sse41.pmovzx") || // Added in 3.9

282

Name.startswith("avx2.pmovsx") || // Added in 3.9

283

Name.startswith("avx2.pmovzx") || // Added in 3.9

284

Name.startswith("avx512.mask.pmovsx") || // Added in 4.0

285

Name.startswith("avx512.mask.pmovzx") || // Added in 4.0

286

Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0

287

Name.startswith("avx512.mask.pternlog.") || // Added in 7.0

288

Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0

289

Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0

290

Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0

291

Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0

292

Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0

293

Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0

294

Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0

295

Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0

296

Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0

297

Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0

298

Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0

299

Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0

300

Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0

301

Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0

302

Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0

303

Name.startswith("avx512.mask.vpshld.") || // Added in 7.0

304

Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0

305

Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0

306

Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0

307

Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0

308

Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0

309

Name.startswith("avx512.vpshld.") || // Added in 8.0

310

Name.startswith("avx512.vpshrd.") || // Added in 8.0

311

Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0

312

Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0

313

Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0

314

Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0

315

Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0

316

Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0

317

Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0

318

Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0

319

Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0

320

Name.startswith("avx512.mask.conflict.") || // Added in 9.0

321

Name == "avx512.mask.pmov.qd.256" || // Added in 9.0

322

Name == "avx512.mask.pmov.qd.512" || // Added in 9.0

323

Name == "avx512.mask.pmov.wb.256" || // Added in 9.0

324

Name == "avx512.mask.pmov.wb.512" || // Added in 9.0

325

Name == "sse.cvtsi2ss" || // Added in 7.0

326

Name == "sse.cvtsi642ss" || // Added in 7.0

327

Name == "sse2.cvtsi2sd" || // Added in 7.0

328

Name == "sse2.cvtsi642sd" || // Added in 7.0

329

Name == "sse2.cvtss2sd" || // Added in 7.0

330

Name == "sse2.cvtdq2pd" || // Added in 3.9

331

Name == "sse2.cvtdq2ps" || // Added in 7.0

332

Name == "sse2.cvtps2pd" || // Added in 3.9

333

Name == "avx.cvtdq2.pd.256" || // Added in 3.9

334

Name == "avx.cvtdq2.ps.256" || // Added in 7.0

335

Name == "avx.cvt.ps2.pd.256" || // Added in 3.9

336

Name.startswith("vcvtph2ps.") || // Added in 11.0

337

Name.startswith("avx.vinsertf128.") || // Added in 3.7

338

Name == "avx2.vinserti128" || // Added in 3.7

339

Name.startswith("avx512.mask.insert") || // Added in 4.0

340

Name.startswith("avx.vextractf128.") || // Added in 3.7

341

Name == "avx2.vextracti128" || // Added in 3.7

342

Name.startswith("avx512.mask.vextract") || // Added in 4.0

343

Name.startswith("sse4a.movnt.") || // Added in 3.9

344

Name.startswith("avx.movnt.") || // Added in 3.2

345

Name.startswith("avx512.storent.") || // Added in 3.9

346

Name == "sse41.movntdqa" || // Added in 5.0

347

Name == "avx2.movntdqa" || // Added in 5.0

348

Name == "avx512.movntdqa" || // Added in 5.0

349

Name == "sse2.storel.dq" || // Added in 3.9

350

Name.startswith("sse.storeu.") || // Added in 3.9

351

Name.startswith("sse2.storeu.") || // Added in 3.9

352

Name.startswith("avx.storeu.") || // Added in 3.9

353

Name.startswith("avx512.mask.storeu.") || // Added in 3.9

354

Name.startswith("avx512.mask.store.p") || // Added in 3.9

355

Name.startswith("avx512.mask.store.b.") || // Added in 3.9

356

Name.startswith("avx512.mask.store.w.") || // Added in 3.9

357

Name.startswith("avx512.mask.store.d.") || // Added in 3.9

358

Name.startswith("avx512.mask.store.q.") || // Added in 3.9

359

Name == "avx512.mask.store.ss" || // Added in 7.0

360

Name.startswith("avx512.mask.loadu.") || // Added in 3.9

361

Name.startswith("avx512.mask.load.") || // Added in 3.9

362

Name.startswith("avx512.mask.expand.load.") || // Added in 7.0

363

Name.startswith("avx512.mask.compress.store.") || // Added in 7.0

364

Name.startswith("avx512.mask.expand.b") || // Added in 9.0

365

Name.startswith("avx512.mask.expand.w") || // Added in 9.0

366

Name.startswith("avx512.mask.expand.d") || // Added in 9.0

367

Name.startswith("avx512.mask.expand.q") || // Added in 9.0

368

Name.startswith("avx512.mask.expand.p") || // Added in 9.0

369

Name.startswith("avx512.mask.compress.b") || // Added in 9.0

370

Name.startswith("avx512.mask.compress.w") || // Added in 9.0

371

Name.startswith("avx512.mask.compress.d") || // Added in 9.0

372

Name.startswith("avx512.mask.compress.q") || // Added in 9.0

373

Name.startswith("avx512.mask.compress.p") || // Added in 9.0

374

Name == "sse42.crc32.64.8" || // Added in 3.4

375

Name.startswith("avx.vbroadcast.s") || // Added in 3.5

376

Name.startswith("avx512.vbroadcast.s") || // Added in 7.0

377

Name.startswith("avx512.mask.palignr.") || // Added in 3.9

378

Name.startswith("avx512.mask.valign.") || // Added in 4.0

379

Name.startswith("sse2.psll.dq") || // Added in 3.7

380

Name.startswith("sse2.psrl.dq") || // Added in 3.7

381

Name.startswith("avx2.psll.dq") || // Added in 3.7

382

Name.startswith("avx2.psrl.dq") || // Added in 3.7

383

Name.startswith("avx512.psll.dq") || // Added in 3.9

384

Name.startswith("avx512.psrl.dq") || // Added in 3.9

385

Name == "sse41.pblendw" || // Added in 3.7

386

Name.startswith("sse41.blendp") || // Added in 3.7

387

Name.startswith("avx.blend.p") || // Added in 3.7

388

Name == "avx2.pblendw" || // Added in 3.7

389

Name.startswith("avx2.pblendd.") || // Added in 3.7

390

Name.startswith("avx.vbroadcastf128") || // Added in 4.0

391

Name == "avx2.vbroadcasti128" || // Added in 3.7

392

Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0

393

Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0

394

Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0

395

Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0

396

Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0

397

Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0

398

Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0

399

Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0

400

Name == "xop.vpcmov" || // Added in 3.8

401

Name == "xop.vpcmov.256" || // Added in 5.0

402

Name.startswith("avx512.mask.move.s") || // Added in 4.0

403

Name.startswith("avx512.cvtmask2") || // Added in 5.0

404

Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0

405

Name.startswith("xop.vprot") || // Added in 8.0

406

Name.startswith("avx512.prol") || // Added in 8.0

407

Name.startswith("avx512.pror") || // Added in 8.0

408

Name.startswith("avx512.mask.prorv.") || // Added in 8.0

409

Name.startswith("avx512.mask.pror.") || // Added in 8.0

410

Name.startswith("avx512.mask.prolv.") || // Added in 8.0

411

Name.startswith("avx512.mask.prol.") || // Added in 8.0

412

Name.startswith("avx512.ptestm") || //Added in 6.0

413

Name.startswith("avx512.ptestnm") || //Added in 6.0

414

Name.startswith("avx512.mask.pavg")) // Added in 6.0

415

return true;

416

417

return false;

418

}

419

420

static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,

421

Function *&NewFn) {

422

// Only handle intrinsics that start with "x86.".

423

if (!Name.startswith("x86."))

424

return false;

425

// Remove "x86." prefix.

426

Name = Name.substr(4);

427

428

if (ShouldUpgradeX86Intrinsic(F, Name)) {

429

NewFn = nullptr;

430

return true;

431

}

432

433

if (Name == "rdtscp") { // Added in 8.0

434

// If this intrinsic has 0 operands, it's the new version.

435

if (F->getFunctionType()->getNumParams() == 0)

436

return false;

437

438

rename(F);

439

NewFn = Intrinsic::getDeclaration(F->getParent(),

440

Intrinsic::x86_rdtscp);

441

return true;

442

}

443

444

// SSE4.1 ptest functions may have an old signature.

445

if (Name.startswith("sse41.ptest")) { // Added in 3.2

446

if (Name.substr(11) == "c")

447

return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);

448

if (Name.substr(11) == "z")

449

return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);

450

if (Name.substr(11) == "nzc")

451

return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);

452

}

453

// Several blend and other instructions with masks used the wrong number of

454

// bits.

455

if (Name == "sse41.insertps") // Added in 3.6

456

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,

457

NewFn);

458

if (Name == "sse41.dppd") // Added in 3.6

459

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,

460

NewFn);

461

if (Name == "sse41.dpps") // Added in 3.6

462

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,

463

NewFn);

464

if (Name == "sse41.mpsadbw") // Added in 3.6

465

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,

466

NewFn);

467

if (Name == "avx.dp.ps.256") // Added in 3.6

468

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,

469

NewFn);

470

if (Name == "avx2.mpsadbw") // Added in 3.6

471

return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,

472

NewFn);

473

if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0

474

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,

475

NewFn);

476

if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0

477

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,

478

NewFn);

479

if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0

480

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,

481

NewFn);

482

if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0

483

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,

484

NewFn);

485

if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0

486

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,

487

NewFn);

488

if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0

489

return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,

490

NewFn);

491

492

// frcz.ss/sd may need to have an argument dropped. Added in 3.2

493

if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {

494

rename(F);

495

NewFn = Intrinsic::getDeclaration(F->getParent(),

496

Intrinsic::x86_xop_vfrcz_ss);

497

return true;

498

}

499

if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {

500

rename(F);

501

NewFn = Intrinsic::getDeclaration(F->getParent(),

502

Intrinsic::x86_xop_vfrcz_sd);

503

return true;

504

}

505

// Upgrade any XOP PERMIL2 index operand still using a float/double vector.

506

if (Name.startswith("xop.vpermil2")) { // Added in 3.9

507

auto Idx = F->getFunctionType()->getParamType(2);

508

if (Idx->isFPOrFPVectorTy()) {

509

rename(F);

510

unsigned IdxSize = Idx->getPrimitiveSizeInBits();

511

unsigned EltSize = Idx->getScalarSizeInBits();

512

Intrinsic::ID Permil2ID;

513

if (EltSize == 64 && IdxSize == 128)

514

Permil2ID = Intrinsic::x86_xop_vpermil2pd;

515

else if (EltSize == 32 && IdxSize == 128)

516

Permil2ID = Intrinsic::x86_xop_vpermil2ps;

517

else if (EltSize == 64 && IdxSize == 256)

518

Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;

519

else

520

Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;

521

NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);

522

return true;

523

}

524

}

525

526

if (Name == "seh.recoverfp") {

527

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);

528

return true;

529

}

530

531

return false;

532

}

533

534

static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {

535

assert(F && "Illegal to upgrade a non-existent Function.")(static_cast <bool> (F && "Illegal to upgrade a non-existent Function."
) ? void (0) : __assert_fail ("F && \"Illegal to upgrade a non-existent Function.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 535, __extension__ __PRETTY_FUNCTION__));

536

537

// Quickly eliminate it, if it's not a candidate.

538

StringRef Name = F->getName();

539

if (Name.size() <= 8 || !Name.startswith("llvm."))

540

return false;

541

Name = Name.substr(5); // Strip off "llvm."

542

543

switch (Name[0]) {

544

default: break;

545

case 'a': {

546

if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {

547

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,

548

F->arg_begin()->getType());

549

return true;

550

}

551

if (Name.startswith("aarch64.neon.frintn")) {

552

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,

553

F->arg_begin()->getType());

554

return true;

555

}

556

if (Name.startswith("aarch64.neon.rbit")) {

557

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,

558

F->arg_begin()->getType());

559

return true;

560

}

561

if (Name.startswith("arm.neon.vclz")) {

562

Type* args[2] = {

563

F->arg_begin()->getType(),

564

Type::getInt1Ty(F->getContext())

565

};

566

// Can't use Intrinsic::getDeclaration here as it adds a ".i1" to

567

// the end of the name. Change name from llvm.arm.neon.vclz.* to

568

// llvm.ctlz.*

569

FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);

570

NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),

571

"llvm.ctlz." + Name.substr(14), F->getParent());

572

return true;

573

}

574

if (Name.startswith("arm.neon.vcnt")) {

575

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,

576

F->arg_begin()->getType());

577

return true;

578

}

579

static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");

580

if (vldRegex.match(Name)) {

581

auto fArgs = F->getFunctionType()->params();

582

SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());

583

// Can't use Intrinsic::getDeclaration here as the return types might

584

// then only be structurally equal.

585

FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);

586

NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),

587

"llvm." + Name + ".p0i8", F->getParent());

588

return true;

589

}

590

static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");

591

if (vstRegex.match(Name)) {

592

static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,

593

Intrinsic::arm_neon_vst2,

594

Intrinsic::arm_neon_vst3,

595

Intrinsic::arm_neon_vst4};

596

597

static const Intrinsic::ID StoreLaneInts[] = {

598

Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,

599

Intrinsic::arm_neon_vst4lane

600

};

601

602

auto fArgs = F->getFunctionType()->params();

603

Type *Tys[] = {fArgs[0], fArgs[1]};

604

if (Name.find("lane") == StringRef::npos)

605

NewFn = Intrinsic::getDeclaration(F->getParent(),

606

StoreInts[fArgs.size() - 3], Tys);

607

else

608

NewFn = Intrinsic::getDeclaration(F->getParent(),

609

StoreLaneInts[fArgs.size() - 5], Tys);

610

return true;

611

}

612

if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {

613

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);

614

return true;

615

}

616

if (Name.startswith("arm.neon.vqadds.")) {

617

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,

618

F->arg_begin()->getType());

619

return true;

620

}

621

if (Name.startswith("arm.neon.vqaddu.")) {

622

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,

623

F->arg_begin()->getType());

624

return true;

625

}

626

if (Name.startswith("arm.neon.vqsubs.")) {

627

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,

628

F->arg_begin()->getType());

629

return true;

630

}

631

if (Name.startswith("arm.neon.vqsubu.")) {

632

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,

633

F->arg_begin()->getType());

634

return true;

635

}

636

if (Name.startswith("aarch64.neon.addp")) {

637

if (F->arg_size() != 2)

638

break; // Invalid IR.

639

VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());

640

if (Ty && Ty->getElementType()->isFloatingPointTy()) {

641

NewFn = Intrinsic::getDeclaration(F->getParent(),

642

Intrinsic::aarch64_neon_faddp, Ty);

643

return true;

644

}

645

}

646

647

// Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8

648

// respectively

649

if ((Name.startswith("arm.neon.bfdot.") ||

650

Name.startswith("aarch64.neon.bfdot.")) &&

651

Name.endswith("i8")) {

652

Intrinsic::ID IID =

653

StringSwitch<Intrinsic::ID>(Name)

654

.Cases("arm.neon.bfdot.v2f32.v8i8",

655

"arm.neon.bfdot.v4f32.v16i8",

656

Intrinsic::arm_neon_bfdot)

657

.Cases("aarch64.neon.bfdot.v2f32.v8i8",

658

"aarch64.neon.bfdot.v4f32.v16i8",

659

Intrinsic::aarch64_neon_bfdot)

660

.Default(Intrinsic::not_intrinsic);

661

if (IID == Intrinsic::not_intrinsic)

662

break;

663

664

size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();

665

assert((OperandWidth == 64 || OperandWidth == 128) &&(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 666, __extension__ __PRETTY_FUNCTION__))

666

"Unexpected operand width")(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 666, __extension__ __PRETTY_FUNCTION__));

667

LLVMContext &Ctx = F->getParent()->getContext();

668

std::array<Type *, 2> Tys {{

669

F->getReturnType(),

670

FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)

671

}};

672

NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);

673

return true;

674

}

675

676

// Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore

677

// and accept v8bf16 instead of v16i8

678

if ((Name.startswith("arm.neon.bfm") ||

679

Name.startswith("aarch64.neon.bfm")) &&

680

Name.endswith(".v4f32.v16i8")) {

681

Intrinsic::ID IID =

682

StringSwitch<Intrinsic::ID>(Name)

683

.Case("arm.neon.bfmmla.v4f32.v16i8",

684

Intrinsic::arm_neon_bfmmla)

685

.Case("arm.neon.bfmlalb.v4f32.v16i8",

686

Intrinsic::arm_neon_bfmlalb)

687

.Case("arm.neon.bfmlalt.v4f32.v16i8",

688

Intrinsic::arm_neon_bfmlalt)

689

.Case("aarch64.neon.bfmmla.v4f32.v16i8",

690

Intrinsic::aarch64_neon_bfmmla)

691

.Case("aarch64.neon.bfmlalb.v4f32.v16i8",

692

Intrinsic::aarch64_neon_bfmlalb)

693

.Case("aarch64.neon.bfmlalt.v4f32.v16i8",

694

Intrinsic::aarch64_neon_bfmlalt)

695

.Default(Intrinsic::not_intrinsic);

696

if (IID == Intrinsic::not_intrinsic)

697

break;

698

699

std::array<Type *, 0> Tys;

700

NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);

701

return true;

702

}

703

break;

704

}

705

706

case 'c': {

707

if (Name.startswith("ctlz.") && F->arg_size() == 1) {

708

rename(F);

709

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,

710

F->arg_begin()->getType());

711

return true;

712

}

713

if (Name.startswith("cttz.") && F->arg_size() == 1) {

714

rename(F);

715

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,

716

F->arg_begin()->getType());

717

return true;

718

}

719

break;

720

}

721

case 'd': {

722

if (Name == "dbg.value" && F->arg_size() == 4) {

723

rename(F);

724

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);

725

return true;

726

}

727

break;

728

}

729

case 'e': {

730

SmallVector<StringRef, 2> Groups;

731

static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");

732

if (R.match(Name, &Groups)) {

733

Intrinsic::ID ID;

734

ID = StringSwitch<Intrinsic::ID>(Groups[1])

735

.Case("add", Intrinsic::vector_reduce_add)

736

.Case("mul", Intrinsic::vector_reduce_mul)

737

.Case("and", Intrinsic::vector_reduce_and)

738

.Case("or", Intrinsic::vector_reduce_or)

739

.Case("xor", Intrinsic::vector_reduce_xor)

740

.Case("smax", Intrinsic::vector_reduce_smax)

741

.Case("smin", Intrinsic::vector_reduce_smin)

742

.Case("umax", Intrinsic::vector_reduce_umax)

743

.Case("umin", Intrinsic::vector_reduce_umin)

744

.Case("fmax", Intrinsic::vector_reduce_fmax)

745

.Case("fmin", Intrinsic::vector_reduce_fmin)

746

.Default(Intrinsic::not_intrinsic);

747

if (ID != Intrinsic::not_intrinsic) {

748

rename(F);

749

auto Args = F->getFunctionType()->params();

750

NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});

751

return true;

752

}

753

}

754

static const Regex R2(

755

"^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");

756

Groups.clear();

757

if (R2.match(Name, &Groups)) {

758

Intrinsic::ID ID = Intrinsic::not_intrinsic;

759

if (Groups[1] == "fadd")

760

ID = Intrinsic::vector_reduce_fadd;

761

if (Groups[1] == "fmul")

762

ID = Intrinsic::vector_reduce_fmul;

763

if (ID != Intrinsic::not_intrinsic) {

764

rename(F);

765

auto Args = F->getFunctionType()->params();

766

Type *Tys[] = {Args[1]};

767

NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);

768

return true;

769

}

770

}

771

break;

772

}

773

case 'i':

774

case 'l': {

775

bool IsLifetimeStart = Name.startswith("lifetime.start");

776

if (IsLifetimeStart || Name.startswith("invariant.start")) {

777

Intrinsic::ID ID = IsLifetimeStart ?

778

Intrinsic::lifetime_start : Intrinsic::invariant_start;

779

auto Args = F->getFunctionType()->params();

780

Type* ObjectPtr[1] = {Args[1]};

781

if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {

782

rename(F);

783

NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);

784

return true;

785

}

786

}

787

788

bool IsLifetimeEnd = Name.startswith("lifetime.end");

789

if (IsLifetimeEnd || Name.startswith("invariant.end")) {

790

Intrinsic::ID ID = IsLifetimeEnd ?

791

Intrinsic::lifetime_end : Intrinsic::invariant_end;

792

793

auto Args = F->getFunctionType()->params();

794

Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};

795

if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) {

796

rename(F);

797

NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);

798

return true;

799

}

800

}

801

if (Name.startswith("invariant.group.barrier")) {

802

// Rename invariant.group.barrier to launder.invariant.group

803

auto Args = F->getFunctionType()->params();

804

Type* ObjectPtr[1] = {Args[0]};

805

rename(F);

806

NewFn = Intrinsic::getDeclaration(F->getParent(),

807

Intrinsic::launder_invariant_group, ObjectPtr);

808

return true;

809

810

}

811

812

break;

813

}

814

case 'm': {

815

if (Name.startswith("masked.load.")) {

816

Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };

817

if (F->getName() !=

818

Intrinsic::getName(Intrinsic::masked_load, Tys, F->getParent())) {

819

rename(F);

820

NewFn = Intrinsic::getDeclaration(F->getParent(),

821

Intrinsic::masked_load,

822

Tys);

823

return true;

824

}

825

}

826

if (Name.startswith("masked.store.")) {

827

auto Args = F->getFunctionType()->params();

828

Type *Tys[] = { Args[0], Args[1] };

829

if (F->getName() !=

830

Intrinsic::getName(Intrinsic::masked_store, Tys, F->getParent())) {

831

rename(F);

832

NewFn = Intrinsic::getDeclaration(F->getParent(),

833

Intrinsic::masked_store,

834

Tys);

835

return true;

836

}

837

}

838

// Renaming gather/scatter intrinsics with no address space overloading

839

// to the new overload which includes an address space

840

if (Name.startswith("masked.gather.")) {

841

Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};

842

if (F->getName() !=

843

Intrinsic::getName(Intrinsic::masked_gather, Tys, F->getParent())) {

844

rename(F);

845

NewFn = Intrinsic::getDeclaration(F->getParent(),

846

Intrinsic::masked_gather, Tys);

847

return true;

848

}

849

}

850

if (Name.startswith("masked.scatter.")) {

851

auto Args = F->getFunctionType()->params();

852

Type *Tys[] = {Args[0], Args[1]};

853

if (F->getName() !=

854

Intrinsic::getName(Intrinsic::masked_scatter, Tys, F->getParent())) {

855

rename(F);

856

NewFn = Intrinsic::getDeclaration(F->getParent(),

857

Intrinsic::masked_scatter, Tys);

858

return true;

859

}

860

}

861

// Updating the memory intrinsics (memcpy/memmove/memset) that have an

862

// alignment parameter to embedding the alignment as an attribute of

863

// the pointer args.

864

if (Name.startswith("memcpy.") && F->arg_size() == 5) {

865

rename(F);

866

// Get the types of dest, src, and len

867

ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);

868

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,

869

ParamTypes);

870

return true;

871

}

872

if (Name.startswith("memmove.") && F->arg_size() == 5) {

873

rename(F);

874

// Get the types of dest, src, and len

875

ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);

876

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,

877

ParamTypes);

878

return true;

879

}

880

if (Name.startswith("memset.") && F->arg_size() == 5) {

881

rename(F);

882

// Get the types of dest, and len

883

const auto *FT = F->getFunctionType();

884

Type *ParamTypes[2] = {

885

FT->getParamType(0), // Dest

886

FT->getParamType(2) // len

887

};

888

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,

889

ParamTypes);

890

return true;

891

}

892

break;

893

}

894

case 'n': {

895

if (Name.startswith("nvvm.")) {

896

Name = Name.substr(5);

897

898

// The following nvvm intrinsics correspond exactly to an LLVM intrinsic.

899

Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)

900

.Cases("brev32", "brev64", Intrinsic::bitreverse)

901

.Case("clz.i", Intrinsic::ctlz)

902

.Case("popc.i", Intrinsic::ctpop)

903

.Default(Intrinsic::not_intrinsic);

904

if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {

905

NewFn = Intrinsic::getDeclaration(F->getParent(), IID,

906

{F->getReturnType()});

907

return true;

908

}

909

910

// The following nvvm intrinsics correspond exactly to an LLVM idiom, but

911

// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.

912

//

913

// TODO: We could add lohi.i2d.

914

bool Expand = StringSwitch<bool>(Name)

915

.Cases("abs.i", "abs.ll", true)

916

.Cases("clz.ll", "popc.ll", "h2f", true)

917

.Cases("max.i", "max.ll", "max.ui", "max.ull", true)

918

.Cases("min.i", "min.ll", "min.ui", "min.ull", true)

919

.StartsWith("atomic.load.add.f32.p", true)

920

.StartsWith("atomic.load.add.f64.p", true)

921

.Default(false);

922

if (Expand) {

923

NewFn = nullptr;

924

return true;

925

}

926

}

927

break;

928

}

929

case 'o':

930

// We only need to change the name to match the mangling including the

931

// address space.

932

if (Name.startswith("objectsize.")) {

933

Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };

934

if (F->arg_size() == 2 || F->arg_size() == 3 ||

935

F->getName() !=

936

Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) {

937

rename(F);

938

NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,

939

Tys);

940

return true;

941

}

942

}

943

break;

944

945

case 'p':

946

if (Name == "prefetch") {

947

// Handle address space overloading.

948

Type *Tys[] = {F->arg_begin()->getType()};

949

if (F->getName() !=

950

Intrinsic::getName(Intrinsic::prefetch, Tys, F->getParent())) {

951

rename(F);

952

NewFn =

953

Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);

954

return true;

955

}

956

} else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {

957

rename(F);

958

NewFn = Intrinsic::getDeclaration(F->getParent(),

959

Intrinsic::ptr_annotation,

960

F->arg_begin()->getType());

961

return true;

962

}

963

break;

964

965

case 's':

966

if (Name == "stackprotectorcheck") {

967

NewFn = nullptr;

968

return true;

969

}

970

break;

971

972

case 'v': {

973

if (Name == "var.annotation" && F->arg_size() == 4) {

974

rename(F);

975

NewFn = Intrinsic::getDeclaration(F->getParent(),

976

Intrinsic::var_annotation);

977

return true;

978

}

979

break;

980

}

981

982

case 'x':

983

if (UpgradeX86IntrinsicFunction(F, Name, NewFn))

984

return true;

985

}

986

// Remangle our intrinsic since we upgrade the mangling

987

auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);

988

if (Result != None) {

989

NewFn = Result.getValue();

990

return true;

991

}

992

993

// This may not belong here. This function is effectively being overloaded

994

// to both detect an intrinsic which needs upgrading, and to provide the

995

// upgraded form of the intrinsic. We should perhaps have two separate

996

// functions for this.

997

return false;

998

}

999

1000

bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {

1001

NewFn = nullptr;

1002

bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);

1003

assert(F != NewFn && "Intrinsic function upgraded to the same function")(static_cast <bool> (F != NewFn && "Intrinsic function upgraded to the same function"
) ? void (0) : __assert_fail ("F != NewFn && \"Intrinsic function upgraded to the same function\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1003, __extension__ __PRETTY_FUNCTION__));

1004

1005

// Upgrade intrinsic attributes. This does not change the function.

1006

if (NewFn)

1007

F = NewFn;

1008

if (Intrinsic::ID id = F->getIntrinsicID())

1009

F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));

1010

return Upgraded;

1011

}

1012

1013

GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {

1014

if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||

1015

GV->getName() == "llvm.global_dtors")) ||

1016

!GV->hasInitializer())

1017

return nullptr;

1018

ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());

1019

if (!ATy)

1020

return nullptr;

1021

StructType *STy = dyn_cast<StructType>(ATy->getElementType());

1022

if (!STy || STy->getNumElements() != 2)

1023

return nullptr;

1024

1025

LLVMContext &C = GV->getContext();

1026

IRBuilder<> IRB(C);

1027

auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),

1028

IRB.getInt8PtrTy());

1029

Constant *Init = GV->getInitializer();

1030

unsigned N = Init->getNumOperands();

1031

std::vector<Constant *> NewCtors(N);

1032

for (unsigned i = 0; i != N; ++i) {

1033

auto Ctor = cast<Constant>(Init->getOperand(i));

1034

NewCtors[i] = ConstantStruct::get(

1035

EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),

1036

Constant::getNullValue(IRB.getInt8PtrTy()));

1037

}

1038

Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);

1039

1040

return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),

1041

NewInit, GV->getName());

1042

}

1043

1044

// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them

1045

// to byte shuffles.

1046

static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,

1047

Value *Op, unsigned Shift) {

1048

auto *ResultTy = cast<FixedVectorType>(Op->getType());

1049

unsigned NumElts = ResultTy->getNumElements() * 8;

1050

1051

// Bitcast from a 64-bit element type to a byte element type.

1052

Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);

1053

Op = Builder.CreateBitCast(Op, VecTy, "cast");

1054

1055

// We'll be shuffling in zeroes.

1056

Value *Res = Constant::getNullValue(VecTy);

1057

1058

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

1059

// we'll just return the zero vector.

1060

if (Shift < 16) {

1061

int Idxs[64];

1062

// 256/512-bit version is split into 2/4 16-byte lanes.

1063

for (unsigned l = 0; l != NumElts; l += 16)

1064

for (unsigned i = 0; i != 16; ++i) {

1065

unsigned Idx = NumElts + i - Shift;

1066

if (Idx < NumElts)

1067

Idx -= NumElts - 16; // end of lane, switch operand.

1068

Idxs[l + i] = Idx + l;

1069

}

1070

1071

Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));

1072

}

1073

1074

// Bitcast back to a 64-bit element type.

1075

return Builder.CreateBitCast(Res, ResultTy, "cast");

1076

}

1077

1078

// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them

1079

// to byte shuffles.

1080

static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,

1081

unsigned Shift) {

1082

auto *ResultTy = cast<FixedVectorType>(Op->getType());

1083

unsigned NumElts = ResultTy->getNumElements() * 8;

1084

1085

// Bitcast from a 64-bit element type to a byte element type.

1086

Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);

1087

Op = Builder.CreateBitCast(Op, VecTy, "cast");

1088

1089

// We'll be shuffling in zeroes.

1090

Value *Res = Constant::getNullValue(VecTy);

1091

1092

// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,

1093

// we'll just return the zero vector.

1094

if (Shift < 16) {

1095

int Idxs[64];

1096

// 256/512-bit version is split into 2/4 16-byte lanes.

1097

for (unsigned l = 0; l != NumElts; l += 16)

1098

for (unsigned i = 0; i != 16; ++i) {

1099

unsigned Idx = i + Shift;

1100

if (Idx >= 16)

1101

Idx += NumElts - 16; // end of lane, switch operand.

1102

Idxs[l + i] = Idx + l;

1103

}

1104

1105

Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));

1106

}

1107

1108

// Bitcast back to a 64-bit element type.

1109

return Builder.CreateBitCast(Res, ResultTy, "cast");

1110

}

1111

1112

static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,

1113

unsigned NumElts) {

1114

assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements")(static_cast <bool> (isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements"
) ? void (0) : __assert_fail ("isPowerOf2_32(NumElts) && \"Expected power-of-2 mask elements\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1114, __extension__ __PRETTY_FUNCTION__));

1115

llvm::VectorType *MaskTy = FixedVectorType::get(

1116

Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());

1117

Mask = Builder.CreateBitCast(Mask, MaskTy);

1118

1119

// If we have less than 8 elements (1, 2 or 4), then the starting mask was an

1120

// i8 and we need to extract down to the right number of elements.

1121

if (NumElts <= 4) {

1122

int Indices[4];

1123

for (unsigned i = 0; i != NumElts; ++i)

1124

Indices[i] = i;

1125

Mask = Builder.CreateShuffleVector(

1126

Mask, Mask, makeArrayRef(Indices, NumElts), "extract");

1127

}

1128

1129

return Mask;

1130

}

1131

1132

static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,

1133

Value *Op0, Value *Op1) {

1134

// If the mask is all ones just emit the first operation.

1135

if (const auto *C = dyn_cast<Constant>(Mask))

1136

if (C->isAllOnesValue())

1137

return Op0;

1138

1139

Mask = getX86MaskVec(Builder, Mask,

1140

cast<FixedVectorType>(Op0->getType())->getNumElements());

1141

return Builder.CreateSelect(Mask, Op0, Op1);

1142

}

1143

1144

static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,

1145

Value *Op0, Value *Op1) {

1146

// If the mask is all ones just emit the first operation.

1147

if (const auto *C = dyn_cast<Constant>(Mask))

1148

if (C->isAllOnesValue())

1149

return Op0;

1150

1151

auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),

1152

Mask->getType()->getIntegerBitWidth());

1153

Mask = Builder.CreateBitCast(Mask, MaskTy);

1154

Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);

1155

return Builder.CreateSelect(Mask, Op0, Op1);

1156

}

1157

1158

// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.

1159

// PALIGNR handles large immediates by shifting while VALIGN masks the immediate

1160

// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.

1161

static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,

1162

Value *Op1, Value *Shift,

1163

Value *Passthru, Value *Mask,

1164

bool IsVALIGN) {

1165

unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();

1166

1167

unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();

1168

assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!")(static_cast <bool> ((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!") ? void (0) : __assert_fail (
"(IsVALIGN || NumElts % 16 == 0) && \"Illegal NumElts for PALIGNR!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1168, __extension__ __PRETTY_FUNCTION__));

1169

assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!")(static_cast <bool> ((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!") ? void (0) : __assert_fail (
"(!IsVALIGN || NumElts <= 16) && \"NumElts too large for VALIGN!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1169, __extension__ __PRETTY_FUNCTION__));

1170

assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!")(static_cast <bool> (isPowerOf2_32(NumElts) && "NumElts not a power of 2!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NumElts) && \"NumElts not a power of 2!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__));

1171

1172

// Mask the immediate for VALIGN.

1173

if (IsVALIGN)

1174

ShiftVal &= (NumElts - 1);

1175

1176

// If palignr is shifting the pair of vectors more than the size of two

1177

// lanes, emit zero.

1178

if (ShiftVal >= 32)

1179

return llvm::Constant::getNullValue(Op0->getType());

1180

1181

// If palignr is shifting the pair of input vectors more than one lane,

1182

// but less than two lanes, convert to shifting in zeroes.

1183

if (ShiftVal > 16) {

1184

ShiftVal -= 16;

1185

Op1 = Op0;

1186

Op0 = llvm::Constant::getNullValue(Op0->getType());

1187

}

1188

1189

int Indices[64];

1190

// 256-bit palignr operates on 128-bit lanes so we need to handle that

1191

for (unsigned l = 0; l < NumElts; l += 16) {

1192

for (unsigned i = 0; i != 16; ++i) {

1193

unsigned Idx = ShiftVal + i;

1194

if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.

1195

Idx += NumElts - 16; // End of lane, switch operand.

1196

Indices[l + i] = Idx + l;

1197

}

1198

}

1199

1200

Value *Align = Builder.CreateShuffleVector(Op1, Op0,

1201

makeArrayRef(Indices, NumElts),

1202

"palignr");

1203

1204

return EmitX86Select(Builder, Mask, Align, Passthru);

1205

}

1206

1207

static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,

1208

bool ZeroMask, bool IndexForm) {

1209

Type *Ty = CI.getType();

1210

unsigned VecWidth = Ty->getPrimitiveSizeInBits();

1211

unsigned EltWidth = Ty->getScalarSizeInBits();

1212

bool IsFloat = Ty->isFPOrFPVectorTy();

1213

Intrinsic::ID IID;

1214

if (VecWidth == 128 && EltWidth == 32 && IsFloat)

1215

IID = Intrinsic::x86_avx512_vpermi2var_ps_128;

1216

else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)

1217

IID = Intrinsic::x86_avx512_vpermi2var_d_128;

1218

else if (VecWidth == 128 && EltWidth == 64 && IsFloat)

1219

IID = Intrinsic::x86_avx512_vpermi2var_pd_128;

1220

else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)

1221

IID = Intrinsic::x86_avx512_vpermi2var_q_128;

1222

else if (VecWidth == 256 && EltWidth == 32 && IsFloat)

1223

IID = Intrinsic::x86_avx512_vpermi2var_ps_256;

1224

else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)

1225

IID = Intrinsic::x86_avx512_vpermi2var_d_256;

1226

else if (VecWidth == 256 && EltWidth == 64 && IsFloat)

1227

IID = Intrinsic::x86_avx512_vpermi2var_pd_256;

1228

else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)

1229

IID = Intrinsic::x86_avx512_vpermi2var_q_256;

1230

else if (VecWidth == 512 && EltWidth == 32 && IsFloat)

1231

IID = Intrinsic::x86_avx512_vpermi2var_ps_512;

1232

else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)

1233

IID = Intrinsic::x86_avx512_vpermi2var_d_512;

1234

else if (VecWidth == 512 && EltWidth == 64 && IsFloat)

1235

IID = Intrinsic::x86_avx512_vpermi2var_pd_512;

1236

else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)

1237

IID = Intrinsic::x86_avx512_vpermi2var_q_512;

1238

else if (VecWidth == 128 && EltWidth == 16)

1239

IID = Intrinsic::x86_avx512_vpermi2var_hi_128;

1240

else if (VecWidth == 256 && EltWidth == 16)

1241

IID = Intrinsic::x86_avx512_vpermi2var_hi_256;

1242

else if (VecWidth == 512 && EltWidth == 16)

1243

IID = Intrinsic::x86_avx512_vpermi2var_hi_512;

1244

else if (VecWidth == 128 && EltWidth == 8)

1245

IID = Intrinsic::x86_avx512_vpermi2var_qi_128;

1246

else if (VecWidth == 256 && EltWidth == 8)

1247

IID = Intrinsic::x86_avx512_vpermi2var_qi_256;

1248

else if (VecWidth == 512 && EltWidth == 8)

1249

IID = Intrinsic::x86_avx512_vpermi2var_qi_512;

1250

else

1251

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1251);

1252

1253

Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),

1254

CI.getArgOperand(2) };

1255

1256

// If this isn't index form we need to swap operand 0 and 1.

1257

if (!IndexForm)

1258

std::swap(Args[0], Args[1]);

1259

1260

Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),

1261

Args);

1262

Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)

1263

: Builder.CreateBitCast(CI.getArgOperand(1),

1264

Ty);

1265

return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);

1266

}

1267

1268

static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,

1269

Intrinsic::ID IID) {

1270

Type *Ty = CI.getType();

1271

Value *Op0 = CI.getOperand(0);

1272

Value *Op1 = CI.getOperand(1);

1273

Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);

1274

Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});

1275

1276

if (CI.getNumArgOperands() == 4) { // For masked intrinsics.

1277

Value *VecSrc = CI.getOperand(2);

1278

Value *Mask = CI.getOperand(3);

1279

Res = EmitX86Select(Builder, Mask, Res, VecSrc);

1280

}

1281

return Res;

1282

}

1283

1284

static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,

1285

bool IsRotateRight) {

1286

Type *Ty = CI.getType();

1287

Value *Src = CI.getArgOperand(0);

1288

Value *Amt = CI.getArgOperand(1);

1289

1290

// Amount may be scalar immediate, in which case create a splat vector.

1291

// Funnel shifts amounts are treated as modulo and types are all power-of-2 so

1292

// we only care about the lowest log2 bits anyway.

1293

if (Amt->getType() != Ty) {

1294

unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();

1295

Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);

1296

Amt = Builder.CreateVectorSplat(NumElts, Amt);

1297

}

1298

1299

Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;

1300

Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);

1301

Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});

1302

1303

if (CI.getNumArgOperands() == 4) { // For masked intrinsics.

1304

Value *VecSrc = CI.getOperand(2);

1305

Value *Mask = CI.getOperand(3);

1306

Res = EmitX86Select(Builder, Mask, Res, VecSrc);

1307

}

1308

return Res;

1309

}

1310

1311

static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,

1312

bool IsSigned) {

1313

Type *Ty = CI.getType();

1314

Value *LHS = CI.getArgOperand(0);

1315

Value *RHS = CI.getArgOperand(1);

1316

1317

CmpInst::Predicate Pred;

1318

switch (Imm) {

1319

case 0x0:

1320

Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;

1321

break;

1322

case 0x1:

1323

Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;

1324

break;

1325

case 0x2:

1326

Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;

1327

break;

1328

case 0x3:

1329

Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;

1330

break;

1331

case 0x4:

1332

Pred = ICmpInst::ICMP_EQ;

1333

break;

1334

case 0x5:

1335

Pred = ICmpInst::ICMP_NE;

1336

break;

1337

case 0x6:

1338

return Constant::getNullValue(Ty); // FALSE

1339

case 0x7:

1340

return Constant::getAllOnesValue(Ty); // TRUE

1341

default:

1342

llvm_unreachable("Unknown XOP vpcom/vpcomu predicate")::llvm::llvm_unreachable_internal("Unknown XOP vpcom/vpcomu predicate"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1342);

1343

}

1344

1345

Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);

1346

Value *Ext = Builder.CreateSExt(Cmp, Ty);

1347

return Ext;

1348

}

1349

1350

static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,

1351

bool IsShiftRight, bool ZeroMask) {

1352

Type *Ty = CI.getType();

1353

Value *Op0 = CI.getArgOperand(0);

1354

Value *Op1 = CI.getArgOperand(1);

1355

Value *Amt = CI.getArgOperand(2);

1356

1357

if (IsShiftRight)

1358

std::swap(Op0, Op1);

1359

1360

// Amount may be scalar immediate, in which case create a splat vector.

1361

// Funnel shifts amounts are treated as modulo and types are all power-of-2 so

1362

// we only care about the lowest log2 bits anyway.

1363

if (Amt->getType() != Ty) {

1364

unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();

1365

Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);

1366

Amt = Builder.CreateVectorSplat(NumElts, Amt);

1367

}

1368

1369

Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;

1370

Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);

1371

Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});

1372

1373

unsigned NumArgs = CI.getNumArgOperands();

1374

if (NumArgs >= 4) { // For masked intrinsics.

1375

Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :

1376

ZeroMask ? ConstantAggregateZero::get(CI.getType()) :

1377

CI.getArgOperand(0);

1378

Value *Mask = CI.getOperand(NumArgs - 1);

1379

Res = EmitX86Select(Builder, Mask, Res, VecSrc);

1380

}

1381

return Res;

1382

}

1383

1384

static Value *UpgradeMaskedStore(IRBuilder<> &Builder,

1385

Value *Ptr, Value *Data, Value *Mask,

1386

bool Aligned) {

1387

// Cast the pointer to the right type.

1388

Ptr = Builder.CreateBitCast(Ptr,

1389

llvm::PointerType::getUnqual(Data->getType()));

1390

const Align Alignment =

1391

Aligned

1392

? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)

1393

: Align(1);

1394

1395

// If the mask is all ones just emit a regular store.

1396

if (const auto *C = dyn_cast<Constant>(Mask))

1397

if (C->isAllOnesValue())

1398

return Builder.CreateAlignedStore(Data, Ptr, Alignment);

1399

1400

// Convert the mask from an integer type to a vector of i1.

1401

unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();

1402

Mask = getX86MaskVec(Builder, Mask, NumElts);

1403

return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);

1404

}

1405

1406

static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,

1407

Value *Ptr, Value *Passthru, Value *Mask,

1408

bool Aligned) {

1409

Type *ValTy = Passthru->getType();

1410

// Cast the pointer to the right type.

1411

Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));

1412

const Align Alignment =

1413

Aligned

1414

? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /

1415

8)

1416

: Align(1);

1417

1418

// If the mask is all ones just emit a regular store.

1419

if (const auto *C = dyn_cast<Constant>(Mask))

1420

if (C->isAllOnesValue())

1421

return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);

1422

1423

// Convert the mask from an integer type to a vector of i1.

1424

unsigned NumElts =

1425

cast<FixedVectorType>(Passthru->getType())->getNumElements();

1426

Mask = getX86MaskVec(Builder, Mask, NumElts);

1427

return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);

1428

}

1429

1430

static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {

1431

Type *Ty = CI.getType();

1432

Value *Op0 = CI.getArgOperand(0);

1433

Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);

1434

Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});

1435

if (CI.getNumArgOperands() == 3)

1436

Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));

1437

return Res;

1438

}

1439

1440

static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {

1441

Type *Ty = CI.getType();

1442

1443

// Arguments have a vXi32 type so cast to vXi64.

1444

Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);

1445

Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);

1446

1447

if (IsSigned) {

1448

// Shift left then arithmetic shift right.

1449

Constant *ShiftAmt = ConstantInt::get(Ty, 32);

1450

LHS = Builder.CreateShl(LHS, ShiftAmt);

1451

LHS = Builder.CreateAShr(LHS, ShiftAmt);

1452

RHS = Builder.CreateShl(RHS, ShiftAmt);

1453

RHS = Builder.CreateAShr(RHS, ShiftAmt);

1454

} else {

1455

// Clear the upper bits.

1456

Constant *Mask = ConstantInt::get(Ty, 0xffffffff);

1457

LHS = Builder.CreateAnd(LHS, Mask);

1458

RHS = Builder.CreateAnd(RHS, Mask);

1459

}

1460

1461

Value *Res = Builder.CreateMul(LHS, RHS);

1462

1463

if (CI.getNumArgOperands() == 4)

1464

Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));

1465

1466

return Res;

1467

}

1468

1469

// Applying mask on vector of i1's and make sure result is at least 8 bits wide.

1470

static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,

1471

Value *Mask) {

1472

unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();

1473

if (Mask) {

1474

const auto *C = dyn_cast<Constant>(Mask);

1475

if (!C || !C->isAllOnesValue())

1476

Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));

1477

}

1478

1479

if (NumElts < 8) {

1480

int Indices[8];

1481

for (unsigned i = 0; i != NumElts; ++i)

1482

Indices[i] = i;

1483

for (unsigned i = NumElts; i != 8; ++i)

1484

Indices[i] = NumElts + i % NumElts;

1485

Vec = Builder.CreateShuffleVector(Vec,

1486

Constant::getNullValue(Vec->getType()),

1487

Indices);

1488

}

1489

return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));

1490

}

1491

1492

static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,

1493

unsigned CC, bool Signed) {

1494

Value *Op0 = CI.getArgOperand(0);

1495

unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();

1496

1497

Value *Cmp;

1498

if (CC == 3) {

1499

Cmp = Constant::getNullValue(

1500

FixedVectorType::get(Builder.getInt1Ty(), NumElts));

1501

} else if (CC == 7) {

1502

Cmp = Constant::getAllOnesValue(

1503

FixedVectorType::get(Builder.getInt1Ty(), NumElts));

1504

} else {

1505

ICmpInst::Predicate Pred;

1506

switch (CC) {

1507

default: llvm_unreachable("Unknown condition code")::llvm::llvm_unreachable_internal("Unknown condition code", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1507);

1508

case 0: Pred = ICmpInst::ICMP_EQ; break;

1509

case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;

1510

case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;

1511

case 4: Pred = ICmpInst::ICMP_NE; break;

1512

case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;

1513

case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;

1514

}

1515

Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));

1516

}

1517

1518

Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);

1519

1520

return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);

1521

}

1522

1523

// Replace a masked intrinsic with an older unmasked intrinsic.

1524

static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,

1525

Intrinsic::ID IID) {

1526

Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);

1527

Value *Rep = Builder.CreateCall(Intrin,

1528

{ CI.getArgOperand(0), CI.getArgOperand(1) });

1529

return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));

1530

}

1531

1532

static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {

1533

Value* A = CI.getArgOperand(0);

1534

Value* B = CI.getArgOperand(1);

1535

Value* Src = CI.getArgOperand(2);

1536

Value* Mask = CI.getArgOperand(3);

1537

1538

Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));

1539

Value* Cmp = Builder.CreateIsNotNull(AndNode);

1540

Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);

1541

Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);

1542

Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);

1543

return Builder.CreateInsertElement(A, Select, (uint64_t)0);

1544

}

1545

1546

1547

static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {

1548

Value* Op = CI.getArgOperand(0);

1549

Type* ReturnOp = CI.getType();

1550

unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();

1551

Value *Mask = getX86MaskVec(Builder, Op, NumElts);

1552

return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");

1553

}

1554

1555

// Replace intrinsic with unmasked version and a select.

1556

static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,

1557

CallInst &CI, Value *&Rep) {

1558

Name = Name.substr(12); // Remove avx512.mask.

1559

1560

unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();

1561

unsigned EltWidth = CI.getType()->getScalarSizeInBits();

1562

Intrinsic::ID IID;

1563

if (Name.startswith("max.p")) {

1564

if (VecWidth == 128 && EltWidth == 32)

1565

IID = Intrinsic::x86_sse_max_ps;

1566

else if (VecWidth == 128 && EltWidth == 64)

1567

IID = Intrinsic::x86_sse2_max_pd;

1568

else if (VecWidth == 256 && EltWidth == 32)

1569

IID = Intrinsic::x86_avx_max_ps_256;

1570

else if (VecWidth == 256 && EltWidth == 64)

1571

IID = Intrinsic::x86_avx_max_pd_256;

1572

else

1573

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1573);

1574

} else if (Name.startswith("min.p")) {

1575

if (VecWidth == 128 && EltWidth == 32)

1576

IID = Intrinsic::x86_sse_min_ps;

1577

else if (VecWidth == 128 && EltWidth == 64)

1578

IID = Intrinsic::x86_sse2_min_pd;

1579

else if (VecWidth == 256 && EltWidth == 32)

1580

IID = Intrinsic::x86_avx_min_ps_256;

1581

else if (VecWidth == 256 && EltWidth == 64)

1582

IID = Intrinsic::x86_avx_min_pd_256;

1583

else

1584

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1584);

1585

} else if (Name.startswith("pshuf.b.")) {

1586

if (VecWidth == 128)

1587

IID = Intrinsic::x86_ssse3_pshuf_b_128;

1588

else if (VecWidth == 256)

1589

IID = Intrinsic::x86_avx2_pshuf_b;

1590

else if (VecWidth == 512)

1591

IID = Intrinsic::x86_avx512_pshuf_b_512;

1592

else

1593

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1593);

1594

} else if (Name.startswith("pmul.hr.sw.")) {

1595

if (VecWidth == 128)

1596

IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;

1597

else if (VecWidth == 256)

1598

IID = Intrinsic::x86_avx2_pmul_hr_sw;

1599

else if (VecWidth == 512)

1600

IID = Intrinsic::x86_avx512_pmul_hr_sw_512;

1601

else

1602

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1602);

1603

} else if (Name.startswith("pmulh.w.")) {

1604

if (VecWidth == 128)

1605

IID = Intrinsic::x86_sse2_pmulh_w;

1606

else if (VecWidth == 256)

1607

IID = Intrinsic::x86_avx2_pmulh_w;

1608

else if (VecWidth == 512)

1609

IID = Intrinsic::x86_avx512_pmulh_w_512;

1610

else

1611

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1611);

1612

} else if (Name.startswith("pmulhu.w.")) {

1613

if (VecWidth == 128)

1614

IID = Intrinsic::x86_sse2_pmulhu_w;

1615

else if (VecWidth == 256)

1616

IID = Intrinsic::x86_avx2_pmulhu_w;

1617

else if (VecWidth == 512)

1618

IID = Intrinsic::x86_avx512_pmulhu_w_512;

1619

else

1620

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1620);

1621

} else if (Name.startswith("pmaddw.d.")) {

1622

if (VecWidth == 128)

1623

IID = Intrinsic::x86_sse2_pmadd_wd;

1624

else if (VecWidth == 256)

1625

IID = Intrinsic::x86_avx2_pmadd_wd;

1626

else if (VecWidth == 512)

1627

IID = Intrinsic::x86_avx512_pmaddw_d_512;

1628

else

1629

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1629);

1630

} else if (Name.startswith("pmaddubs.w.")) {

1631

if (VecWidth == 128)

1632

IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;

1633

else if (VecWidth == 256)

1634

IID = Intrinsic::x86_avx2_pmadd_ub_sw;

1635

else if (VecWidth == 512)

1636

IID = Intrinsic::x86_avx512_pmaddubs_w_512;

1637

else

1638

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1638);

1639

} else if (Name.startswith("packsswb.")) {

1640

if (VecWidth == 128)

1641

IID = Intrinsic::x86_sse2_packsswb_128;

1642

else if (VecWidth == 256)

1643

IID = Intrinsic::x86_avx2_packsswb;

1644

else if (VecWidth == 512)

1645

IID = Intrinsic::x86_avx512_packsswb_512;

1646

else

1647

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1647);

1648

} else if (Name.startswith("packssdw.")) {

1649

if (VecWidth == 128)

1650

IID = Intrinsic::x86_sse2_packssdw_128;

1651

else if (VecWidth == 256)

1652

IID = Intrinsic::x86_avx2_packssdw;

1653

else if (VecWidth == 512)

1654

IID = Intrinsic::x86_avx512_packssdw_512;

1655

else

1656

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1656);

1657

} else if (Name.startswith("packuswb.")) {

1658

if (VecWidth == 128)

1659

IID = Intrinsic::x86_sse2_packuswb_128;

1660

else if (VecWidth == 256)

1661

IID = Intrinsic::x86_avx2_packuswb;

1662

else if (VecWidth == 512)

1663

IID = Intrinsic::x86_avx512_packuswb_512;

1664

else

1665

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1665);

1666

} else if (Name.startswith("packusdw.")) {

1667

if (VecWidth == 128)

1668

IID = Intrinsic::x86_sse41_packusdw;

1669

else if (VecWidth == 256)

1670

IID = Intrinsic::x86_avx2_packusdw;

1671

else if (VecWidth == 512)

1672

IID = Intrinsic::x86_avx512_packusdw_512;

1673

else

1674

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1674);

1675

} else if (Name.startswith("vpermilvar.")) {

1676

if (VecWidth == 128 && EltWidth == 32)

1677

IID = Intrinsic::x86_avx_vpermilvar_ps;

1678

else if (VecWidth == 128 && EltWidth == 64)

1679

IID = Intrinsic::x86_avx_vpermilvar_pd;

1680

else if (VecWidth == 256 && EltWidth == 32)

1681

IID = Intrinsic::x86_avx_vpermilvar_ps_256;

1682

else if (VecWidth == 256 && EltWidth == 64)

1683

IID = Intrinsic::x86_avx_vpermilvar_pd_256;

1684

else if (VecWidth == 512 && EltWidth == 32)

1685

IID = Intrinsic::x86_avx512_vpermilvar_ps_512;

1686

else if (VecWidth == 512 && EltWidth == 64)

1687

IID = Intrinsic::x86_avx512_vpermilvar_pd_512;

1688

else

1689

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1689);

1690

} else if (Name == "cvtpd2dq.256") {

1691

IID = Intrinsic::x86_avx_cvt_pd2dq_256;

1692

} else if (Name == "cvtpd2ps.256") {

1693

IID = Intrinsic::x86_avx_cvt_pd2_ps_256;

1694

} else if (Name == "cvttpd2dq.256") {

1695

IID = Intrinsic::x86_avx_cvtt_pd2dq_256;

1696

} else if (Name == "cvttps2dq.128") {

1697

IID = Intrinsic::x86_sse2_cvttps2dq;

1698

} else if (Name == "cvttps2dq.256") {

1699

IID = Intrinsic::x86_avx_cvtt_ps2dq_256;

1700

} else if (Name.startswith("permvar.")) {

1701

bool IsFloat = CI.getType()->isFPOrFPVectorTy();

1702

if (VecWidth == 256 && EltWidth == 32 && IsFloat)

1703

IID = Intrinsic::x86_avx2_permps;

1704

else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)

1705

IID = Intrinsic::x86_avx2_permd;

1706

else if (VecWidth == 256 && EltWidth == 64 && IsFloat)

1707

IID = Intrinsic::x86_avx512_permvar_df_256;

1708

else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)

1709

IID = Intrinsic::x86_avx512_permvar_di_256;

1710

else if (VecWidth == 512 && EltWidth == 32 && IsFloat)

1711

IID = Intrinsic::x86_avx512_permvar_sf_512;

1712

else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)

1713

IID = Intrinsic::x86_avx512_permvar_si_512;

1714

else if (VecWidth == 512 && EltWidth == 64 && IsFloat)

1715

IID = Intrinsic::x86_avx512_permvar_df_512;

1716

else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)

1717

IID = Intrinsic::x86_avx512_permvar_di_512;

1718

else if (VecWidth == 128 && EltWidth == 16)

1719

IID = Intrinsic::x86_avx512_permvar_hi_128;

1720

else if (VecWidth == 256 && EltWidth == 16)

1721

IID = Intrinsic::x86_avx512_permvar_hi_256;

1722

else if (VecWidth == 512 && EltWidth == 16)

1723

IID = Intrinsic::x86_avx512_permvar_hi_512;

1724

else if (VecWidth == 128 && EltWidth == 8)

1725

IID = Intrinsic::x86_avx512_permvar_qi_128;

1726

else if (VecWidth == 256 && EltWidth == 8)

1727

IID = Intrinsic::x86_avx512_permvar_qi_256;

1728

else if (VecWidth == 512 && EltWidth == 8)

1729

IID = Intrinsic::x86_avx512_permvar_qi_512;

1730

else

1731

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1731);

1732

} else if (Name.startswith("dbpsadbw.")) {

1733

if (VecWidth == 128)

1734

IID = Intrinsic::x86_avx512_dbpsadbw_128;

1735

else if (VecWidth == 256)

1736

IID = Intrinsic::x86_avx512_dbpsadbw_256;

1737

else if (VecWidth == 512)

1738

IID = Intrinsic::x86_avx512_dbpsadbw_512;

1739

else

1740

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1740);

1741

} else if (Name.startswith("pmultishift.qb.")) {

1742

if (VecWidth == 128)

1743

IID = Intrinsic::x86_avx512_pmultishift_qb_128;

1744

else if (VecWidth == 256)

1745

IID = Intrinsic::x86_avx512_pmultishift_qb_256;

1746

else if (VecWidth == 512)

1747

IID = Intrinsic::x86_avx512_pmultishift_qb_512;

1748

else

1749

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1749);

1750

} else if (Name.startswith("conflict.")) {

1751

if (Name[9] == 'd' && VecWidth == 128)

1752

IID = Intrinsic::x86_avx512_conflict_d_128;

1753

else if (Name[9] == 'd' && VecWidth == 256)

1754

IID = Intrinsic::x86_avx512_conflict_d_256;

1755

else if (Name[9] == 'd' && VecWidth == 512)

1756

IID = Intrinsic::x86_avx512_conflict_d_512;

1757

else if (Name[9] == 'q' && VecWidth == 128)

1758

IID = Intrinsic::x86_avx512_conflict_q_128;

1759

else if (Name[9] == 'q' && VecWidth == 256)

1760

IID = Intrinsic::x86_avx512_conflict_q_256;

1761

else if (Name[9] == 'q' && VecWidth == 512)

1762

IID = Intrinsic::x86_avx512_conflict_q_512;

1763

else

1764

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1764);

1765

} else if (Name.startswith("pavg.")) {

1766

if (Name[5] == 'b' && VecWidth == 128)

1767

IID = Intrinsic::x86_sse2_pavg_b;

1768

else if (Name[5] == 'b' && VecWidth == 256)

1769

IID = Intrinsic::x86_avx2_pavg_b;

1770

else if (Name[5] == 'b' && VecWidth == 512)

1771

IID = Intrinsic::x86_avx512_pavg_b_512;

1772

else if (Name[5] == 'w' && VecWidth == 128)

1773

IID = Intrinsic::x86_sse2_pavg_w;

1774

else if (Name[5] == 'w' && VecWidth == 256)

1775

IID = Intrinsic::x86_avx2_pavg_w;

1776

else if (Name[5] == 'w' && VecWidth == 512)

1777

IID = Intrinsic::x86_avx512_pavg_w_512;

1778

else

1779

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1779);

1780

} else

1781

return false;

1782

1783

SmallVector<Value *, 4> Args(CI.arg_operands().begin(),

1784

CI.arg_operands().end());

1785

Args.pop_back();

1786

Args.pop_back();

1787

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),

1788

Args);

1789

unsigned NumArgs = CI.getNumArgOperands();

1790

Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,

1791

CI.getArgOperand(NumArgs - 2));

1792

return true;

1793

}

1794

1795

/// Upgrade comment in call to inline asm that represents an objc retain release

1796

/// marker.

1797

void llvm::UpgradeInlineAsmString(std::string *AsmStr) {

1798

size_t Pos;

1799

if (AsmStr->find("mov\tfp") == 0 &&

1800

AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&

1801

(Pos = AsmStr->find("# marker")) != std::string::npos) {

1802

AsmStr->replace(Pos, 1, ";");

1803

}

1804

}

1805

1806

/// Upgrade a call to an old intrinsic. All argument and return casting must be

1807

/// provided to seamlessly integrate with existing context.

1808

void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {

1809

Function *F = CI->getCalledFunction();

1810

LLVMContext &C = CI->getContext();

1811

IRBuilder<> Builder(C);

1812

Builder.SetInsertPoint(CI->getParent(), CI->getIterator());

1813

1814

assert(F && "Intrinsic call is not direct?")(static_cast <bool> (F && "Intrinsic call is not direct?"
) ? void (0) : __assert_fail ("F && \"Intrinsic call is not direct?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1814, __extension__ __PRETTY_FUNCTION__));

1815

1816

if (!NewFn) {

1817

// Get the Function's name.

1818

StringRef Name = F->getName();

1819

1820

assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'")(static_cast <bool> (Name.startswith("llvm.") &&
"Intrinsic doesn't start with 'llvm.'") ? void (0) : __assert_fail
("Name.startswith(\"llvm.\") && \"Intrinsic doesn't start with 'llvm.'\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 1820, __extension__ __PRETTY_FUNCTION__));

1821

Name = Name.substr(5);

1822

1823

bool IsX86 = Name.startswith("x86.");

1824

if (IsX86)

1825

Name = Name.substr(4);

1826

bool IsNVVM = Name.startswith("nvvm.");

1827

if (IsNVVM)

1828

Name = Name.substr(5);

1829

1830

if (IsX86 && Name.startswith("sse4a.movnt.")) {

1831

Module *M = F->getParent();

1832

SmallVector<Metadata *, 1> Elts;

1833

Elts.push_back(

1834

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

1835

MDNode *Node = MDNode::get(C, Elts);

1836

1837

Value *Arg0 = CI->getArgOperand(0);

1838

Value *Arg1 = CI->getArgOperand(1);

1839

1840

// Nontemporal (unaligned) store of the 0'th element of the float/double

1841

// vector.

1842

Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();

1843

PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);

1844

Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");

1845

Value *Extract =

1846

Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");

1847

1848

StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));

1849

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

1850

1851

// Remove intrinsic.

1852

CI->eraseFromParent();

1853

return;

1854

}

1855

1856

if (IsX86 && (Name.startswith("avx.movnt.") ||

1857

Name.startswith("avx512.storent."))) {

1858

Module *M = F->getParent();

1859

SmallVector<Metadata *, 1> Elts;

1860

Elts.push_back(

1861

ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

1862

MDNode *Node = MDNode::get(C, Elts);

1863

1864

Value *Arg0 = CI->getArgOperand(0);

1865

Value *Arg1 = CI->getArgOperand(1);

1866

1867

// Convert the type of the pointer to a pointer to the stored type.

1868

Value *BC = Builder.CreateBitCast(Arg0,

1869

PointerType::getUnqual(Arg1->getType()),

1870

"cast");

1871

StoreInst *SI = Builder.CreateAlignedStore(

1872

Arg1, BC,

1873

Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));

1874

SI->setMetadata(M->getMDKindID("nontemporal"), Node);

1875

1876

// Remove intrinsic.

1877

CI->eraseFromParent();

1878

return;

1879

}

1880

1881

if (IsX86 && Name == "sse2.storel.dq") {

1882

Value *Arg0 = CI->getArgOperand(0);

1883

Value *Arg1 = CI->getArgOperand(1);

1884

1885

auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);

1886

Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

1887

Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);

1888

Value *BC = Builder.CreateBitCast(Arg0,

1889

PointerType::getUnqual(Elt->getType()),

1890

"cast");

1891

Builder.CreateAlignedStore(Elt, BC, Align(1));

1892

1893

// Remove intrinsic.

1894

CI->eraseFromParent();

1895

return;

1896

}

1897

1898

if (IsX86 && (Name.startswith("sse.storeu.") ||

1899

Name.startswith("sse2.storeu.") ||

1900

Name.startswith("avx.storeu."))) {

1901

Value *Arg0 = CI->getArgOperand(0);

1902

Value *Arg1 = CI->getArgOperand(1);

1903

1904

Arg0 = Builder.CreateBitCast(Arg0,

1905

PointerType::getUnqual(Arg1->getType()),

1906

"cast");

1907

Builder.CreateAlignedStore(Arg1, Arg0, Align(1));

1908

1909

// Remove intrinsic.

1910

CI->eraseFromParent();

1911

return;

1912

}

1913

1914

if (IsX86 && Name == "avx512.mask.store.ss") {

1915

Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));

1916

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

1917

Mask, false);

1918

1919

// Remove intrinsic.

1920

CI->eraseFromParent();

1921

return;

1922

}

1923

1924

if (IsX86 && (Name.startswith("avx512.mask.store"))) {

1925

// "avx512.mask.storeu." or "avx512.mask.store."

1926

bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".

1927

UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),

1928

CI->getArgOperand(2), Aligned);

1929

1930

// Remove intrinsic.

1931

CI->eraseFromParent();

1932

return;

1933

}

1934

1935

Value *Rep;

1936

// Upgrade packed integer vector compare intrinsics to compare instructions.

1937

if (IsX86 && (Name.startswith("sse2.pcmp") ||

1938

Name.startswith("avx2.pcmp"))) {

1939

// "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."

1940

bool CmpEq = Name[9] == 'e';

1941

Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,

1942

CI->getArgOperand(0), CI->getArgOperand(1));

1943

Rep = Builder.CreateSExt(Rep, CI->getType(), "");

1944

} else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {

1945

Type *ExtTy = Type::getInt32Ty(C);

1946

if (CI->getOperand(0)->getType()->isIntegerTy(8))

1947

ExtTy = Type::getInt64Ty(C);

1948

unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /

1949

ExtTy->getPrimitiveSizeInBits();

1950

Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);

1951

Rep = Builder.CreateVectorSplat(NumElts, Rep);

1952

} else if (IsX86 && (Name == "sse.sqrt.ss" ||

1953

Name == "sse2.sqrt.sd")) {

1954

Value *Vec = CI->getArgOperand(0);

1955

Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);

1956

Function *Intr = Intrinsic::getDeclaration(F->getParent(),

1957

Intrinsic::sqrt, Elt0->getType());

1958

Elt0 = Builder.CreateCall(Intr, Elt0);

1959

Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);

1960

} else if (IsX86 && (Name.startswith("avx.sqrt.p") ||

1961

Name.startswith("sse2.sqrt.p") ||

1962

Name.startswith("sse.sqrt.p"))) {

1963

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),

1964

Intrinsic::sqrt,

1965

CI->getType()),

1966

{CI->getArgOperand(0)});

1967

} else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {

1968

if (CI->getNumArgOperands() == 4 &&

1969

(!isa<ConstantInt>(CI->getArgOperand(3)) ||

1970

cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {

1971

Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512

1972

: Intrinsic::x86_avx512_sqrt_pd_512;

1973

1974

Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };

1975

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),

1976

IID), Args);

1977

} else {

1978

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),

1979

Intrinsic::sqrt,

1980

CI->getType()),

1981

{CI->getArgOperand(0)});

1982

}

1983

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

1984

CI->getArgOperand(1));

1985

} else if (IsX86 && (Name.startswith("avx512.ptestm") ||

1986

Name.startswith("avx512.ptestnm"))) {

1987

Value *Op0 = CI->getArgOperand(0);

1988

Value *Op1 = CI->getArgOperand(1);

1989

Value *Mask = CI->getArgOperand(2);

1990

Rep = Builder.CreateAnd(Op0, Op1);

1991

llvm::Type *Ty = Op0->getType();

1992

Value *Zero = llvm::Constant::getNullValue(Ty);

1993

ICmpInst::Predicate Pred =

1994

Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;

1995

Rep = Builder.CreateICmp(Pred, Rep, Zero);

1996

Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);

1997

} else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){

1998

unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())

1999

->getNumElements();

2000

Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));

2001

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2002

CI->getArgOperand(1));

2003

} else if (IsX86 && (Name.startswith("avx512.kunpck"))) {

2004

unsigned NumElts = CI->getType()->getScalarSizeInBits();

2005

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);

2006

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);

2007

int Indices[64];

2008

for (unsigned i = 0; i != NumElts; ++i)

2009

Indices[i] = i;

2010

2011

// First extract half of each vector. This gives better codegen than

2012

// doing it in a single shuffle.

2013

LHS = Builder.CreateShuffleVector(LHS, LHS,

2014

makeArrayRef(Indices, NumElts / 2));

2015

RHS = Builder.CreateShuffleVector(RHS, RHS,

2016

makeArrayRef(Indices, NumElts / 2));

2017

// Concat the vectors.

2018

// NOTE: Operands have to be swapped to match intrinsic definition.

2019

Rep = Builder.CreateShuffleVector(RHS, LHS,

2020

makeArrayRef(Indices, NumElts));

2021

Rep = Builder.CreateBitCast(Rep, CI->getType());

2022

} else if (IsX86 && Name == "avx512.kand.w") {

2023

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2024

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2025

Rep = Builder.CreateAnd(LHS, RHS);

2026

Rep = Builder.CreateBitCast(Rep, CI->getType());

2027

} else if (IsX86 && Name == "avx512.kandn.w") {

2028

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2029

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2030

LHS = Builder.CreateNot(LHS);

2031

Rep = Builder.CreateAnd(LHS, RHS);

2032

Rep = Builder.CreateBitCast(Rep, CI->getType());

2033

} else if (IsX86 && Name == "avx512.kor.w") {

2034

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2035

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2036

Rep = Builder.CreateOr(LHS, RHS);

2037

Rep = Builder.CreateBitCast(Rep, CI->getType());

2038

} else if (IsX86 && Name == "avx512.kxor.w") {

2039

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2040

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2041

Rep = Builder.CreateXor(LHS, RHS);

2042

Rep = Builder.CreateBitCast(Rep, CI->getType());

2043

} else if (IsX86 && Name == "avx512.kxnor.w") {

2044

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2045

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2046

LHS = Builder.CreateNot(LHS);

2047

Rep = Builder.CreateXor(LHS, RHS);

2048

Rep = Builder.CreateBitCast(Rep, CI->getType());

2049

} else if (IsX86 && Name == "avx512.knot.w") {

2050

Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2051

Rep = Builder.CreateNot(Rep);

2052

Rep = Builder.CreateBitCast(Rep, CI->getType());

2053

} else if (IsX86 &&

2054

(Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {

2055

Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);

2056

Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);

2057

Rep = Builder.CreateOr(LHS, RHS);

2058

Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());

2059

Value *C;

2060

if (Name[14] == 'c')

2061

C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());

2062

else

2063

C = ConstantInt::getNullValue(Builder.getInt16Ty());

2064

Rep = Builder.CreateICmpEQ(Rep, C);

2065

Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());

2066

} else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||

2067

Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||

2068

Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||

2069

Name == "sse.div.ss" || Name == "sse2.div.sd")) {

2070

Type *I32Ty = Type::getInt32Ty(C);

2071

Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),

2072

ConstantInt::get(I32Ty, 0));

2073

Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),

2074

ConstantInt::get(I32Ty, 0));

2075

Value *EltOp;

2076

if (Name.contains(".add."))

2077

EltOp = Builder.CreateFAdd(Elt0, Elt1);

2078

else if (Name.contains(".sub."))

2079

EltOp = Builder.CreateFSub(Elt0, Elt1);

2080

else if (Name.contains(".mul."))

2081

EltOp = Builder.CreateFMul(Elt0, Elt1);

2082

else

2083

EltOp = Builder.CreateFDiv(Elt0, Elt1);

2084

Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,

2085

ConstantInt::get(I32Ty, 0));

2086

} else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {

2087

// "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."

2088

bool CmpEq = Name[16] == 'e';

2089

Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);

2090

} else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {

2091

Type *OpTy = CI->getArgOperand(0)->getType();

2092

unsigned VecWidth = OpTy->getPrimitiveSizeInBits();

2093

Intrinsic::ID IID;

2094

switch (VecWidth) {

2095

default: llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2095);

2096

case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;

2097

case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;

2098

case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;

2099

}

2100

2101

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2102

{ CI->getOperand(0), CI->getArgOperand(1) });

2103

Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));

2104

} else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {

2105

Type *OpTy = CI->getArgOperand(0)->getType();

2106

unsigned VecWidth = OpTy->getPrimitiveSizeInBits();

2107

unsigned EltWidth = OpTy->getScalarSizeInBits();

2108

Intrinsic::ID IID;

2109

if (VecWidth == 128 && EltWidth == 32)

2110

IID = Intrinsic::x86_avx512_fpclass_ps_128;

2111

else if (VecWidth == 256 && EltWidth == 32)

2112

IID = Intrinsic::x86_avx512_fpclass_ps_256;

2113

else if (VecWidth == 512 && EltWidth == 32)

2114

IID = Intrinsic::x86_avx512_fpclass_ps_512;

2115

else if (VecWidth == 128 && EltWidth == 64)

2116

IID = Intrinsic::x86_avx512_fpclass_pd_128;

2117

else if (VecWidth == 256 && EltWidth == 64)

2118

IID = Intrinsic::x86_avx512_fpclass_pd_256;

2119

else if (VecWidth == 512 && EltWidth == 64)

2120

IID = Intrinsic::x86_avx512_fpclass_pd_512;

2121

else

2122

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2122);

2123

2124

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2125

{ CI->getOperand(0), CI->getArgOperand(1) });

2126

Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));

2127

} else if (IsX86 && Name.startswith("avx512.cmp.p")) {

2128

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

2129

CI->arg_operands().end());

2130

Type *OpTy = Args[0]->getType();

2131

unsigned VecWidth = OpTy->getPrimitiveSizeInBits();

2132

unsigned EltWidth = OpTy->getScalarSizeInBits();

2133

Intrinsic::ID IID;

2134

if (VecWidth == 128 && EltWidth == 32)

2135

IID = Intrinsic::x86_avx512_mask_cmp_ps_128;

2136

else if (VecWidth == 256 && EltWidth == 32)

2137

IID = Intrinsic::x86_avx512_mask_cmp_ps_256;

2138

else if (VecWidth == 512 && EltWidth == 32)

2139

IID = Intrinsic::x86_avx512_mask_cmp_ps_512;

2140

else if (VecWidth == 128 && EltWidth == 64)

2141

IID = Intrinsic::x86_avx512_mask_cmp_pd_128;

2142

else if (VecWidth == 256 && EltWidth == 64)

2143

IID = Intrinsic::x86_avx512_mask_cmp_pd_256;

2144

else if (VecWidth == 512 && EltWidth == 64)

2145

IID = Intrinsic::x86_avx512_mask_cmp_pd_512;

2146

else

2147

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2147);

2148

2149

Value *Mask = Constant::getAllOnesValue(CI->getType());

2150

if (VecWidth == 512)

2151

std::swap(Mask, Args.back());

2152

Args.push_back(Mask);

2153

2154

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2155

Args);

2156

} else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {

2157

// Integer compare intrinsics.

2158

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2159

Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);

2160

} else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {

2161

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2162

Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);

2163

} else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||

2164

Name.startswith("avx512.cvtw2mask.") ||

2165

Name.startswith("avx512.cvtd2mask.") ||

2166

Name.startswith("avx512.cvtq2mask."))) {

2167

Value *Op = CI->getArgOperand(0);

2168

Value *Zero = llvm::Constant::getNullValue(Op->getType());

2169

Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);

2170

Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);

2171

} else if(IsX86 && (Name == "ssse3.pabs.b.128" ||

2172

Name == "ssse3.pabs.w.128" ||

2173

Name == "ssse3.pabs.d.128" ||

2174

Name.startswith("avx2.pabs") ||

2175

Name.startswith("avx512.mask.pabs"))) {

2176

Rep = upgradeAbs(Builder, *CI);

2177

} else if (IsX86 && (Name == "sse41.pmaxsb" ||

2178

Name == "sse2.pmaxs.w" ||

2179

Name == "sse41.pmaxsd" ||

2180

Name.startswith("avx2.pmaxs") ||

2181

Name.startswith("avx512.mask.pmaxs"))) {

2182

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);

2183

} else if (IsX86 && (Name == "sse2.pmaxu.b" ||

2184

Name == "sse41.pmaxuw" ||

2185

Name == "sse41.pmaxud" ||

2186

Name.startswith("avx2.pmaxu") ||

2187

Name.startswith("avx512.mask.pmaxu"))) {

2188

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);

2189

} else if (IsX86 && (Name == "sse41.pminsb" ||

2190

Name == "sse2.pmins.w" ||

2191

Name == "sse41.pminsd" ||

2192

Name.startswith("avx2.pmins") ||

2193

Name.startswith("avx512.mask.pmins"))) {

2194

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);

2195

} else if (IsX86 && (Name == "sse2.pminu.b" ||

2196

Name == "sse41.pminuw" ||

2197

Name == "sse41.pminud" ||

2198

Name.startswith("avx2.pminu") ||

2199

Name.startswith("avx512.mask.pminu"))) {

2200

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);

2201

} else if (IsX86 && (Name == "sse2.pmulu.dq" ||

2202

Name == "avx2.pmulu.dq" ||

2203

Name == "avx512.pmulu.dq.512" ||

2204

Name.startswith("avx512.mask.pmulu.dq."))) {

2205

Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);

2206

} else if (IsX86 && (Name == "sse41.pmuldq" ||

2207

Name == "avx2.pmul.dq" ||

2208

Name == "avx512.pmul.dq.512" ||

2209

Name.startswith("avx512.mask.pmul.dq."))) {

2210

Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);

2211

} else if (IsX86 && (Name == "sse.cvtsi2ss" ||

2212

Name == "sse2.cvtsi2sd" ||

2213

Name == "sse.cvtsi642ss" ||

2214

Name == "sse2.cvtsi642sd")) {

2215

Rep = Builder.CreateSIToFP(

2216

CI->getArgOperand(1),

2217

cast<VectorType>(CI->getType())->getElementType());

2218

Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);

2219

} else if (IsX86 && Name == "avx512.cvtusi2sd") {

2220

Rep = Builder.CreateUIToFP(

2221

CI->getArgOperand(1),

2222

cast<VectorType>(CI->getType())->getElementType());

2223

Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);

2224

} else if (IsX86 && Name == "sse2.cvtss2sd") {

2225

Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);

2226

Rep = Builder.CreateFPExt(

2227

Rep, cast<VectorType>(CI->getType())->getElementType());

2228

Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);

2229

} else if (IsX86 && (Name == "sse2.cvtdq2pd" ||

2230

Name == "sse2.cvtdq2ps" ||

2231

Name == "avx.cvtdq2.pd.256" ||

2232

Name == "avx.cvtdq2.ps.256" ||

2233

Name.startswith("avx512.mask.cvtdq2pd.") ||

2234

Name.startswith("avx512.mask.cvtudq2pd.") ||

2235

Name.startswith("avx512.mask.cvtdq2ps.") ||

2236

Name.startswith("avx512.mask.cvtudq2ps.") ||

2237

Name.startswith("avx512.mask.cvtqq2pd.") ||

2238

Name.startswith("avx512.mask.cvtuqq2pd.") ||

2239

Name == "avx512.mask.cvtqq2ps.256" ||

2240

Name == "avx512.mask.cvtqq2ps.512" ||

2241

Name == "avx512.mask.cvtuqq2ps.256" ||

2242

Name == "avx512.mask.cvtuqq2ps.512" ||

2243

Name == "sse2.cvtps2pd" ||

2244

Name == "avx.cvt.ps2.pd.256" ||

2245

Name == "avx512.mask.cvtps2pd.128" ||

2246

Name == "avx512.mask.cvtps2pd.256")) {

2247

auto *DstTy = cast<FixedVectorType>(CI->getType());

2248

Rep = CI->getArgOperand(0);

2249

auto *SrcTy = cast<FixedVectorType>(Rep->getType());

2250

2251

unsigned NumDstElts = DstTy->getNumElements();

2252

if (NumDstElts < SrcTy->getNumElements()) {

2253

assert(NumDstElts == 2 && "Unexpected vector size")(static_cast <bool> (NumDstElts == 2 && "Unexpected vector size"
) ? void (0) : __assert_fail ("NumDstElts == 2 && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2253, __extension__ __PRETTY_FUNCTION__));

2254

Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});

2255

}

2256

2257

bool IsPS2PD = SrcTy->getElementType()->isFloatTy();

2258

bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));

2259

if (IsPS2PD)

2260

Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");

2261

else if (CI->getNumArgOperands() == 4 &&

2262

(!isa<ConstantInt>(CI->getArgOperand(3)) ||

2263

cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {

2264

Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round

2265

: Intrinsic::x86_avx512_sitofp_round;

2266

Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,

2267

{ DstTy, SrcTy });

2268

Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });

2269

} else {

2270

Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")

2271

: Builder.CreateSIToFP(Rep, DstTy, "cvt");

2272

}

2273

2274

if (CI->getNumArgOperands() >= 3)

2275

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2276

CI->getArgOperand(1));

2277

} else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||

2278

Name.startswith("vcvtph2ps."))) {

2279

auto *DstTy = cast<FixedVectorType>(CI->getType());

2280

Rep = CI->getArgOperand(0);

2281

auto *SrcTy = cast<FixedVectorType>(Rep->getType());

2282

unsigned NumDstElts = DstTy->getNumElements();

2283

if (NumDstElts != SrcTy->getNumElements()) {

2284

assert(NumDstElts == 4 && "Unexpected vector size")(static_cast <bool> (NumDstElts == 4 && "Unexpected vector size"
) ? void (0) : __assert_fail ("NumDstElts == 4 && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2284, __extension__ __PRETTY_FUNCTION__));

2285

Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});

2286

}

2287

Rep = Builder.CreateBitCast(

2288

Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));

2289

Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");

2290

if (CI->getNumArgOperands() >= 3)

2291

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2292

CI->getArgOperand(1));

2293

} else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {

2294

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

2295

CI->getArgOperand(1), CI->getArgOperand(2),

2296

/*Aligned*/false);

2297

} else if (IsX86 && (Name.startswith("avx512.mask.load."))) {

2298

Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),

2299

CI->getArgOperand(1),CI->getArgOperand(2),

2300

/*Aligned*/true);

2301

} else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {

2302

auto *ResultTy = cast<FixedVectorType>(CI->getType());

2303

Type *PtrTy = ResultTy->getElementType();

2304

2305

// Cast the pointer to element type.

2306

Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),

2307

llvm::PointerType::getUnqual(PtrTy));

2308

2309

Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),

2310

ResultTy->getNumElements());

2311

2312

Function *ELd = Intrinsic::getDeclaration(F->getParent(),

2313

Intrinsic::masked_expandload,

2314

ResultTy);

2315

Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });

2316

} else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {

2317

auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());

2318

Type *PtrTy = ResultTy->getElementType();

2319

2320

// Cast the pointer to element type.

2321

Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),

2322

llvm::PointerType::getUnqual(PtrTy));

2323

2324

Value *MaskVec =

2325

getX86MaskVec(Builder, CI->getArgOperand(2),

2326

cast<FixedVectorType>(ResultTy)->getNumElements());

2327

2328

Function *CSt = Intrinsic::getDeclaration(F->getParent(),

2329

Intrinsic::masked_compressstore,

2330

ResultTy);

2331

Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });

2332

} else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||

2333

Name.startswith("avx512.mask.expand."))) {

2334

auto *ResultTy = cast<FixedVectorType>(CI->getType());

2335

2336

Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),

2337

ResultTy->getNumElements());

2338

2339

bool IsCompress = Name[12] == 'c';

2340

Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress

2341

: Intrinsic::x86_avx512_mask_expand;

2342

Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);

2343

Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),

2344

MaskVec });

2345

} else if (IsX86 && Name.startswith("xop.vpcom")) {

2346

bool IsSigned;

2347

if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||

2348

Name.endswith("uq"))

2349

IsSigned = false;

2350

else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||

2351

Name.endswith("q"))

2352

IsSigned = true;

2353

else

2354

llvm_unreachable("Unknown suffix")::llvm::llvm_unreachable_internal("Unknown suffix", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2354);

2355

2356

unsigned Imm;

2357

if (CI->getNumArgOperands() == 3) {

2358

Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2359

} else {

2360

Name = Name.substr(9); // strip off "xop.vpcom"

2361

if (Name.startswith("lt"))

2362

Imm = 0;

2363

else if (Name.startswith("le"))

2364

Imm = 1;

2365

else if (Name.startswith("gt"))

2366

Imm = 2;

2367

else if (Name.startswith("ge"))

2368

Imm = 3;

2369

else if (Name.startswith("eq"))

2370

Imm = 4;

2371

else if (Name.startswith("ne"))

2372

Imm = 5;

2373

else if (Name.startswith("false"))

2374

Imm = 6;

2375

else if (Name.startswith("true"))

2376

Imm = 7;

2377

else

2378

llvm_unreachable("Unknown condition")::llvm::llvm_unreachable_internal("Unknown condition", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 2378);

2379

}

2380

2381

Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);

2382

} else if (IsX86 && Name.startswith("xop.vpcmov")) {

2383

Value *Sel = CI->getArgOperand(2);

2384

Value *NotSel = Builder.CreateNot(Sel);

2385

Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);

2386

Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);

2387

Rep = Builder.CreateOr(Sel0, Sel1);

2388

} else if (IsX86 && (Name.startswith("xop.vprot") ||

2389

Name.startswith("avx512.prol") ||

2390

Name.startswith("avx512.mask.prol"))) {

2391

Rep = upgradeX86Rotate(Builder, *CI, false);

2392

} else if (IsX86 && (Name.startswith("avx512.pror") ||

2393

Name.startswith("avx512.mask.pror"))) {

2394

Rep = upgradeX86Rotate(Builder, *CI, true);

2395

} else if (IsX86 && (Name.startswith("avx512.vpshld.") ||

2396

Name.startswith("avx512.mask.vpshld") ||

2397

Name.startswith("avx512.maskz.vpshld"))) {

2398

bool ZeroMask = Name[11] == 'z';

2399

Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);

2400

} else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||

2401

Name.startswith("avx512.mask.vpshrd") ||

2402

Name.startswith("avx512.maskz.vpshrd"))) {

2403

bool ZeroMask = Name[11] == 'z';

2404

Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);

2405

} else if (IsX86 && Name == "sse42.crc32.64.8") {

2406

Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),

2407

Intrinsic::x86_sse42_crc32_32_8);

2408

Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));

2409

Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});

2410

Rep = Builder.CreateZExt(Rep, CI->getType(), "");

2411

} else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||

2412

Name.startswith("avx512.vbroadcast.s"))) {

2413

// Replace broadcasts with a series of insertelements.

2414

auto *VecTy = cast<FixedVectorType>(CI->getType());

2415

Type *EltTy = VecTy->getElementType();

2416

unsigned EltNum = VecTy->getNumElements();

2417

Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),

2418

EltTy->getPointerTo());

2419

Value *Load = Builder.CreateLoad(EltTy, Cast);

2420

Type *I32Ty = Type::getInt32Ty(C);

2421

Rep = UndefValue::get(VecTy);

2422

for (unsigned I = 0; I < EltNum; ++I)

2423

Rep = Builder.CreateInsertElement(Rep, Load,

2424

ConstantInt::get(I32Ty, I));

2425

} else if (IsX86 && (Name.startswith("sse41.pmovsx") ||

2426

Name.startswith("sse41.pmovzx") ||

2427

Name.startswith("avx2.pmovsx") ||

2428

Name.startswith("avx2.pmovzx") ||

2429

Name.startswith("avx512.mask.pmovsx") ||

2430

Name.startswith("avx512.mask.pmovzx"))) {

2431

auto *DstTy = cast<FixedVectorType>(CI->getType());

2432

unsigned NumDstElts = DstTy->getNumElements();

2433

2434

// Extract a subvector of the first NumDstElts lanes and sign/zero extend.

2435

SmallVector<int, 8> ShuffleMask(NumDstElts);

2436

for (unsigned i = 0; i != NumDstElts; ++i)

2437

ShuffleMask[i] = i;

2438

2439

Value *SV =

2440

Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);

2441

2442

bool DoSext = (StringRef::npos != Name.find("pmovsx"));

2443

Rep = DoSext ? Builder.CreateSExt(SV, DstTy)

2444

: Builder.CreateZExt(SV, DstTy);

2445

// If there are 3 arguments, it's a masked intrinsic so we need a select.

2446

if (CI->getNumArgOperands() == 3)

2447

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2448

CI->getArgOperand(1));

2449

} else if (Name == "avx512.mask.pmov.qd.256" ||

2450

Name == "avx512.mask.pmov.qd.512" ||

2451

Name == "avx512.mask.pmov.wb.256" ||

2452

Name == "avx512.mask.pmov.wb.512") {

2453

Type *Ty = CI->getArgOperand(1)->getType();

2454

Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);

2455

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2456

CI->getArgOperand(1));

2457

} else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||

2458

Name == "avx2.vbroadcasti128")) {

2459

// Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.

2460

Type *EltTy = cast<VectorType>(CI->getType())->getElementType();

2461

unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();

2462

auto *VT = FixedVectorType::get(EltTy, NumSrcElts);

2463

Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),

2464

PointerType::getUnqual(VT));

2465

Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));

2466

if (NumSrcElts == 2)

2467

Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});

2468

else

2469

Rep = Builder.CreateShuffleVector(

2470

Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});

2471

} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||

2472

Name.startswith("avx512.mask.shuf.f"))) {

2473

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2474

Type *VT = CI->getType();

2475

unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;

2476

unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();

2477

unsigned ControlBitsMask = NumLanes - 1;

2478

unsigned NumControlBits = NumLanes / 2;

2479

SmallVector<int, 8> ShuffleMask(0);

2480

2481

for (unsigned l = 0; l != NumLanes; ++l) {

2482

unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;

2483

// We actually need the other source.

2484

if (l >= NumLanes / 2)

2485

LaneMask += NumLanes;

2486

for (unsigned i = 0; i != NumElementsInLane; ++i)

2487

ShuffleMask.push_back(LaneMask * NumElementsInLane + i);

2488

}

2489

Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),

2490

CI->getArgOperand(1), ShuffleMask);

2491

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,

2492

CI->getArgOperand(3));

2493

}else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||

2494

Name.startswith("avx512.mask.broadcasti"))) {

2495

unsigned NumSrcElts =

2496

cast<FixedVectorType>(CI->getArgOperand(0)->getType())

2497

->getNumElements();

2498

unsigned NumDstElts =

2499

cast<FixedVectorType>(CI->getType())->getNumElements();

2500

2501

SmallVector<int, 8> ShuffleMask(NumDstElts);

2502

for (unsigned i = 0; i != NumDstElts; ++i)

2503

ShuffleMask[i] = i % NumSrcElts;

2504

2505

Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),

2506

CI->getArgOperand(0),

2507

ShuffleMask);

2508

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2509

CI->getArgOperand(1));

2510

} else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||

2511

Name.startswith("avx2.vbroadcast") ||

2512

Name.startswith("avx512.pbroadcast") ||

2513

Name.startswith("avx512.mask.broadcast.s"))) {

2514

// Replace vp?broadcasts with a vector shuffle.

2515

Value *Op = CI->getArgOperand(0);

2516

ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();

2517

Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);

2518

SmallVector<int, 8> M;

2519

ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);

2520

Rep = Builder.CreateShuffleVector(Op, M);

2521

2522

if (CI->getNumArgOperands() == 3)

2523

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2524

CI->getArgOperand(1));

2525

} else if (IsX86 && (Name.startswith("sse2.padds.") ||

2526

Name.startswith("avx2.padds.") ||

2527

Name.startswith("avx512.padds.") ||

2528

Name.startswith("avx512.mask.padds."))) {

2529

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);

2530

} else if (IsX86 && (Name.startswith("sse2.psubs.") ||

2531

Name.startswith("avx2.psubs.") ||

2532

Name.startswith("avx512.psubs.") ||

2533

Name.startswith("avx512.mask.psubs."))) {

2534

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);

2535

} else if (IsX86 && (Name.startswith("sse2.paddus.") ||

2536

Name.startswith("avx2.paddus.") ||

2537

Name.startswith("avx512.mask.paddus."))) {

2538

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);

2539

} else if (IsX86 && (Name.startswith("sse2.psubus.") ||

2540

Name.startswith("avx2.psubus.") ||

2541

Name.startswith("avx512.mask.psubus."))) {

2542

Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);

2543

} else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {

2544

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

2545

CI->getArgOperand(1),

2546

CI->getArgOperand(2),

2547

CI->getArgOperand(3),

2548

CI->getArgOperand(4),

2549

false);

2550

} else if (IsX86 && Name.startswith("avx512.mask.valign.")) {

2551

Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),

2552

CI->getArgOperand(1),

2553

CI->getArgOperand(2),

2554

CI->getArgOperand(3),

2555

CI->getArgOperand(4),

2556

true);

2557

} else if (IsX86 && (Name == "sse2.psll.dq" ||

2558

Name == "avx2.psll.dq")) {

2559

// 128/256-bit shift left specified in bits.

2560

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2561

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),

2562

Shift / 8); // Shift is in bits.

2563

} else if (IsX86 && (Name == "sse2.psrl.dq" ||

2564

Name == "avx2.psrl.dq")) {

2565

// 128/256-bit shift right specified in bits.

2566

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2567

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),

2568

Shift / 8); // Shift is in bits.

2569

} else if (IsX86 && (Name == "sse2.psll.dq.bs" ||

2570

Name == "avx2.psll.dq.bs" ||

2571

Name == "avx512.psll.dq.512")) {

2572

// 128/256/512-bit shift left specified in bytes.

2573

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2574

Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

2575

} else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||

2576

Name == "avx2.psrl.dq.bs" ||

2577

Name == "avx512.psrl.dq.512")) {

2578

// 128/256/512-bit shift right specified in bytes.

2579

unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2580

Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);

2581

} else if (IsX86 && (Name == "sse41.pblendw" ||

2582

Name.startswith("sse41.blendp") ||

2583

Name.startswith("avx.blend.p") ||

2584

Name == "avx2.pblendw" ||

2585

Name.startswith("avx2.pblendd."))) {

2586

Value *Op0 = CI->getArgOperand(0);

2587

Value *Op1 = CI->getArgOperand(1);

2588

unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2589

auto *VecTy = cast<FixedVectorType>(CI->getType());

2590

unsigned NumElts = VecTy->getNumElements();

2591

2592

SmallVector<int, 16> Idxs(NumElts);

2593

for (unsigned i = 0; i != NumElts; ++i)

2594

Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;

2595

2596

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

2597

} else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||

2598

Name == "avx2.vinserti128" ||

2599

Name.startswith("avx512.mask.insert"))) {

2600

Value *Op0 = CI->getArgOperand(0);

2601

Value *Op1 = CI->getArgOperand(1);

2602

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2603

unsigned DstNumElts =

2604

cast<FixedVectorType>(CI->getType())->getNumElements();

2605

unsigned SrcNumElts =

2606

cast<FixedVectorType>(Op1->getType())->getNumElements();

2607

unsigned Scale = DstNumElts / SrcNumElts;

2608

2609

// Mask off the high bits of the immediate value; hardware ignores those.

2610

Imm = Imm % Scale;

2611

2612

// Extend the second operand into a vector the size of the destination.

2613

SmallVector<int, 8> Idxs(DstNumElts);

2614

for (unsigned i = 0; i != SrcNumElts; ++i)

2615

Idxs[i] = i;

2616

for (unsigned i = SrcNumElts; i != DstNumElts; ++i)

2617

Idxs[i] = SrcNumElts;

2618

Rep = Builder.CreateShuffleVector(Op1, Idxs);

2619

2620

// Insert the second operand into the first operand.

2621

2622

// Note that there is no guarantee that instruction lowering will actually

2623

// produce a vinsertf128 instruction for the created shuffles. In

2624

// particular, the 0 immediate case involves no lane changes, so it can

2625

// be handled as a blend.

2626

2627

// Example of shuffle mask for 32-bit elements:

2628

// Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>

2629

// Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >

2630

2631

// First fill with identify mask.

2632

for (unsigned i = 0; i != DstNumElts; ++i)

2633

Idxs[i] = i;

2634

// Then replace the elements where we need to insert.

2635

for (unsigned i = 0; i != SrcNumElts; ++i)

2636

Idxs[i + Imm * SrcNumElts] = i + DstNumElts;

2637

Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);

2638

2639

// If the intrinsic has a mask operand, handle that.

2640

if (CI->getNumArgOperands() == 5)

2641

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,

2642

CI->getArgOperand(3));

2643

} else if (IsX86 && (Name.startswith("avx.vextractf128.") ||

2644

Name == "avx2.vextracti128" ||

2645

Name.startswith("avx512.mask.vextract"))) {

2646

Value *Op0 = CI->getArgOperand(0);

2647

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2648

unsigned DstNumElts =

2649

cast<FixedVectorType>(CI->getType())->getNumElements();

2650

unsigned SrcNumElts =

2651

cast<FixedVectorType>(Op0->getType())->getNumElements();

2652

unsigned Scale = SrcNumElts / DstNumElts;

2653

2654

// Mask off the high bits of the immediate value; hardware ignores those.

2655

Imm = Imm % Scale;

2656

2657

// Get indexes for the subvector of the input vector.

2658

SmallVector<int, 8> Idxs(DstNumElts);

2659

for (unsigned i = 0; i != DstNumElts; ++i) {

2660

Idxs[i] = i + (Imm * DstNumElts);

2661

}

2662

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2663

2664

// If the intrinsic has a mask operand, handle that.

2665

if (CI->getNumArgOperands() == 4)

2666

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2667

CI->getArgOperand(2));

2668

} else if (!IsX86 && Name == "stackprotectorcheck") {

2669

Rep = nullptr;

2670

} else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||

2671

Name.startswith("avx512.mask.perm.di."))) {

2672

Value *Op0 = CI->getArgOperand(0);

2673

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2674

auto *VecTy = cast<FixedVectorType>(CI->getType());

2675

unsigned NumElts = VecTy->getNumElements();

2676

2677

SmallVector<int, 8> Idxs(NumElts);

2678

for (unsigned i = 0; i != NumElts; ++i)

2679

Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);

2680

2681

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2682

2683

if (CI->getNumArgOperands() == 4)

2684

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2685

CI->getArgOperand(2));

2686

} else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||

2687

Name == "avx2.vperm2i128")) {

2688

// The immediate permute control byte looks like this:

2689

// [1:0] - select 128 bits from sources for low half of destination

2690

// [2] - ignore

2691

// [3] - zero low half of destination

2692

// [5:4] - select 128 bits from sources for high half of destination

2693

// [6] - ignore

2694

// [7] - zero high half of destination

2695

2696

uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2697

2698

unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2699

unsigned HalfSize = NumElts / 2;

2700

SmallVector<int, 8> ShuffleMask(NumElts);

2701

2702

// Determine which operand(s) are actually in use for this instruction.

2703

Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);

2704

Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);

2705

2706

// If needed, replace operands based on zero mask.

2707

V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;

2708

V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;

2709

2710

// Permute low half of result.

2711

unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;

2712

for (unsigned i = 0; i < HalfSize; ++i)

2713

ShuffleMask[i] = StartIndex + i;

2714

2715

// Permute high half of result.

2716

StartIndex = (Imm & 0x10) ? HalfSize : 0;

2717

for (unsigned i = 0; i < HalfSize; ++i)

2718

ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;

2719

2720

Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);

2721

2722

} else if (IsX86 && (Name.startswith("avx.vpermil.") ||

2723

Name == "sse2.pshuf.d" ||

2724

Name.startswith("avx512.mask.vpermil.p") ||

2725

Name.startswith("avx512.mask.pshuf.d."))) {

2726

Value *Op0 = CI->getArgOperand(0);

2727

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2728

auto *VecTy = cast<FixedVectorType>(CI->getType());

2729

unsigned NumElts = VecTy->getNumElements();

2730

// Calculate the size of each index in the immediate.

2731

unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();

2732

unsigned IdxMask = ((1 << IdxSize) - 1);

2733

2734

SmallVector<int, 8> Idxs(NumElts);

2735

// Lookup the bits for this element, wrapping around the immediate every

2736

// 8-bits. Elements are grouped into sets of 2 or 4 elements so we need

2737

// to offset by the first index of each group.

2738

for (unsigned i = 0; i != NumElts; ++i)

2739

Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);

2740

2741

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2742

2743

if (CI->getNumArgOperands() == 4)

2744

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2745

CI->getArgOperand(2));

2746

} else if (IsX86 && (Name == "sse2.pshufl.w" ||

2747

Name.startswith("avx512.mask.pshufl.w."))) {

2748

Value *Op0 = CI->getArgOperand(0);

2749

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2750

unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2751

2752

SmallVector<int, 16> Idxs(NumElts);

2753

for (unsigned l = 0; l != NumElts; l += 8) {

2754

for (unsigned i = 0; i != 4; ++i)

2755

Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;

2756

for (unsigned i = 4; i != 8; ++i)

2757

Idxs[i + l] = i + l;

2758

}

2759

2760

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2761

2762

if (CI->getNumArgOperands() == 4)

2763

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2764

CI->getArgOperand(2));

2765

} else if (IsX86 && (Name == "sse2.pshufh.w" ||

2766

Name.startswith("avx512.mask.pshufh.w."))) {

2767

Value *Op0 = CI->getArgOperand(0);

2768

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();

2769

unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2770

2771

SmallVector<int, 16> Idxs(NumElts);

2772

for (unsigned l = 0; l != NumElts; l += 8) {

2773

for (unsigned i = 0; i != 4; ++i)

2774

Idxs[i + l] = i + l;

2775

for (unsigned i = 0; i != 4; ++i)

2776

Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;

2777

}

2778

2779

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2780

2781

if (CI->getNumArgOperands() == 4)

2782

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2783

CI->getArgOperand(2));

2784

} else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {

2785

Value *Op0 = CI->getArgOperand(0);

2786

Value *Op1 = CI->getArgOperand(1);

2787

unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

2788

unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2789

2790

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

2791

unsigned HalfLaneElts = NumLaneElts / 2;

2792

2793

SmallVector<int, 16> Idxs(NumElts);

2794

for (unsigned i = 0; i != NumElts; ++i) {

2795

// Base index is the starting element of the lane.

2796

Idxs[i] = i - (i % NumLaneElts);

2797

// If we are half way through the lane switch to the other source.

2798

if ((i % NumLaneElts) >= HalfLaneElts)

2799

Idxs[i] += NumElts;

2800

// Now select the specific element. By adding HalfLaneElts bits from

2801

// the immediate. Wrapping around the immediate every 8-bits.

2802

Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);

2803

}

2804

2805

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

2806

2807

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,

2808

CI->getArgOperand(3));

2809

} else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||

2810

Name.startswith("avx512.mask.movshdup") ||

2811

Name.startswith("avx512.mask.movsldup"))) {

2812

Value *Op0 = CI->getArgOperand(0);

2813

unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2814

unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

2815

2816

unsigned Offset = 0;

2817

if (Name.startswith("avx512.mask.movshdup."))

2818

Offset = 1;

2819

2820

SmallVector<int, 16> Idxs(NumElts);

2821

for (unsigned l = 0; l != NumElts; l += NumLaneElts)

2822

for (unsigned i = 0; i != NumLaneElts; i += 2) {

2823

Idxs[i + l + 0] = i + l + Offset;

2824

Idxs[i + l + 1] = i + l + Offset;

2825

}

2826

2827

Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);

2828

2829

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2830

CI->getArgOperand(1));

2831

} else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||

2832

Name.startswith("avx512.mask.unpckl."))) {

2833

Value *Op0 = CI->getArgOperand(0);

2834

Value *Op1 = CI->getArgOperand(1);

2835

int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2836

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

2837

2838

SmallVector<int, 64> Idxs(NumElts);

2839

for (int l = 0; l != NumElts; l += NumLaneElts)

2840

for (int i = 0; i != NumLaneElts; ++i)

2841

Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);

2842

2843

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

2844

2845

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2846

CI->getArgOperand(2));

2847

} else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||

2848

Name.startswith("avx512.mask.unpckh."))) {

2849

Value *Op0 = CI->getArgOperand(0);

2850

Value *Op1 = CI->getArgOperand(1);

2851

int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

2852

int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();

2853

2854

SmallVector<int, 64> Idxs(NumElts);

2855

for (int l = 0; l != NumElts; l += NumLaneElts)

2856

for (int i = 0; i != NumLaneElts; ++i)

2857

Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);

2858

2859

Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);

2860

2861

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2862

CI->getArgOperand(2));

2863

} else if (IsX86 && (Name.startswith("avx512.mask.and.") ||

2864

Name.startswith("avx512.mask.pand."))) {

2865

VectorType *FTy = cast<VectorType>(CI->getType());

2866

VectorType *ITy = VectorType::getInteger(FTy);

2867

Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

2868

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

2869

Rep = Builder.CreateBitCast(Rep, FTy);

2870

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2871

CI->getArgOperand(2));

2872

} else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||

2873

Name.startswith("avx512.mask.pandn."))) {

2874

VectorType *FTy = cast<VectorType>(CI->getType());

2875

VectorType *ITy = VectorType::getInteger(FTy);

2876

Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));

2877

Rep = Builder.CreateAnd(Rep,

2878

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

2879

Rep = Builder.CreateBitCast(Rep, FTy);

2880

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2881

CI->getArgOperand(2));

2882

} else if (IsX86 && (Name.startswith("avx512.mask.or.") ||

2883

Name.startswith("avx512.mask.por."))) {

2884

VectorType *FTy = cast<VectorType>(CI->getType());

2885

VectorType *ITy = VectorType::getInteger(FTy);

2886

Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

2887

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

2888

Rep = Builder.CreateBitCast(Rep, FTy);

2889

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2890

CI->getArgOperand(2));

2891

} else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||

2892

Name.startswith("avx512.mask.pxor."))) {

2893

VectorType *FTy = cast<VectorType>(CI->getType());

2894

VectorType *ITy = VectorType::getInteger(FTy);

2895

Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),

2896

Builder.CreateBitCast(CI->getArgOperand(1), ITy));

2897

Rep = Builder.CreateBitCast(Rep, FTy);

2898

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2899

CI->getArgOperand(2));

2900

} else if (IsX86 && Name.startswith("avx512.mask.padd.")) {

2901

Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));

2902

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2903

CI->getArgOperand(2));

2904

} else if (IsX86 && Name.startswith("avx512.mask.psub.")) {

2905

Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));

2906

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2907

CI->getArgOperand(2));

2908

} else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {

2909

Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));

2910

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2911

CI->getArgOperand(2));

2912

} else if (IsX86 && Name.startswith("avx512.mask.add.p")) {

2913

if (Name.endswith(".512")) {

2914

Intrinsic::ID IID;

2915

if (Name[17] == 's')

2916

IID = Intrinsic::x86_avx512_add_ps_512;

2917

else

2918

IID = Intrinsic::x86_avx512_add_pd_512;

2919

2920

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2921

{ CI->getArgOperand(0), CI->getArgOperand(1),

2922

CI->getArgOperand(4) });

2923

} else {

2924

Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));

2925

}

2926

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2927

CI->getArgOperand(2));

2928

} else if (IsX86 && Name.startswith("avx512.mask.div.p")) {

2929

if (Name.endswith(".512")) {

2930

Intrinsic::ID IID;

2931

if (Name[17] == 's')

2932

IID = Intrinsic::x86_avx512_div_ps_512;

2933

else

2934

IID = Intrinsic::x86_avx512_div_pd_512;

2935

2936

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2937

{ CI->getArgOperand(0), CI->getArgOperand(1),

2938

CI->getArgOperand(4) });

2939

} else {

2940

Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));

2941

}

2942

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2943

CI->getArgOperand(2));

2944

} else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {

2945

if (Name.endswith(".512")) {

2946

Intrinsic::ID IID;

2947

if (Name[17] == 's')

2948

IID = Intrinsic::x86_avx512_mul_ps_512;

2949

else

2950

IID = Intrinsic::x86_avx512_mul_pd_512;

2951

2952

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2953

{ CI->getArgOperand(0), CI->getArgOperand(1),

2954

CI->getArgOperand(4) });

2955

} else {

2956

Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));

2957

}

2958

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2959

CI->getArgOperand(2));

2960

} else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {

2961

if (Name.endswith(".512")) {

2962

Intrinsic::ID IID;

2963

if (Name[17] == 's')

2964

IID = Intrinsic::x86_avx512_sub_ps_512;

2965

else

2966

IID = Intrinsic::x86_avx512_sub_pd_512;

2967

2968

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2969

{ CI->getArgOperand(0), CI->getArgOperand(1),

2970

CI->getArgOperand(4) });

2971

} else {

2972

Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));

2973

}

2974

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2975

CI->getArgOperand(2));

2976

} else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||

2977

Name.startswith("avx512.mask.min.p")) &&

2978

Name.drop_front(18) == ".512") {

2979

bool IsDouble = Name[17] == 'd';

2980

bool IsMin = Name[13] == 'i';

2981

static const Intrinsic::ID MinMaxTbl[2][2] = {

2982

{ Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },

2983

{ Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }

2984

};

2985

Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];

2986

2987

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

2988

{ CI->getArgOperand(0), CI->getArgOperand(1),

2989

CI->getArgOperand(4) });

2990

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,

2991

CI->getArgOperand(2));

2992

} else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {

2993

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),

2994

Intrinsic::ctlz,

2995

CI->getType()),

2996

{ CI->getArgOperand(0), Builder.getInt1(false) });

2997

Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,

2998

CI->getArgOperand(1));

2999

} else if (IsX86 && Name.startswith("avx512.mask.psll")) {

3000

bool IsImmediate = Name[16] == 'i' ||

3001

(Name.size() > 18 && Name[18] == 'i');

3002

bool IsVariable = Name[16] == 'v';

3003

char Size = Name[16] == '.' ? Name[17] :

3004

Name[17] == '.' ? Name[18] :

3005

Name[18] == '.' ? Name[19] :

3006

Name[20];

3007

3008

Intrinsic::ID IID;

3009

if (IsVariable && Name[17] != '.') {

3010

if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di

3011

IID = Intrinsic::x86_avx2_psllv_q;

3012

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di

3013

IID = Intrinsic::x86_avx2_psllv_q_256;

3014

else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si

3015

IID = Intrinsic::x86_avx2_psllv_d;

3016

else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si

3017

IID = Intrinsic::x86_avx2_psllv_d_256;

3018

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi

3019

IID = Intrinsic::x86_avx512_psllv_w_128;

3020

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi

3021

IID = Intrinsic::x86_avx512_psllv_w_256;

3022

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi

3023

IID = Intrinsic::x86_avx512_psllv_w_512;

3024

else

3025

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3025);

3026

} else if (Name.endswith(".128")) {

3027

if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128

3028

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d

3029

: Intrinsic::x86_sse2_psll_d;

3030

else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128

3031

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q

3032

: Intrinsic::x86_sse2_psll_q;

3033

else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128

3034

IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w

3035

: Intrinsic::x86_sse2_psll_w;

3036

else

3037

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3037);

3038

} else if (Name.endswith(".256")) {

3039

if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256

3040

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d

3041

: Intrinsic::x86_avx2_psll_d;

3042

else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256

3043

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q

3044

: Intrinsic::x86_avx2_psll_q;

3045

else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256

3046

IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w

3047

: Intrinsic::x86_avx2_psll_w;

3048

else

3049

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3049);

3050

} else {

3051

if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512

3052

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :

3053

IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :

3054

Intrinsic::x86_avx512_psll_d_512;

3055

else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512

3056

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :

3057

IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :

3058

Intrinsic::x86_avx512_psll_q_512;

3059

else if (Size == 'w') // psll.wi.512, pslli.w, psll.w

3060

IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512

3061

: Intrinsic::x86_avx512_psll_w_512;

3062

else

3063

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3063);

3064

}

3065

3066

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

3067

} else if (IsX86 && Name.startswith("avx512.mask.psrl")) {

3068

bool IsImmediate = Name[16] == 'i' ||

3069

(Name.size() > 18 && Name[18] == 'i');

3070

bool IsVariable = Name[16] == 'v';

3071

char Size = Name[16] == '.' ? Name[17] :

3072

Name[17] == '.' ? Name[18] :

3073

Name[18] == '.' ? Name[19] :

3074

Name[20];

3075

3076

Intrinsic::ID IID;

3077

if (IsVariable && Name[17] != '.') {

3078

if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di

3079

IID = Intrinsic::x86_avx2_psrlv_q;

3080

else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di

3081

IID = Intrinsic::x86_avx2_psrlv_q_256;

3082

else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si

3083

IID = Intrinsic::x86_avx2_psrlv_d;

3084

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si

3085

IID = Intrinsic::x86_avx2_psrlv_d_256;

3086

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi

3087

IID = Intrinsic::x86_avx512_psrlv_w_128;

3088

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi

3089

IID = Intrinsic::x86_avx512_psrlv_w_256;

3090

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi

3091

IID = Intrinsic::x86_avx512_psrlv_w_512;

3092

else

3093

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3093);

3094

} else if (Name.endswith(".128")) {

3095

if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128

3096

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d

3097

: Intrinsic::x86_sse2_psrl_d;

3098

else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128

3099

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q

3100

: Intrinsic::x86_sse2_psrl_q;

3101

else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128

3102

IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w

3103

: Intrinsic::x86_sse2_psrl_w;

3104

else

3105

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3105);

3106

} else if (Name.endswith(".256")) {

3107

if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256

3108

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d

3109

: Intrinsic::x86_avx2_psrl_d;

3110

else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256

3111

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q

3112

: Intrinsic::x86_avx2_psrl_q;

3113

else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256

3114

IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w

3115

: Intrinsic::x86_avx2_psrl_w;

3116

else

3117

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3117);

3118

} else {

3119

if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512

3120

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :

3121

IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :

3122

Intrinsic::x86_avx512_psrl_d_512;

3123

else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512

3124

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :

3125

IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :

3126

Intrinsic::x86_avx512_psrl_q_512;

3127

else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)

3128

IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512

3129

: Intrinsic::x86_avx512_psrl_w_512;

3130

else

3131

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3131);

3132

}

3133

3134

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

3135

} else if (IsX86 && Name.startswith("avx512.mask.psra")) {

3136

bool IsImmediate = Name[16] == 'i' ||

3137

(Name.size() > 18 && Name[18] == 'i');

3138

bool IsVariable = Name[16] == 'v';

3139

char Size = Name[16] == '.' ? Name[17] :

3140

Name[17] == '.' ? Name[18] :

3141

Name[18] == '.' ? Name[19] :

3142

Name[20];

3143

3144

Intrinsic::ID IID;

3145

if (IsVariable && Name[17] != '.') {

3146

if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si

3147

IID = Intrinsic::x86_avx2_psrav_d;

3148

else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si

3149

IID = Intrinsic::x86_avx2_psrav_d_256;

3150

else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi

3151

IID = Intrinsic::x86_avx512_psrav_w_128;

3152

else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi

3153

IID = Intrinsic::x86_avx512_psrav_w_256;

3154

else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi

3155

IID = Intrinsic::x86_avx512_psrav_w_512;

3156

else

3157

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3157);

3158

} else if (Name.endswith(".128")) {

3159

if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128

3160

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d

3161

: Intrinsic::x86_sse2_psra_d;

3162

else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128

3163

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :

3164

IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :

3165

Intrinsic::x86_avx512_psra_q_128;

3166

else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128

3167

IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w

3168

: Intrinsic::x86_sse2_psra_w;

3169

else

3170

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3170);

3171

} else if (Name.endswith(".256")) {

3172

if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256

3173

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d

3174

: Intrinsic::x86_avx2_psra_d;

3175

else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256

3176

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :

3177

IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :

3178

Intrinsic::x86_avx512_psra_q_256;

3179

else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256

3180

IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w

3181

: Intrinsic::x86_avx2_psra_w;

3182

else

3183

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3183);

3184

} else {

3185

if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512

3186

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :

3187

IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :

3188

Intrinsic::x86_avx512_psra_d_512;

3189

else if (Size == 'q') // psra.qi.512, psrai.q, psra.q

3190

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :

3191

IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :

3192

Intrinsic::x86_avx512_psra_q_512;

3193

else if (Size == 'w') // psra.wi.512, psrai.w, psra.w

3194

IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512

3195

: Intrinsic::x86_avx512_psra_w_512;

3196

else

3197

llvm_unreachable("Unexpected size")::llvm::llvm_unreachable_internal("Unexpected size", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3197);

3198

}

3199

3200

Rep = UpgradeX86MaskedShift(Builder, *CI, IID);

3201

} else if (IsX86 && Name.startswith("avx512.mask.move.s")) {

3202

Rep = upgradeMaskedMove(Builder, *CI);

3203

} else if (IsX86 && Name.startswith("avx512.cvtmask2")) {

3204

Rep = UpgradeMaskToInt(Builder, *CI);

3205

} else if (IsX86 && Name.endswith(".movntdqa")) {

3206

Module *M = F->getParent();

3207

MDNode *Node = MDNode::get(

3208

C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));

3209

3210

Value *Ptr = CI->getArgOperand(0);

3211

3212

// Convert the type of the pointer to a pointer to the stored type.

3213

Value *BC = Builder.CreateBitCast(

3214

Ptr, PointerType::getUnqual(CI->getType()), "cast");

3215

LoadInst *LI = Builder.CreateAlignedLoad(

3216

CI->getType(), BC,

3217

Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));

3218

LI->setMetadata(M->getMDKindID("nontemporal"), Node);

3219

Rep = LI;

3220

} else if (IsX86 && (Name.startswith("fma.vfmadd.") ||

3221

Name.startswith("fma.vfmsub.") ||

3222

Name.startswith("fma.vfnmadd.") ||

3223

Name.startswith("fma.vfnmsub."))) {

3224

bool NegMul = Name[6] == 'n';

3225

bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';

3226

bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';

3227

3228

Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3229

CI->getArgOperand(2) };

3230

3231

if (IsScalar) {

3232

Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);

3233

Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);

3234

Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);

3235

}

3236

3237

if (NegMul && !IsScalar)

3238

Ops[0] = Builder.CreateFNeg(Ops[0]);

3239

if (NegMul && IsScalar)

3240

Ops[1] = Builder.CreateFNeg(Ops[1]);

3241

if (NegAcc)

3242

Ops[2] = Builder.CreateFNeg(Ops[2]);

3243

3244

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),

3245

Intrinsic::fma,

3246

Ops[0]->getType()),

3247

Ops);

3248

3249

if (IsScalar)

3250

Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,

3251

(uint64_t)0);

3252

} else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {

3253

Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3254

CI->getArgOperand(2) };

3255

3256

Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);

3257

Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);

3258

Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);

3259

3260

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),

3261

Intrinsic::fma,

3262

Ops[0]->getType()),

3263

Ops);

3264

3265

Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),

3266

Rep, (uint64_t)0);

3267

} else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||

3268

Name.startswith("avx512.maskz.vfmadd.s") ||

3269

Name.startswith("avx512.mask3.vfmadd.s") ||

3270

Name.startswith("avx512.mask3.vfmsub.s") ||

3271

Name.startswith("avx512.mask3.vfnmsub.s"))) {

3272

bool IsMask3 = Name[11] == '3';

3273

bool IsMaskZ = Name[11] == 'z';

3274

// Drop the "avx512.mask." to make it easier.

3275

Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

3276

bool NegMul = Name[2] == 'n';

3277

bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';

3278

3279

Value *A = CI->getArgOperand(0);

3280

Value *B = CI->getArgOperand(1);

3281

Value *C = CI->getArgOperand(2);

3282

3283

if (NegMul && (IsMask3 || IsMaskZ))

3284

A = Builder.CreateFNeg(A);

3285

if (NegMul && !(IsMask3 || IsMaskZ))

3286

B = Builder.CreateFNeg(B);

3287

if (NegAcc)

3288

C = Builder.CreateFNeg(C);

3289

3290

A = Builder.CreateExtractElement(A, (uint64_t)0);

3291

B = Builder.CreateExtractElement(B, (uint64_t)0);

3292

C = Builder.CreateExtractElement(C, (uint64_t)0);

3293

3294

if (!isa<ConstantInt>(CI->getArgOperand(4)) ||

3295

cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {

3296

Value *Ops[] = { A, B, C, CI->getArgOperand(4) };

3297

3298

Intrinsic::ID IID;

3299

if (Name.back() == 'd')

3300

IID = Intrinsic::x86_avx512_vfmadd_f64;

3301

else

3302

IID = Intrinsic::x86_avx512_vfmadd_f32;

3303

Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);

3304

Rep = Builder.CreateCall(FMA, Ops);

3305

} else {

3306

Function *FMA = Intrinsic::getDeclaration(CI->getModule(),

3307

Intrinsic::fma,

3308

A->getType());

3309

Rep = Builder.CreateCall(FMA, { A, B, C });

3310

}

3311

3312

Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :

3313

IsMask3 ? C : A;

3314

3315

// For Mask3 with NegAcc, we need to create a new extractelement that

3316

// avoids the negation above.

3317

if (NegAcc && IsMask3)

3318

PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),

3319

(uint64_t)0);

3320

3321

Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),

3322

Rep, PassThru);

3323

Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),

3324

Rep, (uint64_t)0);

3325

} else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||

3326

Name.startswith("avx512.mask.vfnmadd.p") ||

3327

Name.startswith("avx512.mask.vfnmsub.p") ||

3328

Name.startswith("avx512.mask3.vfmadd.p") ||

3329

Name.startswith("avx512.mask3.vfmsub.p") ||

3330

Name.startswith("avx512.mask3.vfnmsub.p") ||

3331

Name.startswith("avx512.maskz.vfmadd.p"))) {

3332

bool IsMask3 = Name[11] == '3';

3333

bool IsMaskZ = Name[11] == 'z';

3334

// Drop the "avx512.mask." to make it easier.

3335

Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

3336

bool NegMul = Name[2] == 'n';

3337

bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';

3338

3339

Value *A = CI->getArgOperand(0);

3340

Value *B = CI->getArgOperand(1);

3341

Value *C = CI->getArgOperand(2);

3342

3343

if (NegMul && (IsMask3 || IsMaskZ))

3344

A = Builder.CreateFNeg(A);

3345

if (NegMul && !(IsMask3 || IsMaskZ))

3346

B = Builder.CreateFNeg(B);

3347

if (NegAcc)

3348

C = Builder.CreateFNeg(C);

3349

3350

if (CI->getNumArgOperands() == 5 &&

3351

(!isa<ConstantInt>(CI->getArgOperand(4)) ||

3352

cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {

3353

Intrinsic::ID IID;

3354

// Check the character before ".512" in string.

3355

if (Name[Name.size()-5] == 's')

3356

IID = Intrinsic::x86_avx512_vfmadd_ps_512;

3357

else

3358

IID = Intrinsic::x86_avx512_vfmadd_pd_512;

3359

3360

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

3361

{ A, B, C, CI->getArgOperand(4) });

3362

} else {

3363

Function *FMA = Intrinsic::getDeclaration(CI->getModule(),

3364

Intrinsic::fma,

3365

A->getType());

3366

Rep = Builder.CreateCall(FMA, { A, B, C });

3367

}

3368

3369

Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :

3370

IsMask3 ? CI->getArgOperand(2) :

3371

CI->getArgOperand(0);

3372

3373

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);

3374

} else if (IsX86 && Name.startswith("fma.vfmsubadd.p")) {

3375

unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();

3376

unsigned EltWidth = CI->getType()->getScalarSizeInBits();

3377

Intrinsic::ID IID;

3378

if (VecWidth == 128 && EltWidth == 32)

3379

IID = Intrinsic::x86_fma_vfmaddsub_ps;

3380

else if (VecWidth == 256 && EltWidth == 32)

3381

IID = Intrinsic::x86_fma_vfmaddsub_ps_256;

3382

else if (VecWidth == 128 && EltWidth == 64)

3383

IID = Intrinsic::x86_fma_vfmaddsub_pd;

3384

else if (VecWidth == 256 && EltWidth == 64)

3385

IID = Intrinsic::x86_fma_vfmaddsub_pd_256;

3386

else

3387

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3387);

3388

3389

Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3390

CI->getArgOperand(2) };

3391

Ops[2] = Builder.CreateFNeg(Ops[2]);

3392

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

3393

Ops);

3394

} else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||

3395

Name.startswith("avx512.mask3.vfmaddsub.p") ||

3396

Name.startswith("avx512.maskz.vfmaddsub.p") ||

3397

Name.startswith("avx512.mask3.vfmsubadd.p"))) {

3398

bool IsMask3 = Name[11] == '3';

3399

bool IsMaskZ = Name[11] == 'z';

3400

// Drop the "avx512.mask." to make it easier.

3401

Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);

3402

bool IsSubAdd = Name[3] == 's';

3403

if (CI->getNumArgOperands() == 5) {

3404

Intrinsic::ID IID;

3405

// Check the character before ".512" in string.

3406

if (Name[Name.size()-5] == 's')

3407

IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;

3408

else

3409

IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;

3410

3411

Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3412

CI->getArgOperand(2), CI->getArgOperand(4) };

3413

if (IsSubAdd)

3414

Ops[2] = Builder.CreateFNeg(Ops[2]);

3415

3416

Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),

3417

Ops);

3418

} else {

3419

int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();

3420

3421

Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3422

CI->getArgOperand(2) };

3423

3424

Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,

3425

Ops[0]->getType());

3426

Value *Odd = Builder.CreateCall(FMA, Ops);

3427

Ops[2] = Builder.CreateFNeg(Ops[2]);

3428

Value *Even = Builder.CreateCall(FMA, Ops);

3429

3430

if (IsSubAdd)

3431

std::swap(Even, Odd);

3432

3433

SmallVector<int, 32> Idxs(NumElts);

3434

for (int i = 0; i != NumElts; ++i)

3435

Idxs[i] = i + (i % 2) * NumElts;

3436

3437

Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);

3438

}

3439

3440

Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :

3441

IsMask3 ? CI->getArgOperand(2) :

3442

CI->getArgOperand(0);

3443

3444

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);

3445

} else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||

3446

Name.startswith("avx512.maskz.pternlog."))) {

3447

bool ZeroMask = Name[11] == 'z';

3448

unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();

3449

unsigned EltWidth = CI->getType()->getScalarSizeInBits();

3450

Intrinsic::ID IID;

3451

if (VecWidth == 128 && EltWidth == 32)

3452

IID = Intrinsic::x86_avx512_pternlog_d_128;

3453

else if (VecWidth == 256 && EltWidth == 32)

3454

IID = Intrinsic::x86_avx512_pternlog_d_256;

3455

else if (VecWidth == 512 && EltWidth == 32)

3456

IID = Intrinsic::x86_avx512_pternlog_d_512;

3457

else if (VecWidth == 128 && EltWidth == 64)

3458

IID = Intrinsic::x86_avx512_pternlog_q_128;

3459

else if (VecWidth == 256 && EltWidth == 64)

3460

IID = Intrinsic::x86_avx512_pternlog_q_256;

3461

else if (VecWidth == 512 && EltWidth == 64)

3462

IID = Intrinsic::x86_avx512_pternlog_q_512;

3463

else

3464

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3464);

3465

3466

Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),

3467

CI->getArgOperand(2), CI->getArgOperand(3) };

3468

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),

3469

Args);

3470

Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())

3471

: CI->getArgOperand(0);

3472

Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);

3473

} else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||

3474

Name.startswith("avx512.maskz.vpmadd52"))) {

3475

bool ZeroMask = Name[11] == 'z';

3476

bool High = Name[20] == 'h' || Name[21] == 'h';

3477

unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();

3478

Intrinsic::ID IID;

3479

if (VecWidth == 128 && !High)

3480

IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;

3481

else if (VecWidth == 256 && !High)

3482

IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;

3483

else if (VecWidth == 512 && !High)

3484

IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;

3485

else if (VecWidth == 128 && High)

3486

IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;

3487

else if (VecWidth == 256 && High)

3488

IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;

3489

else if (VecWidth == 512 && High)

3490

IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;

3491

else

3492

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3492);

3493

3494

Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),

3495

CI->getArgOperand(2) };

3496

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),

3497

Args);

3498

Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())

3499

: CI->getArgOperand(0);

3500

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);

3501

} else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||

3502

Name.startswith("avx512.mask.vpermt2var.") ||

3503

Name.startswith("avx512.maskz.vpermt2var."))) {

3504

bool ZeroMask = Name[11] == 'z';

3505

bool IndexForm = Name[17] == 'i';

3506

Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);

3507

} else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||

3508

Name.startswith("avx512.maskz.vpdpbusd.") ||

3509

Name.startswith("avx512.mask.vpdpbusds.") ||

3510

Name.startswith("avx512.maskz.vpdpbusds."))) {

3511

bool ZeroMask = Name[11] == 'z';

3512

bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';

3513

unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();

3514

Intrinsic::ID IID;

3515

if (VecWidth == 128 && !IsSaturating)

3516

IID = Intrinsic::x86_avx512_vpdpbusd_128;

3517

else if (VecWidth == 256 && !IsSaturating)

3518

IID = Intrinsic::x86_avx512_vpdpbusd_256;

3519

else if (VecWidth == 512 && !IsSaturating)

3520

IID = Intrinsic::x86_avx512_vpdpbusd_512;

3521

else if (VecWidth == 128 && IsSaturating)

3522

IID = Intrinsic::x86_avx512_vpdpbusds_128;

3523

else if (VecWidth == 256 && IsSaturating)

3524

IID = Intrinsic::x86_avx512_vpdpbusds_256;

3525

else if (VecWidth == 512 && IsSaturating)

3526

IID = Intrinsic::x86_avx512_vpdpbusds_512;

3527

else

3528

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3528);

3529

3530

Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3531

CI->getArgOperand(2) };

3532

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),

3533

Args);

3534

Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())

3535

: CI->getArgOperand(0);

3536

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);

3537

} else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||

3538

Name.startswith("avx512.maskz.vpdpwssd.") ||

3539

Name.startswith("avx512.mask.vpdpwssds.") ||

3540

Name.startswith("avx512.maskz.vpdpwssds."))) {

3541

bool ZeroMask = Name[11] == 'z';

3542

bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';

3543

unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();

3544

Intrinsic::ID IID;

3545

if (VecWidth == 128 && !IsSaturating)

3546

IID = Intrinsic::x86_avx512_vpdpwssd_128;

3547

else if (VecWidth == 256 && !IsSaturating)

3548

IID = Intrinsic::x86_avx512_vpdpwssd_256;

3549

else if (VecWidth == 512 && !IsSaturating)

3550

IID = Intrinsic::x86_avx512_vpdpwssd_512;

3551

else if (VecWidth == 128 && IsSaturating)

3552

IID = Intrinsic::x86_avx512_vpdpwssds_128;

3553

else if (VecWidth == 256 && IsSaturating)

3554

IID = Intrinsic::x86_avx512_vpdpwssds_256;

3555

else if (VecWidth == 512 && IsSaturating)

3556

IID = Intrinsic::x86_avx512_vpdpwssds_512;

3557

else

3558

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3558);

3559

3560

Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3561

CI->getArgOperand(2) };

3562

Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),

3563

Args);

3564

Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())

3565

: CI->getArgOperand(0);

3566

Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);

3567

} else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||

3568

Name == "addcarry.u32" || Name == "addcarry.u64" ||

3569

Name == "subborrow.u32" || Name == "subborrow.u64")) {

3570

Intrinsic::ID IID;

3571

if (Name[0] == 'a' && Name.back() == '2')

3572

IID = Intrinsic::x86_addcarry_32;

3573

else if (Name[0] == 'a' && Name.back() == '4')

3574

IID = Intrinsic::x86_addcarry_64;

3575

else if (Name[0] == 's' && Name.back() == '2')

3576

IID = Intrinsic::x86_subborrow_32;

3577

else if (Name[0] == 's' && Name.back() == '4')

3578

IID = Intrinsic::x86_subborrow_64;

3579

else

3580

llvm_unreachable("Unexpected intrinsic")::llvm::llvm_unreachable_internal("Unexpected intrinsic", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3580);

3581

3582

// Make a call with 3 operands.

3583

Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),

3584

CI->getArgOperand(2)};

3585

Value *NewCall = Builder.CreateCall(

3586

Intrinsic::getDeclaration(CI->getModule(), IID),

3587

Args);

3588

3589

// Extract the second result and store it.

3590

Value *Data = Builder.CreateExtractValue(NewCall, 1);

3591

// Cast the pointer to the right type.

3592

Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),

3593

llvm::PointerType::getUnqual(Data->getType()));

3594

Builder.CreateAlignedStore(Data, Ptr, Align(1));

3595

// Replace the original call result with the first result of the new call.

3596

Value *CF = Builder.CreateExtractValue(NewCall, 0);

3597

3598

CI->replaceAllUsesWith(CF);

3599

Rep = nullptr;

3600

} else if (IsX86 && Name.startswith("avx512.mask.") &&

3601

upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {

3602

// Rep will be updated by the call in the condition.

3603

} else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {

3604

Value *Arg = CI->getArgOperand(0);

3605

Value *Neg = Builder.CreateNeg(Arg, "neg");

3606

Value *Cmp = Builder.CreateICmpSGE(

3607

Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");

3608

Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");

3609

} else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||

3610

Name.startswith("atomic.load.add.f64.p"))) {

3611

Value *Ptr = CI->getArgOperand(0);

3612

Value *Val = CI->getArgOperand(1);

3613

Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),

3614

AtomicOrdering::SequentiallyConsistent);

3615

} else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||

3616

Name == "max.ui" || Name == "max.ull")) {

3617

Value *Arg0 = CI->getArgOperand(0);

3618

Value *Arg1 = CI->getArgOperand(1);

3619

Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")

3620

? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")

3621

: Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");

3622

Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");

3623

} else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||

3624

Name == "min.ui" || Name == "min.ull")) {

3625

Value *Arg0 = CI->getArgOperand(0);

3626

Value *Arg1 = CI->getArgOperand(1);

3627

Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")

3628

? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")

3629

: Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");

3630

Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");

3631

} else if (IsNVVM && Name == "clz.ll") {

3632

// llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.

3633

Value *Arg = CI->getArgOperand(0);

3634

Value *Ctlz = Builder.CreateCall(

3635

Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,

3636

{Arg->getType()}),

3637

{Arg, Builder.getFalse()}, "ctlz");

3638

Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");

3639

} else if (IsNVVM && Name == "popc.ll") {

3640

// llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an

3641

// i64.

3642

Value *Arg = CI->getArgOperand(0);

3643

Value *Popc = Builder.CreateCall(

3644

Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,

3645

{Arg->getType()}),

3646

Arg, "ctpop");

3647

Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");

3648

} else if (IsNVVM && Name == "h2f") {

3649

Rep = Builder.CreateCall(Intrinsic::getDeclaration(

3650

F->getParent(), Intrinsic::convert_from_fp16,

3651

{Builder.getFloatTy()}),

3652

CI->getArgOperand(0), "h2f");

3653

} else {

3654

llvm_unreachable("Unknown function for CallInst upgrade.")::llvm::llvm_unreachable_internal("Unknown function for CallInst upgrade."
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3654);

3655

}

3656

3657

if (Rep)

3658

CI->replaceAllUsesWith(Rep);

3659

CI->eraseFromParent();

3660

return;

3661

}

3662

3663

const auto &DefaultCase = [&NewFn, &CI]() -> void {

3664

// Handle generic mangling change, but nothing else

3665

assert((static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3667, __extension__ __PRETTY_FUNCTION__))

3666

(CI->getCalledFunction()->getName() != NewFn->getName()) &&(static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3667, __extension__ __PRETTY_FUNCTION__))

3667

"Unknown function for CallInst upgrade and isn't just a name change")(static_cast <bool> ((CI->getCalledFunction()->getName
() != NewFn->getName()) && "Unknown function for CallInst upgrade and isn't just a name change"
) ? void (0) : __assert_fail ("(CI->getCalledFunction()->getName() != NewFn->getName()) && \"Unknown function for CallInst upgrade and isn't just a name change\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3667, __extension__ __PRETTY_FUNCTION__));

3668

CI->setCalledFunction(NewFn);

3669

};

3670

CallInst *NewCall = nullptr;

3671

switch (NewFn->getIntrinsicID()) {

3672

default: {

3673

DefaultCase();

3674

return;

3675

}

3676

case Intrinsic::arm_neon_vld1:

3677

case Intrinsic::arm_neon_vld2:

3678

case Intrinsic::arm_neon_vld3:

3679

case Intrinsic::arm_neon_vld4:

3680

case Intrinsic::arm_neon_vld2lane:

3681

case Intrinsic::arm_neon_vld3lane:

3682

case Intrinsic::arm_neon_vld4lane:

3683

case Intrinsic::arm_neon_vst1:

3684

case Intrinsic::arm_neon_vst2:

3685

case Intrinsic::arm_neon_vst3:

3686

case Intrinsic::arm_neon_vst4:

3687

case Intrinsic::arm_neon_vst2lane:

3688

case Intrinsic::arm_neon_vst3lane:

3689

case Intrinsic::arm_neon_vst4lane: {

3690

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3691

CI->arg_operands().end());

3692

NewCall = Builder.CreateCall(NewFn, Args);

3693

break;

3694

}

3695

3696

case Intrinsic::arm_neon_bfdot:

3697

case Intrinsic::arm_neon_bfmmla:

3698

case Intrinsic::arm_neon_bfmlalb:

3699

case Intrinsic::arm_neon_bfmlalt:

3700

case Intrinsic::aarch64_neon_bfdot:

3701

case Intrinsic::aarch64_neon_bfmmla:

3702

case Intrinsic::aarch64_neon_bfmlalb:

3703

case Intrinsic::aarch64_neon_bfmlalt: {

3704

SmallVector<Value *, 3> Args;

3705

assert(CI->getNumArgOperands() == 3 &&(static_cast <bool> (CI->getNumArgOperands() == 3 &&
"Mismatch between function args and call args") ? void (0) :
__assert_fail ("CI->getNumArgOperands() == 3 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3706, __extension__ __PRETTY_FUNCTION__))

3706

"Mismatch between function args and call args")(static_cast <bool> (CI->getNumArgOperands() == 3 &&
"Mismatch between function args and call args") ? void (0) :
__assert_fail ("CI->getNumArgOperands() == 3 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3706, __extension__ __PRETTY_FUNCTION__));

3707

size_t OperandWidth =

3708

CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();

3709

assert((OperandWidth == 64 || OperandWidth == 128) &&(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3710, __extension__ __PRETTY_FUNCTION__))

3710

"Unexpected operand width")(static_cast <bool> ((OperandWidth == 64 || OperandWidth
== 128) && "Unexpected operand width") ? void (0) : __assert_fail
("(OperandWidth == 64 || OperandWidth == 128) && \"Unexpected operand width\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3710, __extension__ __PRETTY_FUNCTION__));

3711

Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);

3712

auto Iter = CI->arg_operands().begin();

3713

Args.push_back(*Iter++);

3714

Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));

3715

Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));

3716

NewCall = Builder.CreateCall(NewFn, Args);

3717

break;

3718

}

3719

3720

case Intrinsic::bitreverse:

3721

NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

3722

break;

3723

3724

case Intrinsic::ctlz:

3725

case Intrinsic::cttz:

3726

assert(CI->getNumArgOperands() == 1 &&(static_cast <bool> (CI->getNumArgOperands() == 1 &&
"Mismatch between function args and call args") ? void (0) :
__assert_fail ("CI->getNumArgOperands() == 1 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3727, __extension__ __PRETTY_FUNCTION__))

3727

"Mismatch between function args and call args")(static_cast <bool> (CI->getNumArgOperands() == 1 &&
"Mismatch between function args and call args") ? void (0) :
__assert_fail ("CI->getNumArgOperands() == 1 && \"Mismatch between function args and call args\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3727, __extension__ __PRETTY_FUNCTION__));

3728

NewCall =

3729

Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});

3730

break;

3731

3732

case Intrinsic::objectsize: {

3733

Value *NullIsUnknownSize = CI->getNumArgOperands() == 2

3734

? Builder.getFalse()

3735

: CI->getArgOperand(2);

3736

Value *Dynamic =

3737

CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);

3738

NewCall = Builder.CreateCall(

3739

NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});

3740

break;

3741

}

3742

3743

case Intrinsic::ctpop:

3744

NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

3745

break;

3746

3747

case Intrinsic::convert_from_fp16:

3748

NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});

3749

break;

3750

3751

case Intrinsic::dbg_value:

3752

// Upgrade from the old version that had an extra offset argument.

3753

assert(CI->getNumArgOperands() == 4)(static_cast <bool> (CI->getNumArgOperands() == 4) ?
void (0) : __assert_fail ("CI->getNumArgOperands() == 4",
"/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3753, __extension__ __PRETTY_FUNCTION__));

3754

// Drop nonzero offsets instead of attempting to upgrade them.

3755

if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))

3756

if (Offset->isZeroValue()) {

3757

NewCall = Builder.CreateCall(

3758

NewFn,

3759

{CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});

3760

break;

3761

}

3762

CI->eraseFromParent();

3763

return;

3764

3765

case Intrinsic::ptr_annotation:

3766

// Upgrade from versions that lacked the annotation attribute argument.

3767

assert(CI->getNumArgOperands() == 4 &&(static_cast <bool> (CI->getNumArgOperands() == 4 &&
"Before LLVM 12.0 this intrinsic took four arguments") ? void
(0) : __assert_fail ("CI->getNumArgOperands() == 4 && \"Before LLVM 12.0 this intrinsic took four arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3768, __extension__ __PRETTY_FUNCTION__))

3768

"Before LLVM 12.0 this intrinsic took four arguments")(static_cast <bool> (CI->getNumArgOperands() == 4 &&
"Before LLVM 12.0 this intrinsic took four arguments") ? void
(0) : __assert_fail ("CI->getNumArgOperands() == 4 && \"Before LLVM 12.0 this intrinsic took four arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3768, __extension__ __PRETTY_FUNCTION__));

3769

// Create a new call with an added null annotation attribute argument.

3770

NewCall = Builder.CreateCall(

3771

NewFn,

3772

{CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),

3773

CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});

3774

NewCall->takeName(CI);

3775

CI->replaceAllUsesWith(NewCall);

3776

CI->eraseFromParent();

3777

return;

3778

3779

case Intrinsic::var_annotation:

3780

// Upgrade from versions that lacked the annotation attribute argument.

3781

assert(CI->getNumArgOperands() == 4 &&(static_cast <bool> (CI->getNumArgOperands() == 4 &&
"Before LLVM 12.0 this intrinsic took four arguments") ? void
(0) : __assert_fail ("CI->getNumArgOperands() == 4 && \"Before LLVM 12.0 this intrinsic took four arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3782, __extension__ __PRETTY_FUNCTION__))

3782

"Before LLVM 12.0 this intrinsic took four arguments")(static_cast <bool> (CI->getNumArgOperands() == 4 &&
"Before LLVM 12.0 this intrinsic took four arguments") ? void
(0) : __assert_fail ("CI->getNumArgOperands() == 4 && \"Before LLVM 12.0 this intrinsic took four arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3782, __extension__ __PRETTY_FUNCTION__));

3783

// Create a new call with an added null annotation attribute argument.

3784

NewCall = Builder.CreateCall(

3785

NewFn,

3786

{CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),

3787

CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});

3788

CI->eraseFromParent();

3789

return;

3790

3791

case Intrinsic::x86_xop_vfrcz_ss:

3792

case Intrinsic::x86_xop_vfrcz_sd:

3793

NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});

3794

break;

3795

3796

case Intrinsic::x86_xop_vpermil2pd:

3797

case Intrinsic::x86_xop_vpermil2ps:

3798

case Intrinsic::x86_xop_vpermil2pd_256:

3799

case Intrinsic::x86_xop_vpermil2ps_256: {

3800

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3801

CI->arg_operands().end());

3802

VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());

3803

VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);

3804

Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);

3805

NewCall = Builder.CreateCall(NewFn, Args);

3806

break;

3807

}

3808

3809

case Intrinsic::x86_sse41_ptestc:

3810

case Intrinsic::x86_sse41_ptestz:

3811

case Intrinsic::x86_sse41_ptestnzc: {

3812

// The arguments for these intrinsics used to be v4f32, and changed

3813

// to v2i64. This is purely a nop, since those are bitwise intrinsics.

3814

// So, the only thing required is a bitcast for both arguments.

3815

// First, check the arguments have the old type.

3816

Value *Arg0 = CI->getArgOperand(0);

3817

if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))

3818

return;

3819

3820

// Old intrinsic, add bitcasts

3821

Value *Arg1 = CI->getArgOperand(1);

3822

3823

auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);

3824

3825

Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");

3826

Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");

3827

3828

NewCall = Builder.CreateCall(NewFn, {BC0, BC1});

3829

break;

3830

}

3831

3832

case Intrinsic::x86_rdtscp: {

3833

// This used to take 1 arguments. If we have no arguments, it is already

3834

// upgraded.

3835

if (CI->getNumOperands() == 0)

3836

return;

3837

3838

NewCall = Builder.CreateCall(NewFn);

3839

// Extract the second result and store it.

3840

Value *Data = Builder.CreateExtractValue(NewCall, 1);

3841

// Cast the pointer to the right type.

3842

Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),

3843

llvm::PointerType::getUnqual(Data->getType()));

3844

Builder.CreateAlignedStore(Data, Ptr, Align(1));

3845

// Replace the original call result with the first result of the new call.

3846

Value *TSC = Builder.CreateExtractValue(NewCall, 0);

3847

3848

NewCall->takeName(CI);

3849

CI->replaceAllUsesWith(TSC);

3850

CI->eraseFromParent();

3851

return;

3852

}

3853

3854

case Intrinsic::x86_sse41_insertps:

3855

case Intrinsic::x86_sse41_dppd:

3856

case Intrinsic::x86_sse41_dpps:

3857

case Intrinsic::x86_sse41_mpsadbw:

3858

case Intrinsic::x86_avx_dp_ps_256:

3859

case Intrinsic::x86_avx2_mpsadbw: {

3860

// Need to truncate the last argument from i32 to i8 -- this argument models

3861

// an inherently 8-bit immediate operand to these x86 instructions.

3862

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3863

CI->arg_operands().end());

3864

3865

// Replace the last argument with a trunc.

3866

Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");

3867

NewCall = Builder.CreateCall(NewFn, Args);

3868

break;

3869

}

3870

3871

case Intrinsic::x86_avx512_mask_cmp_pd_128:

3872

case Intrinsic::x86_avx512_mask_cmp_pd_256:

3873

case Intrinsic::x86_avx512_mask_cmp_pd_512:

3874

case Intrinsic::x86_avx512_mask_cmp_ps_128:

3875

case Intrinsic::x86_avx512_mask_cmp_ps_256:

3876

case Intrinsic::x86_avx512_mask_cmp_ps_512: {

3877

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3878

CI->arg_operands().end());

3879

unsigned NumElts =

3880

cast<FixedVectorType>(Args[0]->getType())->getNumElements();

3881

Args[3] = getX86MaskVec(Builder, Args[3], NumElts);

3882

3883

NewCall = Builder.CreateCall(NewFn, Args);

3884

Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);

3885

3886

NewCall->takeName(CI);

3887

CI->replaceAllUsesWith(Res);

3888

CI->eraseFromParent();

3889

return;

3890

}

3891

3892

case Intrinsic::thread_pointer: {

3893

NewCall = Builder.CreateCall(NewFn, {});

3894

break;

3895

}

3896

3897

case Intrinsic::invariant_start:

3898

case Intrinsic::invariant_end: {

3899

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3900

CI->arg_operands().end());

3901

NewCall = Builder.CreateCall(NewFn, Args);

3902

break;

3903

}

3904

case Intrinsic::masked_load:

3905

case Intrinsic::masked_store:

3906

case Intrinsic::masked_gather:

3907

case Intrinsic::masked_scatter: {

3908

SmallVector<Value *, 4> Args(CI->arg_operands().begin(),

3909

CI->arg_operands().end());

3910

NewCall = Builder.CreateCall(NewFn, Args);

3911

NewCall->copyMetadata(*CI);

3912

break;

3913

}

3914

3915

case Intrinsic::memcpy:

3916

case Intrinsic::memmove:

3917

case Intrinsic::memset: {

3918

// We have to make sure that the call signature is what we're expecting.

3919

// We only want to change the old signatures by removing the alignment arg:

3920

// @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)

3921

// -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)

3922

// @llvm.memset...(i8*, i8, i[32|64], i32, i1)

3923

// -> @llvm.memset...(i8*, i8, i[32|64], i1)

3924

// Note: i8*'s in the above can be any pointer type

3925

if (CI->getNumArgOperands() != 5) {

3926

DefaultCase();

3927

return;

3928

}

3929

// Remove alignment argument (3), and add alignment attributes to the

3930

// dest/src pointers.

3931

Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),

3932

CI->getArgOperand(2), CI->getArgOperand(4)};

3933

NewCall = Builder.CreateCall(NewFn, Args);

3934

auto *MemCI = cast<MemIntrinsic>(NewCall);

3935

// All mem intrinsics support dest alignment.

3936

const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));

3937

MemCI->setDestAlignment(Align->getMaybeAlignValue());

3938

// Memcpy/Memmove also support source alignment.

3939

if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))

3940

MTI->setSourceAlignment(Align->getMaybeAlignValue());

3941

break;

3942

}

3943

}

3944

assert(NewCall && "Should have either set this variable or returned through "(static_cast <bool> (NewCall && "Should have either set this variable or returned through "
"the default case") ? void (0) : __assert_fail ("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3945, __extension__ __PRETTY_FUNCTION__))

3945

"the default case")(static_cast <bool> (NewCall && "Should have either set this variable or returned through "
"the default case") ? void (0) : __assert_fail ("NewCall && \"Should have either set this variable or returned through \" \"the default case\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3945, __extension__ __PRETTY_FUNCTION__));

3946

NewCall->takeName(CI);

3947

CI->replaceAllUsesWith(NewCall);

3948

CI->eraseFromParent();

3949

}

3950

3951

void llvm::UpgradeCallsToIntrinsic(Function *F) {

3952

assert(F && "Illegal attempt to upgrade a non-existent intrinsic.")(static_cast <bool> (F && "Illegal attempt to upgrade a non-existent intrinsic."
) ? void (0) : __assert_fail ("F && \"Illegal attempt to upgrade a non-existent intrinsic.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 3952, __extension__ __PRETTY_FUNCTION__));

3953

3954

// Check if this function should be upgraded and get the replacement function

3955

// if there is one.

3956

Function *NewFn;

3957

if (UpgradeIntrinsicFunction(F, NewFn)) {

3958

// Replace all users of the old function with the new function or new

3959

// instructions. This is not a range loop because the call is deleted.

3960

for (User *U : make_early_inc_range(F->users()))

3961

if (CallInst *CI = dyn_cast<CallInst>(U))

3962

UpgradeIntrinsicCall(CI, NewFn);

3963

3964

// Remove old function, no longer used, from the module.

3965

F->eraseFromParent();

3966

}

3967

}

3968

3969

MDNode *llvm::UpgradeTBAANode(MDNode &MD) {

3970

// Check if the tag uses struct-path aware TBAA format.

3971

if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)

3972

return &MD;

3973

3974

auto &Context = MD.getContext();

3975

if (MD.getNumOperands() == 3) {

3976

Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};

3977

MDNode *ScalarType = MDNode::get(Context, Elts);

3978

// Create a MDNode <ScalarType, ScalarType, offset 0, const>

3979

Metadata *Elts2[] = {ScalarType, ScalarType,

3980

ConstantAsMetadata::get(

3981

Constant::getNullValue(Type::getInt64Ty(Context))),

3982

MD.getOperand(2)};

3983

return MDNode::get(Context, Elts2);

3984

}

3985

// Create a MDNode <MD, MD, offset 0>

3986

Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(

3987

Type::getInt64Ty(Context)))};

3988

return MDNode::get(Context, Elts);

3989

}

3990

3991

Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,

3992

Instruction *&Temp) {

3993

if (Opc != Instruction::BitCast)

3994

return nullptr;

3995

3996

Temp = nullptr;

3997

Type *SrcTy = V->getType();

3998

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

3999

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

4000

LLVMContext &Context = V->getContext();

4001

4002

// We have no information about target data layout, so we assume that

4003

// the maximum pointer size is 64bit.

4004

Type *MidTy = Type::getInt64Ty(Context);

4005

Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);

4006

4007

return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);

4008

}

4009

4010

return nullptr;

4011

}

4012

4013

Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {

4014

if (Opc != Instruction::BitCast)

4015

return nullptr;

4016

4017

Type *SrcTy = C->getType();

4018

if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&

4019

SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {

4020

LLVMContext &Context = C->getContext();

4021

4022

// We have no information about target data layout, so we assume that

4023

// the maximum pointer size is 64bit.

4024

Type *MidTy = Type::getInt64Ty(Context);

4025

4026

return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),

4027

DestTy);

4028

}

4029

4030

return nullptr;

4031

}

4032

4033

/// Check the debug info version number, if it is out-dated, drop the debug

4034

/// info. Return true if module is modified.

4035

bool llvm::UpgradeDebugInfo(Module &M) {

4036

unsigned Version = getDebugMetadataVersionFromModule(M);

4037

if (Version == DEBUG_METADATA_VERSION) {

4038

bool BrokenDebugInfo = false;

4039

if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))

4040

report_fatal_error("Broken module found, compilation aborted!");

4041

if (!BrokenDebugInfo)

4042

// Everything is ok.

4043

return false;

4044

else {

4045

// Diagnose malformed debug info.

4046

DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);

4047

M.getContext().diagnose(Diag);

4048

}

4049

}

4050

bool Modified = StripDebugInfo(M);

4051

if (Modified && Version != DEBUG_METADATA_VERSION) {

4052

// Diagnose a version mismatch.

4053

DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);

4054

M.getContext().diagnose(DiagVersion);

4055

}

4056

return Modified;

4057

}

4058

4059

/// This checks for objc retain release marker which should be upgraded. It

4060

/// returns true if module is modified.

4061

static bool UpgradeRetainReleaseMarker(Module &M) {

4062

bool Changed = false;

4063

const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";

4064

NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);

4065

if (ModRetainReleaseMarker) {

4066

MDNode *Op = ModRetainReleaseMarker->getOperand(0);

4067

if (Op) {

4068

MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));

4069

if (ID) {

4070

SmallVector<StringRef, 4> ValueComp;

4071

ID->getString().split(ValueComp, "#");

4072

if (ValueComp.size() == 2) {

4073

std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();

4074

ID = MDString::get(M.getContext(), NewValue);

4075

}

4076

M.addModuleFlag(Module::Error, MarkerKey, ID);

4077

M.eraseNamedMetadata(ModRetainReleaseMarker);

4078

Changed = true;

4079

}

4080

}

4081

}

4082

return Changed;

4083

}

4084

4085

void llvm::UpgradeARCRuntime(Module &M) {

4086

// This lambda converts normal function calls to ARC runtime functions to

4087

// intrinsic calls.

4088

auto UpgradeToIntrinsic = [&](const char *OldFunc,

4089

llvm::Intrinsic::ID IntrinsicFunc) {

4090

Function *Fn = M.getFunction(OldFunc);

4091

4092

if (!Fn)

4093

return;

4094

4095

Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);

4096

4097

for (User *U : make_early_inc_range(Fn->users())) {

4098

CallInst *CI = dyn_cast<CallInst>(U);

4099

if (!CI || CI->getCalledFunction() != Fn)

4100

continue;

4101

4102

IRBuilder<> Builder(CI->getParent(), CI->getIterator());

4103

FunctionType *NewFuncTy = NewFn->getFunctionType();

4104

SmallVector<Value *, 2> Args;

4105

4106

// Don't upgrade the intrinsic if it's not valid to bitcast the return

4107

// value to the return type of the old function.

4108

if (NewFuncTy->getReturnType() != CI->getType() &&

4109

!CastInst::castIsValid(Instruction::BitCast, CI,

4110

NewFuncTy->getReturnType()))

4111

continue;

4112

4113

bool InvalidCast = false;

4114

4115

for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {

4116

Value *Arg = CI->getArgOperand(I);

4117

4118

// Bitcast argument to the parameter type of the new function if it's

4119

// not a variadic argument.

4120

if (I < NewFuncTy->getNumParams()) {

4121

// Don't upgrade the intrinsic if it's not valid to bitcast the argument

4122

// to the parameter type of the new function.

4123

if (!CastInst::castIsValid(Instruction::BitCast, Arg,

4124

NewFuncTy->getParamType(I))) {

4125

InvalidCast = true;

4126

break;

4127

}

4128

Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));

4129

}

4130

Args.push_back(Arg);

4131

}

4132

4133

if (InvalidCast)

4134

continue;

4135

4136

// Create a call instruction that calls the new function.

4137

CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);

4138

NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());

4139

NewCall->takeName(CI);

4140

4141

// Bitcast the return value back to the type of the old call.

4142

Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());

4143

4144

if (!CI->use_empty())

4145

CI->replaceAllUsesWith(NewRetVal);

4146

CI->eraseFromParent();

4147

}

4148

4149

if (Fn->use_empty())

4150

Fn->eraseFromParent();

4151

};

4152

4153

// Unconditionally convert a call to "clang.arc.use" to a call to

4154

// "llvm.objc.clang.arc.use".

4155

UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);

4156

4157

// Upgrade the retain release marker. If there is no need to upgrade

4158

// the marker, that means either the module is already new enough to contain

4159

// new intrinsics or it is not ARC. There is no need to upgrade runtime call.

4160

if (!UpgradeRetainReleaseMarker(M))

4161

return;

4162

4163

std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {

4164

{"objc_autorelease", llvm::Intrinsic::objc_autorelease},

4165

{"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},

4166

{"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},

4167

{"objc_autoreleaseReturnValue",

4168

llvm::Intrinsic::objc_autoreleaseReturnValue},

4169

{"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},

4170

{"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},

4171

{"objc_initWeak", llvm::Intrinsic::objc_initWeak},

4172

{"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},

4173

{"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},

4174

{"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},

4175

{"objc_release", llvm::Intrinsic::objc_release},

4176

{"objc_retain", llvm::Intrinsic::objc_retain},

4177

{"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},

4178

{"objc_retainAutoreleaseReturnValue",

4179

llvm::Intrinsic::objc_retainAutoreleaseReturnValue},

4180

{"objc_retainAutoreleasedReturnValue",

4181

llvm::Intrinsic::objc_retainAutoreleasedReturnValue},

4182

{"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},

4183

{"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},

4184

{"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},

4185

{"objc_unsafeClaimAutoreleasedReturnValue",

4186

llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},

4187

{"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},

4188

{"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},

4189

{"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},

4190

{"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},

4191

{"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},

4192

{"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},

4193

{"objc_arc_annotation_topdown_bbstart",

4194

llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},

4195

{"objc_arc_annotation_topdown_bbend",

4196

llvm::Intrinsic::objc_arc_annotation_topdown_bbend},

4197

{"objc_arc_annotation_bottomup_bbstart",

4198

llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},

4199

{"objc_arc_annotation_bottomup_bbend",

4200

llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};

4201

4202

for (auto &I : RuntimeFuncs)

4203

UpgradeToIntrinsic(I.first, I.second);

4204

}

4205

4206

bool llvm::UpgradeModuleFlags(Module &M) {

4207

NamedMDNode *ModFlags = M.getModuleFlagsMetadata();

4208

if (!ModFlags)

4209

return false;

4210

4211

bool HasObjCFlag = false, HasClassProperties = false, Changed = false;

4212

bool HasSwiftVersionFlag = false;

4213

uint8_t SwiftMajorVersion, SwiftMinorVersion;

4214

uint32_t SwiftABIVersion;

4215

auto Int8Ty = Type::getInt8Ty(M.getContext());

4216

auto Int32Ty = Type::getInt32Ty(M.getContext());

4217

4218

for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {

4219

MDNode *Op = ModFlags->getOperand(I);

4220

if (Op->getNumOperands() != 3)

4221

continue;

4222

MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));

4223

if (!ID)

4224

continue;

4225

if (ID->getString() == "Objective-C Image Info Version")

4226

HasObjCFlag = true;

4227

if (ID->getString() == "Objective-C Class Properties")

4228

HasClassProperties = true;

4229

// Upgrade PIC/PIE Module Flags. The module flag behavior for these two

4230

// field was Error and now they are Max.

4231

if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {

4232

if (auto *Behavior =

4233

mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {

4234

if (Behavior->getLimitedValue() == Module::Error) {

4235

Type *Int32Ty = Type::getInt32Ty(M.getContext());

4236

Metadata *Ops[3] = {

4237

ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),

4238

MDString::get(M.getContext(), ID->getString()),

4239

Op->getOperand(2)};

4240

ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));

4241

Changed = true;

4242

}

4243

}

4244

}

4245

// Upgrade Objective-C Image Info Section. Removed the whitespce in the

4246

// section name so that llvm-lto will not complain about mismatching

4247

// module flags that is functionally the same.

4248

if (ID->getString() == "Objective-C Image Info Section") {

4249

if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {

4250

SmallVector<StringRef, 4> ValueComp;

4251

Value->getString().split(ValueComp, " ");

4252

if (ValueComp.size() != 1) {

4253

std::string NewValue;

4254

for (auto &S : ValueComp)

4255

NewValue += S.str();

4256

Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),

4257

MDString::get(M.getContext(), NewValue)};

4258

ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));

4259

Changed = true;

4260

}

4261

}

4262

}

4263

4264

// IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.

4265

// If the higher bits are set, it adds new module flag for swift info.

4266

if (ID->getString() == "Objective-C Garbage Collection") {

4267

auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));

4268

if (Md) {

4269

assert(Md->getValue() && "Expected non-empty metadata")(static_cast <bool> (Md->getValue() && "Expected non-empty metadata"
) ? void (0) : __assert_fail ("Md->getValue() && \"Expected non-empty metadata\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 4269, __extension__ __PRETTY_FUNCTION__));

4270

auto Type = Md->getValue()->getType();

4271

if (Type == Int8Ty)

4272

continue;

4273

unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();

4274

if ((Val & 0xff) != Val) {

4275

HasSwiftVersionFlag = true;

4276

SwiftABIVersion = (Val & 0xff00) >> 8;

4277

SwiftMajorVersion = (Val & 0xff000000) >> 24;

4278

SwiftMinorVersion = (Val & 0xff0000) >> 16;

4279

}

4280

Metadata *Ops[3] = {

4281

ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),

4282

Op->getOperand(1),

4283

ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};

4284

ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));

4285

Changed = true;

4286

}

4287

}

4288

}

4289

4290

// "Objective-C Class Properties" is recently added for Objective-C. We

4291

// upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module

4292

// flag of value 0, so we can correclty downgrade this flag when trying to

4293

// link an ObjC bitcode without this module flag with an ObjC bitcode with

4294

// this module flag.

4295

if (HasObjCFlag && !HasClassProperties) {

4296

M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",

4297

(uint32_t)0);

4298

Changed = true;

4299

}

4300

4301

if (HasSwiftVersionFlag) {

4302

M.addModuleFlag(Module::Error, "Swift ABI Version",

4303

SwiftABIVersion);

4304

M.addModuleFlag(Module::Error, "Swift Major Version",

4305

ConstantInt::get(Int8Ty, SwiftMajorVersion));

4306

M.addModuleFlag(Module::Error, "Swift Minor Version",

4307

ConstantInt::get(Int8Ty, SwiftMinorVersion));

4308

Changed = true;

4309

}

4310

4311

return Changed;

4312

}

4313

4314

void llvm::UpgradeSectionAttributes(Module &M) {

4315

auto TrimSpaces = [](StringRef Section) -> std::string {

4316

SmallVector<StringRef, 5> Components;

4317

Section.split(Components, ',');

4318

4319

SmallString<32> Buffer;

4320

raw_svector_ostream OS(Buffer);

4321

4322

for (auto Component : Components)

4323

OS << ',' << Component.trim();

4324

4325

return std::string(OS.str().substr(1));

4326

};

4327

4328

for (auto &GV : M.globals()) {

4329

if (!GV.hasSection())

4330

continue;

4331

4332

StringRef Section = GV.getSection();

4333

4334

if (!Section.startswith("__DATA, __objc_catlist"))

4335

continue;

4336

4337

// __DATA, __objc_catlist, regular, no_dead_strip

4338

// __DATA,__objc_catlist,regular,no_dead_strip

4339

GV.setSection(TrimSpaces(Section));

4340

}

4341

}

4342

4343

namespace {

4344

// Prior to LLVM 10.0, the strictfp attribute could be used on individual

4345

// callsites within a function that did not also have the strictfp attribute.

4346

// Since 10.0, if strict FP semantics are needed within a function, the

4347

// function must have the strictfp attribute and all calls within the function

4348

// must also have the strictfp attribute. This latter restriction is

4349

// necessary to prevent unwanted libcall simplification when a function is

4350

// being cloned (such as for inlining).

4351

//

4352

// The "dangling" strictfp attribute usage was only used to prevent constant

4353

// folding and other libcall simplification. The nobuiltin attribute on the

4354

// callsite has the same effect.

4355

struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {

4356

StrictFPUpgradeVisitor() {}

4357

4358

void visitCallBase(CallBase &Call) {

4359

if (!Call.isStrictFP())

4360

return;

4361

if (isa<ConstrainedFPIntrinsic>(&Call))

4362

return;

4363

// If we get here, the caller doesn't have the strictfp attribute

4364

// but this callsite does. Replace the strictfp attribute with nobuiltin.

4365

Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);

4366

Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);

4367

}

4368

};

4369

} // namespace

4370

4371

void llvm::UpgradeFunctionAttributes(Function &F) {

4372

// If a function definition doesn't have the strictfp attribute,

4373

// convert any callsite strictfp attributes to nobuiltin.

4374

if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {

4375

StrictFPUpgradeVisitor SFPV;

4376

SFPV.visit(F);

4377

}

4378

4379

if (F.getCallingConv() == CallingConv::X86_INTR &&

4380

!F.arg_empty() && !F.hasParamAttribute(0, Attribute::ByVal)) {

4381

Type *ByValTy = cast<PointerType>(F.getArg(0)->getType())->getElementType();

4382

Attribute NewAttr = Attribute::getWithByValType(F.getContext(), ByValTy);

4383

F.addParamAttr(0, NewAttr);

4384

}

4385

4386

// Remove all incompatibile attributes from function.

4387

F.removeAttributes(AttributeList::ReturnIndex,

4388

AttributeFuncs::typeIncompatible(F.getReturnType()));

4389

for (auto &Arg : F.args())

4390

Arg.removeAttrs(AttributeFuncs::typeIncompatible(Arg.getType()));

4391

}

4392

4393

static bool isOldLoopArgument(Metadata *MD) {

4394

auto *T = dyn_cast_or_null<MDTuple>(MD);

4395

if (!T)

4396

return false;

4397

if (T->getNumOperands() < 1)

4398

return false;

4399

auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));

4400

if (!S)

4401

return false;

4402

return S->getString().startswith("llvm.vectorizer.");

4403

}

4404

4405

static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {

4406

StringRef OldPrefix = "llvm.vectorizer.";

4407

assert(OldTag.startswith(OldPrefix) && "Expected old prefix")(static_cast <bool> (OldTag.startswith(OldPrefix) &&
"Expected old prefix") ? void (0) : __assert_fail ("OldTag.startswith(OldPrefix) && \"Expected old prefix\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/IR/AutoUpgrade.cpp"
, 4407, __extension__ __PRETTY_FUNCTION__));

4408

4409

if (OldTag == "llvm.vectorizer.unroll")

4410

return MDString::get(C, "llvm.loop.interleave.count");

4411

4412

return MDString::get(

4413

C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))

4414

.str());

4415

}

4416

4417

static Metadata *upgradeLoopArgument(Metadata *MD) {

4418

auto *T = dyn_cast_or_null<MDTuple>(MD);

4419

if (!T)

4420

return MD;

4421

if (T->getNumOperands() < 1)

4422

return MD;

4423

auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));

4424

if (!OldTag)

4425

return MD;

4426

if (!OldTag->getString().startswith("llvm.vectorizer."))

4427

return MD;

4428

4429

// This has an old tag. Upgrade it.

4430

SmallVector<Metadata *, 8> Ops;

4431

Ops.reserve(T->getNumOperands());

4432

Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));

4433

for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)

4434

Ops.push_back(T->getOperand(I));

4435

4436

return MDTuple::get(T->getContext(), Ops);

4437

}

4438

4439

MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {

4440

auto *T = dyn_cast<MDTuple>(&N);

4441

if (!T)

4442

return &N;

4443

4444

if (none_of(T->operands(), isOldLoopArgument))

4445

return &N;

4446

4447

SmallVector<Metadata *, 8> Ops;

4448

Ops.reserve(T->getNumOperands());

4449

for (Metadata *MD : T->operands())

4450

Ops.push_back(upgradeLoopArgument(MD));

4451

4452

return MDTuple::get(T->getContext(), Ops);

4453

}

4454

4455

std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {

4456

Triple T(TT);

4457

// For AMDGPU we uprgrade older DataLayouts to include the default globals

4458

// address space of 1.

4459

if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {

4460

return DL.empty() ? std::string("G1") : (DL + "-G1").str();

4461

}

4462

4463

std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";

4464

// If X86, and the datalayout matches the expected format, add pointer size

4465

// address spaces to the datalayout.

4466

if (!T.isX86() || DL.contains(AddrSpaces))

4467

return std::string(DL);

4468

4469

SmallVector<StringRef, 4> Groups;

4470

Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");

4471

if (!R.match(DL, &Groups))

4472

return std::string(DL);

4473

4474

return (Groups[1] + AddrSpaces + Groups[3]).str();

4475

}

4476

4477

void llvm::UpgradeAttributes(AttrBuilder &B) {

4478

StringRef FramePointer;

4479

if (B.contains("no-frame-pointer-elim")) {

4480

// The value can be "true" or "false".

4481

for (const auto &I : B.td_attrs())

4482

if (I.first == "no-frame-pointer-elim")

4483

FramePointer = I.second == "true" ? "all" : "none";

4484

B.removeAttribute("no-frame-pointer-elim");

4485

}

4486

if (B.contains("no-frame-pointer-elim-non-leaf")) {

4487

// The value is ignored. "no-frame-pointer-elim"="true" takes priority.

4488

if (FramePointer != "all")

4489

FramePointer = "non-leaf";

4490

B.removeAttribute("no-frame-pointer-elim-non-leaf");

4491

}

4492

if (!FramePointer.empty())

4493

B.addAttribute("frame-pointer", FramePointer);

4494

4495

if (B.contains("null-pointer-is-valid")) {

4496

// The value can be "true" or "false".

4497

bool NullPointerIsValid = false;

4498

for (const auto &I : B.td_attrs())

4499

if (I.first == "null-pointer-is-valid")

4500

NullPointerIsValid = I.second == "true";

4501

B.removeAttribute("null-pointer-is-valid");

4502

if (NullPointerIsValid)

4503

B.addAttribute(Attribute::NullPointerIsValid);

4504

}

4505

}

File:	llvm/lib/IR/AutoUpgrade.cpp
Warning:	line 3784, column 5 Value stored to 'NewCall' is never read

Bug Summary

Annotated Source Code