/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1

//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements routines for translating from LLVM IR into SelectionDAG IR.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "SelectionDAGBuilder.h"

14

#include "SDNodeDbgValue.h"

15

#include "llvm/ADT/APFloat.h"

16

#include "llvm/ADT/APInt.h"

17

#include "llvm/ADT/BitVector.h"

18

#include "llvm/ADT/None.h"

19

#include "llvm/ADT/Optional.h"

20

#include "llvm/ADT/STLExtras.h"

21

#include "llvm/ADT/SmallPtrSet.h"

22

#include "llvm/ADT/SmallSet.h"

23

#include "llvm/ADT/StringRef.h"

24

#include "llvm/ADT/Triple.h"

25

#include "llvm/ADT/Twine.h"

26

#include "llvm/Analysis/AliasAnalysis.h"

27

#include "llvm/Analysis/BlockFrequencyInfo.h"

28

#include "llvm/Analysis/BranchProbabilityInfo.h"

29

#include "llvm/Analysis/ConstantFolding.h"

30

#include "llvm/Analysis/EHPersonalities.h"

31

#include "llvm/Analysis/Loads.h"

32

#include "llvm/Analysis/MemoryLocation.h"

33

#include "llvm/Analysis/ProfileSummaryInfo.h"

34

#include "llvm/Analysis/TargetLibraryInfo.h"

35

#include "llvm/Analysis/ValueTracking.h"

36

#include "llvm/Analysis/VectorUtils.h"

37

#include "llvm/CodeGen/Analysis.h"

38

#include "llvm/CodeGen/FunctionLoweringInfo.h"

39

#include "llvm/CodeGen/GCMetadata.h"

40

#include "llvm/CodeGen/MachineBasicBlock.h"

41

#include "llvm/CodeGen/MachineFrameInfo.h"

42

#include "llvm/CodeGen/MachineFunction.h"

43

#include "llvm/CodeGen/MachineInstr.h"

44

#include "llvm/CodeGen/MachineInstrBuilder.h"

45

#include "llvm/CodeGen/MachineJumpTableInfo.h"

46

#include "llvm/CodeGen/MachineMemOperand.h"

47

#include "llvm/CodeGen/MachineModuleInfo.h"

48

#include "llvm/CodeGen/MachineOperand.h"

49

#include "llvm/CodeGen/MachineRegisterInfo.h"

50

#include "llvm/CodeGen/RuntimeLibcalls.h"

51

#include "llvm/CodeGen/SelectionDAG.h"

52

#include "llvm/CodeGen/SelectionDAGTargetInfo.h"

53

#include "llvm/CodeGen/StackMaps.h"

54

#include "llvm/CodeGen/SwiftErrorValueTracking.h"

55

#include "llvm/CodeGen/TargetFrameLowering.h"

56

#include "llvm/CodeGen/TargetInstrInfo.h"

57

#include "llvm/CodeGen/TargetOpcodes.h"

58

#include "llvm/CodeGen/TargetRegisterInfo.h"

59

#include "llvm/CodeGen/TargetSubtargetInfo.h"

60

#include "llvm/CodeGen/WinEHFuncInfo.h"

61

#include "llvm/IR/Argument.h"

62

#include "llvm/IR/Attributes.h"

63

#include "llvm/IR/BasicBlock.h"

64

#include "llvm/IR/CFG.h"

65

#include "llvm/IR/CallingConv.h"

66

#include "llvm/IR/Constant.h"

67

#include "llvm/IR/ConstantRange.h"

68

#include "llvm/IR/Constants.h"

69

#include "llvm/IR/DataLayout.h"

70

#include "llvm/IR/DebugInfoMetadata.h"

71

#include "llvm/IR/DerivedTypes.h"

72

#include "llvm/IR/Function.h"

73

#include "llvm/IR/GetElementPtrTypeIterator.h"

74

#include "llvm/IR/InlineAsm.h"

75

#include "llvm/IR/InstrTypes.h"

76

#include "llvm/IR/Instructions.h"

77

#include "llvm/IR/IntrinsicInst.h"

78

#include "llvm/IR/Intrinsics.h"

79

#include "llvm/IR/IntrinsicsAArch64.h"

80

#include "llvm/IR/IntrinsicsWebAssembly.h"

81

#include "llvm/IR/LLVMContext.h"

82

#include "llvm/IR/Metadata.h"

83

#include "llvm/IR/Module.h"

84

#include "llvm/IR/Operator.h"

85

#include "llvm/IR/PatternMatch.h"

86

#include "llvm/IR/Statepoint.h"

87

#include "llvm/IR/Type.h"

88

#include "llvm/IR/User.h"

89

#include "llvm/IR/Value.h"

90

#include "llvm/MC/MCContext.h"

91

#include "llvm/MC/MCSymbol.h"

92

#include "llvm/Support/AtomicOrdering.h"

93

#include "llvm/Support/Casting.h"

94

#include "llvm/Support/CommandLine.h"

95

#include "llvm/Support/Compiler.h"

96

#include "llvm/Support/Debug.h"

97

#include "llvm/Support/MathExtras.h"

98

#include "llvm/Support/raw_ostream.h"

99

#include "llvm/Target/TargetIntrinsicInfo.h"

100

#include "llvm/Target/TargetMachine.h"

101

#include "llvm/Target/TargetOptions.h"

102

#include "llvm/Transforms/Utils/Local.h"

103

#include <cstddef>

104

#include <cstring>

105

#include <iterator>

106

#include <limits>

107

#include <numeric>

108

#include <tuple>

109

110

using namespace llvm;

111

using namespace PatternMatch;

112

using namespace SwitchCG;

113

114

#define DEBUG_TYPE"isel" "isel"

115

116

/// LimitFloatPrecision - Generate low-precision inline sequences for

117

/// some float libcalls (6, 8 or 12 bits).

118

static unsigned LimitFloatPrecision;

119

120

static cl::opt<bool>

121

InsertAssertAlign("insert-assert-align", cl::init(true),

122

cl::desc("Insert the experimental `assertalign` node."),

123

cl::ReallyHidden);

124

125

static cl::opt<unsigned, true>

126

LimitFPPrecision("limit-float-precision",

127

cl::desc("Generate low-precision inline sequences "

128

"for some float libcalls"),

129

cl::location(LimitFloatPrecision), cl::Hidden,

130

cl::init(0));

131

132

static cl::opt<unsigned> SwitchPeelThreshold(

133

"switch-peel-threshold", cl::Hidden, cl::init(66),

134

cl::desc("Set the case probability threshold for peeling the case from a "

135

"switch statement. A value greater than 100 will void this "

136

"optimization"));

137

138

// Limit the width of DAG chains. This is important in general to prevent

139

// DAG-based analysis from blowing up. For example, alias analysis and

140

// load clustering may not complete in reasonable time. It is difficult to

141

// recognize and avoid this situation within each individual analysis, and

142

// future analyses are likely to have the same behavior. Limiting DAG width is

143

// the safe approach and will be especially important with global DAGs.

144

//

145

// MaxParallelChains default is arbitrarily high to avoid affecting

146

// optimization, but could be lowered to improve compile time. Any ld-ld-st-st

147

// sequence over this should have been converted to llvm.memcpy by the

148

// frontend. It is easy to induce this behavior with .ll code such as:

149

// %buffer = alloca [4096 x i8]

150

// %data = load [4096 x i8]* %argPtr

151

// store [4096 x i8] %data, [4096 x i8]* %buffer

152

static const unsigned MaxParallelChains = 64;

153

154

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

155

const SDValue *Parts, unsigned NumParts,

156

MVT PartVT, EVT ValueVT, const Value *V,

157

Optional<CallingConv::ID> CC);

158

159

/// getCopyFromParts - Create a value that contains the specified legal parts

160

/// combined into the value they represent. If the parts combine to a type

161

/// larger than ValueVT then AssertOp can be used to specify whether the extra

162

/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT

163

/// (ISD::AssertSext).

164

static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,

165

const SDValue *Parts, unsigned NumParts,

166

MVT PartVT, EVT ValueVT, const Value *V,

167

Optional<CallingConv::ID> CC = None,

168

Optional<ISD::NodeType> AssertOp = None) {

169

// Let the target assemble the parts if it wants to

170

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

171

if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,

172

PartVT, ValueVT, CC))

173

return Val;

174

175

if (ValueVT.isVector())

176

return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,

177

CC);

178

179

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 179, __extension__ __PRETTY_FUNCTION__));

180

SDValue Val = Parts[0];

181

182

if (NumParts > 1) {

183

// Assemble the value from multiple parts.

184

if (ValueVT.isInteger()) {

185

unsigned PartBits = PartVT.getSizeInBits();

186

unsigned ValueBits = ValueVT.getSizeInBits();

187

188

// Assemble the power of 2 part.

189

unsigned RoundParts =

190

(NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;

191

unsigned RoundBits = PartBits * RoundParts;

192

EVT RoundVT = RoundBits == ValueBits ?

193

ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);

194

SDValue Lo, Hi;

195

196

EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

197

198

if (RoundParts > 2) {

199

Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,

200

PartVT, HalfVT, V);

201

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,

202

RoundParts / 2, PartVT, HalfVT, V);

203

} else {

204

Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);

205

Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);

206

}

207

208

if (DAG.getDataLayout().isBigEndian())

209

std::swap(Lo, Hi);

210

211

Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

212

213

if (RoundParts < NumParts) {

214

// Assemble the trailing non-power-of-2 part.

215

unsigned OddParts = NumParts - RoundParts;

216

EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);

217

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,

218

OddVT, V, CC);

219

220

// Combine the round and odd parts.

221

Lo = Val;

222

if (DAG.getDataLayout().isBigEndian())

223

std::swap(Lo, Hi);

224

EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

225

Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);

226

Hi =

227

DAG.getNode(ISD::SHL, DL, TotalVT, Hi,

228

DAG.getConstant(Lo.getValueSizeInBits(), DL,

229

TLI.getPointerTy(DAG.getDataLayout())));

230

Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);

231

Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);

232

}

233

} else if (PartVT.isFloatingPoint()) {

234

// FP split into multiple FP parts (for ppcf128)

235

assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __extension__ __PRETTY_FUNCTION__))

236

"Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __extension__ __PRETTY_FUNCTION__));

237

SDValue Lo, Hi;

238

Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);

239

Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);

240

if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

241

std::swap(Lo, Hi);

242

Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);

243

} else {

244

// FP split into integer parts (soft fp)

245

assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __extension__ __PRETTY_FUNCTION__))

246

!PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __extension__ __PRETTY_FUNCTION__));

247

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

248

Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);

249

}

250

}

251

252

// There is now one part, held in Val. Correct it to match ValueVT.

253

// PartEVT is the type of the register class that holds the value.

254

// ValueVT is the type of the inline asm operation.

255

EVT PartEVT = Val.getValueType();

256

257

if (PartEVT == ValueVT)

258

return Val;

259

260

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&

261

ValueVT.bitsLT(PartEVT)) {

262

// For an FP value in an integer part, we need to truncate to the right

263

// width first.

264

PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

265

Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);

266

}

267

268

// Handle types that have the same size.

269

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())

270

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

271

272

// Handle types with different sizes.

273

if (PartEVT.isInteger() && ValueVT.isInteger()) {

274

if (ValueVT.bitsLT(PartEVT)) {

275

// For a truncate, see if we have any information to

276

// indicate whether the truncated bits will always be

277

// zero or sign-extension.

278

if (AssertOp.hasValue())

279

Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,

280

DAG.getValueType(ValueVT));

281

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

282

}

283

return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

284

}

285

286

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

287

// FP_ROUND's are always exact here.

288

if (ValueVT.bitsLT(Val.getValueType()))

289

return DAG.getNode(

290

ISD::FP_ROUND, DL, ValueVT, Val,

291

DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

292

293

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);

294

}

295

296

// Handle MMX to a narrower integer type by bitcasting MMX to integer and

297

// then truncating.

298

if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&

299

ValueVT.bitsLT(PartEVT)) {

300

Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);

301

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

302

}

303

304

report_fatal_error("Unknown mismatch in getCopyFromParts!");

305

}

306

307

static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,

308

const Twine &ErrMsg) {

309

const Instruction *I = dyn_cast_or_null<Instruction>(V);

310

if (!V)

311

return Ctx.emitError(ErrMsg);

312

313

const char *AsmError = ", possible invalid constraint for vector type";

314

if (const CallInst *CI = dyn_cast<CallInst>(I))

315

if (CI->isInlineAsm())

316

return Ctx.emitError(I, ErrMsg + AsmError);

317

318

return Ctx.emitError(I, ErrMsg);

319

}

320

321

/// getCopyFromPartsVector - Create a value that contains the specified legal

322

/// parts combined into the value they represent. If the parts combine to a

323

/// type larger than ValueVT then AssertOp can be used to specify whether the

324

/// extra bits are known to be zero (ISD::AssertZext) or sign extended from

325

/// ValueVT (ISD::AssertSext).

326

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

327

const SDValue *Parts, unsigned NumParts,

328

MVT PartVT, EVT ValueVT, const Value *V,

329

Optional<CallingConv::ID> CallConv) {

330

assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 330, __extension__ __PRETTY_FUNCTION__));

331

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 331, __extension__ __PRETTY_FUNCTION__));

332

const bool IsABIRegCopy = CallConv.hasValue();

333

334

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

335

SDValue Val = Parts[0];

336

337

// Handle a multi-element vector.

338

if (NumParts > 1) {

339

EVT IntermediateVT;

340

MVT RegisterVT;

341

unsigned NumIntermediates;

342

unsigned NumRegs;

343

344

if (IsABIRegCopy) {

345

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

346

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

347

NumIntermediates, RegisterVT);

348

} else {

349

NumRegs =

350

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

351

NumIntermediates, RegisterVT);

352

}

353

354

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 354, __extension__ __PRETTY_FUNCTION__));

355

NumParts = NumRegs; // Silence a compiler warning.

356

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 356, __extension__ __PRETTY_FUNCTION__));

357

assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __extension__ __PRETTY_FUNCTION__))

358

Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __extension__ __PRETTY_FUNCTION__))

359

"Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __extension__ __PRETTY_FUNCTION__));

360

361

// Assemble the parts into intermediate operands.

362

SmallVector<SDValue, 8> Ops(NumIntermediates);

363

if (NumIntermediates == NumParts) {

364

// If the register was not expanded, truncate or copy the value,

365

// as appropriate.

366

for (unsigned i = 0; i != NumParts; ++i)

367

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,

368

PartVT, IntermediateVT, V, CallConv);

369

} else if (NumParts > 0) {

370

// If the intermediate type was expanded, build the intermediate

371

// operands from the parts.

372

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __extension__ __PRETTY_FUNCTION__))

373

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __extension__ __PRETTY_FUNCTION__));

374

unsigned Factor = NumParts / NumIntermediates;

375

for (unsigned i = 0; i != NumIntermediates; ++i)

376

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,

377

PartVT, IntermediateVT, V, CallConv);

378

}

379

380

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the

381

// intermediate operands.

382

EVT BuiltVectorTy =

383

IntermediateVT.isVector()

384

? EVT::getVectorVT(

385

*DAG.getContext(), IntermediateVT.getScalarType(),

386

IntermediateVT.getVectorElementCount() * NumParts)

387

: EVT::getVectorVT(*DAG.getContext(),

388

IntermediateVT.getScalarType(),

389

NumIntermediates);

390

Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS

391

: ISD::BUILD_VECTOR,

392

DL, BuiltVectorTy, Ops);

393

}

394

395

// There is now one part, held in Val. Correct it to match ValueVT.

396

EVT PartEVT = Val.getValueType();

397

398

if (PartEVT == ValueVT)

399

return Val;

400

401

if (PartEVT.isVector()) {

402

// If the element type of the source/dest vectors are the same, but the

403

// parts vector has more elements than the value vector, then we have a

404

// vector widening case (e.g. <2 x float> -> <4 x float>). Extract the

405

// elements we want.

406

if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {

407

assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __extension__ __PRETTY_FUNCTION__))

408

ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __extension__ __PRETTY_FUNCTION__))

409

(PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __extension__ __PRETTY_FUNCTION__))

410

ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __extension__ __PRETTY_FUNCTION__))

411

"Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __extension__ __PRETTY_FUNCTION__));

412

return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,

413

DAG.getVectorIdxConstant(0, DL));

414

}

415

416

// Vector/Vector bitcast.

417

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())

418

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

419

420

assert(PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() &&(static_cast <bool> (PartEVT.getVectorElementCount() ==
ValueVT.getVectorElementCount() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __extension__ __PRETTY_FUNCTION__))

421

"Cannot handle this kind of promotion")(static_cast <bool> (PartEVT.getVectorElementCount() ==
ValueVT.getVectorElementCount() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __extension__ __PRETTY_FUNCTION__));

422

// Promoted vector extract

423

return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);

424

425

}

426

427

// Trivial bitcast if the types are the same size and the destination

428

// vector type is legal.

429

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&

430

TLI.isTypeLegal(ValueVT))

431

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

432

433

if (ValueVT.getVectorNumElements() != 1) {

434

// Certain ABIs require that vectors are passed as integers. For vectors

435

// are the same size, this is an obvious bitcast.

436

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {

437

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

438

} else if (ValueVT.bitsLT(PartEVT)) {

439

const uint64_t ValueSize = ValueVT.getFixedSizeInBits();

440

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

441

// Drop the extra bits.

442

Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);

443

return DAG.getBitcast(ValueVT, Val);

444

}

445

446

diagnosePossiblyInvalidConstraint(

447

*DAG.getContext(), V, "non-trivial scalar-to-vector conversion");

448

return DAG.getUNDEF(ValueVT);

449

}

450

451

// Handle cases such as i8 -> <1 x i1>

452

EVT ValueSVT = ValueVT.getVectorElementType();

453

if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {

454

if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())

455

Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);

456

else

457

Val = ValueVT.isFloatingPoint()

458

? DAG.getFPExtendOrRound(Val, DL, ValueSVT)

459

: DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);

460

}

461

462

return DAG.getBuildVector(ValueVT, DL, Val);

463

}

464

465

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,

466

SDValue Val, SDValue *Parts, unsigned NumParts,

467

MVT PartVT, const Value *V,

468

Optional<CallingConv::ID> CallConv);

469

470

/// getCopyToParts - Create a series of nodes that contain the specified value

471

/// split into legal parts. If the parts contain more bits than Val, then, for

472

/// integers, ExtendKind can be used to specify how to generate the extra bits.

473

static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,

474

SDValue *Parts, unsigned NumParts, MVT PartVT,

475

const Value *V,

476

Optional<CallingConv::ID> CallConv = None,

477

ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {

478

// Let the target split the parts if it wants to

479

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

480

if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,

481

CallConv))

482

return;

483

EVT ValueVT = Val.getValueType();

484

485

// Handle the vector case separately.

486

if (ValueVT.isVector())

487

return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,

488

CallConv);

489

490

unsigned PartBits = PartVT.getSizeInBits();

491

unsigned OrigNumParts = NumParts;

492

assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __extension__ __PRETTY_FUNCTION__))

493

"Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __extension__ __PRETTY_FUNCTION__));

494

495

if (NumParts == 0)

496

return;

497

498

assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 498, __extension__ __PRETTY_FUNCTION__));

499

EVT PartEVT = PartVT;

500

if (PartEVT == ValueVT) {

501

assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __extension__ __PRETTY_FUNCTION__));

502

Parts[0] = Val;

503

return;

504

}

505

506

if (NumParts * PartBits > ValueVT.getSizeInBits()) {

507

// If the parts cover more bits than the value has, promote the value.

508

if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

509

assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 509, __extension__ __PRETTY_FUNCTION__));

510

Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);

511

} else {

512

if (ValueVT.isFloatingPoint()) {

513

// FP values need to be bitcast, then extended if they are being put

514

// into a larger container.

515

ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

516

Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

517

}

518

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))

519

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))

520

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __extension__ __PRETTY_FUNCTION__));

521

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

522

Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);

523

if (PartVT == MVT::x86mmx)

524

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

525

}

526

} else if (PartBits == ValueVT.getSizeInBits()) {

527

// Different types of the same size.

528

assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __extension__ __PRETTY_FUNCTION__));

529

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

530

} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {

531

// If the parts cover less bits than value has, truncate the value.

532

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))

533

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))

534

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __extension__ __PRETTY_FUNCTION__));

535

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

536

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

537

if (PartVT == MVT::x86mmx)

538

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

539

}

540

541

// The value may have changed - recompute ValueVT.

542

ValueVT = Val.getValueType();

543

assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __extension__ __PRETTY_FUNCTION__))

544

"Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __extension__ __PRETTY_FUNCTION__));

545

546

if (NumParts == 1) {

547

if (PartEVT != ValueVT) {

548

diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,

549

"scalar-to-vector conversion failed");

550

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

551

}

552

553

Parts[0] = Val;

554

return;

555

}

556

557

// Expand the value into multiple parts.

558

if (NumParts & (NumParts - 1)) {

559

// The number of parts is not a power of 2. Split off and copy the tail.

560

assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __extension__ __PRETTY_FUNCTION__))

561

"Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __extension__ __PRETTY_FUNCTION__));

562

unsigned RoundParts = 1 << Log2_32(NumParts);

563

unsigned RoundBits = RoundParts * PartBits;

564

unsigned OddParts = NumParts - RoundParts;

565

SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,

566

DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));

567

568

getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,

569

CallConv);

570

571

if (DAG.getDataLayout().isBigEndian())

572

// The odd parts were reversed by getCopyToParts - unreverse them.

573

std::reverse(Parts + RoundParts, Parts + NumParts);

574

575

NumParts = RoundParts;

576

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

577

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

578

}

579

580

// The number of parts is a power of 2. Repeatedly bisect the value using

581

// EXTRACT_ELEMENT.

582

Parts[0] = DAG.getNode(ISD::BITCAST, DL,

583

EVT::getIntegerVT(*DAG.getContext(),

584

ValueVT.getSizeInBits()),

585

Val);

586

587

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {

588

for (unsigned i = 0; i < NumParts; i += StepSize) {

589

unsigned ThisBits = StepSize * PartBits / 2;

590

EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);

591

SDValue &Part0 = Parts[i];

592

SDValue &Part1 = Parts[i+StepSize/2];

593

594

Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

595

ThisVT, Part0, DAG.getIntPtrConstant(1, DL));

596

Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

597

ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

598

599

if (ThisBits == PartBits && ThisVT != PartVT) {

600

Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);

601

Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);

602

}

603

}

604

}

605

606

if (DAG.getDataLayout().isBigEndian())

607

std::reverse(Parts, Parts + OrigNumParts);

608

}

609

610

static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,

611

const SDLoc &DL, EVT PartVT) {

612

if (!PartVT.isVector())

613

return SDValue();

614

615

EVT ValueVT = Val.getValueType();

616

ElementCount PartNumElts = PartVT.getVectorElementCount();

617

ElementCount ValueNumElts = ValueVT.getVectorElementCount();

618

619

// We only support widening vectors with equivalent element types and

620

// fixed/scalable properties. If a target needs to widen a fixed-length type

621

// to a scalable one, it should be possible to use INSERT_SUBVECTOR below.

622

if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||

623

PartNumElts.isScalable() != ValueNumElts.isScalable() ||

624

PartVT.getVectorElementType() != ValueVT.getVectorElementType())

625

return SDValue();

626

627

// Widening a scalable vector to another scalable vector is done by inserting

628

// the vector into a larger undef one.

629

if (PartNumElts.isScalable())

630

return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),

631

Val, DAG.getVectorIdxConstant(0, DL));

632

633

EVT ElementVT = PartVT.getVectorElementType();

634

// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in

635

// undef elements.

636

SmallVector<SDValue, 16> Ops;

637

DAG.ExtractVectorElements(Val, Ops);

638

SDValue EltUndef = DAG.getUNDEF(ElementVT);

639

Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);

640

641

// FIXME: Use CONCAT for 2x -> 4x.

642

return DAG.getBuildVector(PartVT, DL, Ops);

643

}

644

645

/// getCopyToPartsVector - Create a series of nodes that contain the specified

646

/// value split into legal parts.

647

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,

648

SDValue Val, SDValue *Parts, unsigned NumParts,

649

MVT PartVT, const Value *V,

650

Optional<CallingConv::ID> CallConv) {

651

EVT ValueVT = Val.getValueType();

652

assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 652, __extension__ __PRETTY_FUNCTION__));

653

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

654

const bool IsABIRegCopy = CallConv.hasValue();

655

656

if (NumParts == 1) {

657

EVT PartEVT = PartVT;

658

if (PartEVT == ValueVT) {

659

// Nothing to do.

660

} else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {

661

// Bitconvert vector->vector case.

662

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

663

} else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {

664

Val = Widened;

665

} else if (PartVT.isVector() &&

666

PartEVT.getVectorElementType().bitsGE(

667

ValueVT.getVectorElementType()) &&

668

PartEVT.getVectorElementCount() ==

669

ValueVT.getVectorElementCount()) {

670

671

// Promoted vector extract

672

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

673

} else {

674

if (ValueVT.getVectorElementCount().isScalar()) {

675

Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,

676

DAG.getVectorIdxConstant(0, DL));

677

} else {

678

uint64_t ValueSize = ValueVT.getFixedSizeInBits();

679

assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 680, __extension__ __PRETTY_FUNCTION__))

680

"lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 680, __extension__ __PRETTY_FUNCTION__));

681

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

682

Val = DAG.getBitcast(IntermediateType, Val);

683

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

684

}

685

}

686

687

assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
"Unexpected vector part value type") ? void (0) : __assert_fail
("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 687, __extension__ __PRETTY_FUNCTION__));

688

Parts[0] = Val;

689

return;

690

}

691

692

// Handle a multi-element vector.

693

EVT IntermediateVT;

694

MVT RegisterVT;

695

unsigned NumIntermediates;

696

unsigned NumRegs;

697

if (IsABIRegCopy) {

698

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

699

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

700

NumIntermediates, RegisterVT);

701

} else {

702

NumRegs =

703

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

704

NumIntermediates, RegisterVT);

705

}

706

707

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 707, __extension__ __PRETTY_FUNCTION__));

708

NumParts = NumRegs; // Silence a compiler warning.

709

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 709, __extension__ __PRETTY_FUNCTION__));

710

711

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 712, __extension__ __PRETTY_FUNCTION__))

712

"Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 712, __extension__ __PRETTY_FUNCTION__));

713

714

Optional<ElementCount> DestEltCnt;

715

716

if (IntermediateVT.isVector())

717

DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;

718

else

719

DestEltCnt = ElementCount::getFixed(NumIntermediates);

720

721

EVT BuiltVectorTy = EVT::getVectorVT(

722

*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());

723

724

if (ValueVT == BuiltVectorTy) {

725

// Nothing to do.

726

} else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {

727

// Bitconvert vector->vector case.

728

Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);

729

} else if (SDValue Widened =

730

widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {

731

Val = Widened;

732

} else if (BuiltVectorTy.getVectorElementType().bitsGE(

733

ValueVT.getVectorElementType()) &&

734

BuiltVectorTy.getVectorElementCount() ==

735

ValueVT.getVectorElementCount()) {

736

// Promoted vector extract

737

Val = DAG.getAnyExtOrTrunc(Val, DL, BuiltVectorTy);

738

}

739

740

assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy
&& "Unexpected vector value type") ? void (0) : __assert_fail
("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 740, __extension__ __PRETTY_FUNCTION__));

741

742

// Split the vector into intermediate operands.

743

SmallVector<SDValue, 8> Ops(NumIntermediates);

744

for (unsigned i = 0; i != NumIntermediates; ++i) {

745

if (IntermediateVT.isVector()) {

746

// This does something sensible for scalable vectors - see the

747

// definition of EXTRACT_SUBVECTOR for further details.

748

unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();

749

Ops[i] =

750

DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,

751

DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));

752

} else {

753

Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,

754

DAG.getVectorIdxConstant(i, DL));

755

}

756

}

757

758

// Split the intermediate operands into legal parts.

759

if (NumParts == NumIntermediates) {

760

// If the register was not expanded, promote or copy the value,

761

// as appropriate.

762

for (unsigned i = 0; i != NumParts; ++i)

763

getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

764

} else if (NumParts > 0) {

765

// If the intermediate type was expanded, split each the value into

766

// legal parts.

767

assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 767, __extension__ __PRETTY_FUNCTION__));

768

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 769, __extension__ __PRETTY_FUNCTION__))

769

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 769, __extension__ __PRETTY_FUNCTION__));

770

unsigned Factor = NumParts / NumIntermediates;

771

for (unsigned i = 0; i != NumIntermediates; ++i)

772

getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,

773

CallConv);

774

}

775

}

776

777

RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,

778

EVT valuevt, Optional<CallingConv::ID> CC)

779

: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),

780

RegCount(1, regs.size()), CallConv(CC) {}

781

782

RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,

783

const DataLayout &DL, unsigned Reg, Type *Ty,

784

Optional<CallingConv::ID> CC) {

785

ComputeValueVTs(TLI, DL, Ty, ValueVTs);

786

787

CallConv = CC;

788

789

for (EVT ValueVT : ValueVTs) {

790

unsigned NumRegs =

791

isABIMangled()

792

? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)

793

: TLI.getNumRegisters(Context, ValueVT);

794

MVT RegisterVT =

795

isABIMangled()

796

? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)

797

: TLI.getRegisterType(Context, ValueVT);

798

for (unsigned i = 0; i != NumRegs; ++i)

799

Regs.push_back(Reg + i);

800

RegVTs.push_back(RegisterVT);

801

RegCount.push_back(NumRegs);

802

Reg += NumRegs;

803

}

804

}

805

806

SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,

807

FunctionLoweringInfo &FuncInfo,

808

const SDLoc &dl, SDValue &Chain,

809

SDValue *Flag, const Value *V) const {

810

// A Value with type {} or [0 x %t] needs no registers.

811

if (ValueVTs.empty())

812

return SDValue();

813

814

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

815

816

// Assemble the legal parts into the final values.

817

SmallVector<SDValue, 4> Values(ValueVTs.size());

818

SmallVector<SDValue, 8> Parts;

819

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

820

// Copy the legal parts from the registers.

821

EVT ValueVT = ValueVTs[Value];

822

unsigned NumRegs = RegCount[Value];

823

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

824

*DAG.getContext(),

825

CallConv.getValue(), RegVTs[Value])

826

: RegVTs[Value];

827

828

Parts.resize(NumRegs);

829

for (unsigned i = 0; i != NumRegs; ++i) {

830

SDValue P;

831

if (!Flag) {

832

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);

833

} else {

834

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);

835

*Flag = P.getValue(2);

836

}

837

838

Chain = P.getValue(1);

839

Parts[i] = P;

840

841

// If the source register was virtual and if we know something about it,

842

// add an assert node.

843

if (!Register::isVirtualRegister(Regs[Part + i]) ||

844

!RegisterVT.isInteger())

845

continue;

846

847

const FunctionLoweringInfo::LiveOutInfo *LOI =

848

FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);

849

if (!LOI)

850

continue;

851

852

unsigned RegSize = RegisterVT.getScalarSizeInBits();

853

unsigned NumSignBits = LOI->NumSignBits;

854

unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

855

856

if (NumZeroBits == RegSize) {

857

// The current value is a zero.

858

// Explicitly express that as it would be easier for

859

// optimizations to kick in.

860

Parts[i] = DAG.getConstant(0, dl, RegisterVT);

861

continue;

862

}

863

864

// FIXME: We capture more information than the dag can represent. For

865

// now, just use the tightest assertzext/assertsext possible.

866

bool isSExt;

867

EVT FromVT(MVT::Other);

868

if (NumZeroBits) {

869

FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);

870

isSExt = false;

871

} else if (NumSignBits > 1) {

872

FromVT =

873

EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);

874

isSExt = true;

875

} else {

876

continue;

877

}

878

// Add an assertion node.

879

assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
__assert_fail ("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 879, __extension__ __PRETTY_FUNCTION__));

880

Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,

881

RegisterVT, P, DAG.getValueType(FromVT));

882

}

883

884

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,

885

RegisterVT, ValueVT, V, CallConv);

886

Part += NumRegs;

887

Parts.clear();

888

}

889

890

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);

891

}

892

893

void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,

894

const SDLoc &dl, SDValue &Chain, SDValue *Flag,

895

const Value *V,

896

ISD::NodeType PreferredExtendType) const {

897

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

898

ISD::NodeType ExtendKind = PreferredExtendType;

899

900

// Get the list of the values's legal parts.

901

unsigned NumRegs = Regs.size();

902

SmallVector<SDValue, 8> Parts(NumRegs);

903

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

904

unsigned NumParts = RegCount[Value];

905

906

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

907

*DAG.getContext(),

908

CallConv.getValue(), RegVTs[Value])

909

: RegVTs[Value];

910

911

if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))

912

ExtendKind = ISD::ZERO_EXTEND;

913

914

getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],

915

NumParts, RegisterVT, V, CallConv, ExtendKind);

916

Part += NumParts;

917

}

918

919

// Copy the parts into the registers.

920

SmallVector<SDValue, 8> Chains(NumRegs);

921

for (unsigned i = 0; i != NumRegs; ++i) {

922

SDValue Part;

923

if (!Flag) {

924

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);

925

} else {

926

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);

927

*Flag = Part.getValue(1);

928

}

929

930

Chains[i] = Part.getValue(0);

931

}

932

933

if (NumRegs == 1 || Flag)

934

// If NumRegs > 1 && Flag is used then the use of the last CopyToReg is

935

// flagged to it. That is the CopyToReg nodes and the user are considered

936

// a single scheduling unit. If we create a TokenFactor and return it as

937

// chain, then the TokenFactor is both a predecessor (operand) of the

938

// user as well as a successor (the TF operands are flagged to the user).

939

// c1, f1 = CopyToReg

940

// c2, f2 = CopyToReg

941

// c3 = TokenFactor c1, c2

942

// ...

943

// = op c3, ..., f2

944

Chain = Chains[NumRegs-1];

945

else

946

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);

947

}

948

949

void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,

950

unsigned MatchingIdx, const SDLoc &dl,

951

SelectionDAG &DAG,

952

std::vector<SDValue> &Ops) const {

953

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

954

955

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());

956

if (HasMatching)

957

Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);

958

else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {

959

// Put the register class of the virtual registers in the flag word. That

960

// way, later passes can recompute register class constraints for inline

961

// assembly as well as normal instructions.

962

// Don't do this for tied operands that can use the regclass information

963

// from the def.

964

const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

965

const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());

966

Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());

967

}

968

969

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);

970

Ops.push_back(Res);

971

972

if (Code == InlineAsm::Kind_Clobber) {

973

// Clobbers should always have a 1:1 mapping with registers, and may

974

// reference registers that have illegal (e.g. vector) types. Hence, we

975

// shouldn't try to apply any sort of splitting logic to them.

976

assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 977, __extension__ __PRETTY_FUNCTION__))

977

"No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 977, __extension__ __PRETTY_FUNCTION__));

978

Register SP = TLI.getStackPointerRegisterToSaveRestore();

979

(void)SP;

980

for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {

981

Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));

982

assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 985, __extension__ __PRETTY_FUNCTION__))

983

(Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 985, __extension__ __PRETTY_FUNCTION__))

984

DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 985, __extension__ __PRETTY_FUNCTION__))

985

"If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 985, __extension__ __PRETTY_FUNCTION__));

986

}

987

return;

988

}

989

990

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {

991

MVT RegisterVT = RegVTs[Value];

992

unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],

993

RegisterVT);

994

for (unsigned i = 0; i != NumRegs; ++i) {

995

assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 995, __extension__ __PRETTY_FUNCTION__));

996

unsigned TheReg = Regs[Reg++];

997

Ops.push_back(DAG.getRegister(TheReg, RegisterVT));

998

}

999

}

1000

}

1001

1002

SmallVector<std::pair<unsigned, TypeSize>, 4>

1003

RegsForValue::getRegsAndSizes() const {

1004

SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;

1005

unsigned I = 0;

1006

for (auto CountAndVT : zip_first(RegCount, RegVTs)) {

1007

unsigned RegCount = std::get<0>(CountAndVT);

1008

MVT RegisterVT = std::get<1>(CountAndVT);

1009

TypeSize RegisterSize = RegisterVT.getSizeInBits();

1010

for (unsigned E = I + RegCount; I != E; ++I)

1011

OutVec.push_back(std::make_pair(Regs[I], RegisterSize));

1012

}

1013

return OutVec;

1014

}

1015

1016

void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,

1017

const TargetLibraryInfo *li) {

1018

AA = aa;

1019

GFI = gfi;

1020

LibInfo = li;

1021

DL = &DAG.getDataLayout();

1022

Context = DAG.getContext();

1023

LPadToCallSiteMap.clear();

1024

SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());

1025

}

1026

1027

void SelectionDAGBuilder::clear() {

1028

NodeMap.clear();

1029

UnusedArgNodeMap.clear();

1030

PendingLoads.clear();

1031

PendingExports.clear();

1032

PendingConstrainedFP.clear();

1033

PendingConstrainedFPStrict.clear();

1034

CurInst = nullptr;

1035

HasTailCall = false;

1036

SDNodeOrder = LowestSDNodeOrder;

1037

StatepointLowering.clear();

1038

}

1039

1040

void SelectionDAGBuilder::clearDanglingDebugInfo() {

1041

DanglingDebugInfoMap.clear();

1042

}

1043

1044

// Update DAG root to include dependencies on Pending chains.

1045

SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {

1046

SDValue Root = DAG.getRoot();

1047

1048

if (Pending.empty())

1049

return Root;

1050

1051

// Add current root to PendingChains, unless we already indirectly

1052

// depend on it.

1053

if (Root.getOpcode() != ISD::EntryToken) {

1054

unsigned i = 0, e = Pending.size();

1055

for (; i != e; ++i) {

1056

assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1056, __extension__ __PRETTY_FUNCTION__));

1057

if (Pending[i].getNode()->getOperand(0) == Root)

1058

break; // Don't add the root if we already indirectly depend on it.

1059

}

1060

1061

if (i == e)

1062

Pending.push_back(Root);

1063

}

1064

1065

if (Pending.size() == 1)

1066

Root = Pending[0];

1067

else

1068

Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

1069

1070

DAG.setRoot(Root);

1071

Pending.clear();

1072

return Root;

1073

}

1074

1075

SDValue SelectionDAGBuilder::getMemoryRoot() {

1076

return updateRoot(PendingLoads);

1077

}

1078

1079

SDValue SelectionDAGBuilder::getRoot() {

1080

// Chain up all pending constrained intrinsics together with all

1081

// pending loads, by simply appending them to PendingLoads and

1082

// then calling getMemoryRoot().

1083

PendingLoads.reserve(PendingLoads.size() +

1084

PendingConstrainedFP.size() +

1085

PendingConstrainedFPStrict.size());

1086

PendingLoads.append(PendingConstrainedFP.begin(),

1087

PendingConstrainedFP.end());

1088

PendingLoads.append(PendingConstrainedFPStrict.begin(),

1089

PendingConstrainedFPStrict.end());

1090

PendingConstrainedFP.clear();

1091

PendingConstrainedFPStrict.clear();

1092

return getMemoryRoot();

1093

}

1094

1095

SDValue SelectionDAGBuilder::getControlRoot() {

1096

// We need to emit pending fpexcept.strict constrained intrinsics,

1097

// so append them to the PendingExports list.

1098

PendingExports.append(PendingConstrainedFPStrict.begin(),

1099

PendingConstrainedFPStrict.end());

1100

PendingConstrainedFPStrict.clear();

1101

return updateRoot(PendingExports);

1102

}

1103

1104

void SelectionDAGBuilder::visit(const Instruction &I) {

1105

// Set up outgoing PHI node register values before emitting the terminator.

1106

if (I.isTerminator()) {

1107

HandlePHINodesInSuccessorBlocks(I.getParent());

1108

}

1109

1110

// Increase the SDNodeOrder if dealing with a non-debug instruction.

1111

if (!isa<DbgInfoIntrinsic>(I))

1112

++SDNodeOrder;

1113

1114

CurInst = &I;

1115

1116

visit(I.getOpcode(), I);

1117

1118

if (!I.isTerminator() && !HasTailCall &&

1119

!isa<GCStatepointInst>(I)) // statepoints handle their exports internally

1120

CopyToExportRegsIfNeeded(&I);

1121

1122

CurInst = nullptr;

1123

}

1124

1125

void SelectionDAGBuilder::visitPHI(const PHINode &) {

1126

llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1126);

1127

}

1128

1129

void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {

1130

// Note: this doesn't use InstVisitor, because it has to work with

1131

// ConstantExpr's in addition to instructions.

1132

switch (Opcode) {

1133

default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1133);

1134

// Build the switch statement using the Instruction.def file.

1135

#define HANDLE_INST(NUM, OPCODE, CLASS) \

1136

case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;

1137

#include "llvm/IR/Instruction.def"

1138

}

1139

}

1140

1141

void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,

1142

DebugLoc DL, unsigned Order) {

1143

// We treat variadic dbg_values differently at this stage.

1144

if (DI->hasArgList()) {

1145

// For variadic dbg_values we will now insert an undef.

1146

// FIXME: We can potentially recover these!

1147

SmallVector<SDDbgOperand, 2> Locs;

1148

for (const Value *V : DI->getValues()) {

1149

auto Undef = UndefValue::get(V->getType());

1150

Locs.push_back(SDDbgOperand::fromConst(Undef));

1151

}

1152

SDDbgValue *SDV = DAG.getDbgValueList(

1153

DI->getVariable(), DI->getExpression(), Locs, {},

1154

/*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);

1155

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1156

} else {

1157

// TODO: Dangling debug info will eventually either be resolved or produce

1158

// an Undef DBG_VALUE. However in the resolution case, a gap may appear

1159

// between the original dbg.value location and its resolved DBG_VALUE,

1160

// which we should ideally fill with an extra Undef DBG_VALUE.

1161

assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1163, __extension__ __PRETTY_FUNCTION__))

1162

"DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1163, __extension__ __PRETTY_FUNCTION__))

1163

"operand.")(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1163, __extension__ __PRETTY_FUNCTION__));

1164

DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);

1165

}

1166

}

1167

1168

void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,

1169

const DIExpression *Expr) {

1170

auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {

1171

const DbgValueInst *DI = DDI.getDI();

1172

DIVariable *DanglingVariable = DI->getVariable();

1173

DIExpression *DanglingExpr = DI->getExpression();

1174

if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {

1175

LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false);

1176

return true;

1177

}

1178

return false;

1179

};

1180

1181

for (auto &DDIMI : DanglingDebugInfoMap) {

1182

DanglingDebugInfoVector &DDIV = DDIMI.second;

1183

1184

// If debug info is to be dropped, run it through final checks to see

1185

// whether it can be salvaged.

1186

for (auto &DDI : DDIV)

1187

if (isMatchingDbgValue(DDI))

1188

salvageUnresolvedDbgValue(DDI);

1189

1190

erase_if(DDIV, isMatchingDbgValue);

1191

}

1192

}

1193

1194

// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,

1195

// generate the debug data structures now that we've seen its definition.

1196

void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,

1197

SDValue Val) {

1198

auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);

1199

if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())

1200

return;

1201

1202

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;

1203

for (auto &DDI : DDIV) {

1204

const DbgValueInst *DI = DDI.getDI();

1205

assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")(static_cast <bool> (!DI->hasArgList() && "Not implemented for variadic dbg_values"
) ? void (0) : __assert_fail ("!DI->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1205, __extension__ __PRETTY_FUNCTION__));

1206

assert(DI && "Ill-formed DanglingDebugInfo")(static_cast <bool> (DI && "Ill-formed DanglingDebugInfo"
) ? void (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1206, __extension__ __PRETTY_FUNCTION__));

1207

DebugLoc dl = DDI.getdl();

1208

unsigned ValSDNodeOrder = Val.getNode()->getIROrder();

1209

unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();

1210

DILocalVariable *Variable = DI->getVariable();

1211

DIExpression *Expr = DI->getExpression();

1212

assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1213, __extension__ __PRETTY_FUNCTION__))

1213

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1213, __extension__ __PRETTY_FUNCTION__));

1214

SDDbgValue *SDV;

1215

if (Val.getNode()) {

1216

// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a

1217

// FuncArgumentDbgValue (it would be hoisted to the function entry, and if

1218

// we couldn't resolve it directly when examining the DbgValue intrinsic

1219

// in the first place we should not be more successful here). Unless we

1220

// have some test case that prove this to be correct we should avoid

1221

// calling EmitFuncArgumentDbgValue here.

1222

if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {

1223

LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)

1224

<< DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false);

1225

LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);

1226

// Increase the SDNodeOrder for the DbgValue here to make sure it is

1227

// inserted after the definition of Val when emitting the instructions

1228

// after ISel. An alternative could be to teach

1229

// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.

1230

LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1231

<< "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1232

<< ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);

1233

SDV = getDbgValue(Val, Variable, Expr, dl,

1234

std::max(DbgSDNodeOrder, ValSDNodeOrder));

1235

DAG.AddDbgValue(SDV, false);

1236

} else

1237

LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)

1238

<< "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false);

1239

} else {

1240

LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false);

1241

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1242

auto SDV =

1243

DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);

1244

DAG.AddDbgValue(SDV, false);

1245

}

1246

}

1247

DDIV.clear();

1248

}

1249

1250

void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {

1251

// TODO: For the variadic implementation, instead of only checking the fail

1252

// state of `handleDebugValue`, we need know specifically which values were

1253

// invalid, so that we attempt to salvage only those values when processing

1254

// a DIArgList.

1255

assert(!DDI.getDI()->hasArgList() &&(static_cast <bool> (!DDI.getDI()->hasArgList() &&
"Not implemented for variadic dbg_values") ? void (0) : __assert_fail
("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1256, __extension__ __PRETTY_FUNCTION__))

1256

"Not implemented for variadic dbg_values")(static_cast <bool> (!DDI.getDI()->hasArgList() &&
"Not implemented for variadic dbg_values") ? void (0) : __assert_fail
("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1256, __extension__ __PRETTY_FUNCTION__));

1257

Value *V = DDI.getDI()->getValue(0);

1258

DILocalVariable *Var = DDI.getDI()->getVariable();

1259

DIExpression *Expr = DDI.getDI()->getExpression();

1260

DebugLoc DL = DDI.getdl();

1261

DebugLoc InstDL = DDI.getDI()->getDebugLoc();

1262

unsigned SDOrder = DDI.getSDNodeOrder();

1263

// Currently we consider only dbg.value intrinsics -- we tell the salvager

1264

// that DW_OP_stack_value is desired.

1265

assert(isa<DbgValueInst>(DDI.getDI()))(static_cast <bool> (isa<DbgValueInst>(DDI.getDI(
))) ? void (0) : __assert_fail ("isa<DbgValueInst>(DDI.getDI())"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1265, __extension__ __PRETTY_FUNCTION__));

1266

bool StackValue = true;

1267

1268

// Can this Value can be encoded without any further work?

1269

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))

1270

return;

1271

1272

// Attempt to salvage back through as many instructions as possible. Bail if

1273

// a non-instruction is seen, such as a constant expression or global

1274

// variable. FIXME: Further work could recover those too.

1275

while (isa<Instruction>(V)) {

1276

Instruction &VAsInst = *cast<Instruction>(V);

1277

// Temporary "0", awaiting real implementation.

1278

SmallVector<Value *, 4> AdditionalValues;

1279

DIExpression *SalvagedExpr =

1280

salvageDebugInfoImpl(VAsInst, Expr, StackValue, 0, AdditionalValues);

1281

1282

// If we cannot salvage any further, and haven't yet found a suitable debug

1283

// expression, bail out.

1284

// TODO: If AdditionalValues isn't empty, then the salvage can only be

1285

// represented with a DBG_VALUE_LIST, so we give up. When we have support

1286

// here for variadic dbg_values, remove that condition.

1287

if (!SalvagedExpr || !AdditionalValues.empty())

1288

break;

1289

1290

// New value and expr now represent this debuginfo.

1291

V = VAsInst.getOperand(0);

1292

Expr = SalvagedExpr;

1293

1294

// Some kind of simplification occurred: check whether the operand of the

1295

// salvaged debug expression can be encoded in this DAG.

1296

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,

1297

/*IsVariadic=*/false)) {

1298

LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false)

1299

<< DDI.getDI() << "\nBy stripping back to:\n " << V)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false);

1300

return;

1301

}

1302

}

1303

1304

// This was the final opportunity to salvage this debug information, and it

1305

// couldn't be done. Place an undef DBG_VALUE at this location to terminate

1306

// any earlier variable location.

1307

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1308

auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);

1309

DAG.AddDbgValue(SDV, false);

1310

1311

LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false)

1312

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false);

1313

LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
)

1314

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
);

1315

}

1316

1317

bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,

1318

DILocalVariable *Var,

1319

DIExpression *Expr, DebugLoc dl,

1320

DebugLoc InstDL, unsigned Order,

1321

bool IsVariadic) {

1322

if (Values.empty())

1323

return true;

1324

SmallVector<SDDbgOperand> LocationOps;

1325

SmallVector<SDNode *> Dependencies;

1326

for (const Value *V : Values) {

1327

// Constant value.

1328

if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||

1329

isa<ConstantPointerNull>(V)) {

1330

LocationOps.emplace_back(SDDbgOperand::fromConst(V));

1331

continue;

1332

}

1333

1334

// If the Value is a frame index, we can create a FrameIndex debug value

1335

// without relying on the DAG at all.

1336

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1337

auto SI = FuncInfo.StaticAllocaMap.find(AI);

1338

if (SI != FuncInfo.StaticAllocaMap.end()) {

1339

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));

1340

continue;

1341

}

1342

}

1343

1344

// Do not use getValue() in here; we don't want to generate code at

1345

// this point if it hasn't been done yet.

1346

SDValue N = NodeMap[V];

1347

if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.

1348

N = UnusedArgNodeMap[V];

1349

if (N.getNode()) {

1350

// Only emit func arg dbg value for non-variadic dbg.values for now.

1351

if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))

1352

return true;

1353

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

1354

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can

1355

// describe stack slot locations.

1356

//

1357

// Consider "int x = 0; int *px = &x;". There are two kinds of

1358

// interesting debug values here after optimization:

1359

//

1360

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

1361

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

1362

//

1363

// Both describe the direct values of their associated variables.

1364

Dependencies.push_back(N.getNode());

1365

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));

1366

continue;

1367

}

1368

LocationOps.emplace_back(

1369

SDDbgOperand::fromNode(N.getNode(), N.getResNo()));

1370

continue;

1371

}

1372

1373

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1374

// Special rules apply for the first dbg.values of parameter variables in a

1375

// function. Identify them by the fact they reference Argument Values, that

1376

// they're parameters, and they are parameters of the current function. We

1377

// need to let them dangle until they get an SDNode.

1378

bool IsParamOfFunc =

1379

isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();

1380

if (IsParamOfFunc)

1381

return false;

1382

1383

// The value is not used in this block yet (or it would have an SDNode).

1384

// We still want the value to appear for the user if possible -- if it has

1385

// an associated VReg, we can refer to that instead.

1386

auto VMI = FuncInfo.ValueMap.find(V);

1387

if (VMI != FuncInfo.ValueMap.end()) {

1388

unsigned Reg = VMI->second;

1389

// If this is a PHI node, it may be split up into several MI PHI nodes

1390

// (in FunctionLoweringInfo::set).

1391

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,

1392

V->getType(), None);

1393

if (RFV.occupiesMultipleRegs()) {

1394

// FIXME: We could potentially support variadic dbg_values here.

1395

if (IsVariadic)

1396

return false;

1397

unsigned Offset = 0;

1398

unsigned BitsToDescribe = 0;

1399

if (auto VarSize = Var->getSizeInBits())

1400

BitsToDescribe = *VarSize;

1401

if (auto Fragment = Expr->getFragmentInfo())

1402

BitsToDescribe = Fragment->SizeInBits;

1403

for (auto RegAndSize : RFV.getRegsAndSizes()) {

1404

// Bail out if all bits are described already.

1405

if (Offset >= BitsToDescribe)

1406

break;

1407

// TODO: handle scalable vectors.

1408

unsigned RegisterSize = RegAndSize.second;

1409

unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)

1410

? BitsToDescribe - Offset

1411

: RegisterSize;

1412

auto FragmentExpr = DIExpression::createFragmentExpression(

1413

Expr, Offset, FragmentSize);

1414

if (!FragmentExpr)

1415

continue;

1416

SDDbgValue *SDV = DAG.getVRegDbgValue(

1417

Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);

1418

DAG.AddDbgValue(SDV, false);

1419

Offset += RegisterSize;

1420

}

1421

return true;

1422

}

1423

// We can use simple vreg locations for variadic dbg_values as well.

1424

LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));

1425

continue;

1426

}

1427

// We failed to create a SDDbgOperand for V.

1428

return false;

1429

}

1430

1431

// We have created a SDDbgOperand for each Value in Values.

1432

// Should use Order instead of SDNodeOrder?

1433

assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) :
__assert_fail ("!LocationOps.empty()", "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1433, __extension__ __PRETTY_FUNCTION__));

1434

SDDbgValue *SDV =

1435

DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,

1436

/*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);

1437

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1438

return true;

1439

}

1440

1441

void SelectionDAGBuilder::resolveOrClearDbgInfo() {

1442

// Try to fixup any remaining dangling debug info -- and drop it if we can't.

1443

for (auto &Pair : DanglingDebugInfoMap)

1444

for (auto &DDI : Pair.second)

1445

salvageUnresolvedDbgValue(DDI);

1446

clearDanglingDebugInfo();

1447

}

1448

1449

/// getCopyFromRegs - If there was virtual register allocated for the value V

1450

/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.

1451

SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {

1452

DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);

1453

SDValue Result;

1454

1455

if (It != FuncInfo.ValueMap.end()) {

1456

Register InReg = It->second;

1457

1458

RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1459

DAG.getDataLayout(), InReg, Ty,

1460

None); // This is not an ABI copy.

1461

SDValue Chain = DAG.getEntryNode();

1462

Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,

1463

V);

1464

resolveDanglingDebugInfo(V, Result);

1465

}

1466

1467

return Result;

1468

}

1469

1470

/// getValue - Return an SDValue for the given Value.

1471

SDValue SelectionDAGBuilder::getValue(const Value *V) {

1472

// If we already have an SDValue for this value, use it. It's important

1473

// to do this first, so that we don't create a CopyFromReg if we already

1474

// have a regular SDValue.

1475

SDValue &N = NodeMap[V];

1476

if (N.getNode()) return N;

1477

1478

// If there's a virtual register allocated and initialized for this

1479

// value, use it.

1480

if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))

1481

return copyFromReg;

1482

1483

// Otherwise create a new SDValue and remember it.

1484

SDValue Val = getValueImpl(V);

1485

NodeMap[V] = Val;

1486

resolveDanglingDebugInfo(V, Val);

1487

return Val;

1488

}

1489

1490

/// getNonRegisterValue - Return an SDValue for the given Value, but

1491

/// don't look in FuncInfo.ValueMap for a virtual register.

1492

SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {

1493

// If we already have an SDValue for this value, use it.

1494

SDValue &N = NodeMap[V];

1495

if (N.getNode()) {

1496

if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {

1497

// Remove the debug location from the node as the node is about to be used

1498

// in a location which may differ from the original debug location. This

1499

// is relevant to Constant and ConstantFP nodes because they can appear

1500

// as constant expressions inside PHI nodes.

1501

N->setDebugLoc(DebugLoc());

1502

}

1503

return N;

1504

}

1505

1506

// Otherwise create a new SDValue and remember it.

1507

SDValue Val = getValueImpl(V);

1508

NodeMap[V] = Val;

1509

resolveDanglingDebugInfo(V, Val);

1510

return Val;

1511

}

1512

1513

/// getValueImpl - Helper function for getValue and getNonRegisterValue.

1514

/// Create an SDValue for the given value.

1515

SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {

1516

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1517

1518

if (const Constant *C = dyn_cast<Constant>(V)) {

1519

EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

1520

1521

if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))

1522

return DAG.getConstant(*CI, getCurSDLoc(), VT);

1523

1524

if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))

1525

return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

1526

1527

if (isa<ConstantPointerNull>(C)) {

1528

unsigned AS = V->getType()->getPointerAddressSpace();

1529

return DAG.getConstant(0, getCurSDLoc(),

1530

TLI.getPointerTy(DAG.getDataLayout(), AS));

1531

}

1532

1533

if (match(C, m_VScale(DAG.getDataLayout())))

1534

return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

1535

1536

if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))

1537

return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

1538

1539

if (isa<UndefValue>(C) && !V->getType()->isAggregateType())

1540

return DAG.getUNDEF(VT);

1541

1542

if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {

1543

visit(CE->getOpcode(), *CE);

1544

SDValue N1 = NodeMap[V];

1545

assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1545, __extension__ __PRETTY_FUNCTION__));

1546

return N1;

1547

}

1548

1549

if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {

1550

SmallVector<SDValue, 4> Constants;

1551

for (const Use &U : C->operands()) {

1552

SDNode *Val = getValue(U).getNode();

1553

// If the operand is an empty aggregate, there are no values.

1554

if (!Val) continue;

1555

// Add each leaf value from the operand to the Constants list

1556

// to form a flattened list of all the values.

1557

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1558

Constants.push_back(SDValue(Val, i));

1559

}

1560

1561

return DAG.getMergeValues(Constants, getCurSDLoc());

1562

}

1563

1564

if (const ConstantDataSequential *CDS =

1565

dyn_cast<ConstantDataSequential>(C)) {

1566

SmallVector<SDValue, 4> Ops;

1567

for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {

1568

SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();

1569

// Add each leaf value from the operand to the Constants list

1570

// to form a flattened list of all the values.

1571

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1572

Ops.push_back(SDValue(Val, i));

1573

}

1574

1575

if (isa<ArrayType>(CDS->getType()))

1576

return DAG.getMergeValues(Ops, getCurSDLoc());

1577

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1578

}

1579

1580

if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {

1581

assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1582, __extension__ __PRETTY_FUNCTION__))

1582

"Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1582, __extension__ __PRETTY_FUNCTION__));

1583

1584

SmallVector<EVT, 4> ValueVTs;

1585

ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);

1586

unsigned NumElts = ValueVTs.size();

1587

if (NumElts == 0)

1588

return SDValue(); // empty struct

1589

SmallVector<SDValue, 4> Constants(NumElts);

1590

for (unsigned i = 0; i != NumElts; ++i) {

1591

EVT EltVT = ValueVTs[i];

1592

if (isa<UndefValue>(C))

1593

Constants[i] = DAG.getUNDEF(EltVT);

1594

else if (EltVT.isFloatingPoint())

1595

Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1596

else

1597

Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);

1598

}

1599

1600

return DAG.getMergeValues(Constants, getCurSDLoc());

1601

}

1602

1603

if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))

1604

return DAG.getBlockAddress(BA, VT);

1605

1606

if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))

1607

return getValue(Equiv->getGlobalValue());

1608

1609

VectorType *VecTy = cast<VectorType>(V->getType());

1610

1611

// Now that we know the number and type of the elements, get that number of

1612

// elements into the Ops array based on what kind of constant it is.

1613

if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {

1614

SmallVector<SDValue, 16> Ops;

1615

unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();

1616

for (unsigned i = 0; i != NumElements; ++i)

1617

Ops.push_back(getValue(CV->getOperand(i)));

1618

1619

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1620

} else if (isa<ConstantAggregateZero>(C)) {

1621

EVT EltVT =

1622

TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

1623

1624

SDValue Op;

1625

if (EltVT.isFloatingPoint())

1626

Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1627

else

1628

Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

1629

1630

if (isa<ScalableVectorType>(VecTy))

1631

return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);

1632

else {

1633

SmallVector<SDValue, 16> Ops;

1634

Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);

1635

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1636

}

1637

}

1638

llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1638);

1639

}

1640

1641

// If this is a static alloca, generate it as the frameindex instead of

1642

// computation.

1643

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1644

DenseMap<const AllocaInst*, int>::iterator SI =

1645

FuncInfo.StaticAllocaMap.find(AI);

1646

if (SI != FuncInfo.StaticAllocaMap.end())

1647

return DAG.getFrameIndex(SI->second,

1648

TLI.getFrameIndexTy(DAG.getDataLayout()));

1649

}

1650

1651

// If this is an instruction which fast-isel has deferred, select it now.

1652

if (const Instruction *Inst = dyn_cast<Instruction>(V)) {

1653

unsigned InReg = FuncInfo.InitializeRegForValue(Inst);

1654

1655

RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,

1656

Inst->getType(), None);

1657

SDValue Chain = DAG.getEntryNode();

1658

return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);

1659

}

1660

1661

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {

1662

return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

1663

}

1664

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1664);

1665

}

1666

1667

void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {

1668

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1669

bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;

1670

bool IsCoreCLR = Pers == EHPersonality::CoreCLR;

1671

bool IsSEH = isAsynchronousEHPersonality(Pers);

1672

MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;

1673

if (!IsSEH)

1674

CatchPadMBB->setIsEHScopeEntry();

1675

// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.

1676

if (IsMSVCCXX || IsCoreCLR)

1677

CatchPadMBB->setIsEHFuncletEntry();

1678

}

1679

1680

void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {

1681

// Update machine-CFG edge.

1682

MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];

1683

FuncInfo.MBB->addSuccessor(TargetMBB);

1684

TargetMBB->setIsEHCatchretTarget(true);

1685

DAG.getMachineFunction().setHasEHCatchret(true);

1686

1687

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1688

bool IsSEH = isAsynchronousEHPersonality(Pers);

1689

if (IsSEH) {

1690

// If this is not a fall-through branch or optimizations are switched off,

1691

// emit the branch.

1692

if (TargetMBB != NextBlock(FuncInfo.MBB) ||

1693

TM.getOptLevel() == CodeGenOpt::None)

1694

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,

1695

getControlRoot(), DAG.getBasicBlock(TargetMBB)));

1696

return;

1697

}

1698

1699

// Figure out the funclet membership for the catchret's successor.

1700

// This will be used by the FuncletLayout pass to determine how to order the

1701

// BB's.

1702

// A 'catchret' returns to the outer scope's color.

1703

Value *ParentPad = I.getCatchSwitchParentPad();

1704

const BasicBlock *SuccessorColor;

1705

if (isa<ConstantTokenNone>(ParentPad))

1706

SuccessorColor = &FuncInfo.Fn->getEntryBlock();

1707

else

1708

SuccessorColor = cast<Instruction>(ParentPad)->getParent();

1709

assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1709, __extension__ __PRETTY_FUNCTION__));

1710

MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];

1711

assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1711, __extension__ __PRETTY_FUNCTION__));

1712

1713

// Create the terminator node.

1714

SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,

1715

getControlRoot(), DAG.getBasicBlock(TargetMBB),

1716

DAG.getBasicBlock(SuccessorColorMBB));

1717

DAG.setRoot(Ret);

1718

}

1719

1720

void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {

1721

// Don't emit any special code for the cleanuppad instruction. It just marks

1722

// the start of an EH scope/funclet.

1723

FuncInfo.MBB->setIsEHScopeEntry();

1724

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1725

if (Pers != EHPersonality::Wasm_CXX) {

1726

FuncInfo.MBB->setIsEHFuncletEntry();

1727

FuncInfo.MBB->setIsCleanupFuncletEntry();

1728

}

1729

}

1730

1731

// In wasm EH, even though a catchpad may not catch an exception if a tag does

1732

// not match, it is OK to add only the first unwind destination catchpad to the

1733

// successors, because there will be at least one invoke instruction within the

1734

// catch scope that points to the next unwind destination, if one exists, so

1735

// CFGSort cannot mess up with BB sorting order.

1736

// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic

1737

// call within them, and catchpads only consisting of 'catch (...)' have a

1738

// '__cxa_end_catch' call within them, both of which generate invokes in case

1739

// the next unwind destination exists, i.e., the next unwind destination is not

1740

// the caller.)

1741

//

1742

// Having at most one EH pad successor is also simpler and helps later

1743

// transformations.

1744

//

1745

// For example,

1746

// current:

1747

// invoke void @foo to ... unwind label %catch.dispatch

1748

// catch.dispatch:

1749

// %0 = catchswitch within ... [label %catch.start] unwind label %next

1750

// catch.start:

1751

// ...

1752

// ... in this BB or some other child BB dominated by this BB there will be an

1753

// invoke that points to 'next' BB as an unwind destination

1754

//

1755

// next: ; We don't need to add this to 'current' BB's successor

1756

// ...

1757

static void findWasmUnwindDestinations(

1758

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1759

BranchProbability Prob,

1760

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1761

&UnwindDests) {

1762

while (EHPadBB) {

1763

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1764

if (isa<CleanupPadInst>(Pad)) {

1765

// Stop on cleanup pads.

1766

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1767

UnwindDests.back().first->setIsEHScopeEntry();

1768

break;

1769

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1770

// Add the catchpad handlers to the possible destinations. We don't

1771

// continue to the unwind destination of the catchswitch for wasm.

1772

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1773

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1774

UnwindDests.back().first->setIsEHScopeEntry();

1775

}

1776

break;

1777

} else {

1778

continue;

1779

}

1780

}

1781

}

1782

1783

/// When an invoke or a cleanupret unwinds to the next EH pad, there are

1784

/// many places it could ultimately go. In the IR, we have a single unwind

1785

/// destination, but in the machine CFG, we enumerate all the possible blocks.

1786

/// This function skips over imaginary basic blocks that hold catchswitch

1787

/// instructions, and finds all the "real" machine

1788

/// basic block destinations. As those destinations may not be successors of

1789

/// EHPadBB, here we also calculate the edge probability to those destinations.

1790

/// The passed-in Prob is the edge probability to EHPadBB.

1791

static void findUnwindDestinations(

1792

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1793

BranchProbability Prob,

1794

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1795

&UnwindDests) {

1796

EHPersonality Personality =

1797

classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1798

bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;

1799

bool IsCoreCLR = Personality == EHPersonality::CoreCLR;

1800

bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;

1801

bool IsSEH = isAsynchronousEHPersonality(Personality);

1802

1803

if (IsWasmCXX) {

1804

findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);

1805

assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1806, __extension__ __PRETTY_FUNCTION__))

1806

"There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1806, __extension__ __PRETTY_FUNCTION__));

1807

return;

1808

}

1809

1810

while (EHPadBB) {

1811

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1812

BasicBlock *NewEHPadBB = nullptr;

1813

if (isa<LandingPadInst>(Pad)) {

1814

// Stop on landingpads. They are not funclets.

1815

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1816

break;

1817

} else if (isa<CleanupPadInst>(Pad)) {

1818

// Stop on cleanup pads. Cleanups are always funclet entries for all known

1819

// personalities.

1820

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1821

UnwindDests.back().first->setIsEHScopeEntry();

1822

UnwindDests.back().first->setIsEHFuncletEntry();

1823

break;

1824

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1825

// Add the catchpad handlers to the possible destinations.

1826

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1827

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1828

// For MSVC++ and the CLR, catchblocks are funclets and need prologues.

1829

if (IsMSVCCXX || IsCoreCLR)

1830

UnwindDests.back().first->setIsEHFuncletEntry();

1831

if (!IsSEH)

1832

UnwindDests.back().first->setIsEHScopeEntry();

1833

}

1834

NewEHPadBB = CatchSwitch->getUnwindDest();

1835

} else {

1836

continue;

1837

}

1838

1839

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1840

if (BPI && NewEHPadBB)

1841

Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);

1842

EHPadBB = NewEHPadBB;

1843

}

1844

}

1845

1846

void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {

1847

// Update successor info.

1848

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

1849

auto UnwindDest = I.getUnwindDest();

1850

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1851

BranchProbability UnwindDestProb =

1852

(BPI && UnwindDest)

1853

? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)

1854

: BranchProbability::getZero();

1855

findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);

1856

for (auto &UnwindDest : UnwindDests) {

1857

UnwindDest.first->setIsEHPad();

1858

addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);

1859

}

1860

FuncInfo.MBB->normalizeSuccProbs();

1861

1862

// Create the terminator node.

1863

SDValue Ret =

1864

DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());

1865

DAG.setRoot(Ret);

1866

}

1867

1868

void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {

1869

report_fatal_error("visitCatchSwitch not yet implemented!");

1870

}

1871

1872

void SelectionDAGBuilder::visitRet(const ReturnInst &I) {

1873

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1874

auto &DL = DAG.getDataLayout();

1875

SDValue Chain = getControlRoot();

1876

SmallVector<ISD::OutputArg, 8> Outs;

1877

SmallVector<SDValue, 8> OutVals;

1878

1879

// Calls to @llvm.experimental.deoptimize don't generate a return value, so

1880

// lower

1881

//

1882

// %val = call <ty> @llvm.experimental.deoptimize()

1883

// ret <ty> %val

1884

//

1885

// differently.

1886

if (I.getParent()->getTerminatingDeoptimizeCall()) {

1887

LowerDeoptimizingReturn();

1888

return;

1889

}

1890

1891

if (!FuncInfo.CanLowerReturn) {

1892

unsigned DemoteReg = FuncInfo.DemoteRegister;

1893

const Function *F = I.getParent()->getParent();

1894

1895

// Emit a store of the return value through the virtual register.

1896

// Leave Outs empty so that LowerReturn won't try to load return

1897

// registers the usual way.

1898

SmallVector<EVT, 1> PtrValueVTs;

1899

ComputeValueVTs(TLI, DL,

1900

F->getReturnType()->getPointerTo(

1901

DAG.getDataLayout().getAllocaAddrSpace()),

1902

PtrValueVTs);

1903

1904

SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),

1905

DemoteReg, PtrValueVTs[0]);

1906

SDValue RetOp = getValue(I.getOperand(0));

1907

1908

SmallVector<EVT, 4> ValueVTs, MemVTs;

1909

SmallVector<uint64_t, 4> Offsets;

1910

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,

1911

&Offsets);

1912

unsigned NumValues = ValueVTs.size();

1913

1914

SmallVector<SDValue, 4> Chains(NumValues);

1915

Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());

1916

for (unsigned i = 0; i != NumValues; ++i) {

1917

// An aggregate return value cannot wrap around the address space, so

1918

// offsets to its parts don't wrap either.

1919

SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,

1920

TypeSize::Fixed(Offsets[i]));

1921

1922

SDValue Val = RetOp.getValue(RetOp.getResNo() + i);

1923

if (MemVTs[i] != ValueVTs[i])

1924

Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);

1925

Chains[i] = DAG.getStore(

1926

Chain, getCurSDLoc(), Val,

1927

// FIXME: better loc info would be nice.

1928

Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),

1929

commonAlignment(BaseAlign, Offsets[i]));

1930

}

1931

1932

Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),

1933

MVT::Other, Chains);

1934

} else if (I.getNumOperands() != 0) {

1935

SmallVector<EVT, 4> ValueVTs;

1936

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);

1937

unsigned NumValues = ValueVTs.size();

1938

if (NumValues) {

1939

SDValue RetOp = getValue(I.getOperand(0));

1940

1941

const Function *F = I.getParent()->getParent();

1942

1943

bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(

1944

I.getOperand(0)->getType(), F->getCallingConv(),

1945

/*IsVarArg*/ false, DL);

1946

1947

ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

1948

if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1949

Attribute::SExt))

1950

ExtendKind = ISD::SIGN_EXTEND;

1951

else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1952

Attribute::ZExt))

1953

ExtendKind = ISD::ZERO_EXTEND;

1954

1955

LLVMContext &Context = F->getContext();

1956

bool RetInReg = F->getAttributes().hasAttribute(

1957

AttributeList::ReturnIndex, Attribute::InReg);

1958

1959

for (unsigned j = 0; j != NumValues; ++j) {

1960

EVT VT = ValueVTs[j];

1961

1962

if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())

1963

VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

1964

1965

CallingConv::ID CC = F->getCallingConv();

1966

1967

unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);

1968

MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);

1969

SmallVector<SDValue, 4> Parts(NumParts);

1970

getCopyToParts(DAG, getCurSDLoc(),

1971

SDValue(RetOp.getNode(), RetOp.getResNo() + j),

1972

&Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

1973

1974

// 'inreg' on function refers to return value

1975

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1976

if (RetInReg)

1977

Flags.setInReg();

1978

1979

if (I.getOperand(0)->getType()->isPointerTy()) {

1980

Flags.setPointer();

1981

Flags.setPointerAddrSpace(

1982

cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());

1983

}

1984

1985

if (NeedsRegBlock) {

1986

Flags.setInConsecutiveRegs();

1987

if (j == NumValues - 1)

1988

Flags.setInConsecutiveRegsLast();

1989

}

1990

1991

// Propagate extension type if any

1992

if (ExtendKind == ISD::SIGN_EXTEND)

1993

Flags.setSExt();

1994

else if (ExtendKind == ISD::ZERO_EXTEND)

1995

Flags.setZExt();

1996

1997

for (unsigned i = 0; i < NumParts; ++i) {

1998

Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),

1999

VT, /*isfixed=*/true, 0, 0));

2000

OutVals.push_back(Parts[i]);

2001

}

2002

}

2003

}

2004

}

2005

2006

// Push in swifterror virtual register as the last element of Outs. This makes

2007

// sure swifterror virtual register will be returned in the swifterror

2008

// physical register.

2009

const Function *F = I.getParent()->getParent();

2010

if (TLI.supportSwiftError() &&

2011

F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {

2012

assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() &&
"Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2012, __extension__ __PRETTY_FUNCTION__));

2013

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

2014

Flags.setSwiftError();

2015

Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,

2016

EVT(TLI.getPointerTy(DL)) /*argvt*/,

2017

true /*isfixed*/, 1 /*origidx*/,

2018

0 /*partOffs*/));

2019

// Create SDNode for the swifterror virtual register.

2020

OutVals.push_back(

2021

DAG.getRegister(SwiftError.getOrCreateVRegUseAt(

2022

&I, FuncInfo.MBB, SwiftError.getFunctionArg()),

2023

EVT(TLI.getPointerTy(DL))));

2024

}

2025

2026

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();

2027

CallingConv::ID CallConv =

2028

DAG.getMachineFunction().getFunction().getCallingConv();

2029

Chain = DAG.getTargetLoweringInfo().LowerReturn(

2030

Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

2031

2032

// Verify that the target's LowerReturn behaved as expected.

2033

assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2034, __extension__ __PRETTY_FUNCTION__))

2034

"LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2034, __extension__ __PRETTY_FUNCTION__));

2035

2036

// Update the DAG with the new chain value resulting from return lowering.

2037

DAG.setRoot(Chain);

2038

}

2039

2040

/// CopyToExportRegsIfNeeded - If the given value has virtual registers

2041

/// created for it, emit nodes to copy the value into the virtual

2042

/// registers.

2043

void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

2044

// Skip empty types

2045

if (V->getType()->isEmptyTy())

2046

return;

2047

2048

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);

2049

if (VMI != FuncInfo.ValueMap.end()) {

2050

assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast <bool> (!V->use_empty() && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2050, __extension__ __PRETTY_FUNCTION__));

2051

CopyValueToVirtualRegister(V, VMI->second);

2052

}

2053

}

2054

2055

/// ExportFromCurrentBlock - If this condition isn't known to be exported from

2056

/// the current basic block, add it to ValueMap now so that we'll get a

2057

/// CopyTo/FromReg.

2058

void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {

2059

// No need to export constants.

2060

if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

2061

2062

// Already exported?

2063

if (FuncInfo.isExportedInst(V)) return;

2064

2065

unsigned Reg = FuncInfo.InitializeRegForValue(V);

2066

CopyValueToVirtualRegister(V, Reg);

2067

}

2068

2069

bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,

2070

const BasicBlock *FromBB) {

2071

// The operands of the setcc have to be in this block. We don't know

2072

// how to export them from some other block.

2073

if (const Instruction *VI = dyn_cast<Instruction>(V)) {

2074

// Can export from current BB.

2075

if (VI->getParent() == FromBB)

2076

return true;

2077

2078

// Is already exported, noop.

2079

return FuncInfo.isExportedInst(V);

2080

}

2081

2082

// If this is an argument, we can export it if the BB is the entry block or

2083

// if it is already exported.

2084

if (isa<Argument>(V)) {

2085

if (FromBB->isEntryBlock())

2086

return true;

2087

2088

// Otherwise, can only export this if it is already exported.

2089

return FuncInfo.isExportedInst(V);

2090

}

2091

2092

// Otherwise, constants can always be exported.

2093

return true;

2094

}

2095

2096

/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.

2097

BranchProbability

2098

SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,

2099

const MachineBasicBlock *Dst) const {

2100

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2101

const BasicBlock *SrcBB = Src->getBasicBlock();

2102

const BasicBlock *DstBB = Dst->getBasicBlock();

2103

if (!BPI) {

2104

// If BPI is not available, set the default probability as 1 / N, where N is

2105

// the number of successors.

2106

auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);

2107

return BranchProbability(1, SuccSize);

2108

}

2109

return BPI->getEdgeProbability(SrcBB, DstBB);

2110

}

2111

2112

void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,

2113

MachineBasicBlock *Dst,

2114

BranchProbability Prob) {

2115

if (!FuncInfo.BPI)

2116

Src->addSuccessorWithoutProb(Dst);

2117

else {

2118

if (Prob.isUnknown())

2119

Prob = getEdgeProbability(Src, Dst);

2120

Src->addSuccessor(Dst, Prob);

2121

}

2122

}

2123

2124

static bool InBlock(const Value *V, const BasicBlock *BB) {

2125

if (const Instruction *I = dyn_cast<Instruction>(V))

2126

return I->getParent() == BB;

2127

return true;

2128

}

2129

2130

/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.

2131

/// This function emits a branch and is used at the leaves of an OR or an

2132

/// AND operator tree.

2133

void

2134

SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,

2135

MachineBasicBlock *TBB,

2136

MachineBasicBlock *FBB,

2137

MachineBasicBlock *CurBB,

2138

MachineBasicBlock *SwitchBB,

2139

BranchProbability TProb,

2140

BranchProbability FProb,

2141

bool InvertCond) {

2142

const BasicBlock *BB = CurBB->getBasicBlock();

2143

2144

// If the leaf of the tree is a comparison, merge the condition into

2145

// the caseblock.

2146

if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {

2147

// The operands of the cmp have to be in this block. We don't know

2148

// how to export them from some other block. If this is the first block

2149

// of the sequence, no exporting is needed.

2150

if (CurBB == SwitchBB ||

2151

(isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&

2152

isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {

2153

ISD::CondCode Condition;

2154

if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {

2155

ICmpInst::Predicate Pred =

2156

InvertCond ? IC->getInversePredicate() : IC->getPredicate();

2157

Condition = getICmpCondCode(Pred);

2158

} else {

2159

const FCmpInst *FC = cast<FCmpInst>(Cond);

2160

FCmpInst::Predicate Pred =

2161

InvertCond ? FC->getInversePredicate() : FC->getPredicate();

2162

Condition = getFCmpCondCode(Pred);

2163

if (TM.Options.NoNaNsFPMath)

2164

Condition = getFCmpCodeWithoutNaN(Condition);

2165

}

2166

2167

CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,

2168

TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2169

SL->SwitchCases.push_back(CB);

2170

return;

2171

}

2172

}

2173

2174

// Create a CaseBlock record representing this branch.

2175

ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;

2176

CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),

2177

nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2178

SL->SwitchCases.push_back(CB);

2179

}

2180

2181

void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,

2182

MachineBasicBlock *TBB,

2183

MachineBasicBlock *FBB,

2184

MachineBasicBlock *CurBB,

2185

MachineBasicBlock *SwitchBB,

2186

Instruction::BinaryOps Opc,

2187

BranchProbability TProb,

2188

BranchProbability FProb,

2189

bool InvertCond) {

2190

// Skip over not part of the tree and remember to invert op and operands at

2191

// next level.

2192

Value *NotCond;

2193

if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&

2194

InBlock(NotCond, CurBB->getBasicBlock())) {

2195

FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,

2196

!InvertCond);

2197

return;

2198

}

2199

2200

const Instruction *BOp = dyn_cast<Instruction>(Cond);

2201

const Value *BOpOp0, *BOpOp1;

2202

// Compute the effective opcode for Cond, taking into account whether it needs

2203

// to be inverted, e.g.

2204

// and (not (or A, B)), C

2205

// gets lowered as

2206

// and (and (not A, not B), C)

2207

Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;

2208

if (BOp) {

2209

BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))

2210

? Instruction::And

2211

: (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))

2212

? Instruction::Or

2213

: (Instruction::BinaryOps)0);

2214

if (InvertCond) {

2215

if (BOpc == Instruction::And)

2216

BOpc = Instruction::Or;

2217

else if (BOpc == Instruction::Or)

2218

BOpc = Instruction::And;

2219

}

2220

}

2221

2222

// If this node is not part of the or/and tree, emit it as a branch.

2223

// Note that all nodes in the tree should have same opcode.

2224

bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();

2225

if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||

2226

!InBlock(BOpOp0, CurBB->getBasicBlock()) ||

2227

!InBlock(BOpOp1, CurBB->getBasicBlock())) {

2228

EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,

2229

TProb, FProb, InvertCond);

2230

return;

2231

}

2232

2233

// Create TmpBB after CurBB.

2234

MachineFunction::iterator BBI(CurBB);

2235

MachineFunction &MF = DAG.getMachineFunction();

2236

MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());

2237

CurBB->getParent()->insert(++BBI, TmpBB);

2238

2239

if (Opc == Instruction::Or) {

2240

// Codegen X | Y as:

2241

// BB1:

2242

// jmp_if_X TBB

2243

// jmp TmpBB

2244

// TmpBB:

2245

// jmp_if_Y TBB

2246

// jmp FBB

2247

//

2248

2249

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2250

// The requirement is that

2251

// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)

2252

// = TrueProb for original BB.

2253

// Assuming the original probabilities are A and B, one choice is to set

2254

// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to

2255

// A/(1+B) and 2B/(1+B). This choice assumes that

2256

// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.

2257

// Another choice is to assume TrueProb for BB1 equals to TrueProb for

2258

// TmpBB, but the math is more complicated.

2259

2260

auto NewTrueProb = TProb / 2;

2261

auto NewFalseProb = TProb / 2 + FProb;

2262

// Emit the LHS condition.

2263

FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,

2264

NewFalseProb, InvertCond);

2265

2266

// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).

2267

SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};

2268

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2269

// Emit the RHS condition into TmpBB.

2270

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2271

Probs[1], InvertCond);

2272

} else {

2273

assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
"Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2273, __extension__ __PRETTY_FUNCTION__));

2274

// Codegen X & Y as:

2275

// BB1:

2276

// jmp_if_X TmpBB

2277

// jmp FBB

2278

// TmpBB:

2279

// jmp_if_Y TBB

2280

// jmp FBB

2281

//

2282

// This requires creation of TmpBB after CurBB.

2283

2284

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2285

// The requirement is that

2286

// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)

2287

// = FalseProb for original BB.

2288

// Assuming the original probabilities are A and B, one choice is to set

2289

// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to

2290

// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==

2291

// TrueProb for BB1 * FalseProb for TmpBB.

2292

2293

auto NewTrueProb = TProb + FProb / 2;

2294

auto NewFalseProb = FProb / 2;

2295

// Emit the LHS condition.

2296

FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,

2297

NewFalseProb, InvertCond);

2298

2299

// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).

2300

SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};

2301

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2302

// Emit the RHS condition into TmpBB.

2303

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2304

Probs[1], InvertCond);

2305

}

2306

}

2307

2308

/// If the set of cases should be emitted as a series of branches, return true.

2309

/// If we should emit this as a bunch of and/or'd together conditions, return

2310

/// false.

2311

bool

2312

SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {

2313

if (Cases.size() != 2) return true;

2314

2315

// If this is two comparisons of the same values or'd or and'd together, they

2316

// will get folded into a single comparison, so don't emit two blocks.

2317

if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&

2318

Cases[0].CmpRHS == Cases[1].CmpRHS) ||

2319

(Cases[0].CmpRHS == Cases[1].CmpLHS &&

2320

Cases[0].CmpLHS == Cases[1].CmpRHS)) {

2321

return false;

2322

}

2323

2324

// Handle: (X != null) | (Y != null) --> (X|Y) != 0

2325

// Handle: (X == null) & (Y == null) --> (X|Y) == 0

2326

if (Cases[0].CmpRHS == Cases[1].CmpRHS &&

2327

Cases[0].CC == Cases[1].CC &&

2328

isa<Constant>(Cases[0].CmpRHS) &&

2329

cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {

2330

if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)

2331

return false;

2332

if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)

2333

return false;

2334

}

2335

2336

return true;

2337

}

2338

2339

void SelectionDAGBuilder::visitBr(const BranchInst &I) {

2340

MachineBasicBlock *BrMBB = FuncInfo.MBB;

2341

2342

// Update machine-CFG edges.

2343

MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

2344

2345

if (I.isUnconditional()) {

2346

// Update machine-CFG edges.

2347

BrMBB->addSuccessor(Succ0MBB);

2348

2349

// If this is not a fall-through branch or optimizations are switched off,

2350

// emit the branch.

2351

if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)

2352

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2353

MVT::Other, getControlRoot(),

2354

DAG.getBasicBlock(Succ0MBB)));

2355

2356

return;

2357

}

2358

2359

// If this condition is one of the special cases we handle, do special stuff

2360

// now.

2361

const Value *CondVal = I.getCondition();

2362

MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

2363

2364

// If this is a series of conditions that are or'd or and'd together, emit

2365

// this as a sequence of branches instead of setcc's with and/or operations.

2366

// As long as jumps are not expensive (exceptions for multi-use logic ops,

2367

// unpredictable branches, and vector extracts because those jumps are likely

2368

// expensive for any target), this should improve performance.

2369

// For example, instead of something like:

2370

// cmp A, B

2371

// C = seteq

2372

// cmp D, E

2373

// F = setle

2374

// or C, F

2375

// jnz foo

2376

// Emit:

2377

// cmp A, B

2378

// je foo

2379

// cmp D, E

2380

// jle foo

2381

const Instruction *BOp = dyn_cast<Instruction>(CondVal);

2382

if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&

2383

BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {

2384

Value *Vec;

2385

const Value *BOp0, *BOp1;

2386

Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;

2387

if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))

2388

Opcode = Instruction::And;

2389

else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))

2390

Opcode = Instruction::Or;

2391

2392

if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&

2393

match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {

2394

FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,

2395

getEdgeProbability(BrMBB, Succ0MBB),

2396

getEdgeProbability(BrMBB, Succ1MBB),

2397

/*InvertCond=*/false);

2398

// If the compares in later blocks need to use values not currently

2399

// exported from this block, export them now. This block should always

2400

// be the first entry.

2401

assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB
&& "Unexpected lowering!") ? void (0) : __assert_fail
("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2401, __extension__ __PRETTY_FUNCTION__));

2402

2403

// Allow some cases to be rejected.

2404

if (ShouldEmitAsBranches(SL->SwitchCases)) {

2405

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {

2406

ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);

2407

ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);

2408

}

2409

2410

// Emit the branch for this block.

2411

visitSwitchCase(SL->SwitchCases[0], BrMBB);

2412

SL->SwitchCases.erase(SL->SwitchCases.begin());

2413

return;

2414

}

2415

2416

// Okay, we decided not to do this, remove any inserted MBB's and clear

2417

// SwitchCases.

2418

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)

2419

FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

2420

2421

SL->SwitchCases.clear();

2422

}

2423

}

2424

2425

// Create a CaseBlock record representing this branch.

2426

CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),

2427

nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

2428

2429

// Use visitSwitchCase to actually insert the fast branch sequence for this

2430

// cond branch.

2431

visitSwitchCase(CB, BrMBB);

2432

}

2433

2434

/// visitSwitchCase - Emits the necessary code to represent a single node in

2435

/// the binary search tree resulting from lowering a switch instruction.

2436

void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,

2437

MachineBasicBlock *SwitchBB) {

2438

SDValue Cond;

2439

SDValue CondLHS = getValue(CB.CmpLHS);

2440

SDLoc dl = CB.DL;

2441

2442

if (CB.CC == ISD::SETTRUE) {

2443

// Branch or fall through to TrueBB.

2444

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2445

SwitchBB->normalizeSuccProbs();

2446

if (CB.TrueBB != NextBlock(SwitchBB)) {

2447

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),

2448

DAG.getBasicBlock(CB.TrueBB)));

2449

}

2450

return;

2451

}

2452

2453

auto &TLI = DAG.getTargetLoweringInfo();

2454

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

2455

2456

// Build the setcc now.

2457

if (!CB.CmpMHS) {

2458

// Fold "(X == true)" to X and "(X == false)" to !X to

2459

// handle common cases produced by branch lowering.

2460

if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&

2461

CB.CC == ISD::SETEQ)

2462

Cond = CondLHS;

2463

else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&

2464

CB.CC == ISD::SETEQ) {

2465

SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());

2466

Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);

2467

} else {

2468

SDValue CondRHS = getValue(CB.CmpRHS);

2469

2470

// If a pointer's DAG type is larger than its memory type then the DAG

2471

// values are zero-extended. This breaks signed comparisons so truncate

2472

// back to the underlying type before doing the compare.

2473

if (CondLHS.getValueType() != MemVT) {

2474

CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);

2475

CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);

2476

}

2477

Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);

2478

}

2479

} else {

2480

assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2480, __extension__ __PRETTY_FUNCTION__));

2481

2482

const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();

2483

const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

2484

2485

SDValue CmpOp = getValue(CB.CmpMHS);

2486

EVT VT = CmpOp.getValueType();

2487

2488

if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {

2489

Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),

2490

ISD::SETLE);

2491

} else {

2492

SDValue SUB = DAG.getNode(ISD::SUB, dl,

2493

VT, CmpOp, DAG.getConstant(Low, dl, VT));

2494

Cond = DAG.getSetCC(dl, MVT::i1, SUB,

2495

DAG.getConstant(High-Low, dl, VT), ISD::SETULE);

2496

}

2497

}

2498

2499

// Update successor info

2500

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2501

// TrueBB and FalseBB are always different unless the incoming IR is

2502

// degenerate. This only happens when running llc on weird IR.

2503

if (CB.TrueBB != CB.FalseBB)

2504

addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);

2505

SwitchBB->normalizeSuccProbs();

2506

2507

// If the lhs block is the next block, invert the condition so that we can

2508

// fall through to the lhs instead of the rhs block.

2509

if (CB.TrueBB == NextBlock(SwitchBB)) {

2510

std::swap(CB.TrueBB, CB.FalseBB);

2511

SDValue True = DAG.getConstant(1, dl, Cond.getValueType());

2512

Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);

2513

}

2514

2515

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2516

MVT::Other, getControlRoot(), Cond,

2517

DAG.getBasicBlock(CB.TrueBB));

2518

2519

// Insert the false branch. Do this even if it's a fall through branch,

2520

// this makes it easier to do DAG optimizations which require inverting

2521

// the branch condition.

2522

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2523

DAG.getBasicBlock(CB.FalseBB));

2524

2525

DAG.setRoot(BrCond);

2526

}

2527

2528

/// visitJumpTable - Emit JumpTable node in the current MBB

2529

void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {

2530

// Emit the code for the jump table

2531

assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2531, __extension__ __PRETTY_FUNCTION__));

2532

EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());

2533

SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),

2534

JT.Reg, PTy);

2535

SDValue Table = DAG.getJumpTable(JT.JTI, PTy);

2536

SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),

2537

MVT::Other, Index.getValue(1),

2538

Table, Index);

2539

DAG.setRoot(BrJumpTable);

2540

}

2541

2542

/// visitJumpTableHeader - This function emits necessary code to produce index

2543

/// in the JumpTable from switch case.

2544

void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,

2545

JumpTableHeader &JTH,

2546

MachineBasicBlock *SwitchBB) {

2547

SDLoc dl = getCurSDLoc();

2548

2549

// Subtract the lowest switch case value from the value being switched on.

2550

SDValue SwitchOp = getValue(JTH.SValue);

2551

EVT VT = SwitchOp.getValueType();

2552

SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,

2553

DAG.getConstant(JTH.First, dl, VT));

2554

2555

// The SDNode we just created, which holds the value being switched on minus

2556

// the smallest case value, needs to be copied to a virtual register so it

2557

// can be used as an index into the jump table in a subsequent basic block.

2558

// This value may be smaller or larger than the target's pointer type, and

2559

// therefore require extension or truncating.

2560

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2561

SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

2562

2563

unsigned JumpTableReg =

2564

FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));

2565

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,

2566

JumpTableReg, SwitchOp);

2567

JT.Reg = JumpTableReg;

2568

2569

if (!JTH.OmitRangeCheck) {

2570

// Emit the range check for the jump table, and branch to the default block

2571

// for the switch statement if the value being switched on exceeds the

2572

// largest case in the switch.

2573

SDValue CMP = DAG.getSetCC(

2574

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2575

Sub.getValueType()),

2576

Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

2577

2578

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2579

MVT::Other, CopyTo, CMP,

2580

DAG.getBasicBlock(JT.Default));

2581

2582

// Avoid emitting unnecessary branches to the next block.

2583

if (JT.MBB != NextBlock(SwitchBB))

2584

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2585

DAG.getBasicBlock(JT.MBB));

2586

2587

DAG.setRoot(BrCond);

2588

} else {

2589

// Avoid emitting unnecessary branches to the next block.

2590

if (JT.MBB != NextBlock(SwitchBB))

2591

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,

2592

DAG.getBasicBlock(JT.MBB)));

2593

else

2594

DAG.setRoot(CopyTo);

2595

}

2596

}

2597

2598

/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global

2599

/// variable if there exists one.

2600

static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,

2601

SDValue &Chain) {

2602

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2603

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2604

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2605

MachineFunction &MF = DAG.getMachineFunction();

2606

Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());

2607

MachineSDNode *Node =

2608

DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);

2609

if (Global) {

2610

MachinePointerInfo MPInfo(Global);

2611

auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |

2612

MachineMemOperand::MODereferenceable;

2613

MachineMemOperand *MemRef = MF.getMachineMemOperand(

2614

MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));

2615

DAG.setNodeMemRefs(Node, {MemRef});

2616

}

2617

if (PtrTy != PtrMemTy)

2618

return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);

2619

return SDValue(Node, 0);

2620

}

2621

2622

/// Codegen a new tail for a stack protector check ParentMBB which has had its

2623

/// tail spliced into a stack protector check success bb.

2624

///

2625

/// For a high level explanation of how this fits into the stack protector

2626

/// generation see the comment on the declaration of class

2627

/// StackProtectorDescriptor.

2628

void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,

2629

MachineBasicBlock *ParentBB) {

2630

2631

// First create the loads to the guard/stack slot for the comparison.

2632

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2633

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2634

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2635

2636

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();

2637

int FI = MFI.getStackProtectorIndex();

2638

2639

SDValue Guard;

2640

SDLoc dl = getCurSDLoc();

2641

SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);

2642

const Module &M = *ParentBB->getParent()->getFunction().getParent();

2643

Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

2644

2645

// Generate code to load the content of the guard slot.

2646

SDValue GuardVal = DAG.getLoad(

2647

PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,

2648

MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,

2649

MachineMemOperand::MOVolatile);

2650

2651

if (TLI.useStackGuardXorFP())

2652

GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

2653

2654

// Retrieve guard check function, nullptr if instrumentation is inlined.

2655

if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {

2656

// The target provides a guard check function to validate the guard value.

2657

// Generate a call to that function with the content of the guard slot as

2658

// argument.

2659

FunctionType *FnTy = GuardCheckFn->getFunctionType();

2660

assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
"Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2660, __extension__ __PRETTY_FUNCTION__));

2661

2662

TargetLowering::ArgListTy Args;

2663

TargetLowering::ArgListEntry Entry;

2664

Entry.Node = GuardVal;

2665

Entry.Ty = FnTy->getParamType(0);

2666

if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))

2667

Entry.IsInReg = true;

2668

Args.push_back(Entry);

2669

2670

TargetLowering::CallLoweringInfo CLI(DAG);

2671

CLI.setDebugLoc(getCurSDLoc())

2672

.setChain(DAG.getEntryNode())

2673

.setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),

2674

getValue(GuardCheckFn), std::move(Args));

2675

2676

std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

2677

DAG.setRoot(Result.second);

2678

return;

2679

}

2680

2681

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.

2682

// Otherwise, emit a volatile load to retrieve the stack guard value.

2683

SDValue Chain = DAG.getEntryNode();

2684

if (TLI.useLoadStackGuardNode()) {

2685

Guard = getLoadStackGuard(DAG, dl, Chain);

2686

} else {

2687

const Value *IRGuard = TLI.getSDagStackGuard(M);

2688

SDValue GuardPtr = getValue(IRGuard);

2689

2690

Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,

2691

MachinePointerInfo(IRGuard, 0), Align,

2692

MachineMemOperand::MOVolatile);

2693

}

2694

2695

// Perform the comparison via a getsetcc.

2696

SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),

2697

*DAG.getContext(),

2698

Guard.getValueType()),

2699

Guard, GuardVal, ISD::SETNE);

2700

2701

// If the guard/stackslot do not equal, branch to failure MBB.

2702

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2703

MVT::Other, GuardVal.getOperand(0),

2704

Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));

2705

// Otherwise branch to success MBB.

2706

SDValue Br = DAG.getNode(ISD::BR, dl,

2707

MVT::Other, BrCond,

2708

DAG.getBasicBlock(SPD.getSuccessMBB()));

2709

2710

DAG.setRoot(Br);

2711

}

2712

2713

/// Codegen the failure basic block for a stack protector check.

2714

///

2715

/// A failure stack protector machine basic block consists simply of a call to

2716

/// __stack_chk_fail().

2717

///

2718

/// For a high level explanation of how this fits into the stack protector

2719

/// generation see the comment on the declaration of class

2720

/// StackProtectorDescriptor.

2721

void

2722

SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {

2723

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2724

TargetLowering::MakeLibCallOptions CallOptions;

2725

CallOptions.setDiscardResult(true);

2726

SDValue Chain =

2727

TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,

2728

None, CallOptions, getCurSDLoc()).second;

2729

// On PS4, the "return address" must still be within the calling function,

2730

// even if it's at the very end, so emit an explicit TRAP here.

2731

// Passing 'true' for doesNotReturn above won't generate the trap for us.

2732

if (TM.getTargetTriple().isPS4CPU())

2733

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2734

// WebAssembly needs an unreachable instruction after a non-returning call,

2735

// because the function return type can be different from __stack_chk_fail's

2736

// return type (void).

2737

if (TM.getTargetTriple().isWasm())

2738

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2739

2740

DAG.setRoot(Chain);

2741

}

2742

2743

/// visitBitTestHeader - This function emits necessary code to produce value

2744

/// suitable for "bit tests"

2745

void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,

2746

MachineBasicBlock *SwitchBB) {

2747

SDLoc dl = getCurSDLoc();

2748

2749

// Subtract the minimum value.

2750

SDValue SwitchOp = getValue(B.SValue);

2751

EVT VT = SwitchOp.getValueType();

2752

SDValue RangeSub =

2753

DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

2754

2755

// Determine the type of the test operands.

2756

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2757

bool UsePtrType = false;

2758

if (!TLI.isTypeLegal(VT)) {

2759

UsePtrType = true;

2760

} else {

2761

for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)

2762

if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {

2763

// Switch table case range are encoded into series of masks.

2764

// Just use pointer type, it's guaranteed to fit.

2765

UsePtrType = true;

2766

break;

2767

}

2768

}

2769

SDValue Sub = RangeSub;

2770

if (UsePtrType) {

2771

VT = TLI.getPointerTy(DAG.getDataLayout());

2772

Sub = DAG.getZExtOrTrunc(Sub, dl, VT);

2773

}

2774

2775

B.RegVT = VT.getSimpleVT();

2776

B.Reg = FuncInfo.CreateReg(B.RegVT);

2777

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

2778

2779

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

2780

2781

if (!B.OmitRangeCheck)

2782

addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);

2783

addSuccessorWithProb(SwitchBB, MBB, B.Prob);

2784

SwitchBB->normalizeSuccProbs();

2785

2786

SDValue Root = CopyTo;

2787

if (!B.OmitRangeCheck) {

2788

// Conditional branch to the default block.

2789

SDValue RangeCmp = DAG.getSetCC(dl,

2790

TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2791

RangeSub.getValueType()),

2792

RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),

2793

ISD::SETUGT);

2794

2795

Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,

2796

DAG.getBasicBlock(B.Default));

2797

}

2798

2799

// Avoid emitting unnecessary branches to the next block.

2800

if (MBB != NextBlock(SwitchBB))

2801

Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

2802

2803

DAG.setRoot(Root);

2804

}

2805

2806

/// visitBitTestCase - this function produces one "bit test"

2807

void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,

2808

MachineBasicBlock* NextMBB,

2809

BranchProbability BranchProbToNext,

2810

unsigned Reg,

2811

BitTestCase &B,

2812

MachineBasicBlock *SwitchBB) {

2813

SDLoc dl = getCurSDLoc();

2814

MVT VT = BB.RegVT;

2815

SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);

2816

SDValue Cmp;

2817

unsigned PopCount = countPopulation(B.Mask);

2818

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2819

if (PopCount == 1) {

2820

// Testing for a single bit; just compare the shift count with what it

2821

// would need to be to shift a 1 bit in that position.

2822

Cmp = DAG.getSetCC(

2823

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2824

ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),

2825

ISD::SETEQ);

2826

} else if (PopCount == BB.Range) {

2827

// There is only one zero bit in the range, test for it directly.

2828

Cmp = DAG.getSetCC(

2829

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2830

ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),

2831

ISD::SETNE);

2832

} else {

2833

// Make desired shift

2834

SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,

2835

DAG.getConstant(1, dl, VT), ShiftOp);

2836

2837

// Emit bit tests and jumps

2838

SDValue AndOp = DAG.getNode(ISD::AND, dl,

2839

VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));

2840

Cmp = DAG.getSetCC(

2841

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2842

AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);

2843

}

2844

2845

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.

2846

addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);

2847

// The branch probability from SwitchBB to NextMBB is BranchProbToNext.

2848

addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);

2849

// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is

2850

// one as they are relative probabilities (and thus work more like weights),

2851

// and hence we need to normalize them to let the sum of them become one.

2852

SwitchBB->normalizeSuccProbs();

2853

2854

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,

2855

MVT::Other, getControlRoot(),

2856

Cmp, DAG.getBasicBlock(B.TargetBB));

2857

2858

// Avoid emitting unnecessary branches to the next block.

2859

if (NextMBB != NextBlock(SwitchBB))

2860

BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,

2861

DAG.getBasicBlock(NextMBB));

2862

2863

DAG.setRoot(BrAnd);

2864

}

2865

2866

void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {

2867

MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

2868

2869

// Retrieve successors. Look through artificial IR level blocks like

2870

// catchswitch for successors.

2871

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];

2872

const BasicBlock *EHPadBB = I.getSuccessor(1);

2873

2874

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2875

// have to do anything here to lower funclet bundles.

2876

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__))

2877

{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__))

2878

LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__))

2879

LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__))

2880

LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__))

2881

"Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2881, __extension__ __PRETTY_FUNCTION__));

2882

2883

const Value *Callee(I.getCalledOperand());

2884

const Function *Fn = dyn_cast<Function>(Callee);

2885

if (isa<InlineAsm>(Callee))

2886

visitInlineAsm(I, EHPadBB);

2887

else if (Fn && Fn->isIntrinsic()) {

2888

switch (Fn->getIntrinsicID()) {

2889

default:

2890

llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2890);

2891

case Intrinsic::donothing:

2892

// Ignore invokes to @llvm.donothing: jump directly to the next BB.

2893

case Intrinsic::seh_try_begin:

2894

case Intrinsic::seh_scope_begin:

2895

case Intrinsic::seh_try_end:

2896

case Intrinsic::seh_scope_end:

2897

break;

2898

case Intrinsic::experimental_patchpoint_void:

2899

case Intrinsic::experimental_patchpoint_i64:

2900

visitPatchpoint(I, EHPadBB);

2901

break;

2902

case Intrinsic::experimental_gc_statepoint:

2903

LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);

2904

break;

2905

case Intrinsic::wasm_rethrow: {

2906

// This is usually done in visitTargetIntrinsic, but this intrinsic is

2907

// special because it can be invoked, so we manually lower it to a DAG

2908

// node here.

2909

SmallVector<SDValue, 8> Ops;

2910

Ops.push_back(getRoot()); // inchain

2911

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2912

Ops.push_back(

2913

DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),

2914

TLI.getPointerTy(DAG.getDataLayout())));

2915

SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain

2916

DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));

2917

break;

2918

}

2919

}

2920

} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {

2921

// Currently we do not lower any intrinsic calls with deopt operand bundles.

2922

// Eventually we will support lowering the @llvm.experimental.deoptimize

2923

// intrinsic, and right now there are no plans to support other intrinsics

2924

// with deopt state.

2925

LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);

2926

} else {

2927

LowerCallTo(I, getValue(Callee), false, false, EHPadBB);

2928

}

2929

2930

// If the value of the invoke is used outside of its defining block, make it

2931

// available as a virtual register.

2932

// We already took care of the exported value for the statepoint instruction

2933

// during call to the LowerStatepoint.

2934

if (!isa<GCStatepointInst>(I)) {

2935

CopyToExportRegsIfNeeded(&I);

2936

}

2937

2938

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

2939

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2940

BranchProbability EHPadBBProb =

2941

BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)

2942

: BranchProbability::getZero();

2943

findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

2944

2945

// Update successor info.

2946

addSuccessorWithProb(InvokeMBB, Return);

2947

for (auto &UnwindDest : UnwindDests) {

2948

UnwindDest.first->setIsEHPad();

2949

addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);

2950

}

2951

InvokeMBB->normalizeSuccProbs();

2952

2953

// Drop into normal successor.

2954

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),

2955

DAG.getBasicBlock(Return)));

2956

}

2957

2958

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {

2959

MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

2960

2961

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2962

// have to do anything here to lower funclet bundles.

2963

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2965, __extension__ __PRETTY_FUNCTION__))

2964

{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2965, __extension__ __PRETTY_FUNCTION__))

2965

"Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2965, __extension__ __PRETTY_FUNCTION__));

2966

2967

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2967, __extension__ __PRETTY_FUNCTION__));

2968

visitInlineAsm(I);

2969

CopyToExportRegsIfNeeded(&I);

2970

2971

// Retrieve successors.

2972

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

2973

2974

// Update successor info.

2975

addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());

2976

for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {

2977

MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];

2978

addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());

2979

Target->setIsInlineAsmBrIndirectTarget();

2980

}

2981

CallBrMBB->normalizeSuccProbs();

2982

2983

// Drop into default successor.

2984

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2985

MVT::Other, getControlRoot(),

2986

DAG.getBasicBlock(Return)));

2987

}

2988

2989

void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {

2990

llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2990);

2991

}

2992

2993

void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {

2994

assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2995, __extension__ __PRETTY_FUNCTION__))

2995

"Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2995, __extension__ __PRETTY_FUNCTION__));

2996

2997

// If there aren't registers to copy the values into (e.g., during SjLj

2998

// exceptions), then don't bother to create these DAG nodes.

2999

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3000

const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();

3001

if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&

3002

TLI.getExceptionSelectorRegister(PersonalityFn) == 0)

3003

return;

3004

3005

// If landingpad's return type is token type, we don't create DAG nodes

3006

// for its exception pointer and selector value. The extraction of exception

3007

// pointer or selector value from token type landingpads is not currently

3008

// supported.

3009

if (LP.getType()->isTokenTy())

3010

return;

3011

3012

SmallVector<EVT, 2> ValueVTs;

3013

SDLoc dl = getCurSDLoc();

3014

ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);

3015

assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3015, __extension__ __PRETTY_FUNCTION__));

3016

3017

// Get the two live-in registers as SDValues. The physregs have already been

3018

// copied into virtual registers.

3019

SDValue Ops[2];

3020

if (FuncInfo.ExceptionPointerVirtReg) {

3021

Ops[0] = DAG.getZExtOrTrunc(

3022

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3023

FuncInfo.ExceptionPointerVirtReg,

3024

TLI.getPointerTy(DAG.getDataLayout())),

3025

dl, ValueVTs[0]);

3026

} else {

3027

Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));

3028

}

3029

Ops[1] = DAG.getZExtOrTrunc(

3030

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3031

FuncInfo.ExceptionSelectorVirtReg,

3032

TLI.getPointerTy(DAG.getDataLayout())),

3033

dl, ValueVTs[1]);

3034

3035

// Merge into one.

3036

SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,

3037

DAG.getVTList(ValueVTs), Ops);

3038

setValue(&LP, Res);

3039

}

3040

3041

void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,

3042

MachineBasicBlock *Last) {

3043

// Update JTCases.

3044

for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)

3045

if (SL->JTCases[i].first.HeaderBB == First)

3046

SL->JTCases[i].first.HeaderBB = Last;

3047

3048

// Update BitTestCases.

3049

for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)

3050

if (SL->BitTestCases[i].Parent == First)

3051

SL->BitTestCases[i].Parent = Last;

3052

}

3053

3054

void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {

3055

MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

3056

3057

// Update machine-CFG edges with unique successors.

3058

SmallSet<BasicBlock*, 32> Done;

3059

for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {

3060

BasicBlock *BB = I.getSuccessor(i);

3061

bool Inserted = Done.insert(BB).second;

3062

if (!Inserted)

3063

continue;

3064

3065

MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];

3066

addSuccessorWithProb(IndirectBrMBB, Succ);

3067

}

3068

IndirectBrMBB->normalizeSuccProbs();

3069

3070

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),

3071

MVT::Other, getControlRoot(),

3072

getValue(I.getAddress())));

3073

}

3074

3075

void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {

3076

if (!DAG.getTarget().Options.TrapUnreachable)

3077

return;

3078

3079

// We may be able to ignore unreachable behind a noreturn call.

3080

if (DAG.getTarget().Options.NoTrapAfterNoreturn) {

3081

const BasicBlock &BB = *I.getParent();

3082

if (&I != &BB.front()) {

3083

BasicBlock::const_iterator PredI =

3084

std::prev(BasicBlock::const_iterator(&I));

3085

if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {

3086

if (Call->doesNotReturn())

3087

return;

3088

}

3089

}

3090

}

3091

3092

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

3093

}

3094

3095

void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {

3096

SDNodeFlags Flags;

3097

3098

SDValue Op = getValue(I.getOperand(0));

3099

SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),

3100

Op, Flags);

3101

setValue(&I, UnNodeValue);

3102

}

3103

3104

void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {

3105

SDNodeFlags Flags;

3106

if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {

3107

Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());

3108

Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());

3109

}

3110

if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))

3111

Flags.setExact(ExactOp->isExact());

3112

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3113

Flags.copyFMF(*FPOp);

3114

3115

SDValue Op1 = getValue(I.getOperand(0));

3116

SDValue Op2 = getValue(I.getOperand(1));

3117

SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),

3118

Op1, Op2, Flags);

3119

setValue(&I, BinNodeValue);

3120

}

3121

3122

void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {

3123

SDValue Op1 = getValue(I.getOperand(0));

3124

SDValue Op2 = getValue(I.getOperand(1));

3125

3126

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(

3127

Op1.getValueType(), DAG.getDataLayout());

3128

3129

// Coerce the shift amount to the right type if we can.

3130

if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {

3131

unsigned ShiftSize = ShiftTy.getSizeInBits();

3132

unsigned Op2Size = Op2.getValueSizeInBits();

3133

SDLoc DL = getCurSDLoc();

3134

3135

// If the operand is smaller than the shift count type, promote it.

3136

if (ShiftSize > Op2Size)

3137

Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);

3138

3139

// If the operand is larger than the shift count type but the shift

3140

// count type has enough bits to represent any shift value, truncate

3141

// it now. This is a common case and it exposes the truncate to

3142

// optimization early.

3143

else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))

3144

Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);

3145

// Otherwise we'll need to temporarily settle for some other convenient

3146

// type. Type legalization will make adjustments once the shiftee is split.

3147

else

3148

Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);

3149

}

3150

3151

bool nuw = false;

3152

bool nsw = false;

3153

bool exact = false;

3154

3155

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

3156

3157

if (const OverflowingBinaryOperator *OFBinOp =

3158

dyn_cast<const OverflowingBinaryOperator>(&I)) {

3159

nuw = OFBinOp->hasNoUnsignedWrap();

3160

nsw = OFBinOp->hasNoSignedWrap();

3161

}

3162

if (const PossiblyExactOperator *ExactOp =

3163

dyn_cast<const PossiblyExactOperator>(&I))

3164

exact = ExactOp->isExact();

3165

}

3166

SDNodeFlags Flags;

3167

Flags.setExact(exact);

3168

Flags.setNoSignedWrap(nsw);

3169

Flags.setNoUnsignedWrap(nuw);

3170

SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,

3171

Flags);

3172

setValue(&I, Res);

3173

}

3174

3175

void SelectionDAGBuilder::visitSDiv(const User &I) {

3176

SDValue Op1 = getValue(I.getOperand(0));

3177

SDValue Op2 = getValue(I.getOperand(1));

3178

3179

SDNodeFlags Flags;

3180

Flags.setExact(isa<PossiblyExactOperator>(&I) &&

3181

cast<PossiblyExactOperator>(&I)->isExact());

3182

setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,

3183

Op2, Flags));

3184

}

3185

3186

void SelectionDAGBuilder::visitICmp(const User &I) {

3187

ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;

3188

if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))

3189

predicate = IC->getPredicate();

3190

else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))

3191

predicate = ICmpInst::Predicate(IC->getPredicate());

3192

SDValue Op1 = getValue(I.getOperand(0));

3193

SDValue Op2 = getValue(I.getOperand(1));

3194

ISD::CondCode Opcode = getICmpCondCode(predicate);

3195

3196

auto &TLI = DAG.getTargetLoweringInfo();

3197

EVT MemVT =

3198

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3199

3200

// If a pointer's DAG type is larger than its memory type then the DAG values

3201

// are zero-extended. This breaks signed comparisons so truncate back to the

3202

// underlying type before doing the compare.

3203

if (Op1.getValueType() != MemVT) {

3204

Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);

3205

Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);

3206

}

3207

3208

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3209

I.getType());

3210

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));

3211

}

3212

3213

void SelectionDAGBuilder::visitFCmp(const User &I) {

3214

FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;

3215

if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))

3216

predicate = FC->getPredicate();

3217

else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))

3218

predicate = FCmpInst::Predicate(FC->getPredicate());

3219

SDValue Op1 = getValue(I.getOperand(0));

3220

SDValue Op2 = getValue(I.getOperand(1));

3221

3222

ISD::CondCode Condition = getFCmpCondCode(predicate);

3223

auto *FPMO = cast<FPMathOperator>(&I);

3224

if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)

3225

Condition = getFCmpCodeWithoutNaN(Condition);

3226

3227

SDNodeFlags Flags;

3228

Flags.copyFMF(*FPMO);

3229

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

3230

3231

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3232

I.getType());

3233

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));

3234

}

3235

3236

// Check if the condition of the select has one use or two users that are both

3237

// selects with the same condition.

3238

static bool hasOnlySelectUsers(const Value *Cond) {

3239

return llvm::all_of(Cond->users(), [](const Value *V) {

3240

return isa<SelectInst>(V);

3241

});

3242

}

3243

3244

void SelectionDAGBuilder::visitSelect(const User &I) {

3245

SmallVector<EVT, 4> ValueVTs;

3246

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),

3247

ValueVTs);

3248

unsigned NumValues = ValueVTs.size();

3249

if (NumValues == 0) return;

3250

3251

SmallVector<SDValue, 4> Values(NumValues);

3252

SDValue Cond = getValue(I.getOperand(0));

3253

SDValue LHSVal = getValue(I.getOperand(1));

3254

SDValue RHSVal = getValue(I.getOperand(2));

3255

SmallVector<SDValue, 1> BaseOps(1, Cond);

3256

ISD::NodeType OpCode =

3257

Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

3258

3259

bool IsUnaryAbs = false;

3260

bool Negate = false;

3261

3262

SDNodeFlags Flags;

3263

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3264

Flags.copyFMF(*FPOp);

3265

3266

// Min/max matching is only viable if all output VTs are the same.

3267

if (is_splat(ValueVTs)) {

3268

EVT VT = ValueVTs[0];

3269

LLVMContext &Ctx = *DAG.getContext();

3270

auto &TLI = DAG.getTargetLoweringInfo();

3271

3272

// We care about the legality of the operation after it has been type

3273

// legalized.

3274

while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)

3275

VT = TLI.getTypeToTransformTo(Ctx, VT);

3276

3277

// If the vselect is legal, assume we want to leave this as a vector setcc +

3278

// vselect. Otherwise, if this is going to be scalarized, we want to see if

3279

// min/max is legal on the scalar type.

3280

bool UseScalarMinMax = VT.isVector() &&

3281

!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

3282

3283

Value *LHS, *RHS;

3284

auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);

3285

ISD::NodeType Opc = ISD::DELETED_NODE;

3286

switch (SPR.Flavor) {

3287

case SPF_UMAX: Opc = ISD::UMAX; break;

3288

case SPF_UMIN: Opc = ISD::UMIN; break;

3289

case SPF_SMAX: Opc = ISD::SMAX; break;

3290

case SPF_SMIN: Opc = ISD::SMIN; break;

3291

case SPF_FMINNUM:

3292

switch (SPR.NaNBehavior) {

3293

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3293);

3294

case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;

3295

case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;

3296

case SPNB_RETURNS_ANY: {

3297

if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))

3298

Opc = ISD::FMINNUM;

3299

else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))

3300

Opc = ISD::FMINIMUM;

3301

else if (UseScalarMinMax)

3302

Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?

3303

ISD::FMINNUM : ISD::FMINIMUM;

3304

break;

3305

}

3306

}

3307

break;

3308

case SPF_FMAXNUM:

3309

switch (SPR.NaNBehavior) {

3310

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3310);

3311

case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;

3312

case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;

3313

case SPNB_RETURNS_ANY:

3314

3315

if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))

3316

Opc = ISD::FMAXNUM;

3317

else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))

3318

Opc = ISD::FMAXIMUM;

3319

else if (UseScalarMinMax)

3320

Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?

3321

ISD::FMAXNUM : ISD::FMAXIMUM;

3322

break;

3323

}

3324

break;

3325

case SPF_NABS:

3326

Negate = true;

3327

LLVM_FALLTHROUGH[[gnu::fallthrough]];

3328

case SPF_ABS:

3329

IsUnaryAbs = true;

3330

Opc = ISD::ABS;

3331

break;

3332

default: break;

3333

}

3334

3335

if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&

3336

(TLI.isOperationLegalOrCustom(Opc, VT) ||

3337

(UseScalarMinMax &&

3338

TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&

3339

// If the underlying comparison instruction is used by any other

3340

// instruction, the consumed instructions won't be destroyed, so it is

3341

// not profitable to convert to a min/max.

3342

hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {

3343

OpCode = Opc;

3344

LHSVal = getValue(LHS);

3345

RHSVal = getValue(RHS);

3346

BaseOps.clear();

3347

}

3348

3349

if (IsUnaryAbs) {

3350

OpCode = Opc;

3351

LHSVal = getValue(LHS);

3352

BaseOps.clear();

3353

}

3354

}

3355

3356

if (IsUnaryAbs) {

3357

for (unsigned i = 0; i != NumValues; ++i) {

3358

SDLoc dl = getCurSDLoc();

3359

EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);

3360

Values[i] =

3361

DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));

3362

if (Negate)

3363

Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),

3364

Values[i]);

3365

}

3366

} else {

3367

for (unsigned i = 0; i != NumValues; ++i) {

3368

SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());

3369

Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3370

Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));

3371

Values[i] = DAG.getNode(

3372

OpCode, getCurSDLoc(),

3373

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);

3374

}

3375

}

3376

3377

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3378

DAG.getVTList(ValueVTs), Values));

3379

}

3380

3381

void SelectionDAGBuilder::visitTrunc(const User &I) {

3382

// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).

3383

SDValue N = getValue(I.getOperand(0));

3384

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3385

I.getType());

3386

setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));

3387

}

3388

3389

void SelectionDAGBuilder::visitZExt(const User &I) {

3390

// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3391

// ZExt also can't be a cast to bool for same reason. So, nothing much to do

3392

SDValue N = getValue(I.getOperand(0));

3393

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3394

I.getType());

3395

setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));

3396

}

3397

3398

void SelectionDAGBuilder::visitSExt(const User &I) {

3399

// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3400

// SExt also can't be a cast to bool for same reason. So, nothing much to do

3401

SDValue N = getValue(I.getOperand(0));

3402

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3403

I.getType());

3404

setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));

3405

}

3406

3407

void SelectionDAGBuilder::visitFPTrunc(const User &I) {

3408

// FPTrunc is never a no-op cast, no need to check

3409

SDValue N = getValue(I.getOperand(0));

3410

SDLoc dl = getCurSDLoc();

3411

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3412

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3413

setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,

3414

DAG.getTargetConstant(

3415

0, dl, TLI.getPointerTy(DAG.getDataLayout()))));

3416

}

3417

3418

void SelectionDAGBuilder::visitFPExt(const User &I) {

3419

// FPExt is never a no-op cast, no need to check

3420

SDValue N = getValue(I.getOperand(0));

3421

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3422

I.getType());

3423

setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));

3424

}

3425

3426

void SelectionDAGBuilder::visitFPToUI(const User &I) {

3427

// FPToUI is never a no-op cast, no need to check

3428

SDValue N = getValue(I.getOperand(0));

3429

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3430

I.getType());

3431

setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));

3432

}

3433

3434

void SelectionDAGBuilder::visitFPToSI(const User &I) {

3435

// FPToSI is never a no-op cast, no need to check

3436

SDValue N = getValue(I.getOperand(0));

3437

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3438

I.getType());

3439

setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));

3440

}

3441

3442

void SelectionDAGBuilder::visitUIToFP(const User &I) {

3443

// UIToFP is never a no-op cast, no need to check

3444

SDValue N = getValue(I.getOperand(0));

3445

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3446

I.getType());

3447

setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));

3448

}

3449

3450

void SelectionDAGBuilder::visitSIToFP(const User &I) {

3451

// SIToFP is never a no-op cast, no need to check

3452

SDValue N = getValue(I.getOperand(0));

3453

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3454

I.getType());

3455

setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));

3456

}

3457

3458

void SelectionDAGBuilder::visitPtrToInt(const User &I) {

3459

// What to do depends on the size of the integer and the size of the pointer.

3460

// We can either truncate, zero extend, or no-op, accordingly.

3461

SDValue N = getValue(I.getOperand(0));

3462

auto &TLI = DAG.getTargetLoweringInfo();

3463

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3464

I.getType());

3465

EVT PtrMemVT =

3466

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3467

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3468

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);

3469

setValue(&I, N);

3470

}

3471

3472

void SelectionDAGBuilder::visitIntToPtr(const User &I) {

3473

// What to do depends on the size of the integer and the size of the pointer.

3474

// We can either truncate, zero extend, or no-op, accordingly.

3475

SDValue N = getValue(I.getOperand(0));

3476

auto &TLI = DAG.getTargetLoweringInfo();

3477

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3478

EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

3479

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3480

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);

3481

setValue(&I, N);

3482

}

3483

3484

void SelectionDAGBuilder::visitBitCast(const User &I) {

3485

SDValue N = getValue(I.getOperand(0));

3486

SDLoc dl = getCurSDLoc();

3487

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3488

I.getType());

3489

3490

// BitCast assures us that source and destination are the same size so this is

3491

// either a BITCAST or a no-op.

3492

if (DestVT != N.getValueType())

3493

setValue(&I, DAG.getNode(ISD::BITCAST, dl,

3494

DestVT, N)); // convert types.

3495

// Check if the original LLVM IR Operand was a ConstantInt, because getValue()

3496

// might fold any kind of constant expression to an integer constant and that

3497

// is not what we are looking for. Only recognize a bitcast of a genuine

3498

// constant integer as an opaque constant.

3499

else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))

3500

setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,

3501

/*isOpaque*/true));

3502

else

3503

setValue(&I, N); // noop cast.

3504

}

3505

3506

void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {

3507

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3508

const Value *SV = I.getOperand(0);

3509

SDValue N = getValue(SV);

3510

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3511

3512

unsigned SrcAS = SV->getType()->getPointerAddressSpace();

3513

unsigned DestAS = I.getType()->getPointerAddressSpace();

3514

3515

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))

3516

N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

3517

3518

setValue(&I, N);

3519

}

3520

3521

void SelectionDAGBuilder::visitInsertElement(const User &I) {

3522

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3523

SDValue InVec = getValue(I.getOperand(0));

3524

SDValue InVal = getValue(I.getOperand(1));

3525

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),

3526

TLI.getVectorIdxTy(DAG.getDataLayout()));

3527

setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),

3528

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3529

InVec, InVal, InIdx));

3530

}

3531

3532

void SelectionDAGBuilder::visitExtractElement(const User &I) {

3533

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3534

SDValue InVec = getValue(I.getOperand(0));

3535

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),

3536

TLI.getVectorIdxTy(DAG.getDataLayout()));

3537

setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),

3538

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3539

InVec, InIdx));

3540

}

3541

3542

void SelectionDAGBuilder::visitShuffleVector(const User &I) {

3543

SDValue Src1 = getValue(I.getOperand(0));

3544

SDValue Src2 = getValue(I.getOperand(1));

3545

ArrayRef<int> Mask;

3546

if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))

3547

Mask = SVI->getShuffleMask();

3548

else

3549

Mask = cast<ConstantExpr>(I).getShuffleMask();

3550

SDLoc DL = getCurSDLoc();

3551

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3552

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3553

EVT SrcVT = Src1.getValueType();

3554

3555

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&

3556

VT.isScalableVector()) {

3557

// Canonical splat form of first element of first input vector.

3558

SDValue FirstElt =

3559

DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,

3560

DAG.getVectorIdxConstant(0, DL));

3561

setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));

3562

return;

3563

}

3564

3565

// For now, we only handle splats for scalable vectors.

3566

// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation

3567

// for targets that support a SPLAT_VECTOR for non-scalable vector types.

3568

assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3568, __extension__ __PRETTY_FUNCTION__));

3569

3570

unsigned SrcNumElts = SrcVT.getVectorNumElements();

3571

unsigned MaskNumElts = Mask.size();

3572

3573

if (SrcNumElts == MaskNumElts) {

3574

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));

3575

return;

3576

}

3577

3578

// Normalize the shuffle vector since mask and vector length don't match.

3579

if (SrcNumElts < MaskNumElts) {

3580

// Mask is longer than the source vectors. We can use concatenate vector to

3581

// make the mask and vectors lengths match.

3582

3583

if (MaskNumElts % SrcNumElts == 0) {

3584

// Mask length is a multiple of the source vector length.

3585

// Check if the shuffle is some kind of concatenation of the input

3586

// vectors.

3587

unsigned NumConcat = MaskNumElts / SrcNumElts;

3588

bool IsConcat = true;

3589

SmallVector<int, 8> ConcatSrcs(NumConcat, -1);

3590

for (unsigned i = 0; i != MaskNumElts; ++i) {

3591

int Idx = Mask[i];

3592

if (Idx < 0)

3593

continue;

3594

// Ensure the indices in each SrcVT sized piece are sequential and that

3595

// the same source is used for the whole piece.

3596

if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

3597

(ConcatSrcs[i / SrcNumElts] >= 0 &&

3598

ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {

3599

IsConcat = false;

3600

break;

3601

}

3602

// Remember which source this index came from.

3603

ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

3604

}

3605

3606

// The shuffle is concatenating multiple vectors together. Just emit

3607

// a CONCAT_VECTORS operation.

3608

if (IsConcat) {

3609

SmallVector<SDValue, 8> ConcatOps;

3610

for (auto Src : ConcatSrcs) {

3611

if (Src < 0)

3612

ConcatOps.push_back(DAG.getUNDEF(SrcVT));

3613

else if (Src == 0)

3614

ConcatOps.push_back(Src1);

3615

else

3616

ConcatOps.push_back(Src2);

3617

}

3618

setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));

3619

return;

3620

}

3621

}

3622

3623

unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);

3624

unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;

3625

EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),

3626

PaddedMaskNumElts);

3627

3628

// Pad both vectors with undefs to make them the same length as the mask.

3629

SDValue UndefVal = DAG.getUNDEF(SrcVT);

3630

3631

SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);

3632

SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);

3633

MOps1[0] = Src1;

3634

MOps2[0] = Src2;

3635

3636

Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);

3637

Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

3638

3639

// Readjust mask for new input vector length.

3640

SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);

3641

for (unsigned i = 0; i != MaskNumElts; ++i) {

3642

int Idx = Mask[i];

3643

if (Idx >= (int)SrcNumElts)

3644

Idx -= SrcNumElts - PaddedMaskNumElts;

3645

MappedOps[i] = Idx;

3646

}

3647

3648

SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

3649

3650

// If the concatenated vector was padded, extract a subvector with the

3651

// correct number of elements.

3652

if (MaskNumElts != PaddedMaskNumElts)

3653

Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,

3654

DAG.getVectorIdxConstant(0, DL));

3655

3656

setValue(&I, Result);

3657

return;

3658

}

3659

3660

if (SrcNumElts > MaskNumElts) {

3661

// Analyze the access pattern of the vector to see if we can extract

3662

// two subvectors and do the shuffle.

3663

int StartIdx[2] = { -1, -1 }; // StartIdx to extract from

3664

bool CanExtract = true;

3665

for (int Idx : Mask) {

3666

unsigned Input = 0;

3667

if (Idx < 0)

3668

continue;

3669

3670

if (Idx >= (int)SrcNumElts) {

3671

Input = 1;

3672

Idx -= SrcNumElts;

3673

}

3674

3675

// If all the indices come from the same MaskNumElts sized portion of

3676

// the sources we can use extract. Also make sure the extract wouldn't

3677

// extract past the end of the source.

3678

int NewStartIdx = alignDown(Idx, MaskNumElts);

3679

if (NewStartIdx + MaskNumElts > SrcNumElts ||

3680

(StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))

3681

CanExtract = false;

3682

// Make sure we always update StartIdx as we use it to track if all

3683

// elements are undef.

3684

StartIdx[Input] = NewStartIdx;

3685

}

3686

3687

if (StartIdx[0] < 0 && StartIdx[1] < 0) {

3688

setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.

3689

return;

3690

}

3691

if (CanExtract) {

3692

// Extract appropriate subvector and generate a vector shuffle

3693

for (unsigned Input = 0; Input < 2; ++Input) {

3694

SDValue &Src = Input == 0 ? Src1 : Src2;

3695

if (StartIdx[Input] < 0)

3696

Src = DAG.getUNDEF(VT);

3697

else {

3698

Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,

3699

DAG.getVectorIdxConstant(StartIdx[Input], DL));

3700

}

3701

}

3702

3703

// Calculate new mask.

3704

SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());

3705

for (int &Idx : MappedOps) {

3706

if (Idx >= (int)SrcNumElts)

3707

Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;

3708

else if (Idx >= 0)

3709

Idx -= StartIdx[0];

3710

}

3711

3712

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));

3713

return;

3714

}

3715

}

3716

3717

// We can't use either concat vectors or extract subvectors so fall back to

3718

// replacing the shuffle with extract and build vector.

3719

// to insert and build vector.

3720

EVT EltVT = VT.getVectorElementType();

3721

SmallVector<SDValue,8> Ops;

3722

for (int Idx : Mask) {

3723

SDValue Res;

3724

3725

if (Idx < 0) {

3726

Res = DAG.getUNDEF(EltVT);

3727

} else {

3728

SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;

3729

if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

3730

3731

Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,

3732

DAG.getVectorIdxConstant(Idx, DL));

3733

}

3734

3735

Ops.push_back(Res);

3736

}

3737

3738

setValue(&I, DAG.getBuildVector(VT, DL, Ops));

3739

}

3740

3741

void SelectionDAGBuilder::visitInsertValue(const User &I) {

3742

ArrayRef<unsigned> Indices;

3743

if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))

3744

Indices = IV->getIndices();

3745

else

3746

Indices = cast<ConstantExpr>(&I)->getIndices();

3747

3748

const Value *Op0 = I.getOperand(0);

3749

const Value *Op1 = I.getOperand(1);

3750

Type *AggTy = I.getType();

3751

Type *ValTy = Op1->getType();

3752

bool IntoUndef = isa<UndefValue>(Op0);

3753

bool FromUndef = isa<UndefValue>(Op1);

3754

3755

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3756

3757

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3758

SmallVector<EVT, 4> AggValueVTs;

3759

ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);

3760

SmallVector<EVT, 4> ValValueVTs;

3761

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3762

3763

unsigned NumAggValues = AggValueVTs.size();

3764

unsigned NumValValues = ValValueVTs.size();

3765

SmallVector<SDValue, 4> Values(NumAggValues);

3766

3767

// Ignore an insertvalue that produces an empty object

3768

if (!NumAggValues) {

3769

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3770

return;

3771

}

3772

3773

SDValue Agg = getValue(Op0);

3774

unsigned i = 0;

3775

// Copy the beginning value(s) from the original aggregate.

3776

for (; i != LinearIndex; ++i)

3777

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3778

SDValue(Agg.getNode(), Agg.getResNo() + i);

3779

// Copy values from the inserted value(s).

3780

if (NumValValues) {

3781

SDValue Val = getValue(Op1);

3782

for (; i != LinearIndex + NumValValues; ++i)

3783

Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3784

SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);

3785

}

3786

// Copy remaining value(s) from the original aggregate.

3787

for (; i != NumAggValues; ++i)

3788

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3789

SDValue(Agg.getNode(), Agg.getResNo() + i);

3790

3791

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3792

DAG.getVTList(AggValueVTs), Values));

3793

}

3794

3795

void SelectionDAGBuilder::visitExtractValue(const User &I) {

3796

ArrayRef<unsigned> Indices;

3797

if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))

3798

Indices = EV->getIndices();

3799

else

3800

Indices = cast<ConstantExpr>(&I)->getIndices();

3801

3802

const Value *Op0 = I.getOperand(0);

3803

Type *AggTy = Op0->getType();

3804

Type *ValTy = I.getType();

3805

bool OutOfUndef = isa<UndefValue>(Op0);

3806

3807

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3808

3809

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3810

SmallVector<EVT, 4> ValValueVTs;

3811

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3812

3813

unsigned NumValValues = ValValueVTs.size();

3814

3815

// Ignore a extractvalue that produces an empty object

3816

if (!NumValValues) {

3817

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3818

return;

3819

}

3820

3821

SmallVector<SDValue, 4> Values(NumValValues);

3822

3823

SDValue Agg = getValue(Op0);

3824

// Copy out the selected value(s).

3825

for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)

3826

Values[i - LinearIndex] =

3827

OutOfUndef ?

3828

DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :

3829

SDValue(Agg.getNode(), Agg.getResNo() + i);

3830

3831

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3832

DAG.getVTList(ValValueVTs), Values));

3833

}

3834

3835

void SelectionDAGBuilder::visitGetElementPtr(const User &I) {

3836

Value *Op0 = I.getOperand(0);

3837

// Note that the pointer operand may be a vector of pointers. Take the scalar

3838

// element which holds a pointer.

3839

unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();

3840

SDValue N = getValue(Op0);

3841

SDLoc dl = getCurSDLoc();

3842

auto &TLI = DAG.getTargetLoweringInfo();

3843

3844

// Normalize Vector GEP - all scalar operands should be converted to the

3845

// splat vector.

3846

bool IsVectorGEP = I.getType()->isVectorTy();

3847

ElementCount VectorElementCount =

3848

IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()

3849

: ElementCount::getFixed(0);

3850

3851

if (IsVectorGEP && !N.getValueType().isVector()) {

3852

LLVMContext &Context = *DAG.getContext();

3853

EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);

3854

if (VectorElementCount.isScalable())

3855

N = DAG.getSplatVector(VT, dl, N);

3856

else

3857

N = DAG.getSplatBuildVector(VT, dl, N);

3858

}

3859

3860

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);

3861

GTI != E; ++GTI) {

3862

const Value *Idx = GTI.getOperand();

3863

if (StructType *StTy = GTI.getStructTypeOrNull()) {

3864

unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();

3865

if (Field) {

3866

// N = N + Offset

3867

uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);

3868

3869

// In an inbounds GEP with an offset that is nonnegative even when

3870

// interpreted as signed, assume there is no unsigned overflow.

3871

SDNodeFlags Flags;

3872

if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())

3873

Flags.setNoUnsignedWrap(true);

3874

3875

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,

3876

DAG.getConstant(Offset, dl, N.getValueType()), Flags);

3877

}

3878

} else {

3879

// IdxSize is the width of the arithmetic according to IR semantics.

3880

// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth

3881

// (and fix up the result later).

3882

unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);

3883

MVT IdxTy = MVT::getIntegerVT(IdxSize);

3884

TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());

3885

// We intentionally mask away the high bits here; ElementSize may not

3886

// fit in IdxTy.

3887

APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());

3888

bool ElementScalable = ElementSize.isScalable();

3889

3890

// If this is a scalar constant or a splat vector of constants,

3891

// handle it quickly.

3892

const auto *C = dyn_cast<Constant>(Idx);

3893

if (C && isa<VectorType>(C->getType()))

3894

C = C->getSplatValue();

3895

3896

const auto *CI = dyn_cast_or_null<ConstantInt>(C);

3897

if (CI && CI->isZero())

3898

continue;

3899

if (CI && !ElementScalable) {

3900

APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);

3901

LLVMContext &Context = *DAG.getContext();

3902

SDValue OffsVal;

3903

if (IsVectorGEP)

3904

OffsVal = DAG.getConstant(

3905

Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));

3906

else

3907

OffsVal = DAG.getConstant(Offs, dl, IdxTy);

3908

3909

// In an inbounds GEP with an offset that is nonnegative even when

3910

// interpreted as signed, assume there is no unsigned overflow.

3911

SDNodeFlags Flags;

3912

if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())

3913

Flags.setNoUnsignedWrap(true);

3914

3915

OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

3916

3917

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);

3918

continue;

3919

}

3920

3921

// N = N + Idx * ElementMul;

3922

SDValue IdxN = getValue(Idx);

3923

3924

if (!IdxN.getValueType().isVector() && IsVectorGEP) {

3925

EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),

3926

VectorElementCount);

3927

if (VectorElementCount.isScalable())

3928

IdxN = DAG.getSplatVector(VT, dl, IdxN);

3929

else

3930

IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);

3931

}

3932

3933

// If the index is smaller or larger than intptr_t, truncate or extend

3934

// it.

3935

IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

3936

3937

if (ElementScalable) {

3938

EVT VScaleTy = N.getValueType().getScalarType();

3939

SDValue VScale = DAG.getNode(

3940

ISD::VSCALE, dl, VScaleTy,

3941

DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));

3942

if (IsVectorGEP)

3943

VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);

3944

IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);

3945

} else {

3946

// If this is a multiply by a power of two, turn it into a shl

3947

// immediately. This is a very common case.

3948

if (ElementMul != 1) {

3949

if (ElementMul.isPowerOf2()) {

3950

unsigned Amt = ElementMul.logBase2();

3951

IdxN = DAG.getNode(ISD::SHL, dl,

3952

N.getValueType(), IdxN,

3953

DAG.getConstant(Amt, dl, IdxN.getValueType()));

3954

} else {

3955

SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,

3956

IdxN.getValueType());

3957

IdxN = DAG.getNode(ISD::MUL, dl,

3958

N.getValueType(), IdxN, Scale);

3959

}

3960

}

3961

}

3962

3963

N = DAG.getNode(ISD::ADD, dl,

3964

N.getValueType(), N, IdxN);

3965

}

3966

}

3967

3968

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);

3969

MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);

3970

if (IsVectorGEP) {

3971

PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);

3972

PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);

3973

}

3974

3975

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())

3976

N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

3977

3978

setValue(&I, N);

3979

}

3980

3981

void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {

3982

// If this is a fixed sized alloca in the entry block of the function,

3983

// allocate it statically on the stack.

3984

if (FuncInfo.StaticAllocaMap.count(&I))

3985

return; // getValue will auto-populate this.

3986

3987

SDLoc dl = getCurSDLoc();

3988

Type *Ty = I.getAllocatedType();

3989

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3990

auto &DL = DAG.getDataLayout();

3991

uint64_t TySize = DL.getTypeAllocSize(Ty);

3992

MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

3993

3994

SDValue AllocSize = getValue(I.getArraySize());

3995

3996

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());

3997

if (AllocSize.getValueType() != IntPtr)

3998

AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

3999

4000

AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,

4001

AllocSize,

4002

DAG.getConstant(TySize, dl, IntPtr));

4003

4004

// Handle alignment. If the requested alignment is less than or equal to

4005

// the stack alignment, ignore it. If the size is greater than or equal to

4006

// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.

4007

Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();

4008

if (*Alignment <= StackAlign)

4009

Alignment = None;

4010

4011

const uint64_t StackAlignMask = StackAlign.value() - 1U;

4012

// Round the size of the allocation up to the stack alignment size

4013

// by add SA-1 to the size. This doesn't overflow because we're computing

4014

// an address inside an alloca.

4015

SDNodeFlags Flags;

4016

Flags.setNoUnsignedWrap(true);

4017

AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,

4018

DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

4019

4020

// Mask out the low bits for alignment purposes.

4021

AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,

4022

DAG.getConstant(~StackAlignMask, dl, IntPtr));

4023

4024

SDValue Ops[] = {

4025

getRoot(), AllocSize,

4026

DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};

4027

SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);

4028

SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);

4029

setValue(&I, DSA);

4030

DAG.setRoot(DSA.getValue(1));

4031

4032

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4032, __extension__ __PRETTY_FUNCTION__));

4033

}

4034

4035

void SelectionDAGBuilder::visitLoad(const LoadInst &I) {

4036

if (I.isAtomic())

4037

return visitAtomicLoad(I);

4038

4039

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4040

const Value *SV = I.getOperand(0);

4041

if (TLI.supportSwiftError()) {

4042

// Swifterror values can come from either a function parameter with

4043

// swifterror attribute or an alloca with swifterror attribute.

4044

if (const Argument *Arg = dyn_cast<Argument>(SV)) {

4045

if (Arg->hasSwiftErrorAttr())

4046

return visitLoadFromSwiftError(I);

4047

}

4048

4049

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {

4050

if (Alloca->isSwiftError())

4051

return visitLoadFromSwiftError(I);

4052

}

4053

}

4054

4055

SDValue Ptr = getValue(SV);

4056

4057

Type *Ty = I.getType();

4058

Align Alignment = I.getAlign();

4059

4060

AAMDNodes AAInfo;

4061

I.getAAMetadata(AAInfo);

4062

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4063

4064

SmallVector<EVT, 4> ValueVTs, MemVTs;

4065

SmallVector<uint64_t, 4> Offsets;

4066

ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);

4067

unsigned NumValues = ValueVTs.size();

4068

if (NumValues == 0)

4069

return;

4070

4071

bool isVolatile = I.isVolatile();

4072

4073

SDValue Root;

4074

bool ConstantMemory = false;

4075

if (isVolatile)

4076

// Serialize volatile loads with other side effects.

4077

Root = getRoot();

4078

else if (NumValues > MaxParallelChains)

4079

Root = getMemoryRoot();

4080

else if (AA &&

4081

AA->pointsToConstantMemory(MemoryLocation(

4082

SV,

4083

LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),

4084

AAInfo))) {

4085

// Do not serialize (non-volatile) loads of constant memory with anything.

4086

Root = DAG.getEntryNode();

4087

ConstantMemory = true;

4088

} else {

4089

// Do not serialize non-volatile loads against each other.

4090

Root = DAG.getRoot();

4091

}

4092

4093

SDLoc dl = getCurSDLoc();

4094

4095

if (isVolatile)

4096

Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

4097

4098

// An aggregate load cannot wrap around the address space, so offsets to its

4099

// parts don't wrap either.

4100

SDNodeFlags Flags;

4101

Flags.setNoUnsignedWrap(true);

4102

4103

SmallVector<SDValue, 4> Values(NumValues);

4104

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4105

EVT PtrVT = Ptr.getValueType();

4106

4107

MachineMemOperand::Flags MMOFlags

4108

= TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4109

4110

unsigned ChainI = 0;

4111

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4112

// Serializing loads here may result in excessive register pressure, and

4113

// TokenFactor places arbitrary choke points on the scheduler. SD scheduling

4114

// could recover a bit by hoisting nodes upward in the chain by recognizing

4115

// they are side-effect free or do not alias. The optimizer should really

4116

// avoid this case by converting large object/array copies to llvm.memcpy

4117

// (MaxParallelChains should always remain as failsafe).

4118

if (ChainI == MaxParallelChains) {

4119

assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4119, __extension__ __PRETTY_FUNCTION__));

4120

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4121

makeArrayRef(Chains.data(), ChainI));

4122

Root = Chain;

4123

ChainI = 0;

4124

}

4125

SDValue A = DAG.getNode(ISD::ADD, dl,

4126

PtrVT, Ptr,

4127

DAG.getConstant(Offsets[i], dl, PtrVT),

4128

Flags);

4129

4130

SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,

4131

MachinePointerInfo(SV, Offsets[i]), Alignment,

4132

MMOFlags, AAInfo, Ranges);

4133

Chains[ChainI] = L.getValue(1);

4134

4135

if (MemVTs[i] != ValueVTs[i])

4136

L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

4137

4138

Values[i] = L;

4139

}

4140

4141

if (!ConstantMemory) {

4142

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4143

makeArrayRef(Chains.data(), ChainI));

4144

if (isVolatile)

4145

DAG.setRoot(Chain);

4146

else

4147

PendingLoads.push_back(Chain);

4148

}

4149

4150

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,

4151

DAG.getVTList(ValueVTs), Values));

4152

}

4153

4154

void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {

4155

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4156, __extension__ __PRETTY_FUNCTION__))

4156

"call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4156, __extension__ __PRETTY_FUNCTION__));

4157

4158

SmallVector<EVT, 4> ValueVTs;

4159

SmallVector<uint64_t, 4> Offsets;

4160

const Value *SrcV = I.getOperand(0);

4161

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4162

SrcV->getType(), ValueVTs, &Offsets);

4163

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4164, __extension__ __PRETTY_FUNCTION__))

4164

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4164, __extension__ __PRETTY_FUNCTION__));

4165

4166

SDValue Src = getValue(SrcV);

4167

// Create a virtual register, then update the virtual register.

4168

Register VReg =

4169

SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());

4170

// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue

4171

// Chain can be getRoot or getControlRoot.

4172

SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,

4173

SDValue(Src.getNode(), Src.getResNo()));

4174

DAG.setRoot(CopyNode);

4175

}

4176

4177

void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {

4178

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4179, __extension__ __PRETTY_FUNCTION__))

4179

"call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4179, __extension__ __PRETTY_FUNCTION__));

4180

4181

assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4184, __extension__ __PRETTY_FUNCTION__))

4182

!I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4184, __extension__ __PRETTY_FUNCTION__))

4183

!I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4184, __extension__ __PRETTY_FUNCTION__))

4184

"Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4184, __extension__ __PRETTY_FUNCTION__));

4185

4186

const Value *SV = I.getOperand(0);

4187

Type *Ty = I.getType();

4188

AAMDNodes AAInfo;

4189

I.getAAMetadata(AAInfo);

4190

assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__))

4191

(!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__))

4192

!AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__))

4193

SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__))

4194

AAInfo))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__))

4195

"load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4195, __extension__ __PRETTY_FUNCTION__));

4196

4197

SmallVector<EVT, 4> ValueVTs;

4198

SmallVector<uint64_t, 4> Offsets;

4199

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,

4200

ValueVTs, &Offsets);

4201

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4202, __extension__ __PRETTY_FUNCTION__))

4202

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4202, __extension__ __PRETTY_FUNCTION__));

4203

4204

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT

4205

SDValue L = DAG.getCopyFromReg(

4206

getRoot(), getCurSDLoc(),

4207

SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

4208

4209

setValue(&I, L);

4210

}

4211

4212

void SelectionDAGBuilder::visitStore(const StoreInst &I) {

4213

if (I.isAtomic())

4214

return visitAtomicStore(I);

4215

4216

const Value *SrcV = I.getOperand(0);

4217

const Value *PtrV = I.getOperand(1);

4218

4219

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4220

if (TLI.supportSwiftError()) {

4221

// Swifterror values can come from either a function parameter with

4222

// swifterror attribute or an alloca with swifterror attribute.

4223

if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {

4224

if (Arg->hasSwiftErrorAttr())

4225

return visitStoreToSwiftError(I);

4226

}

4227

4228

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {

4229

if (Alloca->isSwiftError())

4230

return visitStoreToSwiftError(I);

4231

}

4232

}

4233

4234

SmallVector<EVT, 4> ValueVTs, MemVTs;

4235

SmallVector<uint64_t, 4> Offsets;

4236

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4237

SrcV->getType(), ValueVTs, &MemVTs, &Offsets);

4238

unsigned NumValues = ValueVTs.size();

4239

if (NumValues == 0)

4240

return;

4241

4242

// Get the lowered operands. Note that we do this after

4243

// checking if NumResults is zero, because with zero results

4244

// the operands won't have values in the map.

4245

SDValue Src = getValue(SrcV);

4246

SDValue Ptr = getValue(PtrV);

4247

4248

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();

4249

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4250

SDLoc dl = getCurSDLoc();

4251

Align Alignment = I.getAlign();

4252

AAMDNodes AAInfo;

4253

I.getAAMetadata(AAInfo);

4254

4255

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4256

4257

// An aggregate load cannot wrap around the address space, so offsets to its

4258

// parts don't wrap either.

4259

SDNodeFlags Flags;

4260

Flags.setNoUnsignedWrap(true);

4261

4262

unsigned ChainI = 0;

4263

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4264

// See visitLoad comments.

4265

if (ChainI == MaxParallelChains) {

4266

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4267

makeArrayRef(Chains.data(), ChainI));

4268

Root = Chain;

4269

ChainI = 0;

4270

}

4271

SDValue Add =

4272

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);

4273

SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);

4274

if (MemVTs[i] != ValueVTs[i])

4275

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);

4276

SDValue St =

4277

DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),

4278

Alignment, MMOFlags, AAInfo);

4279

Chains[ChainI] = St;

4280

}

4281

4282

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4283

makeArrayRef(Chains.data(), ChainI));

4284

DAG.setRoot(StoreNode);

4285

}

4286

4287

void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,

4288

bool IsCompressing) {

4289

SDLoc sdl = getCurSDLoc();

4290

4291

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4292

MaybeAlign &Alignment) {

4293

// llvm.masked.store.*(Src0, Ptr, alignment, Mask)

4294

Src0 = I.getArgOperand(0);

4295

Ptr = I.getArgOperand(1);

4296

Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();

4297

Mask = I.getArgOperand(3);

4298

};

4299

auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4300

MaybeAlign &Alignment) {

4301

// llvm.masked.compressstore.*(Src0, Ptr, Mask)

4302

Src0 = I.getArgOperand(0);

4303

Ptr = I.getArgOperand(1);

4304

Mask = I.getArgOperand(2);

4305

Alignment = None;

4306

};

4307

4308

Value *PtrOperand, *MaskOperand, *Src0Operand;

4309

MaybeAlign Alignment;

4310

if (IsCompressing)

4311

getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4312

else

4313

getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4314

4315

SDValue Ptr = getValue(PtrOperand);

4316

SDValue Src0 = getValue(Src0Operand);

4317

SDValue Mask = getValue(MaskOperand);

4318

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4319

4320

EVT VT = Src0.getValueType();

4321

if (!Alignment)

4322

Alignment = DAG.getEVTAlign(VT);

4323

4324

AAMDNodes AAInfo;

4325

I.getAAMetadata(AAInfo);

4326

4327

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4328

MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,

4329

// TODO: Make MachineMemOperands aware of scalable

4330

// vectors.

4331

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);

4332

SDValue StoreNode =

4333

DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,

4334

ISD::UNINDEXED, false /* Truncating */, IsCompressing);

4335

DAG.setRoot(StoreNode);

4336

setValue(&I, StoreNode);

4337

}

4338

4339

// Get a uniform base for the Gather/Scatter intrinsic.

4340

// The first argument of the Gather/Scatter intrinsic is a vector of pointers.

4341

// We try to represent it as a base pointer + vector of indices.

4342

// Usually, the vector of pointers comes from a 'getelementptr' instruction.

4343

// The first operand of the GEP may be a single pointer or a vector of pointers

4344

// Example:

4345

// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind

4346

// or

4347

// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind

4348

// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..

4349

//

4350

// When the first GEP operand is a single pointer - it is the uniform base we

4351

// are looking for. If first operand of the GEP is a splat vector - we

4352

// extract the splat value and use it as a uniform base.

4353

// In all other cases the function returns 'false'.

4354

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,

4355

ISD::MemIndexType &IndexType, SDValue &Scale,

4356

SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {

4357

SelectionDAG& DAG = SDB->DAG;

4358

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4359

const DataLayout &DL = DAG.getDataLayout();

4360

4361

assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Uexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4361, __extension__ __PRETTY_FUNCTION__));

4362

4363

// Handle splat constant pointer.

4364

if (auto *C = dyn_cast<Constant>(Ptr)) {

4365

C = C->getSplatValue();

4366

if (!C)

4367

return false;

4368

4369

Base = SDB->getValue(C);

4370

4371

ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();

4372

EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);

4373

Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);

4374

IndexType = ISD::SIGNED_SCALED;

4375

Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4376

return true;

4377

}

4378

4379

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

4380

if (!GEP || GEP->getParent() != CurBB)

4381

return false;

4382

4383

if (GEP->getNumOperands() != 2)

4384

return false;

4385

4386

const Value *BasePtr = GEP->getPointerOperand();

4387

const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

4388

4389

// Make sure the base is scalar and the index is a vector.

4390

if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())

4391

return false;

4392

4393

Base = SDB->getValue(BasePtr);

4394

Index = SDB->getValue(IndexVal);

4395

IndexType = ISD::SIGNED_SCALED;

4396

Scale = DAG.getTargetConstant(

4397

DL.getTypeAllocSize(GEP->getResultElementType()),

4398

SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4399

return true;

4400

}

4401

4402

void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {

4403

SDLoc sdl = getCurSDLoc();

4404

4405

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)

4406

const Value *Ptr = I.getArgOperand(1);

4407

SDValue Src0 = getValue(I.getArgOperand(0));

4408

SDValue Mask = getValue(I.getArgOperand(3));

4409

EVT VT = Src0.getValueType();

4410

Align Alignment = cast<ConstantInt>(I.getArgOperand(2))

4411

->getMaybeAlignValue()

4412

.getValueOr(DAG.getEVTAlign(VT.getScalarType()));

4413

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4414

4415

AAMDNodes AAInfo;

4416

I.getAAMetadata(AAInfo);

4417

4418

SDValue Base;

4419

SDValue Index;

4420

ISD::MemIndexType IndexType;

4421

SDValue Scale;

4422

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4423

I.getParent());

4424

4425

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4426

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4427

MachinePointerInfo(AS), MachineMemOperand::MOStore,

4428

// TODO: Make MachineMemOperands aware of scalable

4429

// vectors.

4430

MemoryLocation::UnknownSize, Alignment, AAInfo);

4431

if (!UniformBase) {

4432

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4433

Index = getValue(Ptr);

4434

IndexType = ISD::SIGNED_UNSCALED;

4435

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4436

}

4437

4438

EVT IdxVT = Index.getValueType();

4439

EVT EltTy = IdxVT.getVectorElementType();

4440

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4441

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4442

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4443

}

4444

4445

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };

4446

SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,

4447

Ops, MMO, IndexType, false);

4448

DAG.setRoot(Scatter);

4449

setValue(&I, Scatter);

4450

}

4451

4452

void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {

4453

SDLoc sdl = getCurSDLoc();

4454

4455

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4456

MaybeAlign &Alignment) {

4457

// @llvm.masked.load.*(Ptr, alignment, Mask, Src0)

4458

Ptr = I.getArgOperand(0);

4459

Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();

4460

Mask = I.getArgOperand(2);

4461

Src0 = I.getArgOperand(3);

4462

};

4463

auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4464

MaybeAlign &Alignment) {

4465

// @llvm.masked.expandload.*(Ptr, Mask, Src0)

4466

Ptr = I.getArgOperand(0);

4467

Alignment = None;

4468

Mask = I.getArgOperand(1);

4469

Src0 = I.getArgOperand(2);

4470

};

4471

4472

Value *PtrOperand, *MaskOperand, *Src0Operand;

4473

MaybeAlign Alignment;

4474

if (IsExpanding)

4475

getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4476

else

4477

getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4478

4479

SDValue Ptr = getValue(PtrOperand);

4480

SDValue Src0 = getValue(Src0Operand);

4481

SDValue Mask = getValue(MaskOperand);

4482

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4483

4484

EVT VT = Src0.getValueType();

4485

if (!Alignment)

4486

Alignment = DAG.getEVTAlign(VT);

4487

4488

AAMDNodes AAInfo;

4489

I.getAAMetadata(AAInfo);

4490

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4491

4492

// Do not serialize masked loads of constant memory with anything.

4493

MemoryLocation ML;

4494

if (VT.isScalableVector())

4495

ML = MemoryLocation::getAfter(PtrOperand);

4496

else

4497

ML = MemoryLocation(PtrOperand, LocationSize::precise(

4498

DAG.getDataLayout().getTypeStoreSize(I.getType())),

4499

AAInfo);

4500

bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

4501

4502

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

4503

4504

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4505

MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,

4506

// TODO: Make MachineMemOperands aware of scalable

4507

// vectors.

4508

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);

4509

4510

SDValue Load =

4511

DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,

4512

ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);

4513

if (AddToChain)

4514

PendingLoads.push_back(Load.getValue(1));

4515

setValue(&I, Load);

4516

}

4517

4518

void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {

4519

SDLoc sdl = getCurSDLoc();

4520

4521

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)

4522

const Value *Ptr = I.getArgOperand(0);

4523

SDValue Src0 = getValue(I.getArgOperand(3));

4524

SDValue Mask = getValue(I.getArgOperand(2));

4525

4526

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4527

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4528

Align Alignment = cast<ConstantInt>(I.getArgOperand(1))

4529

->getMaybeAlignValue()

4530

.getValueOr(DAG.getEVTAlign(VT.getScalarType()));

4531

4532

AAMDNodes AAInfo;

4533

I.getAAMetadata(AAInfo);

4534

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4535

4536

SDValue Root = DAG.getRoot();

4537

SDValue Base;

4538

SDValue Index;

4539

ISD::MemIndexType IndexType;

4540

SDValue Scale;

4541

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4542

I.getParent());

4543

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4544

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4545

MachinePointerInfo(AS), MachineMemOperand::MOLoad,

4546

// TODO: Make MachineMemOperands aware of scalable

4547

// vectors.

4548

MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);

4549

4550

if (!UniformBase) {

4551

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4552

Index = getValue(Ptr);

4553

IndexType = ISD::SIGNED_UNSCALED;

4554

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4555

}

4556

4557

EVT IdxVT = Index.getValueType();

4558

EVT EltTy = IdxVT.getVectorElementType();

4559

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4560

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4561

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4562

}

4563

4564

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };

4565

SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,

4566

Ops, MMO, IndexType, ISD::NON_EXTLOAD);

4567

4568

PendingLoads.push_back(Gather.getValue(1));

4569

setValue(&I, Gather);

4570

}

4571

4572

void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {

4573

SDLoc dl = getCurSDLoc();

4574

AtomicOrdering SuccessOrdering = I.getSuccessOrdering();

4575

AtomicOrdering FailureOrdering = I.getFailureOrdering();

4576

SyncScope::ID SSID = I.getSyncScopeID();

4577

4578

SDValue InChain = getRoot();

4579

4580

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();

4581

SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

4582

4583

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4584

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4585

4586

MachineFunction &MF = DAG.getMachineFunction();

4587

MachineMemOperand *MMO = MF.getMachineMemOperand(

4588

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4589

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,

4590

FailureOrdering);

4591

4592

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,

4593

dl, MemVT, VTs, InChain,

4594

getValue(I.getPointerOperand()),

4595

getValue(I.getCompareOperand()),

4596

getValue(I.getNewValOperand()), MMO);

4597

4598

SDValue OutChain = L.getValue(2);

4599

4600

setValue(&I, L);

4601

DAG.setRoot(OutChain);

4602

}

4603

4604

void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {

4605

SDLoc dl = getCurSDLoc();

4606

ISD::NodeType NT;

4607

switch (I.getOperation()) {

4608

default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4608);

4609

case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;

4610

case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;

4611

case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;

4612

case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;

4613

case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;

4614

case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;

4615

case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;

4616

case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;

4617

case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;

4618

case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;

4619

case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;

4620

case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;

4621

case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;

4622

}

4623

AtomicOrdering Ordering = I.getOrdering();

4624

SyncScope::ID SSID = I.getSyncScopeID();

4625

4626

SDValue InChain = getRoot();

4627

4628

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();

4629

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4630

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4631

4632

MachineFunction &MF = DAG.getMachineFunction();

4633

MachineMemOperand *MMO = MF.getMachineMemOperand(

4634

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4635

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

4636

4637

SDValue L =

4638

DAG.getAtomic(NT, dl, MemVT, InChain,

4639

getValue(I.getPointerOperand()), getValue(I.getValOperand()),

4640

MMO);

4641

4642

SDValue OutChain = L.getValue(1);

4643

4644

setValue(&I, L);

4645

DAG.setRoot(OutChain);

4646

}

4647

4648

void SelectionDAGBuilder::visitFence(const FenceInst &I) {

4649

SDLoc dl = getCurSDLoc();

4650

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4651

SDValue Ops[3];

4652

Ops[0] = getRoot();

4653

Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,

4654

TLI.getFenceOperandTy(DAG.getDataLayout()));

4655

Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,

4656

TLI.getFenceOperandTy(DAG.getDataLayout()));

4657

DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));

4658

}

4659

4660

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {

4661

SDLoc dl = getCurSDLoc();

4662

AtomicOrdering Order = I.getOrdering();

4663

SyncScope::ID SSID = I.getSyncScopeID();

4664

4665

SDValue InChain = getRoot();

4666

4667

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4668

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4669

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

4670

4671

if (!TLI.supportsUnalignedAtomics() &&

4672

I.getAlignment() < MemVT.getSizeInBits() / 8)

4673

report_fatal_error("Cannot generate unaligned atomic load");

4674

4675

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4676

4677

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4678

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4679

I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

4680

4681

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

4682

4683

SDValue Ptr = getValue(I.getPointerOperand());

4684

4685

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {

4686

// TODO: Once this is better exercised by tests, it should be merged with

4687

// the normal path for loads to prevent future divergence.

4688

SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);

4689

if (MemVT != VT)

4690

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4691

4692

setValue(&I, L);

4693

SDValue OutChain = L.getValue(1);

4694

if (!I.isUnordered())

4695

DAG.setRoot(OutChain);

4696

else

4697

PendingLoads.push_back(OutChain);

4698

return;

4699

}

4700

4701

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,

4702

Ptr, MMO);

4703

4704

SDValue OutChain = L.getValue(1);

4705

if (MemVT != VT)

4706

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4707

4708

setValue(&I, L);

4709

DAG.setRoot(OutChain);

4710

}

4711

4712

void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {

4713

SDLoc dl = getCurSDLoc();

4714

4715

AtomicOrdering Ordering = I.getOrdering();

4716

SyncScope::ID SSID = I.getSyncScopeID();

4717

4718

SDValue InChain = getRoot();

4719

4720

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4721

EVT MemVT =

4722

TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

4723

4724

if (I.getAlignment() < MemVT.getSizeInBits() / 8)

4725

report_fatal_error("Cannot generate unaligned atomic store");

4726

4727

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4728

4729

MachineFunction &MF = DAG.getMachineFunction();

4730

MachineMemOperand *MMO = MF.getMachineMemOperand(

4731

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4732

I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

4733

4734

SDValue Val = getValue(I.getValueOperand());

4735

if (Val.getValueType() != MemVT)

4736

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);

4737

SDValue Ptr = getValue(I.getPointerOperand());

4738

4739

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {

4740

// TODO: Once this is better exercised by tests, it should be merged with

4741

// the normal path for stores to prevent future divergence.

4742

SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);

4743

DAG.setRoot(S);

4744

return;

4745

}

4746

SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,

4747

Ptr, Val, MMO);

4748

4749

4750

DAG.setRoot(OutChain);

4751

}

4752

4753

/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC

4754

/// node.

4755

void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,

4756

unsigned Intrinsic) {

4757

// Ignore the callsite's attributes. A specific call site may be marked with

4758

// readnone, but the lowering code will expect the chain based on the

4759

// definition.

4760

const Function *F = I.getCalledFunction();

4761

bool HasChain = !F->doesNotAccessMemory();

4762

bool OnlyLoad = HasChain && F->onlyReadsMemory();

4763

4764

// Build the operand list.

4765

SmallVector<SDValue, 8> Ops;

4766

if (HasChain) { // If this intrinsic has side-effects, chainify it.

4767

if (OnlyLoad) {

4768

// We don't need to serialize loads against other loads.

4769

Ops.push_back(DAG.getRoot());

4770

} else {

4771

Ops.push_back(getRoot());

4772

}

4773

}

4774

4775

// Info is set by getTgtMemInstrinsic

4776

TargetLowering::IntrinsicInfo Info;

4777

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4778

bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,

4779

DAG.getMachineFunction(),

4780

Intrinsic);

4781

4782

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.

4783

if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||

4784

Info.opc == ISD::INTRINSIC_W_CHAIN)

4785

Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),

4786

TLI.getPointerTy(DAG.getDataLayout())));

4787

4788

// Add all operands of the call to the operand list.

4789

for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {

4790

const Value *Arg = I.getArgOperand(i);

4791

if (!I.paramHasAttr(i, Attribute::ImmArg)) {

4792

Ops.push_back(getValue(Arg));

4793

continue;

4794

}

4795

4796

// Use TargetConstant instead of a regular constant for immarg.

4797

EVT VT = TLI.getValueType(*DL, Arg->getType(), true);

4798

if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {

4799

assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4800, __extension__ __PRETTY_FUNCTION__))

4800

"large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4800, __extension__ __PRETTY_FUNCTION__));

4801

Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));

4802

} else {

4803

Ops.push_back(

4804

DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));

4805

}

4806

}

4807

4808

SmallVector<EVT, 4> ValueVTs;

4809

ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

4810

4811

if (HasChain)

4812

ValueVTs.push_back(MVT::Other);

4813

4814

SDVTList VTs = DAG.getVTList(ValueVTs);

4815

4816

// Propagate fast-math-flags from IR to node(s).

4817

SDNodeFlags Flags;

4818

if (auto *FPMO = dyn_cast<FPMathOperator>(&I))

4819

Flags.copyFMF(*FPMO);

4820

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

4821

4822

// Create the node.

4823

SDValue Result;

4824

if (IsTgtIntrinsic) {

4825

// This is target intrinsic that touches memory

4826

AAMDNodes AAInfo;

4827

I.getAAMetadata(AAInfo);

4828

Result =

4829

DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,

4830

MachinePointerInfo(Info.ptrVal, Info.offset),

4831

Info.align, Info.flags, Info.size, AAInfo);

4832

} else if (!HasChain) {

4833

Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);

4834

} else if (!I.getType()->isVoidTy()) {

4835

Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);

4836

} else {

4837

Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);

4838

}

4839

4840

if (HasChain) {

4841

SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);

4842

if (OnlyLoad)

4843

PendingLoads.push_back(Chain);

4844

else

4845

DAG.setRoot(Chain);

4846

}

4847

4848

if (!I.getType()->isVoidTy()) {

4849

if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {

4850

EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);

4851

Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);

4852

} else

4853

Result = lowerRangeToAssertZExt(DAG, I, Result);

4854

4855

MaybeAlign Alignment = I.getRetAlign();

4856

if (!Alignment)

4857

Alignment = F->getAttributes().getRetAlignment();

4858

// Insert `assertalign` node if there's an alignment.

4859

if (InsertAssertAlign && Alignment) {

4860

Result =

4861

DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());

4862

}

4863

4864

setValue(&I, Result);

4865

}

4866

}

4867

4868

/// GetSignificand - Get the significand and build it into a floating-point

4869

/// number with exponent of 1:

4870

///

4871

/// Op = (Op & 0x007fffff) | 0x3f800000;

4872

///

4873

/// where Op is the hexadecimal representation of floating point value.

4874

static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {

4875

SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4876

DAG.getConstant(0x007fffff, dl, MVT::i32));

4877

SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,

4878

DAG.getConstant(0x3f800000, dl, MVT::i32));

4879

return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);

4880

}

4881

4882

/// GetExponent - Get the exponent:

4883

///

4884

/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);

4885

///

4886

/// where Op is the hexadecimal representation of floating point value.

4887

static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,

4888

const TargetLowering &TLI, const SDLoc &dl) {

4889

SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4890

DAG.getConstant(0x7f800000, dl, MVT::i32));

4891

SDValue t1 = DAG.getNode(

4892

ISD::SRL, dl, MVT::i32, t0,

4893

DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));

4894

SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,

4895

DAG.getConstant(127, dl, MVT::i32));

4896

return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);

4897

}

4898

4899

/// getF32Constant - Get 32-bit floating point constant.

4900

static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,

4901

const SDLoc &dl) {

4902

return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,

4903

MVT::f32);

4904

}

4905

4906

static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,

4907

SelectionDAG &DAG) {

4908

// TODO: What fast-math-flags should be set on the floating-point nodes?

4909

4910

// IntegerPartOfX = ((int32_t)(t0);

4911

SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

4912

4913

// FractionalPartOfX = t0 - (float)IntegerPartOfX;

4914

SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);

4915

SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

4916

4917

// IntegerPartOfX <<= 23;

4918

IntegerPartOfX = DAG.getNode(

4919

ISD::SHL, dl, MVT::i32, IntegerPartOfX,

4920

DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(

4921

DAG.getDataLayout())));

4922

4923

SDValue TwoToFractionalPartOfX;

4924

if (LimitFloatPrecision <= 6) {

4925

// For floating-point precision of 6:

4926

//

4927

// TwoToFractionalPartOfX =

4928

// 0.997535578f +

4929

// (0.735607626f + 0.252464424f * x) * x;

4930

//

4931

// error 0.0144103317, which is 6 bits

4932

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4933

getF32Constant(DAG, 0x3e814304, dl));

4934

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4935

getF32Constant(DAG, 0x3f3c50c8, dl));

4936

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4937

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4938

getF32Constant(DAG, 0x3f7f5e7e, dl));

4939

} else if (LimitFloatPrecision <= 12) {

4940

// For floating-point precision of 12:

4941

//

4942

// TwoToFractionalPartOfX =

4943

// 0.999892986f +

4944

// (0.696457318f +

4945

// (0.224338339f + 0.792043434e-1f * x) * x) * x;

4946

//

4947

// error 0.000107046256, which is 13 to 14 bits

4948

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4949

getF32Constant(DAG, 0x3da235e3, dl));

4950

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4951

getF32Constant(DAG, 0x3e65b8f3, dl));

4952

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4953

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4954

getF32Constant(DAG, 0x3f324b07, dl));

4955

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4956

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4957

getF32Constant(DAG, 0x3f7ff8fd, dl));

4958

} else { // LimitFloatPrecision <= 18

4959

// For floating-point precision of 18:

4960

//

4961

// TwoToFractionalPartOfX =

4962

// 0.999999982f +

4963

// (0.693148872f +

4964

// (0.240227044f +

4965

// (0.554906021e-1f +

4966

// (0.961591928e-2f +

4967

// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;

4968

// error 2.47208000*10^(-7), which is better than 18 bits

4969

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4970

getF32Constant(DAG, 0x3924b03e, dl));

4971

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4972

getF32Constant(DAG, 0x3ab24b87, dl));

4973

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4974

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4975

getF32Constant(DAG, 0x3c1d8c17, dl));

4976

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4977

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4978

getF32Constant(DAG, 0x3d634a1d, dl));

4979

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

4980

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

4981

getF32Constant(DAG, 0x3e75fe14, dl));

4982

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

4983

SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,

4984

getF32Constant(DAG, 0x3f317234, dl));

4985

SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);

4986

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,

4987

getF32Constant(DAG, 0x3f800000, dl));

4988

}

4989

4990

// Add the exponent into the result in integer domain.

4991

SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);

4992

return DAG.getNode(ISD::BITCAST, dl, MVT::f32,

4993

DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));

4994

}

4995

4996

/// expandExp - Lower an exp intrinsic. Handles the special sequences for

4997

/// limited-precision mode.

4998

static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4999

const TargetLowering &TLI, SDNodeFlags Flags) {

5000

if (Op.getValueType() == MVT::f32 &&

5001

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5002

5003

// Put the exponent in the right bit position for later addition to the

5004

// final result:

5005

//

5006

// t0 = Op * log2(e)

5007

5008

// TODO: What fast-math-flags should be set here?

5009

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,

5010

DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));

5011

return getLimitedPrecisionExp2(t0, dl, DAG);

5012

}

5013

5014

// No special expansion.

5015

return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);

5016

}

5017

5018

/// expandLog - Lower a log intrinsic. Handles the special sequences for

5019

/// limited-precision mode.

5020

static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5021

const TargetLowering &TLI, SDNodeFlags Flags) {

5022

// TODO: What fast-math-flags should be set on the floating-point nodes?

5023

5024

if (Op.getValueType() == MVT::f32 &&

5025

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5026

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5027

5028

// Scale the exponent by log(2).

5029

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5030

SDValue LogOfExponent =

5031

DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5032

DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

5033

5034

// Get the significand and build it into a floating-point number with

5035

// exponent of 1.

5036

SDValue X = GetSignificand(DAG, Op1, dl);

5037

5038

SDValue LogOfMantissa;

5039

if (LimitFloatPrecision <= 6) {

5040

// For floating-point precision of 6:

5041

//

5042

// LogofMantissa =

5043

// -1.1609546f +

5044

// (1.4034025f - 0.23903021f * x) * x;

5045

//

5046

// error 0.0034276066, which is better than 8 bits

5047

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5048

getF32Constant(DAG, 0xbe74c456, dl));

5049

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5050

getF32Constant(DAG, 0x3fb3a2b1, dl));

5051

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5052

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5053

getF32Constant(DAG, 0x3f949a29, dl));

5054

} else if (LimitFloatPrecision <= 12) {

5055

// For floating-point precision of 12:

5056

//

5057

// LogOfMantissa =

5058

// -1.7417939f +

5059

// (2.8212026f +

5060

// (-1.4699568f +

5061

// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;

5062

//

5063

// error 0.000061011436, which is 14 bits

5064

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5065

getF32Constant(DAG, 0xbd67b6d6, dl));

5066

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5067

getF32Constant(DAG, 0x3ee4f4b8, dl));

5068

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5069

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5070

getF32Constant(DAG, 0x3fbc278b, dl));

5071

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5072

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5073

getF32Constant(DAG, 0x40348e95, dl));

5074

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5075

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5076

getF32Constant(DAG, 0x3fdef31a, dl));

5077

} else { // LimitFloatPrecision <= 18

5078

// For floating-point precision of 18:

5079

//

5080

// LogOfMantissa =

5081

// -2.1072184f +

5082

// (4.2372794f +

5083

// (-3.7029485f +

5084

// (2.2781945f +

5085

// (-0.87823314f +

5086

// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;

5087

//

5088

// error 0.0000023660568, which is better than 18 bits

5089

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5090

getF32Constant(DAG, 0xbc91e5ac, dl));

5091

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5092

getF32Constant(DAG, 0x3e4350aa, dl));

5093

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5094

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5095

getF32Constant(DAG, 0x3f60d3e3, dl));

5096

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5097

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5098

getF32Constant(DAG, 0x4011cdf0, dl));

5099

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5100

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5101

getF32Constant(DAG, 0x406cfd1c, dl));

5102

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5103

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5104

getF32Constant(DAG, 0x408797cb, dl));

5105

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5106

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5107

getF32Constant(DAG, 0x4006dcab, dl));

5108

}

5109

5110

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);

5111

}

5112

5113

// No special expansion.

5114

return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);

5115

}

5116

5117

/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for

5118

/// limited-precision mode.

5119

static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5120

const TargetLowering &TLI, SDNodeFlags Flags) {

5121

// TODO: What fast-math-flags should be set on the floating-point nodes?

5122

5123

if (Op.getValueType() == MVT::f32 &&

5124

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5125

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5126

5127

// Get the exponent.

5128

SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

5129

5130

// Get the significand and build it into a floating-point number with

5131

// exponent of 1.

5132

SDValue X = GetSignificand(DAG, Op1, dl);

5133

5134

// Different possible minimax approximations of significand in

5135

// floating-point for various degrees of accuracy over [1,2].

5136

SDValue Log2ofMantissa;

5137

if (LimitFloatPrecision <= 6) {

5138

// For floating-point precision of 6:

5139

//

5140

// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;

5141

//

5142

// error 0.0049451742, which is more than 7 bits

5143

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5144

getF32Constant(DAG, 0xbeb08fe0, dl));

5145

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5146

getF32Constant(DAG, 0x40019463, dl));

5147

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5148

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5149

getF32Constant(DAG, 0x3fd6633d, dl));

5150

} else if (LimitFloatPrecision <= 12) {

5151

// For floating-point precision of 12:

5152

//

5153

// Log2ofMantissa =

5154

// -2.51285454f +

5155

// (4.07009056f +

5156

// (-2.12067489f +

5157

// (.645142248f - 0.816157886e-1f * x) * x) * x) * x;

5158

//

5159

// error 0.0000876136000, which is better than 13 bits

5160

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5161

getF32Constant(DAG, 0xbda7262e, dl));

5162

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5163

getF32Constant(DAG, 0x3f25280b, dl));

5164

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5165

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5166

getF32Constant(DAG, 0x4007b923, dl));

5167

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5168

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5169

getF32Constant(DAG, 0x40823e2f, dl));

5170

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5171

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5172

getF32Constant(DAG, 0x4020d29c, dl));

5173

} else { // LimitFloatPrecision <= 18

5174

// For floating-point precision of 18:

5175

//

5176

// Log2ofMantissa =

5177

// -3.0400495f +

5178

// (6.1129976f +

5179

// (-5.3420409f +

5180

// (3.2865683f +

5181

// (-1.2669343f +

5182

// (0.27515199f -

5183

// 0.25691327e-1f * x) * x) * x) * x) * x) * x;

5184

//

5185

// error 0.0000018516, which is better than 18 bits

5186

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5187

getF32Constant(DAG, 0xbcd2769e, dl));

5188

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5189

getF32Constant(DAG, 0x3e8ce0b9, dl));

5190

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5191

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5192

getF32Constant(DAG, 0x3fa22ae7, dl));

5193

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5194

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5195

getF32Constant(DAG, 0x40525723, dl));

5196

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5197

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5198

getF32Constant(DAG, 0x40aaf200, dl));

5199

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5200

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5201

getF32Constant(DAG, 0x40c39dad, dl));

5202

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5203

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5204

getF32Constant(DAG, 0x4042902c, dl));

5205

}

5206

5207

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);

5208

}

5209

5210

// No special expansion.

5211

return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);

5212

}

5213

5214

/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for

5215

/// limited-precision mode.

5216

static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5217

const TargetLowering &TLI, SDNodeFlags Flags) {

5218

// TODO: What fast-math-flags should be set on the floating-point nodes?

5219

5220

if (Op.getValueType() == MVT::f32 &&

5221

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5222

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5223

5224

// Scale the exponent by log10(2) [0.30102999f].

5225

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5226

SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5227

getF32Constant(DAG, 0x3e9a209a, dl));

5228

5229

// Get the significand and build it into a floating-point number with

5230

// exponent of 1.

5231

SDValue X = GetSignificand(DAG, Op1, dl);

5232

5233

SDValue Log10ofMantissa;

5234

if (LimitFloatPrecision <= 6) {

5235

// For floating-point precision of 6:

5236

//

5237

// Log10ofMantissa =

5238

// -0.50419619f +

5239

// (0.60948995f - 0.10380950f * x) * x;

5240

//

5241

// error 0.0014886165, which is 6 bits

5242

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5243

getF32Constant(DAG, 0xbdd49a13, dl));

5244

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5245

getF32Constant(DAG, 0x3f1c0789, dl));

5246

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5247

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5248

getF32Constant(DAG, 0x3f011300, dl));

5249

} else if (LimitFloatPrecision <= 12) {

5250

// For floating-point precision of 12:

5251

//

5252

// Log10ofMantissa =

5253

// -0.64831180f +

5254

// (0.91751397f +

5255

// (-0.31664806f + 0.47637168e-1f * x) * x) * x;

5256

//

5257

// error 0.00019228036, which is better than 12 bits

5258

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5259

getF32Constant(DAG, 0x3d431f31, dl));

5260

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5261

getF32Constant(DAG, 0x3ea21fb2, dl));

5262

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5263

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5264

getF32Constant(DAG, 0x3f6ae232, dl));

5265

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5266

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5267

getF32Constant(DAG, 0x3f25f7c3, dl));

5268

} else { // LimitFloatPrecision <= 18

5269

// For floating-point precision of 18:

5270

//

5271

// Log10ofMantissa =

5272

// -0.84299375f +

5273

// (1.5327582f +

5274

// (-1.0688956f +

5275

// (0.49102474f +

5276

// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;

5277

//

5278

// error 0.0000037995730, which is better than 18 bits

5279

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5280

getF32Constant(DAG, 0x3c5d51ce, dl));

5281

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5282

getF32Constant(DAG, 0x3e00685a, dl));

5283

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5284

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5285

getF32Constant(DAG, 0x3efb6798, dl));

5286

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5287

SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5288

getF32Constant(DAG, 0x3f88d192, dl));

5289

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5290

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5291

getF32Constant(DAG, 0x3fc4316c, dl));

5292

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5293

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,

5294

getF32Constant(DAG, 0x3f57ce70, dl));

5295

}

5296

5297

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);

5298

}

5299

5300

// No special expansion.

5301

return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);

5302

}

5303

5304

/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for

5305

/// limited-precision mode.

5306

static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5307

const TargetLowering &TLI, SDNodeFlags Flags) {

5308

if (Op.getValueType() == MVT::f32 &&

5309

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)

5310

return getLimitedPrecisionExp2(Op, dl, DAG);

5311

5312

// No special expansion.

5313

return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);

5314

}

5315

5316

/// visitPow - Lower a pow intrinsic. Handles the special sequences for

5317

/// limited-precision mode with x == 10.0f.

5318

static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,

5319

SelectionDAG &DAG, const TargetLowering &TLI,

5320

SDNodeFlags Flags) {

5321

bool IsExp10 = false;

5322

if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&

5323

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5324

if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {

5325

APFloat Ten(10.0f);

5326

IsExp10 = LHSC->isExactlyValue(Ten);

5327

}

5328

}

5329

5330

// TODO: What fast-math-flags should be set on the FMUL node?

5331

if (IsExp10) {

5332

// Put the exponent in the right bit position for later addition to the

5333

// final result:

5334

//

5335

// #define LOG2OF10 3.3219281f

5336

// t0 = Op * LOG2OF10;

5337

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,

5338

getF32Constant(DAG, 0x40549a78, dl));

5339

return getLimitedPrecisionExp2(t0, dl, DAG);

5340

}

5341

5342

// No special expansion.

5343

return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);

5344

}

5345

5346

/// ExpandPowI - Expand a llvm.powi intrinsic.

5347

static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,

5348

SelectionDAG &DAG) {

5349

// If RHS is a constant, we can expand this out to a multiplication tree,

5350

// otherwise we end up lowering to a call to __powidf2 (for example). When

5351

// optimizing for size, we only want to do this if the expansion would produce

5352

// a small number of multiplies, otherwise we do the full expansion.

5353

if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {

5354

// Get the exponent as a positive value.

5355

unsigned Val = RHSC->getSExtValue();

5356

if ((int)Val < 0) Val = -Val;

5357

5358

// powi(x, 0) -> 1.0

5359

if (Val == 0)

5360

return DAG.getConstantFP(1.0, DL, LHS.getValueType());

5361

5362

bool OptForSize = DAG.shouldOptForSize();

5363

if (!OptForSize ||

5364

// If optimizing for size, don't insert too many multiplies.

5365

// This inserts up to 5 multiplies.

5366

countPopulation(Val) + Log2_32(Val) < 7) {

5367

// We use the simple binary decomposition method to generate the multiply

5368

// sequence. There are more optimal ways to do this (for example,

5369

// powi(x,15) generates one more multiply than it should), but this has

5370

// the benefit of being both really simple and much better than a libcall.

5371

SDValue Res; // Logically starts equal to 1.0

5372

SDValue CurSquare = LHS;

5373

// TODO: Intrinsics should have fast-math-flags that propagate to these

5374

// nodes.

5375

while (Val) {

5376

if (Val & 1) {

5377

if (Res.getNode())

5378

Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);

5379

else

5380

Res = CurSquare; // 1.0*CurSquare.

5381

}

5382

5383

CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),

5384

CurSquare, CurSquare);

5385

Val >>= 1;

5386

}

5387

5388

// If the original was negative, invert the result, producing 1/(x*x*x).

5389

if (RHSC->getSExtValue() < 0)

5390

Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),

5391

DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);

5392

return Res;

5393

}

5394

}

5395

5396

// Otherwise, expand to a libcall.

5397

return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);

5398

}

5399

5400

static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,

5401

SDValue LHS, SDValue RHS, SDValue Scale,

5402

SelectionDAG &DAG, const TargetLowering &TLI) {

5403

EVT VT = LHS.getValueType();

5404

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

5405

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

5406

LLVMContext &Ctx = *DAG.getContext();

5407

5408

// If the type is legal but the operation isn't, this node might survive all

5409

// the way to operation legalization. If we end up there and we do not have

5410

// the ability to widen the type (if VT*2 is not legal), we cannot expand the

5411

// node.

5412

5413

// Coax the legalizer into expanding the node during type legalization instead

5414

// by bumping the size by one bit. This will force it to Promote, enabling the

5415

// early expansion and avoiding the need to expand later.

5416

5417

// We don't have to do this if Scale is 0; that can always be expanded, unless

5418

// it's a saturating signed operation. Those can experience true integer

5419

// division overflow, a case which we must avoid.

5420

5421

// FIXME: We wouldn't have to do this (or any of the early

5422

// expansion/promotion) if it was possible to expand a libcall of an

5423

// illegal type during operation legalization. But it's not, so things

5424

// get a bit hacky.

5425

unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();

5426

if ((ScaleInt > 0 || (Saturating && Signed)) &&

5427

(TLI.isTypeLegal(VT) ||

5428

(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {

5429

TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(

5430

Opcode, VT, ScaleInt);

5431

if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {

5432

EVT PromVT;

5433

if (VT.isScalarInteger())

5434

PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);

5435

else if (VT.isVector()) {

5436

PromVT = VT.getVectorElementType();

5437

PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);

5438

PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());

5439

} else

5440

llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5440);

5441

if (Signed) {

5442

LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);

5443

RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);

5444

} else {

5445

LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);

5446

RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);

5447

}

5448

EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());

5449

// For saturating operations, we need to shift up the LHS to get the

5450

// proper saturation width, and then shift down again afterwards.

5451

if (Saturating)

5452

LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,

5453

DAG.getConstant(1, DL, ShiftTy));

5454

SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);

5455

if (Saturating)

5456

Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,

5457

DAG.getConstant(1, DL, ShiftTy));

5458

return DAG.getZExtOrTrunc(Res, DL, VT);

5459

}

5460

}

5461

5462

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);

5463

}

5464

5465

// getUnderlyingArgRegs - Find underlying registers used for a truncated,

5466

// bitcasted, or split argument. Returns a list of <Register, size in bits>

5467

static void

5468

getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,

5469

const SDValue &N) {

5470

switch (N.getOpcode()) {

5471

case ISD::CopyFromReg: {

5472

SDValue Op = N.getOperand(1);

5473

Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),

5474

Op.getValueType().getSizeInBits());

5475

return;

5476

}

5477

case ISD::BITCAST:

5478

case ISD::AssertZext:

5479

case ISD::AssertSext:

5480

case ISD::TRUNCATE:

5481

getUnderlyingArgRegs(Regs, N.getOperand(0));

5482

return;

5483

case ISD::BUILD_PAIR:

5484

case ISD::BUILD_VECTOR:

5485

case ISD::CONCAT_VECTORS:

5486

for (SDValue Op : N->op_values())

5487

getUnderlyingArgRegs(Regs, Op);

5488

return;

5489

default:

5490

return;

5491

}

5492

}

5493

5494

/// If the DbgValueInst is a dbg_value of a function argument, create the

5495

/// corresponding DBG_VALUE machine instruction for it now. At the end of

5496

/// instruction selection, they will be inserted to the entry BB.

5497

/// We don't currently support this for variadic dbg_values, as they shouldn't

5498

/// appear for function arguments or in the prologue.

5499

bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(

5500

const Value *V, DILocalVariable *Variable, DIExpression *Expr,

5501

DILocation *DL, bool IsDbgDeclare, const SDValue &N) {

5502

const Argument *Arg = dyn_cast<Argument>(V);

5503

if (!Arg)

5504

return false;

5505

5506

MachineFunction &MF = DAG.getMachineFunction();

5507

const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

5508

5509

// Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind

5510

// we've been asked to pursue.

5511

auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,

5512

bool Indirect) {

5513

if (Reg.isVirtual() && TM.Options.ValueTrackingVariableLocations) {

5514

// For VRegs, in instruction referencing mode, create a DBG_INSTR_REF

5515

// pointing at the VReg, which will be patched up later.

5516

auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);

5517

auto MIB = BuildMI(MF, DL, Inst);

5518

MIB.addReg(Reg, RegState::Debug);

5519

MIB.addImm(0);

5520

MIB.addMetadata(Variable);

5521

auto *NewDIExpr = FragExpr;

5522

// We don't have an "Indirect" field in DBG_INSTR_REF, fold that into

5523

// the DIExpression.

5524

if (Indirect)

5525

NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);

5526

MIB.addMetadata(NewDIExpr);

5527

return MIB;

5528

} else {

5529

// Create a completely standard DBG_VALUE.

5530

auto &Inst = TII->get(TargetOpcode::DBG_VALUE);

5531

return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);

5532

}

5533

};

5534

5535

if (!IsDbgDeclare) {

5536

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5537

// should only emit as ArgDbgValue if the dbg.value intrinsic is found in

5538

// the entry block.

5539

bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();

5540

if (!IsInEntryBlock)

5541

return false;

5542

5543

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5544

// should only emit as ArgDbgValue if the dbg.value intrinsic describes a

5545

// variable that also is a param.

5546

//

5547

// Although, if we are at the top of the entry block already, we can still

5548

// emit using ArgDbgValue. This might catch some situations when the

5549

// dbg.value refers to an argument that isn't used in the entry block, so

5550

// any CopyToReg node would be optimized out and the only way to express

5551

// this DBG_VALUE is by using the physical reg (or FI) as done in this

5552

// method. ArgDbgValues are hoisted to the beginning of the entry block. So

5553

// we should only emit as ArgDbgValue if the Variable is an argument to the

5554

// current function, and the dbg.value intrinsic is found in the entry

5555

// block.

5556

bool VariableIsFunctionInputArg = Variable->isParameter() &&

5557

!DL->getInlinedAt();

5558

bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;

5559

if (!IsInPrologue && !VariableIsFunctionInputArg)

5560

return false;

5561

5562

// Here we assume that a function argument on IR level only can be used to

5563

// describe one input parameter on source level. If we for example have

5564

// source code like this

5565

//

5566

// struct A { long x, y; };

5567

// void foo(struct A a, long b) {

5568

// ...

5569

// b = a.x;

5570

// ...

5571

// }

5572

//

5573

// and IR like this

5574

//

5575

// define void @foo(i32 %a1, i32 %a2, i32 %b) {

5576

// entry:

5577

// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment

5578

// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment

5579

// call void @llvm.dbg.value(metadata i32 %b, "b",

5580

// ...

5581

// call void @llvm.dbg.value(metadata i32 %a1, "b"

5582

// ...

5583

//

5584

// then the last dbg.value is describing a parameter "b" using a value that

5585

// is an argument. But since we already has used %a1 to describe a parameter

5586

// we should not handle that last dbg.value here (that would result in an

5587

// incorrect hoisting of the DBG_VALUE to the function entry).

5588

// Notice that we allow one dbg.value per IR level argument, to accommodate

5589

// for the situation with fragments above.

5590

if (VariableIsFunctionInputArg) {

5591

unsigned ArgNo = Arg->getArgNo();

5592

if (ArgNo >= FuncInfo.DescribedArgs.size())

5593

FuncInfo.DescribedArgs.resize(ArgNo + 1, false);

5594

else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))

5595

return false;

5596

FuncInfo.DescribedArgs.set(ArgNo);

5597

}

5598

}

5599

5600

bool IsIndirect = false;

5601

Optional<MachineOperand> Op;

5602

// Some arguments' frame index is recorded during argument lowering.

5603

int FI = FuncInfo.getArgumentFrameIndex(Arg);

5604

if (FI != std::numeric_limits<int>::max())

5605

Op = MachineOperand::CreateFI(FI);

5606

5607

SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;

5608

if (!Op && N.getNode()) {

5609

getUnderlyingArgRegs(ArgRegsAndSizes, N);

5610

Register Reg;

5611

if (ArgRegsAndSizes.size() == 1)

5612

Reg = ArgRegsAndSizes.front().first;

5613

5614

if (Reg && Reg.isVirtual()) {

5615

MachineRegisterInfo &RegInfo = MF.getRegInfo();

5616

Register PR = RegInfo.getLiveInPhysReg(Reg);

5617

if (PR)

5618

Reg = PR;

5619

}

5620

if (Reg) {

5621

Op = MachineOperand::CreateReg(Reg, false);

5622

IsIndirect = IsDbgDeclare;

5623

}

5624

}

5625

5626

if (!Op && N.getNode()) {

5627

// Check if frame index is available.

5628

SDValue LCandidate = peekThroughBitcasts(N);

5629

if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))

5630

if (FrameIndexSDNode *FINode =

5631

dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))

5632

Op = MachineOperand::CreateFI(FINode->getIndex());

5633

}

5634

5635

if (!Op) {

5636

// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg

5637

auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>

5638

SplitRegs) {

5639

unsigned Offset = 0;

5640

for (auto RegAndSize : SplitRegs) {

5641

// If the expression is already a fragment, the current register

5642

// offset+size might extend beyond the fragment. In this case, only

5643

// the register bits that are inside the fragment are relevant.

5644

int RegFragmentSizeInBits = RegAndSize.second;

5645

if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {

5646

uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;

5647

// The register is entirely outside the expression fragment,

5648

// so is irrelevant for debug info.

5649

if (Offset >= ExprFragmentSizeInBits)

5650

break;

5651

// The register is partially outside the expression fragment, only

5652

// the low bits within the fragment are relevant for debug info.

5653

if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {

5654

RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;

5655

}

5656

}

5657

5658

auto FragmentExpr = DIExpression::createFragmentExpression(

5659

Expr, Offset, RegFragmentSizeInBits);

5660

Offset += RegAndSize.second;

5661

// If a valid fragment expression cannot be created, the variable's

5662

// correct value cannot be determined and so it is set as Undef.

5663

if (!FragmentExpr) {

5664

SDDbgValue *SDV = DAG.getConstantDbgValue(

5665

Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);

5666

DAG.AddDbgValue(SDV, false);

5667

continue;

5668

}

5669

MachineInstr *NewMI =

5670

MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, IsDbgDeclare);

5671

FuncInfo.ArgDbgValues.push_back(NewMI);

5672

}

5673

};

5674

5675

// Check if ValueMap has reg number.

5676

DenseMap<const Value *, Register>::const_iterator

5677

VMI = FuncInfo.ValueMap.find(V);

5678

if (VMI != FuncInfo.ValueMap.end()) {

5679

const auto &TLI = DAG.getTargetLoweringInfo();

5680

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,

5681

V->getType(), None);

5682

if (RFV.occupiesMultipleRegs()) {

5683

splitMultiRegDbgValue(RFV.getRegsAndSizes());

5684

return true;

5685

}

5686

5687

Op = MachineOperand::CreateReg(VMI->second, false);

5688

IsIndirect = IsDbgDeclare;

5689

} else if (ArgRegsAndSizes.size() > 1) {

5690

// This was split due to the calling convention, and no virtual register

5691

// mapping exists for the value.

5692

splitMultiRegDbgValue(ArgRegsAndSizes);

5693

return true;

5694

}

5695

}

5696

5697

if (!Op)

5698

return false;

5699

5700

assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5701, __extension__ __PRETTY_FUNCTION__))

5701

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5701, __extension__ __PRETTY_FUNCTION__));

5702

MachineInstr *NewMI = nullptr;

5703

5704

if (Op->isReg())

5705

NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);

5706

else

5707

NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,

5708

Variable, Expr);

5709

5710

FuncInfo.ArgDbgValues.push_back(NewMI);

5711

return true;

5712

}

5713

5714

/// Return the appropriate SDDbgValue based on N.

5715

SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,

5716

DILocalVariable *Variable,

5717

DIExpression *Expr,

5718

const DebugLoc &dl,

5719

unsigned DbgSDNodeOrder) {

5720

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

5721

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe

5722

// stack slot locations.

5723

//

5724

// Consider "int x = 0; int *px = &x;". There are two kinds of interesting

5725

// debug values here after optimization:

5726

//

5727

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

5728

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

5729

//

5730

// Both describe the direct values of their associated variables.

5731

return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),

5732

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5733

}

5734

return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),

5735

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5736

}

5737

5738

static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {

5739

switch (Intrinsic) {

5740

case Intrinsic::smul_fix:

5741

return ISD::SMULFIX;

5742

case Intrinsic::umul_fix:

5743

return ISD::UMULFIX;

5744

case Intrinsic::smul_fix_sat:

5745

return ISD::SMULFIXSAT;

5746

case Intrinsic::umul_fix_sat:

5747

return ISD::UMULFIXSAT;

5748

case Intrinsic::sdiv_fix:

5749

return ISD::SDIVFIX;

5750

case Intrinsic::udiv_fix:

5751

return ISD::UDIVFIX;

5752

case Intrinsic::sdiv_fix_sat:

5753

return ISD::SDIVFIXSAT;

5754

case Intrinsic::udiv_fix_sat:

5755

return ISD::UDIVFIXSAT;

5756

default:

5757

llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5757);

5758

}

5759

}

5760

5761

void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,

5762

const char *FunctionName) {

5763

assert(FunctionName && "FunctionName must not be nullptr")(static_cast <bool> (FunctionName && "FunctionName must not be nullptr"
) ? void (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5763, __extension__ __PRETTY_FUNCTION__));

5764

SDValue Callee = DAG.getExternalSymbol(

5765

FunctionName,

5766

DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));

5767

LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());

5768

}

5769

5770

/// Given a @llvm.call.preallocated.setup, return the corresponding

5771

/// preallocated call.

5772

static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {

5773

assert(cast<CallBase>(PreallocatedSetup)(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5776, __extension__ __PRETTY_FUNCTION__))

5774

->getCalledFunction()(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5776, __extension__ __PRETTY_FUNCTION__))

5775

->getIntrinsicID() == Intrinsic::call_preallocated_setup &&(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5776, __extension__ __PRETTY_FUNCTION__))

5776

"expected call_preallocated_setup Value")(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5776, __extension__ __PRETTY_FUNCTION__));

5777

for (auto *U : PreallocatedSetup->users()) {

5778

auto *UseCall = cast<CallBase>(U);

5779

const Function *Fn = UseCall->getCalledFunction();

5780

if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {

5781

return UseCall;

5782

}

5783

}

5784

llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg"
, "/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5784);

5785

}

5786

5787

/// Lower the call to the specified intrinsic function.

5788

void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,

5789

unsigned Intrinsic) {

5790

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5791

SDLoc sdl = getCurSDLoc();

5792

DebugLoc dl = getCurDebugLoc();

5793

SDValue Res;

5794

5795

SDNodeFlags Flags;

5796

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

5797

Flags.copyFMF(*FPOp);

5798

5799

switch (Intrinsic) {

5800

default:

5801

// By default, turn this into a target intrinsic node.

5802

visitTargetIntrinsic(I, Intrinsic);

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1114, column 10 Called C++ object pointer is null

Bug Summary

Annotated Source Code

/build/llvm-toolchain-snapshot-13~++20210722111111+b115c038d2d4/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp