/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1

//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements routines for translating from LLVM IR into SelectionDAG IR.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "SelectionDAGBuilder.h"

14

#include "SDNodeDbgValue.h"

15

#include "llvm/ADT/APFloat.h"

16

#include "llvm/ADT/APInt.h"

17

#include "llvm/ADT/BitVector.h"

18

#include "llvm/ADT/None.h"

19

#include "llvm/ADT/Optional.h"

20

#include "llvm/ADT/STLExtras.h"

21

#include "llvm/ADT/SmallPtrSet.h"

22

#include "llvm/ADT/SmallSet.h"

23

#include "llvm/ADT/StringRef.h"

24

#include "llvm/ADT/Triple.h"

25

#include "llvm/ADT/Twine.h"

26

#include "llvm/Analysis/AliasAnalysis.h"

27

#include "llvm/Analysis/BlockFrequencyInfo.h"

28

#include "llvm/Analysis/BranchProbabilityInfo.h"

29

#include "llvm/Analysis/ConstantFolding.h"

30

#include "llvm/Analysis/EHPersonalities.h"

31

#include "llvm/Analysis/Loads.h"

32

#include "llvm/Analysis/MemoryLocation.h"

33

#include "llvm/Analysis/ProfileSummaryInfo.h"

34

#include "llvm/Analysis/TargetLibraryInfo.h"

35

#include "llvm/Analysis/ValueTracking.h"

36

#include "llvm/Analysis/VectorUtils.h"

37

#include "llvm/CodeGen/Analysis.h"

38

#include "llvm/CodeGen/FunctionLoweringInfo.h"

39

#include "llvm/CodeGen/GCMetadata.h"

40

#include "llvm/CodeGen/MachineBasicBlock.h"

41

#include "llvm/CodeGen/MachineFrameInfo.h"

42

#include "llvm/CodeGen/MachineFunction.h"

43

#include "llvm/CodeGen/MachineInstr.h"

44

#include "llvm/CodeGen/MachineInstrBuilder.h"

45

#include "llvm/CodeGen/MachineJumpTableInfo.h"

46

#include "llvm/CodeGen/MachineMemOperand.h"

47

#include "llvm/CodeGen/MachineModuleInfo.h"

48

#include "llvm/CodeGen/MachineOperand.h"

49

#include "llvm/CodeGen/MachineRegisterInfo.h"

50

#include "llvm/CodeGen/RuntimeLibcalls.h"

51

#include "llvm/CodeGen/SelectionDAG.h"

52

#include "llvm/CodeGen/SelectionDAGTargetInfo.h"

53

#include "llvm/CodeGen/StackMaps.h"

54

#include "llvm/CodeGen/SwiftErrorValueTracking.h"

55

#include "llvm/CodeGen/TargetFrameLowering.h"

56

#include "llvm/CodeGen/TargetInstrInfo.h"

57

#include "llvm/CodeGen/TargetOpcodes.h"

58

#include "llvm/CodeGen/TargetRegisterInfo.h"

59

#include "llvm/CodeGen/TargetSubtargetInfo.h"

60

#include "llvm/CodeGen/WinEHFuncInfo.h"

61

#include "llvm/IR/Argument.h"

62

#include "llvm/IR/Attributes.h"

63

#include "llvm/IR/BasicBlock.h"

64

#include "llvm/IR/CFG.h"

65

#include "llvm/IR/CallingConv.h"

66

#include "llvm/IR/Constant.h"

67

#include "llvm/IR/ConstantRange.h"

68

#include "llvm/IR/Constants.h"

69

#include "llvm/IR/DataLayout.h"

70

#include "llvm/IR/DebugInfoMetadata.h"

71

#include "llvm/IR/DerivedTypes.h"

72

#include "llvm/IR/Function.h"

73

#include "llvm/IR/GetElementPtrTypeIterator.h"

74

#include "llvm/IR/InlineAsm.h"

75

#include "llvm/IR/InstrTypes.h"

76

#include "llvm/IR/Instructions.h"

77

#include "llvm/IR/IntrinsicInst.h"

78

#include "llvm/IR/Intrinsics.h"

79

#include "llvm/IR/IntrinsicsAArch64.h"

80

#include "llvm/IR/IntrinsicsWebAssembly.h"

81

#include "llvm/IR/LLVMContext.h"

82

#include "llvm/IR/Metadata.h"

83

#include "llvm/IR/Module.h"

84

#include "llvm/IR/Operator.h"

85

#include "llvm/IR/PatternMatch.h"

86

#include "llvm/IR/Statepoint.h"

87

#include "llvm/IR/Type.h"

88

#include "llvm/IR/User.h"

89

#include "llvm/IR/Value.h"

90

#include "llvm/MC/MCContext.h"

91

#include "llvm/MC/MCSymbol.h"

92

#include "llvm/Support/AtomicOrdering.h"

93

#include "llvm/Support/Casting.h"

94

#include "llvm/Support/CommandLine.h"

95

#include "llvm/Support/Compiler.h"

96

#include "llvm/Support/Debug.h"

97

#include "llvm/Support/MathExtras.h"

98

#include "llvm/Support/raw_ostream.h"

99

#include "llvm/Target/TargetIntrinsicInfo.h"

100

#include "llvm/Target/TargetMachine.h"

101

#include "llvm/Target/TargetOptions.h"

102

#include "llvm/Transforms/Utils/Local.h"

103

#include <cstddef>

104

#include <cstring>

105

#include <iterator>

106

#include <limits>

107

#include <numeric>

108

#include <tuple>

109

110

using namespace llvm;

111

using namespace PatternMatch;

112

using namespace SwitchCG;

113

114

#define DEBUG_TYPE"isel" "isel"

115

116

/// LimitFloatPrecision - Generate low-precision inline sequences for

117

/// some float libcalls (6, 8 or 12 bits).

118

static unsigned LimitFloatPrecision;

119

120

static cl::opt<bool>

121

InsertAssertAlign("insert-assert-align", cl::init(true),

122

cl::desc("Insert the experimental `assertalign` node."),

123

cl::ReallyHidden);

124

125

static cl::opt<unsigned, true>

126

LimitFPPrecision("limit-float-precision",

127

cl::desc("Generate low-precision inline sequences "

128

"for some float libcalls"),

129

cl::location(LimitFloatPrecision), cl::Hidden,

130

cl::init(0));

131

132

static cl::opt<unsigned> SwitchPeelThreshold(

133

"switch-peel-threshold", cl::Hidden, cl::init(66),

134

cl::desc("Set the case probability threshold for peeling the case from a "

135

"switch statement. A value greater than 100 will void this "

136

"optimization"));

137

138

// Limit the width of DAG chains. This is important in general to prevent

139

// DAG-based analysis from blowing up. For example, alias analysis and

140

// load clustering may not complete in reasonable time. It is difficult to

141

// recognize and avoid this situation within each individual analysis, and

142

// future analyses are likely to have the same behavior. Limiting DAG width is

143

// the safe approach and will be especially important with global DAGs.

144

//

145

// MaxParallelChains default is arbitrarily high to avoid affecting

146

// optimization, but could be lowered to improve compile time. Any ld-ld-st-st

147

// sequence over this should have been converted to llvm.memcpy by the

148

// frontend. It is easy to induce this behavior with .ll code such as:

149

// %buffer = alloca [4096 x i8]

150

// %data = load [4096 x i8]* %argPtr

151

// store [4096 x i8] %data, [4096 x i8]* %buffer

152

static const unsigned MaxParallelChains = 64;

153

154

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

155

const SDValue *Parts, unsigned NumParts,

156

MVT PartVT, EVT ValueVT, const Value *V,

157

Optional<CallingConv::ID> CC);

158

159

/// getCopyFromParts - Create a value that contains the specified legal parts

160

/// combined into the value they represent. If the parts combine to a type

161

/// larger than ValueVT then AssertOp can be used to specify whether the extra

162

/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT

163

/// (ISD::AssertSext).

164

static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,

165

const SDValue *Parts, unsigned NumParts,

166

MVT PartVT, EVT ValueVT, const Value *V,

167

Optional<CallingConv::ID> CC = None,

168

Optional<ISD::NodeType> AssertOp = None) {

169

// Let the target assemble the parts if it wants to

170

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

171

if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,

172

PartVT, ValueVT, CC))

173

return Val;

174

175

if (ValueVT.isVector())

176

return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,

177

CC);

178

179

assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 179, __PRETTY_FUNCTION__));

180

SDValue Val = Parts[0];

181

182

if (NumParts > 1) {

183

// Assemble the value from multiple parts.

184

if (ValueVT.isInteger()) {

185

unsigned PartBits = PartVT.getSizeInBits();

186

unsigned ValueBits = ValueVT.getSizeInBits();

187

188

// Assemble the power of 2 part.

189

unsigned RoundParts =

190

(NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;

191

unsigned RoundBits = PartBits * RoundParts;

192

EVT RoundVT = RoundBits == ValueBits ?

193

ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);

194

SDValue Lo, Hi;

195

196

EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

197

198

if (RoundParts > 2) {

199

Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,

200

PartVT, HalfVT, V);

201

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,

202

RoundParts / 2, PartVT, HalfVT, V);

203

} else {

204

Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);

205

Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);

206

}

207

208

if (DAG.getDataLayout().isBigEndian())

209

std::swap(Lo, Hi);

210

211

Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

212

213

if (RoundParts < NumParts) {

214

// Assemble the trailing non-power-of-2 part.

215

unsigned OddParts = NumParts - RoundParts;

216

EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);

217

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,

218

OddVT, V, CC);

219

220

// Combine the round and odd parts.

221

Lo = Val;

222

if (DAG.getDataLayout().isBigEndian())

223

std::swap(Lo, Hi);

224

EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

225

Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);

226

Hi =

227

DAG.getNode(ISD::SHL, DL, TotalVT, Hi,

228

DAG.getConstant(Lo.getValueSizeInBits(), DL,

229

TLI.getPointerTy(DAG.getDataLayout())));

230

Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);

231

Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);

232

}

233

} else if (PartVT.isFloatingPoint()) {

234

// FP split into multiple FP parts (for ppcf128)

235

assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __PRETTY_FUNCTION__))

236

"Unexpected split")((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __PRETTY_FUNCTION__));

237

SDValue Lo, Hi;

238

Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);

239

Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);

240

if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

241

std::swap(Lo, Hi);

242

Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);

243

} else {

244

// FP split into integer parts (soft fp)

245

assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __PRETTY_FUNCTION__))

246

!PartVT.isVector() && "Unexpected split")((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __PRETTY_FUNCTION__));

247

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

248

Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);

249

}

250

}

251

252

// There is now one part, held in Val. Correct it to match ValueVT.

253

// PartEVT is the type of the register class that holds the value.

254

// ValueVT is the type of the inline asm operation.

255

EVT PartEVT = Val.getValueType();

256

257

if (PartEVT == ValueVT)

258

return Val;

259

260

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&

261

ValueVT.bitsLT(PartEVT)) {

262

// For an FP value in an integer part, we need to truncate to the right

263

// width first.

264

PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

265

Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);

266

}

267

268

// Handle types that have the same size.

269

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())

270

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

271

272

// Handle types with different sizes.

273

if (PartEVT.isInteger() && ValueVT.isInteger()) {

274

if (ValueVT.bitsLT(PartEVT)) {

275

// For a truncate, see if we have any information to

276

// indicate whether the truncated bits will always be

277

// zero or sign-extension.

278

if (AssertOp.hasValue())

279

Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,

280

DAG.getValueType(ValueVT));

281

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

282

}

283

return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

284

}

285

286

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

287

// FP_ROUND's are always exact here.

288

if (ValueVT.bitsLT(Val.getValueType()))

289

return DAG.getNode(

290

ISD::FP_ROUND, DL, ValueVT, Val,

291

DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

292

293

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);

294

}

295

296

// Handle MMX to a narrower integer type by bitcasting MMX to integer and

297

// then truncating.

298

if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&

299

ValueVT.bitsLT(PartEVT)) {

300

Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);

301

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

302

}

303

304

report_fatal_error("Unknown mismatch in getCopyFromParts!");

305

}

306

307

static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,

308

const Twine &ErrMsg) {

309

const Instruction *I = dyn_cast_or_null<Instruction>(V);

310

if (!V)

311

return Ctx.emitError(ErrMsg);

312

313

const char *AsmError = ", possible invalid constraint for vector type";

314

if (const CallInst *CI = dyn_cast<CallInst>(I))

315

if (CI->isInlineAsm())

316

return Ctx.emitError(I, ErrMsg + AsmError);

317

318

return Ctx.emitError(I, ErrMsg);

319

}

320

321

/// getCopyFromPartsVector - Create a value that contains the specified legal

322

/// parts combined into the value they represent. If the parts combine to a

323

/// type larger than ValueVT then AssertOp can be used to specify whether the

324

/// extra bits are known to be zero (ISD::AssertZext) or sign extended from

325

/// ValueVT (ISD::AssertSext).

326

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

327

const SDValue *Parts, unsigned NumParts,

328

MVT PartVT, EVT ValueVT, const Value *V,

329

Optional<CallingConv::ID> CallConv) {

330

assert(ValueVT.isVector() && "Not a vector value")((ValueVT.isVector() && "Not a vector value") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 330, __PRETTY_FUNCTION__));

331

assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 331, __PRETTY_FUNCTION__));

332

const bool IsABIRegCopy = CallConv.hasValue();

333

334

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

335

SDValue Val = Parts[0];

336

337

// Handle a multi-element vector.

338

if (NumParts > 1) {

339

EVT IntermediateVT;

340

MVT RegisterVT;

341

unsigned NumIntermediates;

342

unsigned NumRegs;

343

344

if (IsABIRegCopy) {

345

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

346

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

347

NumIntermediates, RegisterVT);

348

} else {

349

NumRegs =

350

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

351

NumIntermediates, RegisterVT);

352

}

353

354

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 354, __PRETTY_FUNCTION__));

355

NumParts = NumRegs; // Silence a compiler warning.

356

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 356, __PRETTY_FUNCTION__));

357

assert(RegisterVT.getSizeInBits() ==((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__))

358

Parts[0].getSimpleValueType().getSizeInBits() &&((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__))

359

"Part type sizes don't match!")((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__));

360

361

// Assemble the parts into intermediate operands.

362

SmallVector<SDValue, 8> Ops(NumIntermediates);

363

if (NumIntermediates == NumParts) {

364

// If the register was not expanded, truncate or copy the value,

365

// as appropriate.

366

for (unsigned i = 0; i != NumParts; ++i)

367

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,

368

PartVT, IntermediateVT, V, CallConv);

369

} else if (NumParts > 0) {

370

// If the intermediate type was expanded, build the intermediate

371

// operands from the parts.

372

assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __PRETTY_FUNCTION__))

373

"Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __PRETTY_FUNCTION__));

374

unsigned Factor = NumParts / NumIntermediates;

375

for (unsigned i = 0; i != NumIntermediates; ++i)

376

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,

377

PartVT, IntermediateVT, V, CallConv);

378

}

379

380

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the

381

// intermediate operands.

382

EVT BuiltVectorTy =

383

IntermediateVT.isVector()

384

? EVT::getVectorVT(

385

*DAG.getContext(), IntermediateVT.getScalarType(),

386

IntermediateVT.getVectorElementCount() * NumParts)

387

: EVT::getVectorVT(*DAG.getContext(),

388

IntermediateVT.getScalarType(),

389

NumIntermediates);

390

Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS

391

: ISD::BUILD_VECTOR,

392

DL, BuiltVectorTy, Ops);

393

}

394

395

// There is now one part, held in Val. Correct it to match ValueVT.

396

EVT PartEVT = Val.getValueType();

397

398

if (PartEVT == ValueVT)

399

return Val;

400

401

if (PartEVT.isVector()) {

402

// If the element type of the source/dest vectors are the same, but the

403

// parts vector has more elements than the value vector, then we have a

404

// vector widening case (e.g. <2 x float> -> <4 x float>). Extract the

405

// elements we want.

406

if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {

407

assert((PartEVT.getVectorElementCount().getKnownMinValue() >(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

408

ValueVT.getVectorElementCount().getKnownMinValue()) &&(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

409

(PartEVT.getVectorElementCount().isScalable() ==(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

410

ValueVT.getVectorElementCount().isScalable()) &&(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

411

"Cannot narrow, it would be a lossy transformation")(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__));

412

return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,

413

DAG.getVectorIdxConstant(0, DL));

414

}

415

416

// Vector/Vector bitcast.

417

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())

418

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

419

420

assert(PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() &&((PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __PRETTY_FUNCTION__))

421

"Cannot handle this kind of promotion")((PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __PRETTY_FUNCTION__));

422

// Promoted vector extract

423

return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);

424

425

}

426

427

// Trivial bitcast if the types are the same size and the destination

428

// vector type is legal.

429

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&

430

TLI.isTypeLegal(ValueVT))

431

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

432

433

if (ValueVT.getVectorNumElements() != 1) {

434

// Certain ABIs require that vectors are passed as integers. For vectors

435

// are the same size, this is an obvious bitcast.

436

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {

437

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

438

} else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {

439

// Bitcast Val back the original type and extract the corresponding

440

// vector we want.

441

unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();

442

EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),

443

ValueVT.getVectorElementType(), Elts);

444

Val = DAG.getBitcast(WiderVecType, Val);

445

return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,

446

DAG.getVectorIdxConstant(0, DL));

447

}

448

449

diagnosePossiblyInvalidConstraint(

450

*DAG.getContext(), V, "non-trivial scalar-to-vector conversion");

451

return DAG.getUNDEF(ValueVT);

452

}

453

454

// Handle cases such as i8 -> <1 x i1>

455

EVT ValueSVT = ValueVT.getVectorElementType();

456

if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {

457

if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())

458

Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);

459

else

460

Val = ValueVT.isFloatingPoint()

461

? DAG.getFPExtendOrRound(Val, DL, ValueSVT)

462

: DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);

463

}

464

465

return DAG.getBuildVector(ValueVT, DL, Val);

466

}

467

468

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,

469

SDValue Val, SDValue *Parts, unsigned NumParts,

470

MVT PartVT, const Value *V,

471

Optional<CallingConv::ID> CallConv);

472

473

/// getCopyToParts - Create a series of nodes that contain the specified value

474

/// split into legal parts. If the parts contain more bits than Val, then, for

475

/// integers, ExtendKind can be used to specify how to generate the extra bits.

476

static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,

477

SDValue *Parts, unsigned NumParts, MVT PartVT,

478

const Value *V,

479

Optional<CallingConv::ID> CallConv = None,

480

ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {

481

// Let the target split the parts if it wants to

482

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

483

if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,

484

CallConv))

485

return;

486

EVT ValueVT = Val.getValueType();

487

488

// Handle the vector case separately.

489

if (ValueVT.isVector())

490

return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,

491

CallConv);

492

493

unsigned PartBits = PartVT.getSizeInBits();

494

unsigned OrigNumParts = NumParts;

495

assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 496, __PRETTY_FUNCTION__))

496

"Copying to an illegal type!")((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 496, __PRETTY_FUNCTION__));

497

498

if (NumParts == 0)

499

return;

500

501

assert(!ValueVT.isVector() && "Vector case handled elsewhere")((!ValueVT.isVector() && "Vector case handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __PRETTY_FUNCTION__));

502

EVT PartEVT = PartVT;

503

if (PartEVT == ValueVT) {

504

assert(NumParts == 1 && "No-op copy with multiple parts!")((NumParts == 1 && "No-op copy with multiple parts!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 504, __PRETTY_FUNCTION__));

505

Parts[0] = Val;

506

return;

507

}

508

509

if (NumParts * PartBits > ValueVT.getSizeInBits()) {

510

// If the parts cover more bits than the value has, promote the value.

511

if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

512

assert(NumParts == 1 && "Do not know what to promote to!")((NumParts == 1 && "Do not know what to promote to!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 512, __PRETTY_FUNCTION__));

513

Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);

514

} else {

515

if (ValueVT.isFloatingPoint()) {

516

// FP values need to be bitcast, then extended if they are being put

517

// into a larger container.

518

ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

519

Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

520

}

521

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __PRETTY_FUNCTION__))

522

ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __PRETTY_FUNCTION__))

523

"Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __PRETTY_FUNCTION__));

524

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

525

Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);

526

if (PartVT == MVT::x86mmx)

527

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

528

}

529

} else if (PartBits == ValueVT.getSizeInBits()) {

530

// Different types of the same size.

531

assert(NumParts == 1 && PartEVT != ValueVT)((NumParts == 1 && PartEVT != ValueVT) ? static_cast<
void> (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 531, __PRETTY_FUNCTION__));

532

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

533

} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {

534

// If the parts cover less bits than value has, truncate the value.

535

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __PRETTY_FUNCTION__))

536

ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __PRETTY_FUNCTION__))

537

"Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __PRETTY_FUNCTION__));

538

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

539

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

540

if (PartVT == MVT::x86mmx)

541

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

542

}

543

544

// The value may have changed - recompute ValueVT.

545

ValueVT = Val.getValueType();

546

assert(NumParts * PartBits == ValueVT.getSizeInBits() &&((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 547, __PRETTY_FUNCTION__))

547

"Failed to tile the value with PartVT!")((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 547, __PRETTY_FUNCTION__));

548

549

if (NumParts == 1) {

550

if (PartEVT != ValueVT) {

551

diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,

552

"scalar-to-vector conversion failed");

553

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

554

}

555

556

Parts[0] = Val;

557

return;

558

}

559

560

// Expand the value into multiple parts.

561

if (NumParts & (NumParts - 1)) {

562

// The number of parts is not a power of 2. Split off and copy the tail.

563

assert(PartVT.isInteger() && ValueVT.isInteger() &&((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 564, __PRETTY_FUNCTION__))

564

"Do not know what to expand to!")((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 564, __PRETTY_FUNCTION__));

565

unsigned RoundParts = 1 << Log2_32(NumParts);

566

unsigned RoundBits = RoundParts * PartBits;

567

unsigned OddParts = NumParts - RoundParts;

568

SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,

569

DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));

570

571

getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,

572

CallConv);

573

574

if (DAG.getDataLayout().isBigEndian())

575

// The odd parts were reversed by getCopyToParts - unreverse them.

576

std::reverse(Parts + RoundParts, Parts + NumParts);

577

578

NumParts = RoundParts;

579

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

580

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

581

}

582

583

// The number of parts is a power of 2. Repeatedly bisect the value using

584

// EXTRACT_ELEMENT.

585

Parts[0] = DAG.getNode(ISD::BITCAST, DL,

586

EVT::getIntegerVT(*DAG.getContext(),

587

ValueVT.getSizeInBits()),

588

Val);

589

590

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {

591

for (unsigned i = 0; i < NumParts; i += StepSize) {

592

unsigned ThisBits = StepSize * PartBits / 2;

593

EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);

594

SDValue &Part0 = Parts[i];

595

SDValue &Part1 = Parts[i+StepSize/2];

596

597

Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

598

ThisVT, Part0, DAG.getIntPtrConstant(1, DL));

599

Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

600

ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

601

602

if (ThisBits == PartBits && ThisVT != PartVT) {

603

Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);

604

Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);

605

}

606

}

607

}

608

609

if (DAG.getDataLayout().isBigEndian())

610

std::reverse(Parts, Parts + OrigNumParts);

611

}

612

613

static SDValue widenVectorToPartType(SelectionDAG &DAG,

614

SDValue Val, const SDLoc &DL, EVT PartVT) {

615

if (!PartVT.isFixedLengthVector())

616

return SDValue();

617

618

EVT ValueVT = Val.getValueType();

619

unsigned PartNumElts = PartVT.getVectorNumElements();

620

unsigned ValueNumElts = ValueVT.getVectorNumElements();

621

if (PartNumElts > ValueNumElts &&

622

PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {

623

EVT ElementVT = PartVT.getVectorElementType();

624

// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in

625

// undef elements.

626

SmallVector<SDValue, 16> Ops;

627

DAG.ExtractVectorElements(Val, Ops);

628

SDValue EltUndef = DAG.getUNDEF(ElementVT);

629

for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)

630

Ops.push_back(EltUndef);

631

632

// FIXME: Use CONCAT for 2x -> 4x.

633

return DAG.getBuildVector(PartVT, DL, Ops);

634

}

635

636

return SDValue();

637

}

638

639

/// getCopyToPartsVector - Create a series of nodes that contain the specified

640

/// value split into legal parts.

641

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,

642

SDValue Val, SDValue *Parts, unsigned NumParts,

643

MVT PartVT, const Value *V,

644

Optional<CallingConv::ID> CallConv) {

645

EVT ValueVT = Val.getValueType();

646

assert(ValueVT.isVector() && "Not a vector")((ValueVT.isVector() && "Not a vector") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 646, __PRETTY_FUNCTION__));

647

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

648

const bool IsABIRegCopy = CallConv.hasValue();

649

650

if (NumParts == 1) {

651

EVT PartEVT = PartVT;

652

if (PartEVT == ValueVT) {

653

// Nothing to do.

654

} else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {

655

// Bitconvert vector->vector case.

656

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

657

} else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {

658

Val = Widened;

659

} else if (PartVT.isVector() &&

660

PartEVT.getVectorElementType().bitsGE(

661

ValueVT.getVectorElementType()) &&

662

PartEVT.getVectorElementCount() ==

663

ValueVT.getVectorElementCount()) {

664

665

// Promoted vector extract

666

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

667

} else {

668

if (ValueVT.getVectorNumElements() == 1) {

669

Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,

670

DAG.getVectorIdxConstant(0, DL));

671

} else {

672

assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&((PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
"lossy conversion of vector to scalar type") ? static_cast<
void> (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 673, __PRETTY_FUNCTION__))

673

"lossy conversion of vector to scalar type")((PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
"lossy conversion of vector to scalar type") ? static_cast<
void> (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 673, __PRETTY_FUNCTION__));

674

EVT IntermediateType =

675

EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

676

Val = DAG.getBitcast(IntermediateType, Val);

677

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

678

}

679

}

680

681

assert(Val.getValueType() == PartVT && "Unexpected vector part value type")((Val.getValueType() == PartVT && "Unexpected vector part value type"
) ? static_cast<void> (0) : __assert_fail ("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 681, __PRETTY_FUNCTION__));

682

Parts[0] = Val;

683

return;

684

}

685

686

// Handle a multi-element vector.

687

EVT IntermediateVT;

688

MVT RegisterVT;

689

unsigned NumIntermediates;

690

unsigned NumRegs;

691

if (IsABIRegCopy) {

692

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

693

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

694

NumIntermediates, RegisterVT);

695

} else {

696

NumRegs =

697

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

698

NumIntermediates, RegisterVT);

699

}

700

701

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 701, __PRETTY_FUNCTION__));

702

NumParts = NumRegs; // Silence a compiler warning.

703

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 703, __PRETTY_FUNCTION__));

704

705

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&((IntermediateVT.isScalableVector() == ValueVT.isScalableVector
() && "Mixing scalable and fixed vectors when copying in parts"
) ? static_cast<void> (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 706, __PRETTY_FUNCTION__))

706

"Mixing scalable and fixed vectors when copying in parts")((IntermediateVT.isScalableVector() == ValueVT.isScalableVector
() && "Mixing scalable and fixed vectors when copying in parts"
) ? static_cast<void> (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 706, __PRETTY_FUNCTION__));

707

708

Optional<ElementCount> DestEltCnt;

709

710

if (IntermediateVT.isVector())

711

DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;

712

else

713

DestEltCnt = ElementCount::getFixed(NumIntermediates);

714

715

EVT BuiltVectorTy = EVT::getVectorVT(

716

*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());

717

if (ValueVT != BuiltVectorTy) {

718

if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))

719

Val = Widened;

720

721

Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);

722

}

723

724

// Split the vector into intermediate operands.

725

SmallVector<SDValue, 8> Ops(NumIntermediates);

726

for (unsigned i = 0; i != NumIntermediates; ++i) {

727

if (IntermediateVT.isVector()) {

728

// This does something sensible for scalable vectors - see the

729

// definition of EXTRACT_SUBVECTOR for further details.

730

unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();

731

Ops[i] =

732

DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,

733

DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));

734

} else {

735

Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,

736

DAG.getVectorIdxConstant(i, DL));

737

}

738

}

739

740

// Split the intermediate operands into legal parts.

741

if (NumParts == NumIntermediates) {

742

// If the register was not expanded, promote or copy the value,

743

// as appropriate.

744

for (unsigned i = 0; i != NumParts; ++i)

745

getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

746

} else if (NumParts > 0) {

747

// If the intermediate type was expanded, split each the value into

748

// legal parts.

749

assert(NumIntermediates != 0 && "division by zero")((NumIntermediates != 0 && "division by zero") ? static_cast
<void> (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 749, __PRETTY_FUNCTION__));

750

assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 751, __PRETTY_FUNCTION__))

751

"Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 751, __PRETTY_FUNCTION__));

752

unsigned Factor = NumParts / NumIntermediates;

753

for (unsigned i = 0; i != NumIntermediates; ++i)

754

getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,

755

CallConv);

756

}

757

}

758

759

RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,

760

EVT valuevt, Optional<CallingConv::ID> CC)

761

: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),

762

RegCount(1, regs.size()), CallConv(CC) {}

763

764

RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,

765

const DataLayout &DL, unsigned Reg, Type *Ty,

766

Optional<CallingConv::ID> CC) {

767

ComputeValueVTs(TLI, DL, Ty, ValueVTs);

768

769

CallConv = CC;

770

771

for (EVT ValueVT : ValueVTs) {

772

unsigned NumRegs =

773

isABIMangled()

774

? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)

775

: TLI.getNumRegisters(Context, ValueVT);

776

MVT RegisterVT =

777

isABIMangled()

778

? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)

779

: TLI.getRegisterType(Context, ValueVT);

780

for (unsigned i = 0; i != NumRegs; ++i)

781

Regs.push_back(Reg + i);

782

RegVTs.push_back(RegisterVT);

783

RegCount.push_back(NumRegs);

784

Reg += NumRegs;

785

}

786

}

787

788

SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,

789

FunctionLoweringInfo &FuncInfo,

790

const SDLoc &dl, SDValue &Chain,

791

SDValue *Flag, const Value *V) const {

792

// A Value with type {} or [0 x %t] needs no registers.

793

if (ValueVTs.empty())

794

return SDValue();

795

796

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

797

798

// Assemble the legal parts into the final values.

799

SmallVector<SDValue, 4> Values(ValueVTs.size());

800

SmallVector<SDValue, 8> Parts;

801

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

802

// Copy the legal parts from the registers.

803

EVT ValueVT = ValueVTs[Value];

804

unsigned NumRegs = RegCount[Value];

805

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

806

*DAG.getContext(),

807

CallConv.getValue(), RegVTs[Value])

808

: RegVTs[Value];

809

810

Parts.resize(NumRegs);

811

for (unsigned i = 0; i != NumRegs; ++i) {

812

SDValue P;

813

if (!Flag) {

814

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);

815

} else {

816

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);

817

*Flag = P.getValue(2);

818

}

819

820

Chain = P.getValue(1);

821

Parts[i] = P;

822

823

// If the source register was virtual and if we know something about it,

824

// add an assert node.

825

if (!Register::isVirtualRegister(Regs[Part + i]) ||

826

!RegisterVT.isInteger())

827

continue;

828

829

const FunctionLoweringInfo::LiveOutInfo *LOI =

830

FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);

831

if (!LOI)

832

continue;

833

834

unsigned RegSize = RegisterVT.getScalarSizeInBits();

835

unsigned NumSignBits = LOI->NumSignBits;

836

unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

837

838

if (NumZeroBits == RegSize) {

839

// The current value is a zero.

840

// Explicitly express that as it would be easier for

841

// optimizations to kick in.

842

Parts[i] = DAG.getConstant(0, dl, RegisterVT);

843

continue;

844

}

845

846

// FIXME: We capture more information than the dag can represent. For

847

// now, just use the tightest assertzext/assertsext possible.

848

bool isSExt;

849

EVT FromVT(MVT::Other);

850

if (NumZeroBits) {

851

FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);

852

isSExt = false;

853

} else if (NumSignBits > 1) {

854

FromVT =

855

EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);

856

isSExt = true;

857

} else {

858

continue;

859

}

860

// Add an assertion node.

861

assert(FromVT != MVT::Other)((FromVT != MVT::Other) ? static_cast<void> (0) : __assert_fail
("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 861, __PRETTY_FUNCTION__));

862

Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,

863

RegisterVT, P, DAG.getValueType(FromVT));

864

}

865

866

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,

867

RegisterVT, ValueVT, V, CallConv);

868

Part += NumRegs;

869

Parts.clear();

870

}

871

872

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);

873

}

874

875

void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,

876

const SDLoc &dl, SDValue &Chain, SDValue *Flag,

877

const Value *V,

878

ISD::NodeType PreferredExtendType) const {

879

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

880

ISD::NodeType ExtendKind = PreferredExtendType;

881

882

// Get the list of the values's legal parts.

883

unsigned NumRegs = Regs.size();

884

SmallVector<SDValue, 8> Parts(NumRegs);

885

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

886

unsigned NumParts = RegCount[Value];

887

888

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

889

*DAG.getContext(),

890

CallConv.getValue(), RegVTs[Value])

891

: RegVTs[Value];

892

893

if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))

894

ExtendKind = ISD::ZERO_EXTEND;

895

896

getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],

897

NumParts, RegisterVT, V, CallConv, ExtendKind);

898

Part += NumParts;

899

}

900

901

// Copy the parts into the registers.

902

SmallVector<SDValue, 8> Chains(NumRegs);

903

for (unsigned i = 0; i != NumRegs; ++i) {

904

SDValue Part;

905

if (!Flag) {

906

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);

907

} else {

908

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);

909

*Flag = Part.getValue(1);

910

}

911

912

Chains[i] = Part.getValue(0);

913

}

914

915

if (NumRegs == 1 || Flag)

916

// If NumRegs > 1 && Flag is used then the use of the last CopyToReg is

917

// flagged to it. That is the CopyToReg nodes and the user are considered

918

// a single scheduling unit. If we create a TokenFactor and return it as

919

// chain, then the TokenFactor is both a predecessor (operand) of the

920

// user as well as a successor (the TF operands are flagged to the user).

921

// c1, f1 = CopyToReg

922

// c2, f2 = CopyToReg

923

// c3 = TokenFactor c1, c2

924

// ...

925

// = op c3, ..., f2

926

Chain = Chains[NumRegs-1];

927

else

928

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);

929

}

930

931

void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,

932

unsigned MatchingIdx, const SDLoc &dl,

933

SelectionDAG &DAG,

934

std::vector<SDValue> &Ops) const {

935

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

936

937

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());

938

if (HasMatching)

939

Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);

940

else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {

941

// Put the register class of the virtual registers in the flag word. That

942

// way, later passes can recompute register class constraints for inline

943

// assembly as well as normal instructions.

944

// Don't do this for tied operands that can use the regclass information

945

// from the def.

946

const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

947

const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());

948

Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());

949

}

950

951

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);

952

Ops.push_back(Res);

953

954

if (Code == InlineAsm::Kind_Clobber) {

955

// Clobbers should always have a 1:1 mapping with registers, and may

956

// reference registers that have illegal (e.g. vector) types. Hence, we

957

// shouldn't try to apply any sort of splitting logic to them.

958

assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&((Regs.size() == RegVTs.size() && Regs.size() == ValueVTs
.size() && "No 1:1 mapping from clobbers to regs?") ?
static_cast<void> (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 959, __PRETTY_FUNCTION__))

959

"No 1:1 mapping from clobbers to regs?")((Regs.size() == RegVTs.size() && Regs.size() == ValueVTs
.size() && "No 1:1 mapping from clobbers to regs?") ?
static_cast<void> (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 959, __PRETTY_FUNCTION__));

960

unsigned SP = TLI.getStackPointerRegisterToSaveRestore();

961

(void)SP;

962

for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {

963

Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));

964

assert((((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 967, __PRETTY_FUNCTION__))

965

(Regs[I] != SP ||(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 967, __PRETTY_FUNCTION__))

966

DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 967, __PRETTY_FUNCTION__))

967

"If we clobbered the stack pointer, MFI should know about it.")(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 967, __PRETTY_FUNCTION__));

968

}

969

return;

970

}

971

972

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {

973

unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);

974

MVT RegisterVT = RegVTs[Value];

975

for (unsigned i = 0; i != NumRegs; ++i) {

976

assert(Reg < Regs.size() && "Mismatch in # registers expected")((Reg < Regs.size() && "Mismatch in # registers expected"
) ? static_cast<void> (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 976, __PRETTY_FUNCTION__));

977

unsigned TheReg = Regs[Reg++];

978

Ops.push_back(DAG.getRegister(TheReg, RegisterVT));

979

}

980

}

981

}

982

983

SmallVector<std::pair<unsigned, unsigned>, 4>

984

RegsForValue::getRegsAndSizes() const {

985

SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;

986

unsigned I = 0;

987

for (auto CountAndVT : zip_first(RegCount, RegVTs)) {

988

unsigned RegCount = std::get<0>(CountAndVT);

989

MVT RegisterVT = std::get<1>(CountAndVT);

990

unsigned RegisterSize = RegisterVT.getSizeInBits();

991

for (unsigned E = I + RegCount; I != E; ++I)

992

OutVec.push_back(std::make_pair(Regs[I], RegisterSize));

993

}

994

return OutVec;

995

}

996

997

void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,

998

const TargetLibraryInfo *li) {

999

AA = aa;

1000

GFI = gfi;

1001

LibInfo = li;

1002

DL = &DAG.getDataLayout();

1003

Context = DAG.getContext();

1004

LPadToCallSiteMap.clear();

1005

SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());

1006

}

1007

1008

void SelectionDAGBuilder::clear() {

1009

NodeMap.clear();

1010

UnusedArgNodeMap.clear();

1011

PendingLoads.clear();

1012

PendingExports.clear();

1013

PendingConstrainedFP.clear();

1014

PendingConstrainedFPStrict.clear();

1015

CurInst = nullptr;

1016

HasTailCall = false;

1017

SDNodeOrder = LowestSDNodeOrder;

1018

StatepointLowering.clear();

1019

}

1020

1021

void SelectionDAGBuilder::clearDanglingDebugInfo() {

1022

DanglingDebugInfoMap.clear();

1023

}

1024

1025

// Update DAG root to include dependencies on Pending chains.

1026

SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {

1027

SDValue Root = DAG.getRoot();

1028

1029

if (Pending.empty())

1030

return Root;

1031

1032

// Add current root to PendingChains, unless we already indirectly

1033

// depend on it.

1034

if (Root.getOpcode() != ISD::EntryToken) {

1035

unsigned i = 0, e = Pending.size();

1036

for (; i != e; ++i) {

1037

assert(Pending[i].getNode()->getNumOperands() > 1)((Pending[i].getNode()->getNumOperands() > 1) ? static_cast
<void> (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1037, __PRETTY_FUNCTION__));

1038

if (Pending[i].getNode()->getOperand(0) == Root)

1039

break; // Don't add the root if we already indirectly depend on it.

1040

}

1041

1042

if (i == e)

1043

Pending.push_back(Root);

1044

}

1045

1046

if (Pending.size() == 1)

1047

Root = Pending[0];

1048

else

1049

Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

1050

1051

DAG.setRoot(Root);

1052

Pending.clear();

1053

return Root;

1054

}

1055

1056

SDValue SelectionDAGBuilder::getMemoryRoot() {

1057

return updateRoot(PendingLoads);

1058

}

1059

1060

SDValue SelectionDAGBuilder::getRoot() {

1061

// Chain up all pending constrained intrinsics together with all

1062

// pending loads, by simply appending them to PendingLoads and

1063

// then calling getMemoryRoot().

1064

PendingLoads.reserve(PendingLoads.size() +

1065

PendingConstrainedFP.size() +

1066

PendingConstrainedFPStrict.size());

1067

PendingLoads.append(PendingConstrainedFP.begin(),

1068

PendingConstrainedFP.end());

1069

PendingLoads.append(PendingConstrainedFPStrict.begin(),

1070

PendingConstrainedFPStrict.end());

1071

PendingConstrainedFP.clear();

1072

PendingConstrainedFPStrict.clear();

1073

return getMemoryRoot();

1074

}

1075

1076

SDValue SelectionDAGBuilder::getControlRoot() {

1077

// We need to emit pending fpexcept.strict constrained intrinsics,

1078

// so append them to the PendingExports list.

1079

PendingExports.append(PendingConstrainedFPStrict.begin(),

1080

PendingConstrainedFPStrict.end());

1081

PendingConstrainedFPStrict.clear();

1082

return updateRoot(PendingExports);

1083

}

1084

1085

void SelectionDAGBuilder::visit(const Instruction &I) {

1086

// Set up outgoing PHI node register values before emitting the terminator.

1087

if (I.isTerminator()) {

1088

HandlePHINodesInSuccessorBlocks(I.getParent());

1089

}

1090

1091

// Increase the SDNodeOrder if dealing with a non-debug instruction.

1092

if (!isa<DbgInfoIntrinsic>(I))

1093

++SDNodeOrder;

1094

1095

CurInst = &I;

1096

1097

visit(I.getOpcode(), I);

1098

1099

if (!I.isTerminator() && !HasTailCall &&

1100

!isa<GCStatepointInst>(I)) // statepoints handle their exports internally

1101

CopyToExportRegsIfNeeded(&I);

1102

1103

CurInst = nullptr;

1104

}

1105

1106

void SelectionDAGBuilder::visitPHI(const PHINode &) {

1107

llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1107);

1108

}

1109

1110

void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {

1111

// Note: this doesn't use InstVisitor, because it has to work with

1112

// ConstantExpr's in addition to instructions.

1113

switch (Opcode) {

1114

default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1114);

1115

// Build the switch statement using the Instruction.def file.

1116

#define HANDLE_INST(NUM, OPCODE, CLASS) \

1117

case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;

1118

#include "llvm/IR/Instruction.def"

1119

}

1120

}

1121

1122

void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,

1123

const DIExpression *Expr) {

1124

auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {

1125

const DbgValueInst *DI = DDI.getDI();

1126

DIVariable *DanglingVariable = DI->getVariable();

1127

DIExpression *DanglingExpr = DI->getExpression();

1128

if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {

1129

LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false);

1130

return true;

1131

}

1132

return false;

1133

};

1134

1135

for (auto &DDIMI : DanglingDebugInfoMap) {

1136

DanglingDebugInfoVector &DDIV = DDIMI.second;

1137

1138

// If debug info is to be dropped, run it through final checks to see

1139

// whether it can be salvaged.

1140

for (auto &DDI : DDIV)

1141

if (isMatchingDbgValue(DDI))

1142

salvageUnresolvedDbgValue(DDI);

1143

1144

DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());

1145

}

1146

}

1147

1148

// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,

1149

// generate the debug data structures now that we've seen its definition.

1150

void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,

1151

SDValue Val) {

1152

auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);

1153

if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())

1154

return;

1155

1156

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;

1157

for (auto &DDI : DDIV) {

1158

const DbgValueInst *DI = DDI.getDI();

1159

assert(DI && "Ill-formed DanglingDebugInfo")((DI && "Ill-formed DanglingDebugInfo") ? static_cast
<void> (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1159, __PRETTY_FUNCTION__));

1160

DebugLoc dl = DDI.getdl();

1161

unsigned ValSDNodeOrder = Val.getNode()->getIROrder();

1162

unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();

1163

DILocalVariable *Variable = DI->getVariable();

1164

DIExpression *Expr = DI->getExpression();

1165

assert(Variable->isValidLocationForIntrinsic(dl) &&((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1166, __PRETTY_FUNCTION__))

1166

"Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1166, __PRETTY_FUNCTION__));

1167

SDDbgValue *SDV;

1168

if (Val.getNode()) {

1169

// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a

1170

// FuncArgumentDbgValue (it would be hoisted to the function entry, and if

1171

// we couldn't resolve it directly when examining the DbgValue intrinsic

1172

// in the first place we should not be more successful here). Unless we

1173

// have some test case that prove this to be correct we should avoid

1174

// calling EmitFuncArgumentDbgValue here.

1175

if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {

1176

LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)

1177

<< DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false);

1178

LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);

1179

// Increase the SDNodeOrder for the DbgValue here to make sure it is

1180

// inserted after the definition of Val when emitting the instructions

1181

// after ISel. An alternative could be to teach

1182

// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.

1183

LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1184

<< "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1185

<< ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);

1186

SDV = getDbgValue(Val, Variable, Expr, dl,

1187

std::max(DbgSDNodeOrder, ValSDNodeOrder));

1188

DAG.AddDbgValue(SDV, Val.getNode(), false);

1189

} else

1190

LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)

1191

<< "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false);

1192

} else {

1193

LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false);

1194

auto Undef =

1195

UndefValue::get(DDI.getDI()->getVariableLocation()->getType());

1196

auto SDV =

1197

DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);

1198

DAG.AddDbgValue(SDV, nullptr, false);

1199

}

1200

}

1201

DDIV.clear();

1202

}

1203

1204

void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {

1205

Value *V = DDI.getDI()->getValue();

1206

DILocalVariable *Var = DDI.getDI()->getVariable();

1207

DIExpression *Expr = DDI.getDI()->getExpression();

1208

DebugLoc DL = DDI.getdl();

1209

DebugLoc InstDL = DDI.getDI()->getDebugLoc();

1210

unsigned SDOrder = DDI.getSDNodeOrder();

1211

1212

// Currently we consider only dbg.value intrinsics -- we tell the salvager

1213

// that DW_OP_stack_value is desired.

1214

assert(isa<DbgValueInst>(DDI.getDI()))((isa<DbgValueInst>(DDI.getDI())) ? static_cast<void
> (0) : __assert_fail ("isa<DbgValueInst>(DDI.getDI())"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1214, __PRETTY_FUNCTION__));

1215

bool StackValue = true;

1216

1217

// Can this Value can be encoded without any further work?

1218

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))

1219

return;

1220

1221

// Attempt to salvage back through as many instructions as possible. Bail if

1222

// a non-instruction is seen, such as a constant expression or global

1223

// variable. FIXME: Further work could recover those too.

1224

while (isa<Instruction>(V)) {

1225

Instruction &VAsInst = *cast<Instruction>(V);

1226

DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);

1227

1228

// If we cannot salvage any further, and haven't yet found a suitable debug

1229

// expression, bail out.

1230

if (!NewExpr)

1231

break;

1232

1233

// New value and expr now represent this debuginfo.

1234

V = VAsInst.getOperand(0);

1235

Expr = NewExpr;

1236

1237

// Some kind of simplification occurred: check whether the operand of the

1238

// salvaged debug expression can be encoded in this DAG.

1239

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {

1240

LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false)

1241

<< DDI.getDI() << "\nBy stripping back to:\n " << V)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false);

1242

return;

1243

}

1244

}

1245

1246

// This was the final opportunity to salvage this debug information, and it

1247

// couldn't be done. Place an undef DBG_VALUE at this location to terminate

1248

// any earlier variable location.

1249

auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());

1250

auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);

1251

DAG.AddDbgValue(SDV, nullptr, false);

1252

1253

LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false)

1254

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false);

1255

LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
)

1256

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
);

1257

}

1258

1259

bool SelectionDAGBuilder::handleDebugValue(const Value *V, DILocalVariable *Var,

1260

DIExpression *Expr, DebugLoc dl,

1261

DebugLoc InstDL, unsigned Order) {

1262

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1263

SDDbgValue *SDV;

1264

if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||

1265

isa<ConstantPointerNull>(V)) {

1266

SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);

1267

DAG.AddDbgValue(SDV, nullptr, false);

1268

return true;

1269

}

1270

1271

// If the Value is a frame index, we can create a FrameIndex debug value

1272

// without relying on the DAG at all.

1273

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1274

auto SI = FuncInfo.StaticAllocaMap.find(AI);

1275

if (SI != FuncInfo.StaticAllocaMap.end()) {

1276

auto SDV =

1277

DAG.getFrameIndexDbgValue(Var, Expr, SI->second,

1278

/*IsIndirect*/ false, dl, SDNodeOrder);

1279

// Do not attach the SDNodeDbgValue to an SDNode: this variable location

1280

// is still available even if the SDNode gets optimized out.

1281

DAG.AddDbgValue(SDV, nullptr, false);

1282

return true;

1283

}

1284

}

1285

1286

// Do not use getValue() in here; we don't want to generate code at

1287

// this point if it hasn't been done yet.

1288

SDValue N = NodeMap[V];

1289

if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.

1290

N = UnusedArgNodeMap[V];

1291

if (N.getNode()) {

1292

if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))

1293

return true;

1294

SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);

1295

DAG.AddDbgValue(SDV, N.getNode(), false);

1296

return true;

1297

}

1298

1299

// Special rules apply for the first dbg.values of parameter variables in a

1300

// function. Identify them by the fact they reference Argument Values, that

1301

// they're parameters, and they are parameters of the current function. We

1302

// need to let them dangle until they get an SDNode.

1303

bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&

1304

!InstDL.getInlinedAt();

1305

if (!IsParamOfFunc) {

1306

// The value is not used in this block yet (or it would have an SDNode).

1307

// We still want the value to appear for the user if possible -- if it has

1308

// an associated VReg, we can refer to that instead.

1309

auto VMI = FuncInfo.ValueMap.find(V);

1310

if (VMI != FuncInfo.ValueMap.end()) {

1311

unsigned Reg = VMI->second;

1312

// If this is a PHI node, it may be split up into several MI PHI nodes

1313

// (in FunctionLoweringInfo::set).

1314

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,

1315

V->getType(), None);

1316

if (RFV.occupiesMultipleRegs()) {

1317

unsigned Offset = 0;

1318

unsigned BitsToDescribe = 0;

1319

if (auto VarSize = Var->getSizeInBits())

1320

BitsToDescribe = *VarSize;

1321

if (auto Fragment = Expr->getFragmentInfo())

1322

BitsToDescribe = Fragment->SizeInBits;

1323

for (auto RegAndSize : RFV.getRegsAndSizes()) {

1324

unsigned RegisterSize = RegAndSize.second;

1325

// Bail out if all bits are described already.

1326

if (Offset >= BitsToDescribe)

1327

break;

1328

unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)

1329

? BitsToDescribe - Offset

1330

: RegisterSize;

1331

auto FragmentExpr = DIExpression::createFragmentExpression(

1332

Expr, Offset, FragmentSize);

1333

if (!FragmentExpr)

1334

continue;

1335

SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,

1336

false, dl, SDNodeOrder);

1337

DAG.AddDbgValue(SDV, nullptr, false);

1338

Offset += RegisterSize;

1339

}

1340

} else {

1341

SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);

1342

DAG.AddDbgValue(SDV, nullptr, false);

1343

}

1344

return true;

1345

}

1346

}

1347

1348

return false;

1349

}

1350

1351

void SelectionDAGBuilder::resolveOrClearDbgInfo() {

1352

// Try to fixup any remaining dangling debug info -- and drop it if we can't.

1353

for (auto &Pair : DanglingDebugInfoMap)

1354

for (auto &DDI : Pair.second)

1355

salvageUnresolvedDbgValue(DDI);

1356

clearDanglingDebugInfo();

1357

}

1358

1359

/// getCopyFromRegs - If there was virtual register allocated for the value V

1360

/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.

1361

SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {

1362

DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);

1363

SDValue Result;

1364

1365

if (It != FuncInfo.ValueMap.end()) {

1366

Register InReg = It->second;

1367

1368

RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1369

DAG.getDataLayout(), InReg, Ty,

1370

None); // This is not an ABI copy.

1371

SDValue Chain = DAG.getEntryNode();

1372

Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,

1373

V);

1374

resolveDanglingDebugInfo(V, Result);

1375

}

1376

1377

return Result;

1378

}

1379

1380

/// getValue - Return an SDValue for the given Value.

1381

SDValue SelectionDAGBuilder::getValue(const Value *V) {

1382

// If we already have an SDValue for this value, use it. It's important

1383

// to do this first, so that we don't create a CopyFromReg if we already

1384

// have a regular SDValue.

1385

SDValue &N = NodeMap[V];

1386

if (N.getNode()) return N;

1387

1388

// If there's a virtual register allocated and initialized for this

1389

// value, use it.

1390

if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))

1391

return copyFromReg;

1392

1393

// Otherwise create a new SDValue and remember it.

1394

SDValue Val = getValueImpl(V);

1395

NodeMap[V] = Val;

1396

resolveDanglingDebugInfo(V, Val);

1397

return Val;

1398

}

1399

1400

/// getNonRegisterValue - Return an SDValue for the given Value, but

1401

/// don't look in FuncInfo.ValueMap for a virtual register.

1402

SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {

1403

// If we already have an SDValue for this value, use it.

1404

SDValue &N = NodeMap[V];

1405

if (N.getNode()) {

1406

if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {

1407

// Remove the debug location from the node as the node is about to be used

1408

// in a location which may differ from the original debug location. This

1409

// is relevant to Constant and ConstantFP nodes because they can appear

1410

// as constant expressions inside PHI nodes.

1411

N->setDebugLoc(DebugLoc());

1412

}

1413

return N;

1414

}

1415

1416

// Otherwise create a new SDValue and remember it.

1417

SDValue Val = getValueImpl(V);

1418

NodeMap[V] = Val;

1419

resolveDanglingDebugInfo(V, Val);

1420

return Val;

1421

}

1422

1423

/// getValueImpl - Helper function for getValue and getNonRegisterValue.

1424

/// Create an SDValue for the given value.

1425

SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {

1426

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1427

1428

if (const Constant *C = dyn_cast<Constant>(V)) {

1429

EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

1430

1431

if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))

1432

return DAG.getConstant(*CI, getCurSDLoc(), VT);

1433

1434

if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))

1435

return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

1436

1437

if (isa<ConstantPointerNull>(C)) {

1438

unsigned AS = V->getType()->getPointerAddressSpace();

1439

return DAG.getConstant(0, getCurSDLoc(),

1440

TLI.getPointerTy(DAG.getDataLayout(), AS));

1441

}

1442

1443

if (match(C, m_VScale(DAG.getDataLayout())))

1444

return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

1445

1446

if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))

1447

return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

1448

1449

if (isa<UndefValue>(C) && !V->getType()->isAggregateType())

1450

return DAG.getUNDEF(VT);

1451

1452

if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {

1453

visit(CE->getOpcode(), *CE);

1454

SDValue N1 = NodeMap[V];

1455

assert(N1.getNode() && "visit didn't populate the NodeMap!")((N1.getNode() && "visit didn't populate the NodeMap!"
) ? static_cast<void> (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1455, __PRETTY_FUNCTION__));

1456

return N1;

1457

}

1458

1459

if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {

1460

SmallVector<SDValue, 4> Constants;

1461

for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();

1462

OI != OE; ++OI) {

1463

SDNode *Val = getValue(*OI).getNode();

1464

// If the operand is an empty aggregate, there are no values.

1465

if (!Val) continue;

1466

// Add each leaf value from the operand to the Constants list

1467

// to form a flattened list of all the values.

1468

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1469

Constants.push_back(SDValue(Val, i));

1470

}

1471

1472

return DAG.getMergeValues(Constants, getCurSDLoc());

1473

}

1474

1475

if (const ConstantDataSequential *CDS =

1476

dyn_cast<ConstantDataSequential>(C)) {

1477

SmallVector<SDValue, 4> Ops;

1478

for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {

1479

SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();

1480

// Add each leaf value from the operand to the Constants list

1481

// to form a flattened list of all the values.

1482

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1483

Ops.push_back(SDValue(Val, i));

1484

}

1485

1486

if (isa<ArrayType>(CDS->getType()))

1487

return DAG.getMergeValues(Ops, getCurSDLoc());

1488

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1489

}

1490

1491

if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {

1492

assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1493, __PRETTY_FUNCTION__))

1493

"Unknown struct or array constant!")(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1493, __PRETTY_FUNCTION__));

1494

1495

SmallVector<EVT, 4> ValueVTs;

1496

ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);

1497

unsigned NumElts = ValueVTs.size();

1498

if (NumElts == 0)

1499

return SDValue(); // empty struct

1500

SmallVector<SDValue, 4> Constants(NumElts);

1501

for (unsigned i = 0; i != NumElts; ++i) {

1502

EVT EltVT = ValueVTs[i];

1503

if (isa<UndefValue>(C))

1504

Constants[i] = DAG.getUNDEF(EltVT);

1505

else if (EltVT.isFloatingPoint())

1506

Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1507

else

1508

Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);

1509

}

1510

1511

return DAG.getMergeValues(Constants, getCurSDLoc());

1512

}

1513

1514

if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))

1515

return DAG.getBlockAddress(BA, VT);

1516

1517

VectorType *VecTy = cast<VectorType>(V->getType());

1518

1519

// Now that we know the number and type of the elements, get that number of

1520

// elements into the Ops array based on what kind of constant it is.

1521

if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {

1522

SmallVector<SDValue, 16> Ops;

1523

unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();

1524

for (unsigned i = 0; i != NumElements; ++i)

1525

Ops.push_back(getValue(CV->getOperand(i)));

1526

1527

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1528

} else if (isa<ConstantAggregateZero>(C)) {

1529

EVT EltVT =

1530

TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

1531

1532

SDValue Op;

1533

if (EltVT.isFloatingPoint())

1534

Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1535

else

1536

Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

1537

1538

if (isa<ScalableVectorType>(VecTy))

1539

return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);

1540

else {

1541

SmallVector<SDValue, 16> Ops;

1542

Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);

1543

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1544

}

1545

}

1546

llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1546);

1547

}

1548

1549

// If this is a static alloca, generate it as the frameindex instead of

1550

// computation.

1551

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1552

DenseMap<const AllocaInst*, int>::iterator SI =

1553

FuncInfo.StaticAllocaMap.find(AI);

1554

if (SI != FuncInfo.StaticAllocaMap.end())

1555

return DAG.getFrameIndex(SI->second,

1556

TLI.getFrameIndexTy(DAG.getDataLayout()));

1557

}

1558

1559

// If this is an instruction which fast-isel has deferred, select it now.

1560

if (const Instruction *Inst = dyn_cast<Instruction>(V)) {

1561

unsigned InReg = FuncInfo.InitializeRegForValue(Inst);

1562

1563

RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,

1564

Inst->getType(), None);

1565

SDValue Chain = DAG.getEntryNode();

1566

return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);

1567

}

1568

1569

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {

1570

return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

1571

}

1572

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1572);

1573

}

1574

1575

void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {

1576

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1577

bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;

1578

bool IsCoreCLR = Pers == EHPersonality::CoreCLR;

1579

bool IsSEH = isAsynchronousEHPersonality(Pers);

1580

MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;

1581

if (!IsSEH)

1582

CatchPadMBB->setIsEHScopeEntry();

1583

// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.

1584

if (IsMSVCCXX || IsCoreCLR)

1585

CatchPadMBB->setIsEHFuncletEntry();

1586

}

1587

1588

void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {

1589

// Update machine-CFG edge.

1590

MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];

1591

FuncInfo.MBB->addSuccessor(TargetMBB);

1592

1593

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1594

bool IsSEH = isAsynchronousEHPersonality(Pers);

1595

if (IsSEH) {

1596

// If this is not a fall-through branch or optimizations are switched off,

1597

// emit the branch.

1598

if (TargetMBB != NextBlock(FuncInfo.MBB) ||

1599

TM.getOptLevel() == CodeGenOpt::None)

1600

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,

1601

getControlRoot(), DAG.getBasicBlock(TargetMBB)));

1602

return;

1603

}

1604

1605

// Figure out the funclet membership for the catchret's successor.

1606

// This will be used by the FuncletLayout pass to determine how to order the

1607

// BB's.

1608

// A 'catchret' returns to the outer scope's color.

1609

Value *ParentPad = I.getCatchSwitchParentPad();

1610

const BasicBlock *SuccessorColor;

1611

if (isa<ConstantTokenNone>(ParentPad))

1612

SuccessorColor = &FuncInfo.Fn->getEntryBlock();

1613

else

1614

SuccessorColor = cast<Instruction>(ParentPad)->getParent();

1615

assert(SuccessorColor && "No parent funclet for catchret!")((SuccessorColor && "No parent funclet for catchret!"
) ? static_cast<void> (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1615, __PRETTY_FUNCTION__));

1616

MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];

1617

assert(SuccessorColorMBB && "No MBB for SuccessorColor!")((SuccessorColorMBB && "No MBB for SuccessorColor!") ?
static_cast<void> (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1617, __PRETTY_FUNCTION__));

1618

1619

// Create the terminator node.

1620

SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,

1621

getControlRoot(), DAG.getBasicBlock(TargetMBB),

1622

DAG.getBasicBlock(SuccessorColorMBB));

1623

DAG.setRoot(Ret);

1624

}

1625

1626

void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {

1627

// Don't emit any special code for the cleanuppad instruction. It just marks

1628

// the start of an EH scope/funclet.

1629

FuncInfo.MBB->setIsEHScopeEntry();

1630

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1631

if (Pers != EHPersonality::Wasm_CXX) {

1632

FuncInfo.MBB->setIsEHFuncletEntry();

1633

FuncInfo.MBB->setIsCleanupFuncletEntry();

1634

}

1635

}

1636

1637

// For wasm, there's alwyas a single catch pad attached to a catchswitch, and

1638

// the control flow always stops at the single catch pad, as it does for a

1639

// cleanup pad. In case the exception caught is not of the types the catch pad

1640

// catches, it will be rethrown by a rethrow.

1641

static void findWasmUnwindDestinations(

1642

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1643

BranchProbability Prob,

1644

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1645

&UnwindDests) {

1646

while (EHPadBB) {

1647

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1648

if (isa<CleanupPadInst>(Pad)) {

1649

// Stop on cleanup pads.

1650

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1651

UnwindDests.back().first->setIsEHScopeEntry();

1652

break;

1653

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1654

// Add the catchpad handlers to the possible destinations. We don't

1655

// continue to the unwind destination of the catchswitch for wasm.

1656

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1657

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1658

UnwindDests.back().first->setIsEHScopeEntry();

1659

}

1660

break;

1661

} else {

1662

continue;

1663

}

1664

}

1665

}

1666

1667

/// When an invoke or a cleanupret unwinds to the next EH pad, there are

1668

/// many places it could ultimately go. In the IR, we have a single unwind

1669

/// destination, but in the machine CFG, we enumerate all the possible blocks.

1670

/// This function skips over imaginary basic blocks that hold catchswitch

1671

/// instructions, and finds all the "real" machine

1672

/// basic block destinations. As those destinations may not be successors of

1673

/// EHPadBB, here we also calculate the edge probability to those destinations.

1674

/// The passed-in Prob is the edge probability to EHPadBB.

1675

static void findUnwindDestinations(

1676

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1677

BranchProbability Prob,

1678

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1679

&UnwindDests) {

1680

EHPersonality Personality =

1681

classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1682

bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;

1683

bool IsCoreCLR = Personality == EHPersonality::CoreCLR;

1684

bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;

1685

bool IsSEH = isAsynchronousEHPersonality(Personality);

1686

1687

if (IsWasmCXX) {

1688

findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);

1689

assert(UnwindDests.size() <= 1 &&((UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm"
) ? static_cast<void> (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1690, __PRETTY_FUNCTION__))

1690

"There should be at most one unwind destination for wasm")((UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm"
) ? static_cast<void> (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1690, __PRETTY_FUNCTION__));

1691

return;

1692

}

1693

1694

while (EHPadBB) {

1695

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1696

BasicBlock *NewEHPadBB = nullptr;

1697

if (isa<LandingPadInst>(Pad)) {

1698

// Stop on landingpads. They are not funclets.

1699

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1700

break;

1701

} else if (isa<CleanupPadInst>(Pad)) {

1702

// Stop on cleanup pads. Cleanups are always funclet entries for all known

1703

// personalities.

1704

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1705

UnwindDests.back().first->setIsEHScopeEntry();

1706

UnwindDests.back().first->setIsEHFuncletEntry();

1707

break;

1708

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1709

// Add the catchpad handlers to the possible destinations.

1710

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1711

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1712

// For MSVC++ and the CLR, catchblocks are funclets and need prologues.

1713

if (IsMSVCCXX || IsCoreCLR)

1714

UnwindDests.back().first->setIsEHFuncletEntry();

1715

if (!IsSEH)

1716

UnwindDests.back().first->setIsEHScopeEntry();

1717

}

1718

NewEHPadBB = CatchSwitch->getUnwindDest();

1719

} else {

1720

continue;

1721

}

1722

1723

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1724

if (BPI && NewEHPadBB)

1725

Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);

1726

EHPadBB = NewEHPadBB;

1727

}

1728

}

1729

1730

void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {

1731

// Update successor info.

1732

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

1733

auto UnwindDest = I.getUnwindDest();

1734

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1735

BranchProbability UnwindDestProb =

1736

(BPI && UnwindDest)

1737

? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)

1738

: BranchProbability::getZero();

1739

findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);

1740

for (auto &UnwindDest : UnwindDests) {

1741

UnwindDest.first->setIsEHPad();

1742

addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);

1743

}

1744

FuncInfo.MBB->normalizeSuccProbs();

1745

1746

// Create the terminator node.

1747

SDValue Ret =

1748

DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());

1749

DAG.setRoot(Ret);

1750

}

1751

1752

void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {

1753

report_fatal_error("visitCatchSwitch not yet implemented!");

1754

}

1755

1756

void SelectionDAGBuilder::visitRet(const ReturnInst &I) {

1757

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1758

auto &DL = DAG.getDataLayout();

1759

SDValue Chain = getControlRoot();

1760

SmallVector<ISD::OutputArg, 8> Outs;

1761

SmallVector<SDValue, 8> OutVals;

1762

1763

// Calls to @llvm.experimental.deoptimize don't generate a return value, so

1764

// lower

1765

//

1766

// %val = call <ty> @llvm.experimental.deoptimize()

1767

// ret <ty> %val

1768

//

1769

// differently.

1770

if (I.getParent()->getTerminatingDeoptimizeCall()) {

1771

LowerDeoptimizingReturn();

1772

return;

1773

}

1774

1775

if (!FuncInfo.CanLowerReturn) {

1776

unsigned DemoteReg = FuncInfo.DemoteRegister;

1777

const Function *F = I.getParent()->getParent();

1778

1779

// Emit a store of the return value through the virtual register.

1780

// Leave Outs empty so that LowerReturn won't try to load return

1781

// registers the usual way.

1782

SmallVector<EVT, 1> PtrValueVTs;

1783

ComputeValueVTs(TLI, DL,

1784

F->getReturnType()->getPointerTo(

1785

DAG.getDataLayout().getAllocaAddrSpace()),

1786

PtrValueVTs);

1787

1788

SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),

1789

DemoteReg, PtrValueVTs[0]);

1790

SDValue RetOp = getValue(I.getOperand(0));

1791

1792

SmallVector<EVT, 4> ValueVTs, MemVTs;

1793

SmallVector<uint64_t, 4> Offsets;

1794

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,

1795

&Offsets);

1796

unsigned NumValues = ValueVTs.size();

1797

1798

SmallVector<SDValue, 4> Chains(NumValues);

1799

Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());

1800

for (unsigned i = 0; i != NumValues; ++i) {

1801

// An aggregate return value cannot wrap around the address space, so

1802

// offsets to its parts don't wrap either.

1803

SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,

1804

TypeSize::Fixed(Offsets[i]));

1805

1806

SDValue Val = RetOp.getValue(RetOp.getResNo() + i);

1807

if (MemVTs[i] != ValueVTs[i])

1808

Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);

1809

Chains[i] = DAG.getStore(

1810

Chain, getCurSDLoc(), Val,

1811

// FIXME: better loc info would be nice.

1812

Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),

1813

commonAlignment(BaseAlign, Offsets[i]));

1814

}

1815

1816

Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),

1817

MVT::Other, Chains);

1818

} else if (I.getNumOperands() != 0) {

1819

SmallVector<EVT, 4> ValueVTs;

1820

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);

1821

unsigned NumValues = ValueVTs.size();

1822

if (NumValues) {

1823

SDValue RetOp = getValue(I.getOperand(0));

1824

1825

const Function *F = I.getParent()->getParent();

1826

1827

bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(

1828

I.getOperand(0)->getType(), F->getCallingConv(),

1829

/*IsVarArg*/ false);

1830

1831

ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

1832

if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1833

Attribute::SExt))

1834

ExtendKind = ISD::SIGN_EXTEND;

1835

else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1836

Attribute::ZExt))

1837

ExtendKind = ISD::ZERO_EXTEND;

1838

1839

LLVMContext &Context = F->getContext();

1840

bool RetInReg = F->getAttributes().hasAttribute(

1841

AttributeList::ReturnIndex, Attribute::InReg);

1842

1843

for (unsigned j = 0; j != NumValues; ++j) {

1844

EVT VT = ValueVTs[j];

1845

1846

if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())

1847

VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

1848

1849

CallingConv::ID CC = F->getCallingConv();

1850

1851

unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);

1852

MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);

1853

SmallVector<SDValue, 4> Parts(NumParts);

1854

getCopyToParts(DAG, getCurSDLoc(),

1855

SDValue(RetOp.getNode(), RetOp.getResNo() + j),

1856

&Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

1857

1858

// 'inreg' on function refers to return value

1859

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1860

if (RetInReg)

1861

Flags.setInReg();

1862

1863

if (I.getOperand(0)->getType()->isPointerTy()) {

1864

Flags.setPointer();

1865

Flags.setPointerAddrSpace(

1866

cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());

1867

}

1868

1869

if (NeedsRegBlock) {

1870

Flags.setInConsecutiveRegs();

1871

if (j == NumValues - 1)

1872

Flags.setInConsecutiveRegsLast();

1873

}

1874

1875

// Propagate extension type if any

1876

if (ExtendKind == ISD::SIGN_EXTEND)

1877

Flags.setSExt();

1878

else if (ExtendKind == ISD::ZERO_EXTEND)

1879

Flags.setZExt();

1880

1881

for (unsigned i = 0; i < NumParts; ++i) {

1882

Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),

1883

VT, /*isfixed=*/true, 0, 0));

1884

OutVals.push_back(Parts[i]);

1885

}

1886

}

1887

}

1888

}

1889

1890

// Push in swifterror virtual register as the last element of Outs. This makes

1891

// sure swifterror virtual register will be returned in the swifterror

1892

// physical register.

1893

const Function *F = I.getParent()->getParent();

1894

if (TLI.supportSwiftError() &&

1895

F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {

1896

assert(SwiftError.getFunctionArg() && "Need a swift error argument")((SwiftError.getFunctionArg() && "Need a swift error argument"
) ? static_cast<void> (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1896, __PRETTY_FUNCTION__));

1897

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1898

Flags.setSwiftError();

1899

Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,

1900

EVT(TLI.getPointerTy(DL)) /*argvt*/,

1901

true /*isfixed*/, 1 /*origidx*/,

1902

0 /*partOffs*/));

1903

// Create SDNode for the swifterror virtual register.

1904

OutVals.push_back(

1905

DAG.getRegister(SwiftError.getOrCreateVRegUseAt(

1906

&I, FuncInfo.MBB, SwiftError.getFunctionArg()),

1907

EVT(TLI.getPointerTy(DL))));

1908

}

1909

1910

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();

1911

CallingConv::ID CallConv =

1912

DAG.getMachineFunction().getFunction().getCallingConv();

1913

Chain = DAG.getTargetLoweringInfo().LowerReturn(

1914

Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

1915

1916

// Verify that the target's LowerReturn behaved as expected.

1917

assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1918, __PRETTY_FUNCTION__))

1918

"LowerReturn didn't return a valid chain!")((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1918, __PRETTY_FUNCTION__));

1919

1920

// Update the DAG with the new chain value resulting from return lowering.

1921

DAG.setRoot(Chain);

1922

}

1923

1924

/// CopyToExportRegsIfNeeded - If the given value has virtual registers

1925

/// created for it, emit nodes to copy the value into the virtual

1926

/// registers.

1927

void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

1928

// Skip empty types

1929

if (V->getType()->isEmptyTy())

1930

return;

1931

1932

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);

1933

if (VMI != FuncInfo.ValueMap.end()) {

1934

assert(!V->use_empty() && "Unused value assigned virtual registers!")((!V->use_empty() && "Unused value assigned virtual registers!"
) ? static_cast<void> (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1934, __PRETTY_FUNCTION__));

1935

CopyValueToVirtualRegister(V, VMI->second);

1936

}

1937

}

1938

1939

/// ExportFromCurrentBlock - If this condition isn't known to be exported from

1940

/// the current basic block, add it to ValueMap now so that we'll get a

1941

/// CopyTo/FromReg.

1942

void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {

1943

// No need to export constants.

1944

if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

1945

1946

// Already exported?

1947

if (FuncInfo.isExportedInst(V)) return;

1948

1949

unsigned Reg = FuncInfo.InitializeRegForValue(V);

1950

CopyValueToVirtualRegister(V, Reg);

1951

}

1952

1953

bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,

1954

const BasicBlock *FromBB) {

1955

// The operands of the setcc have to be in this block. We don't know

1956

// how to export them from some other block.

1957

if (const Instruction *VI = dyn_cast<Instruction>(V)) {

1958

// Can export from current BB.

1959

if (VI->getParent() == FromBB)

1960

return true;

1961

1962

// Is already exported, noop.

1963

return FuncInfo.isExportedInst(V);

1964

}

1965

1966

// If this is an argument, we can export it if the BB is the entry block or

1967

// if it is already exported.

1968

if (isa<Argument>(V)) {

1969

if (FromBB == &FromBB->getParent()->getEntryBlock())

1970

return true;

1971

1972

// Otherwise, can only export this if it is already exported.

1973

return FuncInfo.isExportedInst(V);

1974

}

1975

1976

// Otherwise, constants can always be exported.

1977

return true;

1978

}

1979

1980

/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.

1981

BranchProbability

1982

SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,

1983

const MachineBasicBlock *Dst) const {

1984

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1985

const BasicBlock *SrcBB = Src->getBasicBlock();

1986

const BasicBlock *DstBB = Dst->getBasicBlock();

1987

if (!BPI) {

1988

// If BPI is not available, set the default probability as 1 / N, where N is

1989

// the number of successors.

1990

auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);

1991

return BranchProbability(1, SuccSize);

1992

}

1993

return BPI->getEdgeProbability(SrcBB, DstBB);

1994

}

1995

1996

void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,

1997

MachineBasicBlock *Dst,

1998

BranchProbability Prob) {

1999

if (!FuncInfo.BPI)

2000

Src->addSuccessorWithoutProb(Dst);

2001

else {

2002

if (Prob.isUnknown())

2003

Prob = getEdgeProbability(Src, Dst);

2004

Src->addSuccessor(Dst, Prob);

2005

}

2006

}

2007

2008

static bool InBlock(const Value *V, const BasicBlock *BB) {

2009

if (const Instruction *I = dyn_cast<Instruction>(V))

2010

return I->getParent() == BB;

2011

return true;

2012

}

2013

2014

/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.

2015

/// This function emits a branch and is used at the leaves of an OR or an

2016

/// AND operator tree.

2017

void

2018

SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,

2019

MachineBasicBlock *TBB,

2020

MachineBasicBlock *FBB,

2021

MachineBasicBlock *CurBB,

2022

MachineBasicBlock *SwitchBB,

2023

BranchProbability TProb,

2024

BranchProbability FProb,

2025

bool InvertCond) {

2026

const BasicBlock *BB = CurBB->getBasicBlock();

2027

2028

// If the leaf of the tree is a comparison, merge the condition into

2029

// the caseblock.

2030

if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {

2031

// The operands of the cmp have to be in this block. We don't know

2032

// how to export them from some other block. If this is the first block

2033

// of the sequence, no exporting is needed.

2034

if (CurBB == SwitchBB ||

2035

(isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&

2036

isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {

2037

ISD::CondCode Condition;

2038

if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {

2039

ICmpInst::Predicate Pred =

2040

InvertCond ? IC->getInversePredicate() : IC->getPredicate();

2041

Condition = getICmpCondCode(Pred);

2042

} else {

2043

const FCmpInst *FC = cast<FCmpInst>(Cond);

2044

FCmpInst::Predicate Pred =

2045

InvertCond ? FC->getInversePredicate() : FC->getPredicate();

2046

Condition = getFCmpCondCode(Pred);

2047

if (TM.Options.NoNaNsFPMath)

2048

Condition = getFCmpCodeWithoutNaN(Condition);

2049

}

2050

2051

CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,

2052

TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2053

SL->SwitchCases.push_back(CB);

2054

return;

2055

}

2056

}

2057

2058

// Create a CaseBlock record representing this branch.

2059

ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;

2060

CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),

2061

nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2062

SL->SwitchCases.push_back(CB);

2063

}

2064

2065

void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,

2066

MachineBasicBlock *TBB,

2067

MachineBasicBlock *FBB,

2068

MachineBasicBlock *CurBB,

2069

MachineBasicBlock *SwitchBB,

2070

Instruction::BinaryOps Opc,

2071

BranchProbability TProb,

2072

BranchProbability FProb,

2073

bool InvertCond) {

2074

// Skip over not part of the tree and remember to invert op and operands at

2075

// next level.

2076

Value *NotCond;

2077

if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&

2078

InBlock(NotCond, CurBB->getBasicBlock())) {

2079

FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,

2080

!InvertCond);

2081

return;

2082

}

2083

2084

const Instruction *BOp = dyn_cast<Instruction>(Cond);

2085

// Compute the effective opcode for Cond, taking into account whether it needs

2086

// to be inverted, e.g.

2087

// and (not (or A, B)), C

2088

// gets lowered as

2089

// and (and (not A, not B), C)

2090

unsigned BOpc = 0;

2091

if (BOp) {

2092

BOpc = BOp->getOpcode();

2093

if (InvertCond) {

2094

if (BOpc == Instruction::And)

2095

BOpc = Instruction::Or;

2096

else if (BOpc == Instruction::Or)

2097

BOpc = Instruction::And;

2098

}

2099

}

2100

2101

// If this node is not part of the or/and tree, emit it as a branch.

2102

if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||

2103

BOpc != unsigned(Opc) || !BOp->hasOneUse() ||

2104

BOp->getParent() != CurBB->getBasicBlock() ||

2105

!InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||

2106

!InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {

2107

EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,

2108

TProb, FProb, InvertCond);

2109

return;

2110

}

2111

2112

// Create TmpBB after CurBB.

2113

MachineFunction::iterator BBI(CurBB);

2114

MachineFunction &MF = DAG.getMachineFunction();

2115

MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());

2116

CurBB->getParent()->insert(++BBI, TmpBB);

2117

2118

if (Opc == Instruction::Or) {

2119

// Codegen X | Y as:

2120

// BB1:

2121

// jmp_if_X TBB

2122

// jmp TmpBB

2123

// TmpBB:

2124

// jmp_if_Y TBB

2125

// jmp FBB

2126

//

2127

2128

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2129

// The requirement is that

2130

// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)

2131

// = TrueProb for original BB.

2132

// Assuming the original probabilities are A and B, one choice is to set

2133

// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to

2134

// A/(1+B) and 2B/(1+B). This choice assumes that

2135

// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.

2136

// Another choice is to assume TrueProb for BB1 equals to TrueProb for

2137

// TmpBB, but the math is more complicated.

2138

2139

auto NewTrueProb = TProb / 2;

2140

auto NewFalseProb = TProb / 2 + FProb;

2141

// Emit the LHS condition.

2142

FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,

2143

NewTrueProb, NewFalseProb, InvertCond);

2144

2145

// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).

2146

SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};

2147

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2148

// Emit the RHS condition into TmpBB.

2149

FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,

2150

Probs[0], Probs[1], InvertCond);

2151

} else {

2152

assert(Opc == Instruction::And && "Unknown merge op!")((Opc == Instruction::And && "Unknown merge op!") ? static_cast
<void> (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2152, __PRETTY_FUNCTION__));

2153

// Codegen X & Y as:

2154

// BB1:

2155

// jmp_if_X TmpBB

2156

// jmp FBB

2157

// TmpBB:

2158

// jmp_if_Y TBB

2159

// jmp FBB

2160

//

2161

// This requires creation of TmpBB after CurBB.

2162

2163

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2164

// The requirement is that

2165

// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)

2166

// = FalseProb for original BB.

2167

// Assuming the original probabilities are A and B, one choice is to set

2168

// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to

2169

// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==

2170

// TrueProb for BB1 * FalseProb for TmpBB.

2171

2172

auto NewTrueProb = TProb + FProb / 2;

2173

auto NewFalseProb = FProb / 2;

2174

// Emit the LHS condition.

2175

FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,

2176

NewTrueProb, NewFalseProb, InvertCond);

2177

2178

// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).

2179

SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};

2180

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2181

// Emit the RHS condition into TmpBB.

2182

FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,

2183

Probs[0], Probs[1], InvertCond);

2184

}

2185

}

2186

2187

/// If the set of cases should be emitted as a series of branches, return true.

2188

/// If we should emit this as a bunch of and/or'd together conditions, return

2189

/// false.

2190

bool

2191

SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {

2192

if (Cases.size() != 2) return true;

2193

2194

// If this is two comparisons of the same values or'd or and'd together, they

2195

// will get folded into a single comparison, so don't emit two blocks.

2196

if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&

2197

Cases[0].CmpRHS == Cases[1].CmpRHS) ||

2198

(Cases[0].CmpRHS == Cases[1].CmpLHS &&

2199

Cases[0].CmpLHS == Cases[1].CmpRHS)) {

2200

return false;

2201

}

2202

2203

// Handle: (X != null) | (Y != null) --> (X|Y) != 0

2204

// Handle: (X == null) & (Y == null) --> (X|Y) == 0

2205

if (Cases[0].CmpRHS == Cases[1].CmpRHS &&

2206

Cases[0].CC == Cases[1].CC &&

2207

isa<Constant>(Cases[0].CmpRHS) &&

2208

cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {

2209

if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)

2210

return false;

2211

if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)

2212

return false;

2213

}

2214

2215

return true;

2216

}

2217

2218

void SelectionDAGBuilder::visitBr(const BranchInst &I) {

2219

MachineBasicBlock *BrMBB = FuncInfo.MBB;

2220

2221

// Update machine-CFG edges.

2222

MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

2223

2224

if (I.isUnconditional()) {

2225

// Update machine-CFG edges.

2226

BrMBB->addSuccessor(Succ0MBB);

2227

2228

// If this is not a fall-through branch or optimizations are switched off,

2229

// emit the branch.

2230

if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)

2231

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2232

MVT::Other, getControlRoot(),

2233

DAG.getBasicBlock(Succ0MBB)));

2234

2235

return;

2236

}

2237

2238

// If this condition is one of the special cases we handle, do special stuff

2239

// now.

2240

const Value *CondVal = I.getCondition();

2241

MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

2242

2243

// If this is a series of conditions that are or'd or and'd together, emit

2244

// this as a sequence of branches instead of setcc's with and/or operations.

2245

// As long as jumps are not expensive (exceptions for multi-use logic ops,

2246

// unpredictable branches, and vector extracts because those jumps are likely

2247

// expensive for any target), this should improve performance.

2248

// For example, instead of something like:

2249

// cmp A, B

2250

// C = seteq

2251

// cmp D, E

2252

// F = setle

2253

// or C, F

2254

// jnz foo

2255

// Emit:

2256

// cmp A, B

2257

// je foo

2258

// cmp D, E

2259

// jle foo

2260

if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {

2261

Instruction::BinaryOps Opcode = BOp->getOpcode();

2262

Value *Vec, *BOp0 = BOp->getOperand(0), *BOp1 = BOp->getOperand(1);

2263

if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&

2264

!I.hasMetadata(LLVMContext::MD_unpredictable) &&

2265

(Opcode == Instruction::And || Opcode == Instruction::Or) &&

2266

!(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&

2267

match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {

2268

FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,

2269

Opcode,

2270

getEdgeProbability(BrMBB, Succ0MBB),

2271

getEdgeProbability(BrMBB, Succ1MBB),

2272

/*InvertCond=*/false);

2273

// If the compares in later blocks need to use values not currently

2274

// exported from this block, export them now. This block should always

2275

// be the first entry.

2276

assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")((SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"
) ? static_cast<void> (0) : __assert_fail ("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2276, __PRETTY_FUNCTION__));

2277

2278

// Allow some cases to be rejected.

2279

if (ShouldEmitAsBranches(SL->SwitchCases)) {

2280

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {

2281

ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);

2282

ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);

2283

}

2284

2285

// Emit the branch for this block.

2286

visitSwitchCase(SL->SwitchCases[0], BrMBB);

2287

SL->SwitchCases.erase(SL->SwitchCases.begin());

2288

return;

2289

}

2290

2291

// Okay, we decided not to do this, remove any inserted MBB's and clear

2292

// SwitchCases.

2293

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)

2294

FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

2295

2296

SL->SwitchCases.clear();

2297

}

2298

}

2299

2300

// Create a CaseBlock record representing this branch.

2301

CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),

2302

nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

2303

2304

// Use visitSwitchCase to actually insert the fast branch sequence for this

2305

// cond branch.

2306

visitSwitchCase(CB, BrMBB);

2307

}

2308

2309

/// visitSwitchCase - Emits the necessary code to represent a single node in

2310

/// the binary search tree resulting from lowering a switch instruction.

2311

void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,

2312

MachineBasicBlock *SwitchBB) {

2313

SDValue Cond;

2314

SDValue CondLHS = getValue(CB.CmpLHS);

2315

SDLoc dl = CB.DL;

2316

2317

if (CB.CC == ISD::SETTRUE) {

2318

// Branch or fall through to TrueBB.

2319

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2320

SwitchBB->normalizeSuccProbs();

2321

if (CB.TrueBB != NextBlock(SwitchBB)) {

2322

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),

2323

DAG.getBasicBlock(CB.TrueBB)));

2324

}

2325

return;

2326

}

2327

2328

auto &TLI = DAG.getTargetLoweringInfo();

2329

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

2330

2331

// Build the setcc now.

2332

if (!CB.CmpMHS) {

2333

// Fold "(X == true)" to X and "(X == false)" to !X to

2334

// handle common cases produced by branch lowering.

2335

if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&

2336

CB.CC == ISD::SETEQ)

2337

Cond = CondLHS;

2338

else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&

2339

CB.CC == ISD::SETEQ) {

2340

SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());

2341

Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);

2342

} else {

2343

SDValue CondRHS = getValue(CB.CmpRHS);

2344

2345

// If a pointer's DAG type is larger than its memory type then the DAG

2346

// values are zero-extended. This breaks signed comparisons so truncate

2347

// back to the underlying type before doing the compare.

2348

if (CondLHS.getValueType() != MemVT) {

2349

CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);

2350

CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);

2351

}

2352

Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);

2353

}

2354

} else {

2355

assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")((CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? static_cast<void> (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2355, __PRETTY_FUNCTION__));

2356

2357

const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();

2358

const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

2359

2360

SDValue CmpOp = getValue(CB.CmpMHS);

2361

EVT VT = CmpOp.getValueType();

2362

2363

if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {

2364

Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),

2365

ISD::SETLE);

2366

} else {

2367

SDValue SUB = DAG.getNode(ISD::SUB, dl,

2368

VT, CmpOp, DAG.getConstant(Low, dl, VT));

2369

Cond = DAG.getSetCC(dl, MVT::i1, SUB,

2370

DAG.getConstant(High-Low, dl, VT), ISD::SETULE);

2371

}

2372

}

2373

2374

// Update successor info

2375

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2376

// TrueBB and FalseBB are always different unless the incoming IR is

2377

// degenerate. This only happens when running llc on weird IR.

2378

if (CB.TrueBB != CB.FalseBB)

2379

addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);

2380

SwitchBB->normalizeSuccProbs();

2381

2382

// If the lhs block is the next block, invert the condition so that we can

2383

// fall through to the lhs instead of the rhs block.

2384

if (CB.TrueBB == NextBlock(SwitchBB)) {

2385

std::swap(CB.TrueBB, CB.FalseBB);

2386

SDValue True = DAG.getConstant(1, dl, Cond.getValueType());

2387

Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);

2388

}

2389

2390

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2391

MVT::Other, getControlRoot(), Cond,

2392

DAG.getBasicBlock(CB.TrueBB));

2393

2394

// Insert the false branch. Do this even if it's a fall through branch,

2395

// this makes it easier to do DAG optimizations which require inverting

2396

// the branch condition.

2397

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2398

DAG.getBasicBlock(CB.FalseBB));

2399

2400

DAG.setRoot(BrCond);

2401

}

2402

2403

/// visitJumpTable - Emit JumpTable node in the current MBB

2404

void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {

2405

// Emit the code for the jump table

2406

assert(JT.Reg != -1U && "Should lower JT Header first!")((JT.Reg != -1U && "Should lower JT Header first!") ?
static_cast<void> (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2406, __PRETTY_FUNCTION__));

2407

EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());

2408

SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),

2409

JT.Reg, PTy);

2410

SDValue Table = DAG.getJumpTable(JT.JTI, PTy);

2411

SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),

2412

MVT::Other, Index.getValue(1),

2413

Table, Index);

2414

DAG.setRoot(BrJumpTable);

2415

}

2416

2417

/// visitJumpTableHeader - This function emits necessary code to produce index

2418

/// in the JumpTable from switch case.

2419

void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,

2420

JumpTableHeader &JTH,

2421

MachineBasicBlock *SwitchBB) {

2422

SDLoc dl = getCurSDLoc();

2423

2424

// Subtract the lowest switch case value from the value being switched on.

2425

SDValue SwitchOp = getValue(JTH.SValue);

2426

EVT VT = SwitchOp.getValueType();

2427

SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,

2428

DAG.getConstant(JTH.First, dl, VT));

2429

2430

// The SDNode we just created, which holds the value being switched on minus

2431

// the smallest case value, needs to be copied to a virtual register so it

2432

// can be used as an index into the jump table in a subsequent basic block.

2433

// This value may be smaller or larger than the target's pointer type, and

2434

// therefore require extension or truncating.

2435

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2436

SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

2437

2438

unsigned JumpTableReg =

2439

FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));

2440

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,

2441

JumpTableReg, SwitchOp);

2442

JT.Reg = JumpTableReg;

2443

2444

if (!JTH.OmitRangeCheck) {

2445

// Emit the range check for the jump table, and branch to the default block

2446

// for the switch statement if the value being switched on exceeds the

2447

// largest case in the switch.

2448

SDValue CMP = DAG.getSetCC(

2449

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2450

Sub.getValueType()),

2451

Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

2452

2453

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2454

MVT::Other, CopyTo, CMP,

2455

DAG.getBasicBlock(JT.Default));

2456

2457

// Avoid emitting unnecessary branches to the next block.

2458

if (JT.MBB != NextBlock(SwitchBB))

2459

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2460

DAG.getBasicBlock(JT.MBB));

2461

2462

DAG.setRoot(BrCond);

2463

} else {

2464

// Avoid emitting unnecessary branches to the next block.

2465

if (JT.MBB != NextBlock(SwitchBB))

2466

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,

2467

DAG.getBasicBlock(JT.MBB)));

2468

else

2469

DAG.setRoot(CopyTo);

2470

}

2471

}

2472

2473

/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global

2474

/// variable if there exists one.

2475

static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,

2476

SDValue &Chain) {

2477

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2478

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2479

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2480

MachineFunction &MF = DAG.getMachineFunction();

2481

Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());

2482

MachineSDNode *Node =

2483

DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);

2484

if (Global) {

2485

MachinePointerInfo MPInfo(Global);

2486

auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |

2487

MachineMemOperand::MODereferenceable;

2488

MachineMemOperand *MemRef = MF.getMachineMemOperand(

2489

MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));

2490

DAG.setNodeMemRefs(Node, {MemRef});

2491

}

2492

if (PtrTy != PtrMemTy)

2493

return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);

2494

return SDValue(Node, 0);

2495

}

2496

2497

/// Codegen a new tail for a stack protector check ParentMBB which has had its

2498

/// tail spliced into a stack protector check success bb.

2499

///

2500

/// For a high level explanation of how this fits into the stack protector

2501

/// generation see the comment on the declaration of class

2502

/// StackProtectorDescriptor.

2503

void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,

2504

MachineBasicBlock *ParentBB) {

2505

2506

// First create the loads to the guard/stack slot for the comparison.

2507

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2508

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2509

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2510

2511

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();

2512

int FI = MFI.getStackProtectorIndex();

2513

2514

SDValue Guard;

2515

SDLoc dl = getCurSDLoc();

2516

SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);

2517

const Module &M = *ParentBB->getParent()->getFunction().getParent();

2518

Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

2519

2520

// Generate code to load the content of the guard slot.

2521

SDValue GuardVal = DAG.getLoad(

2522

PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,

2523

MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,

2524

MachineMemOperand::MOVolatile);

2525

2526

if (TLI.useStackGuardXorFP())

2527

GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

2528

2529

// Retrieve guard check function, nullptr if instrumentation is inlined.

2530

if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {

2531

// The target provides a guard check function to validate the guard value.

2532

// Generate a call to that function with the content of the guard slot as

2533

// argument.

2534

FunctionType *FnTy = GuardCheckFn->getFunctionType();

2535

assert(FnTy->getNumParams() == 1 && "Invalid function signature")((FnTy->getNumParams() == 1 && "Invalid function signature"
) ? static_cast<void> (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2535, __PRETTY_FUNCTION__));

2536

2537

TargetLowering::ArgListTy Args;

2538

TargetLowering::ArgListEntry Entry;

2539

Entry.Node = GuardVal;

2540

Entry.Ty = FnTy->getParamType(0);

2541

if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))

2542

Entry.IsInReg = true;

2543

Args.push_back(Entry);

2544

2545

TargetLowering::CallLoweringInfo CLI(DAG);

2546

CLI.setDebugLoc(getCurSDLoc())

2547

.setChain(DAG.getEntryNode())

2548

.setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),

2549

getValue(GuardCheckFn), std::move(Args));

2550

2551

std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

2552

DAG.setRoot(Result.second);

2553

return;

2554

}

2555

2556

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.

2557

// Otherwise, emit a volatile load to retrieve the stack guard value.

2558

SDValue Chain = DAG.getEntryNode();

2559

if (TLI.useLoadStackGuardNode()) {

2560

Guard = getLoadStackGuard(DAG, dl, Chain);

2561

} else {

2562

const Value *IRGuard = TLI.getSDagStackGuard(M);

2563

SDValue GuardPtr = getValue(IRGuard);

2564

2565

Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,

2566

MachinePointerInfo(IRGuard, 0), Align,

2567

MachineMemOperand::MOVolatile);

2568

}

2569

2570

// Perform the comparison via a getsetcc.

2571

SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),

2572

*DAG.getContext(),

2573

Guard.getValueType()),

2574

Guard, GuardVal, ISD::SETNE);

2575

2576

// If the guard/stackslot do not equal, branch to failure MBB.

2577

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2578

MVT::Other, GuardVal.getOperand(0),

2579

Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));

2580

// Otherwise branch to success MBB.

2581

SDValue Br = DAG.getNode(ISD::BR, dl,

2582

MVT::Other, BrCond,

2583

DAG.getBasicBlock(SPD.getSuccessMBB()));

2584

2585

DAG.setRoot(Br);

2586

}

2587

2588

/// Codegen the failure basic block for a stack protector check.

2589

///

2590

/// A failure stack protector machine basic block consists simply of a call to

2591

/// __stack_chk_fail().

2592

///

2593

/// For a high level explanation of how this fits into the stack protector

2594

/// generation see the comment on the declaration of class

2595

/// StackProtectorDescriptor.

2596

void

2597

SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {

2598

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2599

TargetLowering::MakeLibCallOptions CallOptions;

2600

CallOptions.setDiscardResult(true);

2601

SDValue Chain =

2602

TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,

2603

None, CallOptions, getCurSDLoc()).second;

2604

// On PS4, the "return address" must still be within the calling function,

2605

// even if it's at the very end, so emit an explicit TRAP here.

2606

// Passing 'true' for doesNotReturn above won't generate the trap for us.

2607

if (TM.getTargetTriple().isPS4CPU())

2608

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2609

// WebAssembly needs an unreachable instruction after a non-returning call,

2610

// because the function return type can be different from __stack_chk_fail's

2611

// return type (void).

2612

if (TM.getTargetTriple().isWasm())

2613

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2614

2615

DAG.setRoot(Chain);

2616

}

2617

2618

/// visitBitTestHeader - This function emits necessary code to produce value

2619

/// suitable for "bit tests"

2620

void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,

2621

MachineBasicBlock *SwitchBB) {

2622

SDLoc dl = getCurSDLoc();

2623

2624

// Subtract the minimum value.

2625

SDValue SwitchOp = getValue(B.SValue);

2626

EVT VT = SwitchOp.getValueType();

2627

SDValue RangeSub =

2628

DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

2629

2630

// Determine the type of the test operands.

2631

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2632

bool UsePtrType = false;

2633

if (!TLI.isTypeLegal(VT)) {

2634

UsePtrType = true;

2635

} else {

2636

for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)

2637

if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {

2638

// Switch table case range are encoded into series of masks.

2639

// Just use pointer type, it's guaranteed to fit.

2640

UsePtrType = true;

2641

break;

2642

}

2643

}

2644

SDValue Sub = RangeSub;

2645

if (UsePtrType) {

2646

VT = TLI.getPointerTy(DAG.getDataLayout());

2647

Sub = DAG.getZExtOrTrunc(Sub, dl, VT);

2648

}

2649

2650

B.RegVT = VT.getSimpleVT();

2651

B.Reg = FuncInfo.CreateReg(B.RegVT);

2652

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

2653

2654

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

2655

2656

if (!B.OmitRangeCheck)

2657

addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);

2658

addSuccessorWithProb(SwitchBB, MBB, B.Prob);

2659

SwitchBB->normalizeSuccProbs();

2660

2661

SDValue Root = CopyTo;

2662

if (!B.OmitRangeCheck) {

2663

// Conditional branch to the default block.

2664

SDValue RangeCmp = DAG.getSetCC(dl,

2665

TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2666

RangeSub.getValueType()),

2667

RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),

2668

ISD::SETUGT);

2669

2670

Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,

2671

DAG.getBasicBlock(B.Default));

2672

}

2673

2674

// Avoid emitting unnecessary branches to the next block.

2675

if (MBB != NextBlock(SwitchBB))

2676

Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

2677

2678

DAG.setRoot(Root);

2679

}

2680

2681

/// visitBitTestCase - this function produces one "bit test"

2682

void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,

2683

MachineBasicBlock* NextMBB,

2684

BranchProbability BranchProbToNext,

2685

unsigned Reg,

2686

BitTestCase &B,

2687

MachineBasicBlock *SwitchBB) {

2688

SDLoc dl = getCurSDLoc();

2689

MVT VT = BB.RegVT;

2690

SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);

2691

SDValue Cmp;

2692

unsigned PopCount = countPopulation(B.Mask);

2693

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2694

if (PopCount == 1) {

2695

// Testing for a single bit; just compare the shift count with what it

2696

// would need to be to shift a 1 bit in that position.

2697

Cmp = DAG.getSetCC(

2698

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2699

ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),

2700

ISD::SETEQ);

2701

} else if (PopCount == BB.Range) {

2702

// There is only one zero bit in the range, test for it directly.

2703

Cmp = DAG.getSetCC(

2704

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2705

ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),

2706

ISD::SETNE);

2707

} else {

2708

// Make desired shift

2709

SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,

2710

DAG.getConstant(1, dl, VT), ShiftOp);

2711

2712

// Emit bit tests and jumps

2713

SDValue AndOp = DAG.getNode(ISD::AND, dl,

2714

VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));

2715

Cmp = DAG.getSetCC(

2716

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2717

AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);

2718

}

2719

2720

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.

2721

addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);

2722

// The branch probability from SwitchBB to NextMBB is BranchProbToNext.

2723

addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);

2724

// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is

2725

// one as they are relative probabilities (and thus work more like weights),

2726

// and hence we need to normalize them to let the sum of them become one.

2727

SwitchBB->normalizeSuccProbs();

2728

2729

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,

2730

MVT::Other, getControlRoot(),

2731

Cmp, DAG.getBasicBlock(B.TargetBB));

2732

2733

// Avoid emitting unnecessary branches to the next block.

2734

if (NextMBB != NextBlock(SwitchBB))

2735

BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,

2736

DAG.getBasicBlock(NextMBB));

2737

2738

DAG.setRoot(BrAnd);

2739

}

2740

2741

void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {

2742

MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

2743

2744

// Retrieve successors. Look through artificial IR level blocks like

2745

// catchswitch for successors.

2746

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];

2747

const BasicBlock *EHPadBB = I.getSuccessor(1);

2748

2749

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2750

// have to do anything here to lower funclet bundles.

2751

assert(!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt,((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__))

2752

LLVMContext::OB_gc_transition,((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__))

2753

LLVMContext::OB_gc_live,((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__))

2754

LLVMContext::OB_funclet,((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__))

2755

LLVMContext::OB_cfguardtarget}) &&((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__))

2756

"Cannot lower invokes with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan({LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2756, __PRETTY_FUNCTION__));

2757

2758

const Value *Callee(I.getCalledOperand());

2759

const Function *Fn = dyn_cast<Function>(Callee);

2760

if (isa<InlineAsm>(Callee))

2761

visitInlineAsm(I);

2762

else if (Fn && Fn->isIntrinsic()) {

2763

switch (Fn->getIntrinsicID()) {

2764

default:

2765

llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2765);

2766

case Intrinsic::donothing:

2767

// Ignore invokes to @llvm.donothing: jump directly to the next BB.

2768

break;

2769

case Intrinsic::experimental_patchpoint_void:

2770

case Intrinsic::experimental_patchpoint_i64:

2771

visitPatchpoint(I, EHPadBB);

2772

break;

2773

case Intrinsic::experimental_gc_statepoint:

2774

LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);

2775

break;

2776

case Intrinsic::wasm_rethrow_in_catch: {

2777

// This is usually done in visitTargetIntrinsic, but this intrinsic is

2778

// special because it can be invoked, so we manually lower it to a DAG

2779

// node here.

2780

SmallVector<SDValue, 8> Ops;

2781

Ops.push_back(getRoot()); // inchain

2782

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2783

Ops.push_back(

2784

DAG.getTargetConstant(Intrinsic::wasm_rethrow_in_catch, getCurSDLoc(),

2785

TLI.getPointerTy(DAG.getDataLayout())));

2786

SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain

2787

DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));

2788

break;

2789

}

2790

}

2791

} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {

2792

// Currently we do not lower any intrinsic calls with deopt operand bundles.

2793

// Eventually we will support lowering the @llvm.experimental.deoptimize

2794

// intrinsic, and right now there are no plans to support other intrinsics

2795

// with deopt state.

2796

LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);

2797

} else {

2798

LowerCallTo(I, getValue(Callee), false, EHPadBB);

2799

}

2800

2801

// If the value of the invoke is used outside of its defining block, make it

2802

// available as a virtual register.

2803

// We already took care of the exported value for the statepoint instruction

2804

// during call to the LowerStatepoint.

2805

if (!isa<GCStatepointInst>(I)) {

2806

CopyToExportRegsIfNeeded(&I);

2807

}

2808

2809

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

2810

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2811

BranchProbability EHPadBBProb =

2812

BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)

2813

: BranchProbability::getZero();

2814

findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

2815

2816

// Update successor info.

2817

addSuccessorWithProb(InvokeMBB, Return);

2818

for (auto &UnwindDest : UnwindDests) {

2819

UnwindDest.first->setIsEHPad();

2820

addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);

2821

}

2822

InvokeMBB->normalizeSuccProbs();

2823

2824

// Drop into normal successor.

2825

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),

2826

DAG.getBasicBlock(Return)));

2827

}

2828

2829

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {

2830

MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

2831

2832

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2833

// have to do anything here to lower funclet bundles.

2834

assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2836, __PRETTY_FUNCTION__))

2835

{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2836, __PRETTY_FUNCTION__))

2836

"Cannot lower callbrs with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2836, __PRETTY_FUNCTION__));

2837

2838

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")((I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? static_cast<void> (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2838, __PRETTY_FUNCTION__));

2839

visitInlineAsm(I);

2840

CopyToExportRegsIfNeeded(&I);

2841

2842

// Retrieve successors.

2843

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

2844

2845

// Update successor info.

2846

addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());

2847

for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {

2848

MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];

2849

addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());

2850

Target->setIsInlineAsmBrIndirectTarget();

2851

}

2852

CallBrMBB->normalizeSuccProbs();

2853

2854

// Drop into default successor.

2855

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2856

MVT::Other, getControlRoot(),

2857

DAG.getBasicBlock(Return)));

2858

}

2859

2860

void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {

2861

llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2861);

2862

}

2863

2864

void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {

2865

assert(FuncInfo.MBB->isEHPad() &&((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2866, __PRETTY_FUNCTION__))

2866

"Call to landingpad not in landing pad!")((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2866, __PRETTY_FUNCTION__));

2867

2868

// If there aren't registers to copy the values into (e.g., during SjLj

2869

// exceptions), then don't bother to create these DAG nodes.

2870

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2871

const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();

2872

if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&

2873

TLI.getExceptionSelectorRegister(PersonalityFn) == 0)

2874

return;

2875

2876

// If landingpad's return type is token type, we don't create DAG nodes

2877

// for its exception pointer and selector value. The extraction of exception

2878

// pointer or selector value from token type landingpads is not currently

2879

// supported.

2880

if (LP.getType()->isTokenTy())

2881

return;

2882

2883

SmallVector<EVT, 2> ValueVTs;

2884

SDLoc dl = getCurSDLoc();

2885

ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);

2886

assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")((ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2886, __PRETTY_FUNCTION__));

2887

2888

// Get the two live-in registers as SDValues. The physregs have already been

2889

// copied into virtual registers.

2890

SDValue Ops[2];

2891

if (FuncInfo.ExceptionPointerVirtReg) {

2892

Ops[0] = DAG.getZExtOrTrunc(

2893

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

2894

FuncInfo.ExceptionPointerVirtReg,

2895

TLI.getPointerTy(DAG.getDataLayout())),

2896

dl, ValueVTs[0]);

2897

} else {

2898

Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));

2899

}

2900

Ops[1] = DAG.getZExtOrTrunc(

2901

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

2902

FuncInfo.ExceptionSelectorVirtReg,

2903

TLI.getPointerTy(DAG.getDataLayout())),

2904

dl, ValueVTs[1]);

2905

2906

// Merge into one.

2907

SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,

2908

DAG.getVTList(ValueVTs), Ops);

2909

setValue(&LP, Res);

2910

}

2911

2912

void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,

2913

MachineBasicBlock *Last) {

2914

// Update JTCases.

2915

for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)

2916

if (SL->JTCases[i].first.HeaderBB == First)

2917

SL->JTCases[i].first.HeaderBB = Last;

2918

2919

// Update BitTestCases.

2920

for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)

2921

if (SL->BitTestCases[i].Parent == First)

2922

SL->BitTestCases[i].Parent = Last;

2923

}

2924

2925

void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {

2926

MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

2927

2928

// Update machine-CFG edges with unique successors.

2929

SmallSet<BasicBlock*, 32> Done;

2930

for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {

2931

BasicBlock *BB = I.getSuccessor(i);

2932

bool Inserted = Done.insert(BB).second;

2933

if (!Inserted)

2934

continue;

2935

2936

MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];

2937

addSuccessorWithProb(IndirectBrMBB, Succ);

2938

}

2939

IndirectBrMBB->normalizeSuccProbs();

2940

2941

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),

2942

MVT::Other, getControlRoot(),

2943

getValue(I.getAddress())));

2944

}

2945

2946

void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {

2947

if (!DAG.getTarget().Options.TrapUnreachable)

2948

return;

2949

2950

// We may be able to ignore unreachable behind a noreturn call.

2951

if (DAG.getTarget().Options.NoTrapAfterNoreturn) {

2952

const BasicBlock &BB = *I.getParent();

2953

if (&I != &BB.front()) {

2954

BasicBlock::const_iterator PredI =

2955

std::prev(BasicBlock::const_iterator(&I));

2956

if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {

2957

if (Call->doesNotReturn())

2958

return;

2959

}

2960

}

2961

}

2962

2963

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

2964

}

2965

2966

void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {

2967

SDNodeFlags Flags;

2968

2969

SDValue Op = getValue(I.getOperand(0));

2970

SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),

2971

Op, Flags);

2972

setValue(&I, UnNodeValue);

2973

}

2974

2975

void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {

2976

SDNodeFlags Flags;

2977

if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {

2978

Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());

2979

Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());

2980

}

2981

if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))

2982

Flags.setExact(ExactOp->isExact());

2983

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

2984

Flags.copyFMF(*FPOp);

2985

2986

SDValue Op1 = getValue(I.getOperand(0));

2987

SDValue Op2 = getValue(I.getOperand(1));

2988

SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),

2989

Op1, Op2, Flags);

2990

setValue(&I, BinNodeValue);

2991

}

2992

2993

void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {

2994

SDValue Op1 = getValue(I.getOperand(0));

2995

SDValue Op2 = getValue(I.getOperand(1));

2996

2997

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(

2998

Op1.getValueType(), DAG.getDataLayout());

2999

3000

// Coerce the shift amount to the right type if we can.

3001

if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {

3002

unsigned ShiftSize = ShiftTy.getSizeInBits();

3003

unsigned Op2Size = Op2.getValueSizeInBits();

3004

SDLoc DL = getCurSDLoc();

3005

3006

// If the operand is smaller than the shift count type, promote it.

3007

if (ShiftSize > Op2Size)

3008

Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);

3009

3010

// If the operand is larger than the shift count type but the shift

3011

// count type has enough bits to represent any shift value, truncate

3012

// it now. This is a common case and it exposes the truncate to

3013

// optimization early.

3014

else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))

3015

Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);

3016

// Otherwise we'll need to temporarily settle for some other convenient

3017

// type. Type legalization will make adjustments once the shiftee is split.

3018

else

3019

Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);

3020

}

3021

3022

bool nuw = false;

3023

bool nsw = false;

3024

bool exact = false;

3025

3026

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

3027

3028

if (const OverflowingBinaryOperator *OFBinOp =

3029

dyn_cast<const OverflowingBinaryOperator>(&I)) {

3030

nuw = OFBinOp->hasNoUnsignedWrap();

3031

nsw = OFBinOp->hasNoSignedWrap();

3032

}

3033

if (const PossiblyExactOperator *ExactOp =

3034

dyn_cast<const PossiblyExactOperator>(&I))

3035

exact = ExactOp->isExact();

3036

}

3037

SDNodeFlags Flags;

3038

Flags.setExact(exact);

3039

Flags.setNoSignedWrap(nsw);

3040

Flags.setNoUnsignedWrap(nuw);

3041

SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,

3042

Flags);

3043

setValue(&I, Res);

3044

}

3045

3046

void SelectionDAGBuilder::visitSDiv(const User &I) {

3047

SDValue Op1 = getValue(I.getOperand(0));

3048

SDValue Op2 = getValue(I.getOperand(1));

3049

3050

SDNodeFlags Flags;

3051

Flags.setExact(isa<PossiblyExactOperator>(&I) &&

3052

cast<PossiblyExactOperator>(&I)->isExact());

3053

setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,

3054

Op2, Flags));

3055

}

3056

3057

void SelectionDAGBuilder::visitICmp(const User &I) {

3058

ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;

3059

if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))

3060

predicate = IC->getPredicate();

3061

else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))

3062

predicate = ICmpInst::Predicate(IC->getPredicate());

3063

SDValue Op1 = getValue(I.getOperand(0));

3064

SDValue Op2 = getValue(I.getOperand(1));

3065

ISD::CondCode Opcode = getICmpCondCode(predicate);

3066

3067

auto &TLI = DAG.getTargetLoweringInfo();

3068

EVT MemVT =

3069

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3070

3071

// If a pointer's DAG type is larger than its memory type then the DAG values

3072

// are zero-extended. This breaks signed comparisons so truncate back to the

3073

// underlying type before doing the compare.

3074

if (Op1.getValueType() != MemVT) {

3075

Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);

3076

Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);

3077

}

3078

3079

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3080

I.getType());

3081

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));

3082

}

3083

3084

void SelectionDAGBuilder::visitFCmp(const User &I) {

3085

FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;

3086

if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))

3087

predicate = FC->getPredicate();

3088

else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))

3089

predicate = FCmpInst::Predicate(FC->getPredicate());

3090

SDValue Op1 = getValue(I.getOperand(0));

3091

SDValue Op2 = getValue(I.getOperand(1));

3092

3093

ISD::CondCode Condition = getFCmpCondCode(predicate);

3094

auto *FPMO = cast<FPMathOperator>(&I);

3095

if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)

3096

Condition = getFCmpCodeWithoutNaN(Condition);

3097

3098

SDNodeFlags Flags;

3099

Flags.copyFMF(*FPMO);

3100

3101

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3102

I.getType());

3103

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition, Flags));

3104

}

3105

3106

// Check if the condition of the select has one use or two users that are both

3107

// selects with the same condition.

3108

static bool hasOnlySelectUsers(const Value *Cond) {

3109

return llvm::all_of(Cond->users(), [](const Value *V) {

3110

return isa<SelectInst>(V);

3111

});

3112

}

3113

3114

void SelectionDAGBuilder::visitSelect(const User &I) {

3115

SmallVector<EVT, 4> ValueVTs;

3116

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),

3117

ValueVTs);

3118

unsigned NumValues = ValueVTs.size();

3119

if (NumValues == 0) return;

3120

3121

SmallVector<SDValue, 4> Values(NumValues);

3122

SDValue Cond = getValue(I.getOperand(0));

3123

SDValue LHSVal = getValue(I.getOperand(1));

3124

SDValue RHSVal = getValue(I.getOperand(2));

3125

SmallVector<SDValue, 1> BaseOps(1, Cond);

3126

ISD::NodeType OpCode =

3127

Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

3128

3129

bool IsUnaryAbs = false;

3130

3131

SDNodeFlags Flags;

3132

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3133

Flags.copyFMF(*FPOp);

3134

3135

// Min/max matching is only viable if all output VTs are the same.

3136

if (is_splat(ValueVTs)) {

3137

EVT VT = ValueVTs[0];

3138

LLVMContext &Ctx = *DAG.getContext();

3139

auto &TLI = DAG.getTargetLoweringInfo();

3140

3141

// We care about the legality of the operation after it has been type

3142

// legalized.

3143

while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)

3144

VT = TLI.getTypeToTransformTo(Ctx, VT);

3145

3146

// If the vselect is legal, assume we want to leave this as a vector setcc +

3147

// vselect. Otherwise, if this is going to be scalarized, we want to see if

3148

// min/max is legal on the scalar type.

3149

bool UseScalarMinMax = VT.isVector() &&

3150

!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

3151

3152

Value *LHS, *RHS;

3153

auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);

3154

ISD::NodeType Opc = ISD::DELETED_NODE;

3155

switch (SPR.Flavor) {

3156

case SPF_UMAX: Opc = ISD::UMAX; break;

3157

case SPF_UMIN: Opc = ISD::UMIN; break;

3158

case SPF_SMAX: Opc = ISD::SMAX; break;

3159

case SPF_SMIN: Opc = ISD::SMIN; break;

3160

case SPF_FMINNUM:

3161

switch (SPR.NaNBehavior) {

3162

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3162);

3163

case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;

3164

case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;

3165

case SPNB_RETURNS_ANY: {

3166

if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))

3167

Opc = ISD::FMINNUM;

3168

else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))

3169

Opc = ISD::FMINIMUM;

3170

else if (UseScalarMinMax)

3171

Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?

3172

ISD::FMINNUM : ISD::FMINIMUM;

3173

break;

3174

}

3175

}

3176

break;

3177

case SPF_FMAXNUM:

3178

switch (SPR.NaNBehavior) {

3179

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3179);

3180

case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;

3181

case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;

3182

case SPNB_RETURNS_ANY:

3183

3184

if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))

3185

Opc = ISD::FMAXNUM;

3186

else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))

3187

Opc = ISD::FMAXIMUM;

3188

else if (UseScalarMinMax)

3189

Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?

3190

ISD::FMAXNUM : ISD::FMAXIMUM;

3191

break;

3192

}

3193

break;

3194

case SPF_ABS:

3195

IsUnaryAbs = true;

3196

Opc = ISD::ABS;

3197

break;

3198

case SPF_NABS:

3199

// TODO: we need to produce sub(0, abs(X)).

3200

default: break;

3201

}

3202

3203

if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&

3204

(TLI.isOperationLegalOrCustom(Opc, VT) ||

3205

(UseScalarMinMax &&

3206

TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&

3207

// If the underlying comparison instruction is used by any other

3208

// instruction, the consumed instructions won't be destroyed, so it is

3209

// not profitable to convert to a min/max.

3210

hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {

3211

OpCode = Opc;

3212

LHSVal = getValue(LHS);

3213

RHSVal = getValue(RHS);

3214

BaseOps.clear();

3215

}

3216

3217

if (IsUnaryAbs) {

3218

OpCode = Opc;

3219

LHSVal = getValue(LHS);

3220

BaseOps.clear();

3221

}

3222

}

3223

3224

if (IsUnaryAbs) {

3225

for (unsigned i = 0; i != NumValues; ++i) {

3226

Values[i] =

3227

DAG.getNode(OpCode, getCurSDLoc(),

3228

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),

3229

SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3230

}

3231

} else {

3232

for (unsigned i = 0; i != NumValues; ++i) {

3233

SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());

3234

Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3235

Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));

3236

Values[i] = DAG.getNode(

3237

OpCode, getCurSDLoc(),

3238

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);

3239

}

3240

}

3241

3242

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3243

DAG.getVTList(ValueVTs), Values));

3244

}

3245

3246

void SelectionDAGBuilder::visitTrunc(const User &I) {

3247

// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).

3248

SDValue N = getValue(I.getOperand(0));

3249

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3250

I.getType());

3251

setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));

3252

}

3253

3254

void SelectionDAGBuilder::visitZExt(const User &I) {

3255

// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3256

// ZExt also can't be a cast to bool for same reason. So, nothing much to do

3257

SDValue N = getValue(I.getOperand(0));

3258

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3259

I.getType());

3260

setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));

3261

}

3262

3263

void SelectionDAGBuilder::visitSExt(const User &I) {

3264

// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3265

// SExt also can't be a cast to bool for same reason. So, nothing much to do

3266

SDValue N = getValue(I.getOperand(0));

3267

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3268

I.getType());

3269

setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));

3270

}

3271

3272

void SelectionDAGBuilder::visitFPTrunc(const User &I) {

3273

// FPTrunc is never a no-op cast, no need to check

3274

SDValue N = getValue(I.getOperand(0));

3275

SDLoc dl = getCurSDLoc();

3276

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3277

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3278

setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,

3279

DAG.getTargetConstant(

3280

0, dl, TLI.getPointerTy(DAG.getDataLayout()))));

3281

}

3282

3283

void SelectionDAGBuilder::visitFPExt(const User &I) {

3284

// FPExt is never a no-op cast, no need to check

3285

SDValue N = getValue(I.getOperand(0));

3286

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3287

I.getType());

3288

setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));

3289

}

3290

3291

void SelectionDAGBuilder::visitFPToUI(const User &I) {

3292

// FPToUI is never a no-op cast, no need to check

3293

SDValue N = getValue(I.getOperand(0));

3294

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3295

I.getType());

3296

setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));

3297

}

3298

3299

void SelectionDAGBuilder::visitFPToSI(const User &I) {

3300

// FPToSI is never a no-op cast, no need to check

3301

SDValue N = getValue(I.getOperand(0));

3302

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3303

I.getType());

3304

setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));

3305

}

3306

3307

void SelectionDAGBuilder::visitUIToFP(const User &I) {

3308

// UIToFP is never a no-op cast, no need to check

3309

SDValue N = getValue(I.getOperand(0));

3310

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3311

I.getType());

3312

setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));

3313

}

3314

3315

void SelectionDAGBuilder::visitSIToFP(const User &I) {

3316

// SIToFP is never a no-op cast, no need to check

3317

SDValue N = getValue(I.getOperand(0));

3318

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3319

I.getType());

3320

setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));

3321

}

3322

3323

void SelectionDAGBuilder::visitPtrToInt(const User &I) {

3324

// What to do depends on the size of the integer and the size of the pointer.

3325

// We can either truncate, zero extend, or no-op, accordingly.

3326

SDValue N = getValue(I.getOperand(0));

3327

auto &TLI = DAG.getTargetLoweringInfo();

3328

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3329

I.getType());

3330

EVT PtrMemVT =

3331

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3332

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3333

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);

3334

setValue(&I, N);

3335

}

3336

3337

void SelectionDAGBuilder::visitIntToPtr(const User &I) {

3338

// What to do depends on the size of the integer and the size of the pointer.

3339

// We can either truncate, zero extend, or no-op, accordingly.

3340

SDValue N = getValue(I.getOperand(0));

3341

auto &TLI = DAG.getTargetLoweringInfo();

3342

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3343

EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

3344

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3345

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);

3346

setValue(&I, N);

3347

}

3348

3349

void SelectionDAGBuilder::visitBitCast(const User &I) {

3350

SDValue N = getValue(I.getOperand(0));

3351

SDLoc dl = getCurSDLoc();

3352

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3353

I.getType());

3354

3355

// BitCast assures us that source and destination are the same size so this is

3356

// either a BITCAST or a no-op.

3357

if (DestVT != N.getValueType())

3358

setValue(&I, DAG.getNode(ISD::BITCAST, dl,

3359

DestVT, N)); // convert types.

3360

// Check if the original LLVM IR Operand was a ConstantInt, because getValue()

3361

// might fold any kind of constant expression to an integer constant and that

3362

// is not what we are looking for. Only recognize a bitcast of a genuine

3363

// constant integer as an opaque constant.

3364

else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))

3365

setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,

3366

/*isOpaque*/true));

3367

else

3368

setValue(&I, N); // noop cast.

3369

}

3370

3371

void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {

3372

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3373

const Value *SV = I.getOperand(0);

3374

SDValue N = getValue(SV);

3375

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3376

3377

unsigned SrcAS = SV->getType()->getPointerAddressSpace();

3378

unsigned DestAS = I.getType()->getPointerAddressSpace();

3379

3380

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))

3381

N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

3382

3383

setValue(&I, N);

3384

}

3385

3386

void SelectionDAGBuilder::visitInsertElement(const User &I) {

3387

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3388

SDValue InVec = getValue(I.getOperand(0));

3389

SDValue InVal = getValue(I.getOperand(1));

3390

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),

3391

TLI.getVectorIdxTy(DAG.getDataLayout()));

3392

setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),

3393

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3394

InVec, InVal, InIdx));

3395

}

3396

3397

void SelectionDAGBuilder::visitExtractElement(const User &I) {

3398

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3399

SDValue InVec = getValue(I.getOperand(0));

3400

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),

3401

TLI.getVectorIdxTy(DAG.getDataLayout()));

3402

setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),

3403

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3404

InVec, InIdx));

3405

}

3406

3407

void SelectionDAGBuilder::visitShuffleVector(const User &I) {

3408

SDValue Src1 = getValue(I.getOperand(0));

3409

SDValue Src2 = getValue(I.getOperand(1));

3410

ArrayRef<int> Mask;

3411

if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))

3412

Mask = SVI->getShuffleMask();

3413

else

3414

Mask = cast<ConstantExpr>(I).getShuffleMask();

3415

SDLoc DL = getCurSDLoc();

3416

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3417

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3418

EVT SrcVT = Src1.getValueType();

3419

3420

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&

3421

VT.isScalableVector()) {

3422

// Canonical splat form of first element of first input vector.

3423

SDValue FirstElt =

3424

DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,

3425

DAG.getVectorIdxConstant(0, DL));

3426

setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));

3427

return;

3428

}

3429

3430

// For now, we only handle splats for scalable vectors.

3431

// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation

3432

// for targets that support a SPLAT_VECTOR for non-scalable vector types.

3433

assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")((!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? static_cast<void> (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3433, __PRETTY_FUNCTION__));

3434

3435

unsigned SrcNumElts = SrcVT.getVectorNumElements();

3436

unsigned MaskNumElts = Mask.size();

3437

3438

if (SrcNumElts == MaskNumElts) {

3439

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));

3440

return;

3441

}

3442

3443

// Normalize the shuffle vector since mask and vector length don't match.

3444

if (SrcNumElts < MaskNumElts) {

3445

// Mask is longer than the source vectors. We can use concatenate vector to

3446

// make the mask and vectors lengths match.

3447

3448

if (MaskNumElts % SrcNumElts == 0) {

3449

// Mask length is a multiple of the source vector length.

3450

// Check if the shuffle is some kind of concatenation of the input

3451

// vectors.

3452

unsigned NumConcat = MaskNumElts / SrcNumElts;

3453

bool IsConcat = true;

3454

SmallVector<int, 8> ConcatSrcs(NumConcat, -1);

3455

for (unsigned i = 0; i != MaskNumElts; ++i) {

3456

int Idx = Mask[i];

3457

if (Idx < 0)

3458

continue;

3459

// Ensure the indices in each SrcVT sized piece are sequential and that

3460

// the same source is used for the whole piece.

3461

if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

3462

(ConcatSrcs[i / SrcNumElts] >= 0 &&

3463

ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {

3464

IsConcat = false;

3465

break;

3466

}

3467

// Remember which source this index came from.

3468

ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

3469

}

3470

3471

// The shuffle is concatenating multiple vectors together. Just emit

3472

// a CONCAT_VECTORS operation.

3473

if (IsConcat) {

3474

SmallVector<SDValue, 8> ConcatOps;

3475

for (auto Src : ConcatSrcs) {

3476

if (Src < 0)

3477

ConcatOps.push_back(DAG.getUNDEF(SrcVT));

3478

else if (Src == 0)

3479

ConcatOps.push_back(Src1);

3480

else

3481

ConcatOps.push_back(Src2);

3482

}

3483

setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));

3484

return;

3485

}

3486

}

3487

3488

unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);

3489

unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;

3490

EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),

3491

PaddedMaskNumElts);

3492

3493

// Pad both vectors with undefs to make them the same length as the mask.

3494

SDValue UndefVal = DAG.getUNDEF(SrcVT);

3495

3496

SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);

3497

SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);

3498

MOps1[0] = Src1;

3499

MOps2[0] = Src2;

3500

3501

Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);

3502

Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

3503

3504

// Readjust mask for new input vector length.

3505

SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);

3506

for (unsigned i = 0; i != MaskNumElts; ++i) {

3507

int Idx = Mask[i];

3508

if (Idx >= (int)SrcNumElts)

3509

Idx -= SrcNumElts - PaddedMaskNumElts;

3510

MappedOps[i] = Idx;

3511

}

3512

3513

SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

3514

3515

// If the concatenated vector was padded, extract a subvector with the

3516

// correct number of elements.

3517

if (MaskNumElts != PaddedMaskNumElts)

3518

Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,

3519

DAG.getVectorIdxConstant(0, DL));

3520

3521

setValue(&I, Result);

3522

return;

3523

}

3524

3525

if (SrcNumElts > MaskNumElts) {

3526

// Analyze the access pattern of the vector to see if we can extract

3527

// two subvectors and do the shuffle.

3528

int StartIdx[2] = { -1, -1 }; // StartIdx to extract from

3529

bool CanExtract = true;

3530

for (int Idx : Mask) {

3531

unsigned Input = 0;

3532

if (Idx < 0)

3533

continue;

3534

3535

if (Idx >= (int)SrcNumElts) {

3536

Input = 1;

3537

Idx -= SrcNumElts;

3538

}

3539

3540

// If all the indices come from the same MaskNumElts sized portion of

3541

// the sources we can use extract. Also make sure the extract wouldn't

3542

// extract past the end of the source.

3543

int NewStartIdx = alignDown(Idx, MaskNumElts);

3544

if (NewStartIdx + MaskNumElts > SrcNumElts ||

3545

(StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))

3546

CanExtract = false;

3547

// Make sure we always update StartIdx as we use it to track if all

3548

// elements are undef.

3549

StartIdx[Input] = NewStartIdx;

3550

}

3551

3552

if (StartIdx[0] < 0 && StartIdx[1] < 0) {

3553

setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.

3554

return;

3555

}

3556

if (CanExtract) {

3557

// Extract appropriate subvector and generate a vector shuffle

3558

for (unsigned Input = 0; Input < 2; ++Input) {

3559

SDValue &Src = Input == 0 ? Src1 : Src2;

3560

if (StartIdx[Input] < 0)

3561

Src = DAG.getUNDEF(VT);

3562

else {

3563

Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,

3564

DAG.getVectorIdxConstant(StartIdx[Input], DL));

3565

}

3566

}

3567

3568

// Calculate new mask.

3569

SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());

3570

for (int &Idx : MappedOps) {

3571

if (Idx >= (int)SrcNumElts)

3572

Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;

3573

else if (Idx >= 0)

3574

Idx -= StartIdx[0];

3575

}

3576

3577

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));

3578

return;

3579

}

3580

}

3581

3582

// We can't use either concat vectors or extract subvectors so fall back to

3583

// replacing the shuffle with extract and build vector.

3584

// to insert and build vector.

3585

EVT EltVT = VT.getVectorElementType();

3586

SmallVector<SDValue,8> Ops;

3587

for (int Idx : Mask) {

3588

SDValue Res;

3589

3590

if (Idx < 0) {

3591

Res = DAG.getUNDEF(EltVT);

3592

} else {

3593

SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;

3594

if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

3595

3596

Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,

3597

DAG.getVectorIdxConstant(Idx, DL));

3598

}

3599

3600

Ops.push_back(Res);

3601

}

3602

3603

setValue(&I, DAG.getBuildVector(VT, DL, Ops));

3604

}

3605

3606

void SelectionDAGBuilder::visitInsertValue(const User &I) {

3607

ArrayRef<unsigned> Indices;

3608

if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))

3609

Indices = IV->getIndices();

3610

else

3611

Indices = cast<ConstantExpr>(&I)->getIndices();

3612

3613

const Value *Op0 = I.getOperand(0);

3614

const Value *Op1 = I.getOperand(1);

3615

Type *AggTy = I.getType();

3616

Type *ValTy = Op1->getType();

3617

bool IntoUndef = isa<UndefValue>(Op0);

3618

bool FromUndef = isa<UndefValue>(Op1);

3619

3620

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3621

3622

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3623

SmallVector<EVT, 4> AggValueVTs;

3624

ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);

3625

SmallVector<EVT, 4> ValValueVTs;

3626

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3627

3628

unsigned NumAggValues = AggValueVTs.size();

3629

unsigned NumValValues = ValValueVTs.size();

3630

SmallVector<SDValue, 4> Values(NumAggValues);

3631

3632

// Ignore an insertvalue that produces an empty object

3633

if (!NumAggValues) {

3634

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3635

return;

3636

}

3637

3638

SDValue Agg = getValue(Op0);

3639

unsigned i = 0;

3640

// Copy the beginning value(s) from the original aggregate.

3641

for (; i != LinearIndex; ++i)

3642

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3643

SDValue(Agg.getNode(), Agg.getResNo() + i);

3644

// Copy values from the inserted value(s).

3645

if (NumValValues) {

3646

SDValue Val = getValue(Op1);

3647

for (; i != LinearIndex + NumValValues; ++i)

3648

Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3649

SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);

3650

}

3651

// Copy remaining value(s) from the original aggregate.

3652

for (; i != NumAggValues; ++i)

3653

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3654

SDValue(Agg.getNode(), Agg.getResNo() + i);

3655

3656

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3657

DAG.getVTList(AggValueVTs), Values));

3658

}

3659

3660

void SelectionDAGBuilder::visitExtractValue(const User &I) {

3661

ArrayRef<unsigned> Indices;

3662

if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))

3663

Indices = EV->getIndices();

3664

else

3665

Indices = cast<ConstantExpr>(&I)->getIndices();

3666

3667

const Value *Op0 = I.getOperand(0);

3668

Type *AggTy = Op0->getType();

3669

Type *ValTy = I.getType();

3670

bool OutOfUndef = isa<UndefValue>(Op0);

3671

3672

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3673

3674

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3675

SmallVector<EVT, 4> ValValueVTs;

3676

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3677

3678

unsigned NumValValues = ValValueVTs.size();

3679

3680

// Ignore a extractvalue that produces an empty object

3681

if (!NumValValues) {

3682

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3683

return;

3684

}

3685

3686

SmallVector<SDValue, 4> Values(NumValValues);

3687

3688

SDValue Agg = getValue(Op0);

3689

// Copy out the selected value(s).

3690

for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)

3691

Values[i - LinearIndex] =

3692

OutOfUndef ?

3693

DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :

3694

SDValue(Agg.getNode(), Agg.getResNo() + i);

3695

3696

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3697

DAG.getVTList(ValValueVTs), Values));

3698

}

3699

3700

void SelectionDAGBuilder::visitGetElementPtr(const User &I) {

3701

Value *Op0 = I.getOperand(0);

3702

// Note that the pointer operand may be a vector of pointers. Take the scalar

3703

// element which holds a pointer.

3704

unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();

3705

SDValue N = getValue(Op0);

3706

SDLoc dl = getCurSDLoc();

3707

auto &TLI = DAG.getTargetLoweringInfo();

3708

3709

// Normalize Vector GEP - all scalar operands should be converted to the

3710

// splat vector.

3711

bool IsVectorGEP = I.getType()->isVectorTy();

3712

ElementCount VectorElementCount =

3713

IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()

3714

: ElementCount::getFixed(0);

3715

3716

if (IsVectorGEP && !N.getValueType().isVector()) {

3717

LLVMContext &Context = *DAG.getContext();

3718

EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);

3719

if (VectorElementCount.isScalable())

3720

N = DAG.getSplatVector(VT, dl, N);

3721

else

3722

N = DAG.getSplatBuildVector(VT, dl, N);

3723

}

3724

3725

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);

3726

GTI != E; ++GTI) {

3727

const Value *Idx = GTI.getOperand();

3728

if (StructType *StTy = GTI.getStructTypeOrNull()) {

3729

unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();

3730

if (Field) {

3731

// N = N + Offset

3732

uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);

3733

3734

// In an inbounds GEP with an offset that is nonnegative even when

3735

// interpreted as signed, assume there is no unsigned overflow.

3736

SDNodeFlags Flags;

3737

if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())

3738

Flags.setNoUnsignedWrap(true);

3739

3740

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,

3741

DAG.getConstant(Offset, dl, N.getValueType()), Flags);

3742

}

3743

} else {

3744

// IdxSize is the width of the arithmetic according to IR semantics.

3745

// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth

3746

// (and fix up the result later).

3747

unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);

3748

MVT IdxTy = MVT::getIntegerVT(IdxSize);

3749

TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());

3750

// We intentionally mask away the high bits here; ElementSize may not

3751

// fit in IdxTy.

3752

APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());

3753

bool ElementScalable = ElementSize.isScalable();

3754

3755

// If this is a scalar constant or a splat vector of constants,

3756

// handle it quickly.

3757

const auto *C = dyn_cast<Constant>(Idx);

3758

if (C && isa<VectorType>(C->getType()))

3759

C = C->getSplatValue();

3760

3761

const auto *CI = dyn_cast_or_null<ConstantInt>(C);

3762

if (CI && CI->isZero())

3763

continue;

3764

if (CI && !ElementScalable) {

3765

APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);

3766

LLVMContext &Context = *DAG.getContext();

3767

SDValue OffsVal;

3768

if (IsVectorGEP)

3769

OffsVal = DAG.getConstant(

3770

Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));

3771

else

3772

OffsVal = DAG.getConstant(Offs, dl, IdxTy);

3773

3774

// In an inbounds GEP with an offset that is nonnegative even when

3775

// interpreted as signed, assume there is no unsigned overflow.

3776

SDNodeFlags Flags;

3777

if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())

3778

Flags.setNoUnsignedWrap(true);

3779

3780

OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

3781

3782

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);

3783

continue;

3784

}

3785

3786

// N = N + Idx * ElementMul;

3787

SDValue IdxN = getValue(Idx);

3788

3789

if (!IdxN.getValueType().isVector() && IsVectorGEP) {

3790

EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),

3791

VectorElementCount);

3792

if (VectorElementCount.isScalable())

3793

IdxN = DAG.getSplatVector(VT, dl, IdxN);

3794

else

3795

IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);

3796

}

3797

3798

// If the index is smaller or larger than intptr_t, truncate or extend

3799

// it.

3800

IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

3801

3802

if (ElementScalable) {

3803

EVT VScaleTy = N.getValueType().getScalarType();

3804

SDValue VScale = DAG.getNode(

3805

ISD::VSCALE, dl, VScaleTy,

3806

DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));

3807

if (IsVectorGEP)

3808

VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);

3809

IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);

3810

} else {

3811

// If this is a multiply by a power of two, turn it into a shl

3812

// immediately. This is a very common case.

3813

if (ElementMul != 1) {

3814

if (ElementMul.isPowerOf2()) {

3815

unsigned Amt = ElementMul.logBase2();

3816

IdxN = DAG.getNode(ISD::SHL, dl,

3817

N.getValueType(), IdxN,

3818

DAG.getConstant(Amt, dl, IdxN.getValueType()));

3819

} else {

3820

SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,

3821

IdxN.getValueType());

3822

IdxN = DAG.getNode(ISD::MUL, dl,

3823

N.getValueType(), IdxN, Scale);

3824

}

3825

}

3826

}

3827

3828

N = DAG.getNode(ISD::ADD, dl,

3829

N.getValueType(), N, IdxN);

3830

}

3831

}

3832

3833

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);

3834

MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);

3835

if (IsVectorGEP) {

3836

PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);

3837

PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);

3838

}

3839

3840

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())

3841

N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

3842

3843

setValue(&I, N);

3844

}

3845

3846

void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {

3847

// If this is a fixed sized alloca in the entry block of the function,

3848

// allocate it statically on the stack.

3849

if (FuncInfo.StaticAllocaMap.count(&I))

3850

return; // getValue will auto-populate this.

3851

3852

SDLoc dl = getCurSDLoc();

3853

Type *Ty = I.getAllocatedType();

3854

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3855

auto &DL = DAG.getDataLayout();

3856

uint64_t TySize = DL.getTypeAllocSize(Ty);

3857

MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

3858

3859

SDValue AllocSize = getValue(I.getArraySize());

3860

3861

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());

3862

if (AllocSize.getValueType() != IntPtr)

3863

AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

3864

3865

AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,

3866

AllocSize,

3867

DAG.getConstant(TySize, dl, IntPtr));

3868

3869

// Handle alignment. If the requested alignment is less than or equal to

3870

// the stack alignment, ignore it. If the size is greater than or equal to

3871

// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.

3872

Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();

3873

if (*Alignment <= StackAlign)

3874

Alignment = None;

3875

3876

const uint64_t StackAlignMask = StackAlign.value() - 1U;

3877

// Round the size of the allocation up to the stack alignment size

3878

// by add SA-1 to the size. This doesn't overflow because we're computing

3879

// an address inside an alloca.

3880

SDNodeFlags Flags;

3881

Flags.setNoUnsignedWrap(true);

3882

AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,

3883

DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

3884

3885

// Mask out the low bits for alignment purposes.

3886

AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,

3887

DAG.getConstant(~StackAlignMask, dl, IntPtr));

3888

3889

SDValue Ops[] = {

3890

getRoot(), AllocSize,

3891

DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};

3892

SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);

3893

SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);

3894

setValue(&I, DSA);

3895

DAG.setRoot(DSA.getValue(1));

3896

3897

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())((FuncInfo.MF->getFrameInfo().hasVarSizedObjects()) ? static_cast
<void> (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3897, __PRETTY_FUNCTION__));

3898

}

3899

3900

void SelectionDAGBuilder::visitLoad(const LoadInst &I) {

3901

if (I.isAtomic())

3902

return visitAtomicLoad(I);

3903

3904

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3905

const Value *SV = I.getOperand(0);

3906

if (TLI.supportSwiftError()) {

3907

// Swifterror values can come from either a function parameter with

3908

// swifterror attribute or an alloca with swifterror attribute.

3909

if (const Argument *Arg = dyn_cast<Argument>(SV)) {

3910

if (Arg->hasSwiftErrorAttr())

3911

return visitLoadFromSwiftError(I);

3912

}

3913

3914

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {

3915

if (Alloca->isSwiftError())

3916

return visitLoadFromSwiftError(I);

3917

}

3918

}

3919

3920

SDValue Ptr = getValue(SV);

3921

3922

Type *Ty = I.getType();

3923

Align Alignment = I.getAlign();

3924

3925

AAMDNodes AAInfo;

3926

I.getAAMetadata(AAInfo);

3927

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

3928

3929

SmallVector<EVT, 4> ValueVTs, MemVTs;

3930

SmallVector<uint64_t, 4> Offsets;

3931

ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);

3932

unsigned NumValues = ValueVTs.size();

3933

if (NumValues == 0)

3934

return;

3935

3936

bool isVolatile = I.isVolatile();

3937

3938

SDValue Root;

3939

bool ConstantMemory = false;

3940

if (isVolatile)

3941

// Serialize volatile loads with other side effects.

3942

Root = getRoot();

3943

else if (NumValues > MaxParallelChains)

3944

Root = getMemoryRoot();

3945

else if (AA &&

3946

AA->pointsToConstantMemory(MemoryLocation(

3947

SV,

3948

LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),

3949

AAInfo))) {

3950

// Do not serialize (non-volatile) loads of constant memory with anything.

3951

Root = DAG.getEntryNode();

3952

ConstantMemory = true;

3953

} else {

3954

// Do not serialize non-volatile loads against each other.

3955

Root = DAG.getRoot();

3956

}

3957

3958

SDLoc dl = getCurSDLoc();

3959

3960

if (isVolatile)

3961

Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

3962

3963

// An aggregate load cannot wrap around the address space, so offsets to its

3964

// parts don't wrap either.

3965

SDNodeFlags Flags;

3966

Flags.setNoUnsignedWrap(true);

3967

3968

SmallVector<SDValue, 4> Values(NumValues);

3969

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

3970

EVT PtrVT = Ptr.getValueType();

3971

3972

MachineMemOperand::Flags MMOFlags

3973

= TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

3974

3975

unsigned ChainI = 0;

3976

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

3977

// Serializing loads here may result in excessive register pressure, and

3978

// TokenFactor places arbitrary choke points on the scheduler. SD scheduling

3979

// could recover a bit by hoisting nodes upward in the chain by recognizing

3980

// they are side-effect free or do not alias. The optimizer should really

3981

// avoid this case by converting large object/array copies to llvm.memcpy

3982

// (MaxParallelChains should always remain as failsafe).

3983

if (ChainI == MaxParallelChains) {

3984

assert(PendingLoads.empty() && "PendingLoads must be serialized first")((PendingLoads.empty() && "PendingLoads must be serialized first"
) ? static_cast<void> (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3984, __PRETTY_FUNCTION__));

3985

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

3986

makeArrayRef(Chains.data(), ChainI));

3987

Root = Chain;

3988

ChainI = 0;

3989

}

3990

SDValue A = DAG.getNode(ISD::ADD, dl,

3991

PtrVT, Ptr,

3992

DAG.getConstant(Offsets[i], dl, PtrVT),

3993

Flags);

3994

3995

SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,

3996

MachinePointerInfo(SV, Offsets[i]), Alignment,

3997

MMOFlags, AAInfo, Ranges);

3998

Chains[ChainI] = L.getValue(1);

3999

4000

if (MemVTs[i] != ValueVTs[i])

4001

L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

4002

4003

Values[i] = L;

4004

}

4005

4006

if (!ConstantMemory) {

4007

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4008

makeArrayRef(Chains.data(), ChainI));

4009

if (isVolatile)

4010

DAG.setRoot(Chain);

4011

else

4012

PendingLoads.push_back(Chain);

4013

}

4014

4015

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,

4016

DAG.getVTList(ValueVTs), Values));

4017

}

4018

4019

void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {

4020

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4021, __PRETTY_FUNCTION__))

4021

"call visitStoreToSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4021, __PRETTY_FUNCTION__));

4022

4023

SmallVector<EVT, 4> ValueVTs;

4024

SmallVector<uint64_t, 4> Offsets;

4025

const Value *SrcV = I.getOperand(0);

4026

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4027

SrcV->getType(), ValueVTs, &Offsets);

4028

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4029, __PRETTY_FUNCTION__))

4029

"expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4029, __PRETTY_FUNCTION__));

4030

4031

SDValue Src = getValue(SrcV);

4032

// Create a virtual register, then update the virtual register.

4033

Register VReg =

4034

SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());

4035

// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue

4036

// Chain can be getRoot or getControlRoot.

4037

SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,

4038

SDValue(Src.getNode(), Src.getResNo()));

4039

DAG.setRoot(CopyNode);

4040

}

4041

4042

void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {

4043

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4044, __PRETTY_FUNCTION__))

4044

"call visitLoadFromSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4044, __PRETTY_FUNCTION__));

4045

4046

assert(!I.isVolatile() &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4049, __PRETTY_FUNCTION__))

4047

!I.hasMetadata(LLVMContext::MD_nontemporal) &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4049, __PRETTY_FUNCTION__))

4048

!I.hasMetadata(LLVMContext::MD_invariant_load) &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4049, __PRETTY_FUNCTION__))

4049

"Support volatile, non temporal, invariant for load_from_swift_error")((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4049, __PRETTY_FUNCTION__));

4050

4051

const Value *SV = I.getOperand(0);

4052

Type *Ty = I.getType();

4053

AAMDNodes AAInfo;

4054

I.getAAMetadata(AAInfo);

4055

assert((((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__))

4056

(!AA ||(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__))

4057

!AA->pointsToConstantMemory(MemoryLocation((((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__))

4058

SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__))

4059

AAInfo))) &&(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__))

4060

"load_from_swift_error should not be constant memory")(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4060, __PRETTY_FUNCTION__));

4061

4062

SmallVector<EVT, 4> ValueVTs;

4063

SmallVector<uint64_t, 4> Offsets;

4064

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,

4065

ValueVTs, &Offsets);

4066

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4067, __PRETTY_FUNCTION__))

4067

"expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4067, __PRETTY_FUNCTION__));

4068

4069

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT

4070

SDValue L = DAG.getCopyFromReg(

4071

getRoot(), getCurSDLoc(),

4072

SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

4073

4074

setValue(&I, L);

4075

}

4076

4077

void SelectionDAGBuilder::visitStore(const StoreInst &I) {

4078

if (I.isAtomic())

4079

return visitAtomicStore(I);

4080

4081

const Value *SrcV = I.getOperand(0);

4082

const Value *PtrV = I.getOperand(1);

4083

4084

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4085

if (TLI.supportSwiftError()) {

4086

// Swifterror values can come from either a function parameter with

4087

// swifterror attribute or an alloca with swifterror attribute.

4088

if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {

4089

if (Arg->hasSwiftErrorAttr())

4090

return visitStoreToSwiftError(I);

4091

}

4092

4093

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {

4094

if (Alloca->isSwiftError())

4095

return visitStoreToSwiftError(I);

4096

}

4097

}

4098

4099

SmallVector<EVT, 4> ValueVTs, MemVTs;

4100

SmallVector<uint64_t, 4> Offsets;

4101

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4102

SrcV->getType(), ValueVTs, &MemVTs, &Offsets);

4103

unsigned NumValues = ValueVTs.size();

4104

if (NumValues == 0)

4105

return;

4106

4107

// Get the lowered operands. Note that we do this after

4108

// checking if NumResults is zero, because with zero results

4109

// the operands won't have values in the map.

4110

SDValue Src = getValue(SrcV);

4111

SDValue Ptr = getValue(PtrV);

4112

4113

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();

4114

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4115

SDLoc dl = getCurSDLoc();

4116

Align Alignment = I.getAlign();

4117

AAMDNodes AAInfo;

4118

I.getAAMetadata(AAInfo);

4119

4120

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4121

4122

// An aggregate load cannot wrap around the address space, so offsets to its

4123

// parts don't wrap either.

4124

SDNodeFlags Flags;

4125

Flags.setNoUnsignedWrap(true);

4126

4127

unsigned ChainI = 0;

4128

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4129

// See visitLoad comments.

4130

if (ChainI == MaxParallelChains) {

4131

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4132

makeArrayRef(Chains.data(), ChainI));

4133

Root = Chain;

4134

ChainI = 0;

4135

}

4136

SDValue Add =

4137

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);

4138

SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);

4139

if (MemVTs[i] != ValueVTs[i])

4140

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);

4141

SDValue St =

4142

DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),

4143

Alignment, MMOFlags, AAInfo);

4144

Chains[ChainI] = St;

4145

}

4146

4147

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4148

makeArrayRef(Chains.data(), ChainI));

4149

DAG.setRoot(StoreNode);

4150

}

4151

4152

void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,

4153

bool IsCompressing) {

4154

SDLoc sdl = getCurSDLoc();

4155

4156

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4157

MaybeAlign &Alignment) {

4158

// llvm.masked.store.*(Src0, Ptr, alignment, Mask)

4159

Src0 = I.getArgOperand(0);

4160

Ptr = I.getArgOperand(1);

4161

Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();

4162

Mask = I.getArgOperand(3);

4163

};

4164

auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4165

MaybeAlign &Alignment) {

4166

// llvm.masked.compressstore.*(Src0, Ptr, Mask)

4167

Src0 = I.getArgOperand(0);

4168

Ptr = I.getArgOperand(1);

4169

Mask = I.getArgOperand(2);

4170

Alignment = None;

4171

};

4172

4173

Value *PtrOperand, *MaskOperand, *Src0Operand;

4174

MaybeAlign Alignment;

4175

if (IsCompressing)

4176

getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4177

else

4178

getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4179

4180

SDValue Ptr = getValue(PtrOperand);

4181

SDValue Src0 = getValue(Src0Operand);

4182

SDValue Mask = getValue(MaskOperand);

4183

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4184

4185

EVT VT = Src0.getValueType();

4186

if (!Alignment)

4187

Alignment = DAG.getEVTAlign(VT);

4188

4189

AAMDNodes AAInfo;

4190

I.getAAMetadata(AAInfo);

4191

4192

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4193

MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,

4194

// TODO: Make MachineMemOperands aware of scalable

4195

// vectors.

4196

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);

4197

SDValue StoreNode =

4198

DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,

4199

ISD::UNINDEXED, false /* Truncating */, IsCompressing);

4200

DAG.setRoot(StoreNode);

4201

setValue(&I, StoreNode);

4202

}

4203

4204

// Get a uniform base for the Gather/Scatter intrinsic.

4205

// The first argument of the Gather/Scatter intrinsic is a vector of pointers.

4206

// We try to represent it as a base pointer + vector of indices.

4207

// Usually, the vector of pointers comes from a 'getelementptr' instruction.

4208

// The first operand of the GEP may be a single pointer or a vector of pointers

4209

// Example:

4210

// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind

4211

// or

4212

// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind

4213

// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..

4214

//

4215

// When the first GEP operand is a single pointer - it is the uniform base we

4216

// are looking for. If first operand of the GEP is a splat vector - we

4217

// extract the splat value and use it as a uniform base.

4218

// In all other cases the function returns 'false'.

4219

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,

4220

ISD::MemIndexType &IndexType, SDValue &Scale,

4221

SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {

4222

SelectionDAG& DAG = SDB->DAG;

4223

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4224

const DataLayout &DL = DAG.getDataLayout();

4225

4226

assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")((Ptr->getType()->isVectorTy() && "Uexpected pointer type"
) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4226, __PRETTY_FUNCTION__));

4227

4228

// Handle splat constant pointer.

4229

if (auto *C = dyn_cast<Constant>(Ptr)) {

4230

C = C->getSplatValue();

4231

if (!C)

4232

return false;

4233

4234

Base = SDB->getValue(C);

4235

4236

unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();

4237

EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);

4238

Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);

4239

IndexType = ISD::SIGNED_SCALED;

4240

Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4241

return true;

4242

}

4243

4244

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

4245

if (!GEP || GEP->getParent() != CurBB)

4246

return false;

4247

4248

if (GEP->getNumOperands() != 2)

4249

return false;

4250

4251

const Value *BasePtr = GEP->getPointerOperand();

4252

const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

4253

4254

// Make sure the base is scalar and the index is a vector.

4255

if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())

4256

return false;

4257

4258

Base = SDB->getValue(BasePtr);

4259

Index = SDB->getValue(IndexVal);

4260

IndexType = ISD::SIGNED_SCALED;

4261

Scale = DAG.getTargetConstant(

4262

DL.getTypeAllocSize(GEP->getResultElementType()),

4263

SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4264

return true;

4265

}

4266

4267

void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {

4268

SDLoc sdl = getCurSDLoc();

4269

4270

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)

4271

const Value *Ptr = I.getArgOperand(1);

4272

SDValue Src0 = getValue(I.getArgOperand(0));

4273

SDValue Mask = getValue(I.getArgOperand(3));

4274

EVT VT = Src0.getValueType();

4275

Align Alignment = cast<ConstantInt>(I.getArgOperand(2))

4276

->getMaybeAlignValue()

4277

.getValueOr(DAG.getEVTAlign(VT));

4278

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4279

4280

AAMDNodes AAInfo;

4281

I.getAAMetadata(AAInfo);

4282

4283

SDValue Base;

4284

SDValue Index;

4285

ISD::MemIndexType IndexType;

4286

SDValue Scale;

4287

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4288

I.getParent());

4289

4290

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4291

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4292

MachinePointerInfo(AS), MachineMemOperand::MOStore,

4293

// TODO: Make MachineMemOperands aware of scalable

4294

// vectors.

4295

MemoryLocation::UnknownSize, Alignment, AAInfo);

4296

if (!UniformBase) {

4297

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4298

Index = getValue(Ptr);

4299

IndexType = ISD::SIGNED_SCALED;

4300

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4301

}

4302

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };

4303

SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,

4304

Ops, MMO, IndexType);

4305

DAG.setRoot(Scatter);

4306

setValue(&I, Scatter);

4307

}

4308

4309

void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {

4310

SDLoc sdl = getCurSDLoc();

4311

4312

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4313

MaybeAlign &Alignment) {

4314

// @llvm.masked.load.*(Ptr, alignment, Mask, Src0)

4315

Ptr = I.getArgOperand(0);

4316

Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();

4317

Mask = I.getArgOperand(2);

4318

Src0 = I.getArgOperand(3);

4319

};

4320

auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4321

MaybeAlign &Alignment) {

4322

// @llvm.masked.expandload.*(Ptr, Mask, Src0)

4323

Ptr = I.getArgOperand(0);

4324

Alignment = None;

4325

Mask = I.getArgOperand(1);

4326

Src0 = I.getArgOperand(2);

4327

};

4328

4329

Value *PtrOperand, *MaskOperand, *Src0Operand;

4330

MaybeAlign Alignment;

4331

if (IsExpanding)

4332

getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4333

else

4334

getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4335

4336

SDValue Ptr = getValue(PtrOperand);

4337

SDValue Src0 = getValue(Src0Operand);

4338

SDValue Mask = getValue(MaskOperand);

4339

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4340

4341

EVT VT = Src0.getValueType();

4342

if (!Alignment)

4343

Alignment = DAG.getEVTAlign(VT);

4344

4345

AAMDNodes AAInfo;

4346

I.getAAMetadata(AAInfo);

4347

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4348

4349

// Do not serialize masked loads of constant memory with anything.

4350

MemoryLocation ML;

4351

if (VT.isScalableVector())

4352

ML = MemoryLocation(PtrOperand);

4353

else

4354

ML = MemoryLocation(PtrOperand, LocationSize::precise(

4355

DAG.getDataLayout().getTypeStoreSize(I.getType())),

4356

AAInfo);

4357

bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

4358

4359

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

4360

4361

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4362

MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,

4363

// TODO: Make MachineMemOperands aware of scalable

4364

// vectors.

4365

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);

4366

4367

SDValue Load =

4368

DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,

4369

ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);

4370

if (AddToChain)

4371

PendingLoads.push_back(Load.getValue(1));

4372

setValue(&I, Load);

4373

}

4374

4375

void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {

4376

SDLoc sdl = getCurSDLoc();

4377

4378

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)

4379

const Value *Ptr = I.getArgOperand(0);

4380

SDValue Src0 = getValue(I.getArgOperand(3));

4381

SDValue Mask = getValue(I.getArgOperand(2));

4382

4383

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4384

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4385

Align Alignment = cast<ConstantInt>(I.getArgOperand(1))

4386

->getMaybeAlignValue()

4387

.getValueOr(DAG.getEVTAlign(VT));

4388

4389

AAMDNodes AAInfo;

4390

I.getAAMetadata(AAInfo);

4391

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4392

4393

SDValue Root = DAG.getRoot();

4394

SDValue Base;

4395

SDValue Index;

4396

ISD::MemIndexType IndexType;

4397

SDValue Scale;

4398

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4399

I.getParent());

4400

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4401

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4402

MachinePointerInfo(AS), MachineMemOperand::MOLoad,

4403

// TODO: Make MachineMemOperands aware of scalable

4404

// vectors.

4405

MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);

4406

4407

if (!UniformBase) {

4408

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4409

Index = getValue(Ptr);

4410

IndexType = ISD::SIGNED_SCALED;

4411

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4412

}

4413

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };

4414

SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,

4415

Ops, MMO, IndexType);

4416

4417

PendingLoads.push_back(Gather.getValue(1));

4418

setValue(&I, Gather);

4419

}

4420

4421

void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {

4422

SDLoc dl = getCurSDLoc();

4423

AtomicOrdering SuccessOrdering = I.getSuccessOrdering();

4424

AtomicOrdering FailureOrdering = I.getFailureOrdering();

4425

SyncScope::ID SSID = I.getSyncScopeID();

4426

4427

SDValue InChain = getRoot();

4428

4429

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();

4430

SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

4431

4432

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4433

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4434

4435

MachineFunction &MF = DAG.getMachineFunction();

4436

MachineMemOperand *MMO = MF.getMachineMemOperand(

4437

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4438

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,

4439

FailureOrdering);

4440

4441

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,

4442

dl, MemVT, VTs, InChain,

4443

getValue(I.getPointerOperand()),

4444

getValue(I.getCompareOperand()),

4445

getValue(I.getNewValOperand()), MMO);

4446

4447

SDValue OutChain = L.getValue(2);

4448

4449

setValue(&I, L);

4450

DAG.setRoot(OutChain);

4451

}

4452

4453

void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {

4454

SDLoc dl = getCurSDLoc();

4455

ISD::NodeType NT;

4456

switch (I.getOperation()) {

4457

default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4457);

4458

case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;

4459

case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;

4460

case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;

4461

case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;

4462

case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;

4463

case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;

4464

case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;

4465

case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;

4466

case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;

4467

case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;

4468

case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;

4469

case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;

4470

case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;

4471

}

4472

AtomicOrdering Ordering = I.getOrdering();

4473

SyncScope::ID SSID = I.getSyncScopeID();

4474

4475

SDValue InChain = getRoot();

4476

4477

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();

4478

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4479

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4480

4481

MachineFunction &MF = DAG.getMachineFunction();

4482

MachineMemOperand *MMO = MF.getMachineMemOperand(

4483

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4484

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

4485

4486

SDValue L =

4487

DAG.getAtomic(NT, dl, MemVT, InChain,

4488

getValue(I.getPointerOperand()), getValue(I.getValOperand()),

4489

MMO);

4490

4491

SDValue OutChain = L.getValue(1);

4492

4493

setValue(&I, L);

4494

DAG.setRoot(OutChain);

4495

}

4496

4497

void SelectionDAGBuilder::visitFence(const FenceInst &I) {

4498

SDLoc dl = getCurSDLoc();

4499

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4500

SDValue Ops[3];

4501

Ops[0] = getRoot();

4502

Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,

4503

TLI.getFenceOperandTy(DAG.getDataLayout()));

4504

Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,

4505

TLI.getFenceOperandTy(DAG.getDataLayout()));

4506

DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));

4507

}

4508

4509

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {

4510

SDLoc dl = getCurSDLoc();

4511

AtomicOrdering Order = I.getOrdering();

4512

SyncScope::ID SSID = I.getSyncScopeID();

4513

4514

SDValue InChain = getRoot();

4515

4516

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4517

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4518

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

4519

4520

if (!TLI.supportsUnalignedAtomics() &&

4521

I.getAlignment() < MemVT.getSizeInBits() / 8)

4522

report_fatal_error("Cannot generate unaligned atomic load");

4523

4524

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4525

4526

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4527

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4528

I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

4529

4530

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

4531

4532

SDValue Ptr = getValue(I.getPointerOperand());

4533

4534

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {

4535

// TODO: Once this is better exercised by tests, it should be merged with

4536

// the normal path for loads to prevent future divergence.

4537

SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);

4538

if (MemVT != VT)

4539

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4540

4541

setValue(&I, L);

4542

SDValue OutChain = L.getValue(1);

4543

if (!I.isUnordered())

4544

DAG.setRoot(OutChain);

4545

else

4546

PendingLoads.push_back(OutChain);

4547

return;

4548

}

4549

4550

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,

4551

Ptr, MMO);

4552

4553

SDValue OutChain = L.getValue(1);

4554

if (MemVT != VT)

4555

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4556

4557

setValue(&I, L);

4558

DAG.setRoot(OutChain);

4559

}

4560

4561

void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {

4562

SDLoc dl = getCurSDLoc();

4563

4564

AtomicOrdering Ordering = I.getOrdering();

4565

SyncScope::ID SSID = I.getSyncScopeID();

4566

4567

SDValue InChain = getRoot();

4568

4569

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4570

EVT MemVT =

4571

TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

4572

4573

if (I.getAlignment() < MemVT.getSizeInBits() / 8)

4574

report_fatal_error("Cannot generate unaligned atomic store");

4575

4576

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4577

4578

MachineFunction &MF = DAG.getMachineFunction();

4579

MachineMemOperand *MMO = MF.getMachineMemOperand(

4580

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4581

I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

4582

4583

SDValue Val = getValue(I.getValueOperand());

4584

if (Val.getValueType() != MemVT)

4585

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);

4586

SDValue Ptr = getValue(I.getPointerOperand());

4587

4588

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {

4589

// TODO: Once this is better exercised by tests, it should be merged with

4590

// the normal path for stores to prevent future divergence.

4591

SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);

4592

DAG.setRoot(S);

4593

return;

4594

}

4595

SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,

4596

Ptr, Val, MMO);

4597

4598

4599

DAG.setRoot(OutChain);

4600

}

4601

4602

/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC

4603

/// node.

4604

void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,

4605

unsigned Intrinsic) {

4606

// Ignore the callsite's attributes. A specific call site may be marked with

4607

// readnone, but the lowering code will expect the chain based on the

4608

// definition.

4609

const Function *F = I.getCalledFunction();

4610

bool HasChain = !F->doesNotAccessMemory();

4611

bool OnlyLoad = HasChain && F->onlyReadsMemory();

4612

4613

// Build the operand list.

4614

SmallVector<SDValue, 8> Ops;

4615

if (HasChain) { // If this intrinsic has side-effects, chainify it.

4616

if (OnlyLoad) {

4617

// We don't need to serialize loads against other loads.

4618

Ops.push_back(DAG.getRoot());

4619

} else {

4620

Ops.push_back(getRoot());

4621

}

4622

}

4623

4624

// Info is set by getTgtMemInstrinsic

4625

TargetLowering::IntrinsicInfo Info;

4626

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4627

bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,

4628

DAG.getMachineFunction(),

4629

Intrinsic);

4630

4631

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.

4632

if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||

4633

Info.opc == ISD::INTRINSIC_W_CHAIN)

4634

Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),

4635

TLI.getPointerTy(DAG.getDataLayout())));

4636

4637

// Add all operands of the call to the operand list.

4638

for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {

4639

const Value *Arg = I.getArgOperand(i);

4640

if (!I.paramHasAttr(i, Attribute::ImmArg)) {

4641

Ops.push_back(getValue(Arg));

4642

continue;

4643

}

4644

4645

// Use TargetConstant instead of a regular constant for immarg.

4646

EVT VT = TLI.getValueType(*DL, Arg->getType(), true);

4647

if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {

4648

assert(CI->getBitWidth() <= 64 &&((CI->getBitWidth() <= 64 && "large intrinsic immediates not handled"
) ? static_cast<void> (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4649, __PRETTY_FUNCTION__))

4649

"large intrinsic immediates not handled")((CI->getBitWidth() <= 64 && "large intrinsic immediates not handled"
) ? static_cast<void> (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4649, __PRETTY_FUNCTION__));

4650

Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));

4651

} else {

4652

Ops.push_back(

4653

DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));

4654

}

4655

}

4656

4657

SmallVector<EVT, 4> ValueVTs;

4658

ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

4659

4660

if (HasChain)

4661

ValueVTs.push_back(MVT::Other);

4662

4663

SDVTList VTs = DAG.getVTList(ValueVTs);

4664

4665

// Create the node.

4666

SDValue Result;

4667

if (IsTgtIntrinsic) {

4668

// This is target intrinsic that touches memory

4669

AAMDNodes AAInfo;

4670

I.getAAMetadata(AAInfo);

4671

Result =

4672

DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,

4673

MachinePointerInfo(Info.ptrVal, Info.offset),

4674

Info.align, Info.flags, Info.size, AAInfo);

4675

} else if (!HasChain) {

4676

Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);

4677

} else if (!I.getType()->isVoidTy()) {

4678

Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);

4679

} else {

4680

Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);

4681

}

4682

4683

if (HasChain) {

4684

SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);

4685

if (OnlyLoad)

4686

PendingLoads.push_back(Chain);

4687

else

4688

DAG.setRoot(Chain);

4689

}

4690

4691

if (!I.getType()->isVoidTy()) {

4692

if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {

4693

EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);

4694

Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);

4695

} else

4696

Result = lowerRangeToAssertZExt(DAG, I, Result);

4697

4698

MaybeAlign Alignment = I.getRetAlign();

4699

if (!Alignment)

4700

Alignment = F->getAttributes().getRetAlignment();

4701

// Insert `assertalign` node if there's an alignment.

4702

if (InsertAssertAlign && Alignment) {

4703

Result =

4704

DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());

4705

}

4706

4707

setValue(&I, Result);

4708

}

4709

}

4710

4711

/// GetSignificand - Get the significand and build it into a floating-point

4712

/// number with exponent of 1:

4713

///

4714

/// Op = (Op & 0x007fffff) | 0x3f800000;

4715

///

4716

/// where Op is the hexadecimal representation of floating point value.

4717

static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {

4718

SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4719

DAG.getConstant(0x007fffff, dl, MVT::i32));

4720

SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,

4721

DAG.getConstant(0x3f800000, dl, MVT::i32));

4722

return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);

4723

}

4724

4725

/// GetExponent - Get the exponent:

4726

///

4727

/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);

4728

///

4729

/// where Op is the hexadecimal representation of floating point value.

4730

static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,

4731

const TargetLowering &TLI, const SDLoc &dl) {

4732

SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4733

DAG.getConstant(0x7f800000, dl, MVT::i32));

4734

SDValue t1 = DAG.getNode(

4735

ISD::SRL, dl, MVT::i32, t0,

4736

DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));

4737

SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,

4738

DAG.getConstant(127, dl, MVT::i32));

4739

return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);

4740

}

4741

4742

/// getF32Constant - Get 32-bit floating point constant.

4743

static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,

4744

const SDLoc &dl) {

4745

return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,

4746

MVT::f32);

4747

}

4748

4749

static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,

4750

SelectionDAG &DAG) {

4751

// TODO: What fast-math-flags should be set on the floating-point nodes?

4752

4753

// IntegerPartOfX = ((int32_t)(t0);

4754

SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

4755

4756

// FractionalPartOfX = t0 - (float)IntegerPartOfX;

4757

SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);

4758

SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

4759

4760

// IntegerPartOfX <<= 23;

4761

IntegerPartOfX = DAG.getNode(

4762

ISD::SHL, dl, MVT::i32, IntegerPartOfX,

4763

DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(

4764

DAG.getDataLayout())));

4765

4766

SDValue TwoToFractionalPartOfX;

4767

if (LimitFloatPrecision <= 6) {

4768

// For floating-point precision of 6:

4769

//

4770

// TwoToFractionalPartOfX =

4771

// 0.997535578f +

4772

// (0.735607626f + 0.252464424f * x) * x;

4773

//

4774

// error 0.0144103317, which is 6 bits

4775

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4776

getF32Constant(DAG, 0x3e814304, dl));

4777

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4778

getF32Constant(DAG, 0x3f3c50c8, dl));

4779

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4780

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4781

getF32Constant(DAG, 0x3f7f5e7e, dl));

4782

} else if (LimitFloatPrecision <= 12) {

4783

// For floating-point precision of 12:

4784

//

4785

// TwoToFractionalPartOfX =

4786

// 0.999892986f +

4787

// (0.696457318f +

4788

// (0.224338339f + 0.792043434e-1f * x) * x) * x;

4789

//

4790

// error 0.000107046256, which is 13 to 14 bits

4791

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4792

getF32Constant(DAG, 0x3da235e3, dl));

4793

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4794

getF32Constant(DAG, 0x3e65b8f3, dl));

4795

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4796

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4797

getF32Constant(DAG, 0x3f324b07, dl));

4798

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4799

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4800

getF32Constant(DAG, 0x3f7ff8fd, dl));

4801

} else { // LimitFloatPrecision <= 18

4802

// For floating-point precision of 18:

4803

//

4804

// TwoToFractionalPartOfX =

4805

// 0.999999982f +

4806

// (0.693148872f +

4807

// (0.240227044f +

4808

// (0.554906021e-1f +

4809

// (0.961591928e-2f +

4810

// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;

4811

// error 2.47208000*10^(-7), which is better than 18 bits

4812

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4813

getF32Constant(DAG, 0x3924b03e, dl));

4814

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4815

getF32Constant(DAG, 0x3ab24b87, dl));

4816

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4817

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4818

getF32Constant(DAG, 0x3c1d8c17, dl));

4819

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4820

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4821

getF32Constant(DAG, 0x3d634a1d, dl));

4822

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

4823

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

4824

getF32Constant(DAG, 0x3e75fe14, dl));

4825

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

4826

SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,

4827

getF32Constant(DAG, 0x3f317234, dl));

4828

SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);

4829

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,

4830

getF32Constant(DAG, 0x3f800000, dl));

4831

}

4832

4833

// Add the exponent into the result in integer domain.

4834

SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);

4835

return DAG.getNode(ISD::BITCAST, dl, MVT::f32,

4836

DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));

4837

}

4838

4839

/// expandExp - Lower an exp intrinsic. Handles the special sequences for

4840

/// limited-precision mode.

4841

static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4842

const TargetLowering &TLI, SDNodeFlags Flags) {

4843

if (Op.getValueType() == MVT::f32 &&

4844

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

4845

4846

// Put the exponent in the right bit position for later addition to the

4847

// final result:

4848

//

4849

// t0 = Op * log2(e)

4850

4851

// TODO: What fast-math-flags should be set here?

4852

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,

4853

DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));

4854

return getLimitedPrecisionExp2(t0, dl, DAG);

4855

}

4856

4857

// No special expansion.

4858

return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);

4859

}

4860

4861

/// expandLog - Lower a log intrinsic. Handles the special sequences for

4862

/// limited-precision mode.

4863

static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4864

const TargetLowering &TLI, SDNodeFlags Flags) {

4865

// TODO: What fast-math-flags should be set on the floating-point nodes?

4866

4867

if (Op.getValueType() == MVT::f32 &&

4868

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

4869

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

4870

4871

// Scale the exponent by log(2).

4872

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

4873

SDValue LogOfExponent =

4874

DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

4875

DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

4876

4877

// Get the significand and build it into a floating-point number with

4878

// exponent of 1.

4879

SDValue X = GetSignificand(DAG, Op1, dl);

4880

4881

SDValue LogOfMantissa;

4882

if (LimitFloatPrecision <= 6) {

4883

// For floating-point precision of 6:

4884

//

4885

// LogofMantissa =

4886

// -1.1609546f +

4887

// (1.4034025f - 0.23903021f * x) * x;

4888

//

4889

// error 0.0034276066, which is better than 8 bits

4890

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4891

getF32Constant(DAG, 0xbe74c456, dl));

4892

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

4893

getF32Constant(DAG, 0x3fb3a2b1, dl));

4894

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

4895

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

4896

getF32Constant(DAG, 0x3f949a29, dl));

4897

} else if (LimitFloatPrecision <= 12) {

4898

// For floating-point precision of 12:

4899

//

4900

// LogOfMantissa =

4901

// -1.7417939f +

4902

// (2.8212026f +

4903

// (-1.4699568f +

4904

// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;

4905

//

4906

// error 0.000061011436, which is 14 bits

4907

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4908

getF32Constant(DAG, 0xbd67b6d6, dl));

4909

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

4910

getF32Constant(DAG, 0x3ee4f4b8, dl));

4911

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

4912

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

4913

getF32Constant(DAG, 0x3fbc278b, dl));

4914

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4915

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4916

getF32Constant(DAG, 0x40348e95, dl));

4917

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4918

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

4919

getF32Constant(DAG, 0x3fdef31a, dl));

4920

} else { // LimitFloatPrecision <= 18

4921

// For floating-point precision of 18:

4922

//

4923

// LogOfMantissa =

4924

// -2.1072184f +

4925

// (4.2372794f +

4926

// (-3.7029485f +

4927

// (2.2781945f +

4928

// (-0.87823314f +

4929

// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;

4930

//

4931

// error 0.0000023660568, which is better than 18 bits

4932

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4933

getF32Constant(DAG, 0xbc91e5ac, dl));

4934

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

4935

getF32Constant(DAG, 0x3e4350aa, dl));

4936

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

4937

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

4938

getF32Constant(DAG, 0x3f60d3e3, dl));

4939

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4940

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4941

getF32Constant(DAG, 0x4011cdf0, dl));

4942

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4943

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

4944

getF32Constant(DAG, 0x406cfd1c, dl));

4945

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

4946

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

4947

getF32Constant(DAG, 0x408797cb, dl));

4948

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

4949

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

4950

getF32Constant(DAG, 0x4006dcab, dl));

4951

}

4952

4953

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);

4954

}

4955

4956

// No special expansion.

4957

return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);

4958

}

4959

4960

/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for

4961

/// limited-precision mode.

4962

static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4963

const TargetLowering &TLI, SDNodeFlags Flags) {

4964

// TODO: What fast-math-flags should be set on the floating-point nodes?

4965

4966

if (Op.getValueType() == MVT::f32 &&

4967

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

4968

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

4969

4970

// Get the exponent.

4971

SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

4972

4973

// Get the significand and build it into a floating-point number with

4974

// exponent of 1.

4975

SDValue X = GetSignificand(DAG, Op1, dl);

4976

4977

// Different possible minimax approximations of significand in

4978

// floating-point for various degrees of accuracy over [1,2].

4979

SDValue Log2ofMantissa;

4980

if (LimitFloatPrecision <= 6) {

4981

// For floating-point precision of 6:

4982

//

4983

// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;

4984

//

4985

// error 0.0049451742, which is more than 7 bits

4986

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4987

getF32Constant(DAG, 0xbeb08fe0, dl));

4988

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

4989

getF32Constant(DAG, 0x40019463, dl));

4990

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

4991

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

4992

getF32Constant(DAG, 0x3fd6633d, dl));

4993

} else if (LimitFloatPrecision <= 12) {

4994

// For floating-point precision of 12:

4995

//

4996

// Log2ofMantissa =

4997

// -2.51285454f +

4998

// (4.07009056f +

4999

// (-2.12067489f +

5000

// (.645142248f - 0.816157886e-1f * x) * x) * x) * x;

5001

//

5002

// error 0.0000876136000, which is better than 13 bits

5003

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5004

getF32Constant(DAG, 0xbda7262e, dl));

5005

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5006

getF32Constant(DAG, 0x3f25280b, dl));

5007

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5008

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5009

getF32Constant(DAG, 0x4007b923, dl));

5010

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5011

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5012

getF32Constant(DAG, 0x40823e2f, dl));

5013

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5014

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5015

getF32Constant(DAG, 0x4020d29c, dl));

5016

} else { // LimitFloatPrecision <= 18

5017

// For floating-point precision of 18:

5018

//

5019

// Log2ofMantissa =

5020

// -3.0400495f +

5021

// (6.1129976f +

5022

// (-5.3420409f +

5023

// (3.2865683f +

5024

// (-1.2669343f +

5025

// (0.27515199f -

5026

// 0.25691327e-1f * x) * x) * x) * x) * x) * x;

5027

//

5028

// error 0.0000018516, which is better than 18 bits

5029

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5030

getF32Constant(DAG, 0xbcd2769e, dl));

5031

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5032

getF32Constant(DAG, 0x3e8ce0b9, dl));

5033

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5034

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5035

getF32Constant(DAG, 0x3fa22ae7, dl));

5036

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5037

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5038

getF32Constant(DAG, 0x40525723, dl));

5039

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5040

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5041

getF32Constant(DAG, 0x40aaf200, dl));

5042

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5043

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5044

getF32Constant(DAG, 0x40c39dad, dl));

5045

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5046

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5047

getF32Constant(DAG, 0x4042902c, dl));

5048

}

5049

5050

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);

5051

}

5052

5053

// No special expansion.

5054

return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);

5055

}

5056

5057

/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for

5058

/// limited-precision mode.

5059

static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5060

const TargetLowering &TLI, SDNodeFlags Flags) {

5061

// TODO: What fast-math-flags should be set on the floating-point nodes?

5062

5063

if (Op.getValueType() == MVT::f32 &&

5064

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5065

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5066

5067

// Scale the exponent by log10(2) [0.30102999f].

5068

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5069

SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5070

getF32Constant(DAG, 0x3e9a209a, dl));

5071

5072

// Get the significand and build it into a floating-point number with

5073

// exponent of 1.

5074

SDValue X = GetSignificand(DAG, Op1, dl);

5075

5076

SDValue Log10ofMantissa;

5077

if (LimitFloatPrecision <= 6) {

5078

// For floating-point precision of 6:

5079

//

5080

// Log10ofMantissa =

5081

// -0.50419619f +

5082

// (0.60948995f - 0.10380950f * x) * x;

5083

//

5084

// error 0.0014886165, which is 6 bits

5085

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5086

getF32Constant(DAG, 0xbdd49a13, dl));

5087

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5088

getF32Constant(DAG, 0x3f1c0789, dl));

5089

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5090

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5091

getF32Constant(DAG, 0x3f011300, dl));

5092

} else if (LimitFloatPrecision <= 12) {

5093

// For floating-point precision of 12:

5094

//

5095

// Log10ofMantissa =

5096

// -0.64831180f +

5097

// (0.91751397f +

5098

// (-0.31664806f + 0.47637168e-1f * x) * x) * x;

5099

//

5100

// error 0.00019228036, which is better than 12 bits

5101

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5102

getF32Constant(DAG, 0x3d431f31, dl));

5103

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5104

getF32Constant(DAG, 0x3ea21fb2, dl));

5105

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5106

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5107

getF32Constant(DAG, 0x3f6ae232, dl));

5108

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5109

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5110

getF32Constant(DAG, 0x3f25f7c3, dl));

5111

} else { // LimitFloatPrecision <= 18

5112

// For floating-point precision of 18:

5113

//

5114

// Log10ofMantissa =

5115

// -0.84299375f +

5116

// (1.5327582f +

5117

// (-1.0688956f +

5118

// (0.49102474f +

5119

// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;

5120

//

5121

// error 0.0000037995730, which is better than 18 bits

5122

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5123

getF32Constant(DAG, 0x3c5d51ce, dl));

5124

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5125

getF32Constant(DAG, 0x3e00685a, dl));

5126

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5127

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5128

getF32Constant(DAG, 0x3efb6798, dl));

5129

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5130

SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5131

getF32Constant(DAG, 0x3f88d192, dl));

5132

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5133

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5134

getF32Constant(DAG, 0x3fc4316c, dl));

5135

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5136

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,

5137

getF32Constant(DAG, 0x3f57ce70, dl));

5138

}

5139

5140

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);

5141

}

5142

5143

// No special expansion.

5144

return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);

5145

}

5146

5147

/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for

5148

/// limited-precision mode.

5149

static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5150

const TargetLowering &TLI, SDNodeFlags Flags) {

5151

if (Op.getValueType() == MVT::f32 &&

5152

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)

5153

return getLimitedPrecisionExp2(Op, dl, DAG);

5154

5155

// No special expansion.

5156

return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);

5157

}

5158

5159

/// visitPow - Lower a pow intrinsic. Handles the special sequences for

5160

/// limited-precision mode with x == 10.0f.

5161

static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,

5162

SelectionDAG &DAG, const TargetLowering &TLI,

5163

SDNodeFlags Flags) {

5164

bool IsExp10 = false;

5165

if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&

5166

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5167

if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {

5168

APFloat Ten(10.0f);

5169

IsExp10 = LHSC->isExactlyValue(Ten);

5170

}

5171

}

5172

5173

// TODO: What fast-math-flags should be set on the FMUL node?

5174

if (IsExp10) {

5175

// Put the exponent in the right bit position for later addition to the

5176

// final result:

5177

//

5178

// #define LOG2OF10 3.3219281f

5179

// t0 = Op * LOG2OF10;

5180

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,

5181

getF32Constant(DAG, 0x40549a78, dl));

5182

return getLimitedPrecisionExp2(t0, dl, DAG);

5183

}

5184

5185

// No special expansion.

5186

return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);

5187

}

5188

5189

/// ExpandPowI - Expand a llvm.powi intrinsic.

5190

static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,

5191

SelectionDAG &DAG) {

5192

// If RHS is a constant, we can expand this out to a multiplication tree,

5193

// otherwise we end up lowering to a call to __powidf2 (for example). When

5194

// optimizing for size, we only want to do this if the expansion would produce

5195

// a small number of multiplies, otherwise we do the full expansion.

5196

if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {

5197

// Get the exponent as a positive value.

5198

unsigned Val = RHSC->getSExtValue();

5199

if ((int)Val < 0) Val = -Val;

5200

5201

// powi(x, 0) -> 1.0

5202

if (Val == 0)

5203

return DAG.getConstantFP(1.0, DL, LHS.getValueType());

5204

5205

bool OptForSize = DAG.shouldOptForSize();

5206

if (!OptForSize ||

5207

// If optimizing for size, don't insert too many multiplies.

5208

// This inserts up to 5 multiplies.

5209

countPopulation(Val) + Log2_32(Val) < 7) {

5210

// We use the simple binary decomposition method to generate the multiply

5211

// sequence. There are more optimal ways to do this (for example,

5212

// powi(x,15) generates one more multiply than it should), but this has

5213

// the benefit of being both really simple and much better than a libcall.

5214

SDValue Res; // Logically starts equal to 1.0

5215

SDValue CurSquare = LHS;

5216

// TODO: Intrinsics should have fast-math-flags that propagate to these

5217

// nodes.

5218

while (Val) {

5219

if (Val & 1) {

5220

if (Res.getNode())

5221

Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);

5222

else

5223

Res = CurSquare; // 1.0*CurSquare.

5224

}

5225

5226

CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),

5227

CurSquare, CurSquare);

5228

Val >>= 1;

5229

}

5230

5231

// If the original was negative, invert the result, producing 1/(x*x*x).

5232

if (RHSC->getSExtValue() < 0)

5233

Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),

5234

DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);

5235

return Res;

5236

}

5237

}

5238

5239

// Otherwise, expand to a libcall.

5240

return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);

5241

}

5242

5243

static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,

5244

SDValue LHS, SDValue RHS, SDValue Scale,

5245

SelectionDAG &DAG, const TargetLowering &TLI) {

5246

EVT VT = LHS.getValueType();

5247

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

5248

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

5249

LLVMContext &Ctx = *DAG.getContext();

5250

5251

// If the type is legal but the operation isn't, this node might survive all

5252

// the way to operation legalization. If we end up there and we do not have

5253

// the ability to widen the type (if VT*2 is not legal), we cannot expand the

5254

// node.

5255

5256

// Coax the legalizer into expanding the node during type legalization instead

5257

// by bumping the size by one bit. This will force it to Promote, enabling the

5258

// early expansion and avoiding the need to expand later.

5259

5260

// We don't have to do this if Scale is 0; that can always be expanded, unless

5261

// it's a saturating signed operation. Those can experience true integer

5262

// division overflow, a case which we must avoid.

5263

5264

// FIXME: We wouldn't have to do this (or any of the early

5265

// expansion/promotion) if it was possible to expand a libcall of an

5266

// illegal type during operation legalization. But it's not, so things

5267

// get a bit hacky.

5268

unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();

5269

if ((ScaleInt > 0 || (Saturating && Signed)) &&

5270

(TLI.isTypeLegal(VT) ||

5271

(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {

5272

TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(

5273

Opcode, VT, ScaleInt);

5274

if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {

5275

EVT PromVT;

5276

if (VT.isScalarInteger())

5277

PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);

5278

else if (VT.isVector()) {

5279

PromVT = VT.getVectorElementType();

5280

PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);

5281

PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());

5282

} else

5283

llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5283);

5284

if (Signed) {

5285

LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);

5286

RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);

5287

} else {

5288

LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);

5289

RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);

5290

}

5291

EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());

5292

// For saturating operations, we need to shift up the LHS to get the

5293

// proper saturation width, and then shift down again afterwards.

5294

if (Saturating)

5295

LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,

5296

DAG.getConstant(1, DL, ShiftTy));

5297

SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);

5298

if (Saturating)

5299

Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,

5300

DAG.getConstant(1, DL, ShiftTy));

5301

return DAG.getZExtOrTrunc(Res, DL, VT);

5302

}

5303

}

5304

5305

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);

5306

}

5307

5308

// getUnderlyingArgRegs - Find underlying registers used for a truncated,

5309

// bitcasted, or split argument. Returns a list of <Register, size in bits>

5310

static void

5311

getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs,

5312

const SDValue &N) {

5313

switch (N.getOpcode()) {

5314

case ISD::CopyFromReg: {

5315

SDValue Op = N.getOperand(1);

5316

Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),

5317

Op.getValueType().getSizeInBits());

5318

return;

5319

}

5320

case ISD::BITCAST:

5321

case ISD::AssertZext:

5322

case ISD::AssertSext:

5323

case ISD::TRUNCATE:

5324

getUnderlyingArgRegs(Regs, N.getOperand(0));

5325

return;

5326

case ISD::BUILD_PAIR:

5327

case ISD::BUILD_VECTOR:

5328

case ISD::CONCAT_VECTORS:

5329

for (SDValue Op : N->op_values())

5330

getUnderlyingArgRegs(Regs, Op);

5331

return;

5332

default:

5333

return;

5334

}

5335

}

5336

5337

/// If the DbgValueInst is a dbg_value of a function argument, create the

5338

/// corresponding DBG_VALUE machine instruction for it now. At the end of

5339

/// instruction selection, they will be inserted to the entry BB.

5340

bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(

5341

const Value *V, DILocalVariable *Variable, DIExpression *Expr,

5342

DILocation *DL, bool IsDbgDeclare, const SDValue &N) {

5343

const Argument *Arg = dyn_cast<Argument>(V);

5344

if (!Arg)

5345

return false;

5346

5347

if (!IsDbgDeclare) {

5348

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5349

// should only emit as ArgDbgValue if the dbg.value intrinsic is found in

5350

// the entry block.

5351

bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();

5352

if (!IsInEntryBlock)

5353

return false;

5354

5355

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5356

// should only emit as ArgDbgValue if the dbg.value intrinsic describes a

5357

// variable that also is a param.

5358

//

5359

// Although, if we are at the top of the entry block already, we can still

5360

// emit using ArgDbgValue. This might catch some situations when the

5361

// dbg.value refers to an argument that isn't used in the entry block, so

5362

// any CopyToReg node would be optimized out and the only way to express

5363

// this DBG_VALUE is by using the physical reg (or FI) as done in this

5364

// method. ArgDbgValues are hoisted to the beginning of the entry block. So

5365

// we should only emit as ArgDbgValue if the Variable is an argument to the

5366

// current function, and the dbg.value intrinsic is found in the entry

5367

// block.

5368

bool VariableIsFunctionInputArg = Variable->isParameter() &&

5369

!DL->getInlinedAt();

5370

bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;

5371

if (!IsInPrologue && !VariableIsFunctionInputArg)

5372

return false;

5373

5374

// Here we assume that a function argument on IR level only can be used to

5375

// describe one input parameter on source level. If we for example have

5376

// source code like this

5377

//

5378

// struct A { long x, y; };

5379

// void foo(struct A a, long b) {

5380

// ...

5381

// b = a.x;

5382

// ...

5383

// }

5384

//

5385

// and IR like this

5386

//

5387

// define void @foo(i32 %a1, i32 %a2, i32 %b) {

5388

// entry:

5389

// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment

5390

// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment

5391

// call void @llvm.dbg.value(metadata i32 %b, "b",

5392

// ...

5393

// call void @llvm.dbg.value(metadata i32 %a1, "b"

5394

// ...

5395

//

5396

// then the last dbg.value is describing a parameter "b" using a value that

5397

// is an argument. But since we already has used %a1 to describe a parameter

5398

// we should not handle that last dbg.value here (that would result in an

5399

// incorrect hoisting of the DBG_VALUE to the function entry).

5400

// Notice that we allow one dbg.value per IR level argument, to accommodate

5401

// for the situation with fragments above.

5402

if (VariableIsFunctionInputArg) {

5403

unsigned ArgNo = Arg->getArgNo();

5404

if (ArgNo >= FuncInfo.DescribedArgs.size())

5405

FuncInfo.DescribedArgs.resize(ArgNo + 1, false);

5406

else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))

5407

return false;

5408

FuncInfo.DescribedArgs.set(ArgNo);

5409

}

5410

}

5411

5412

MachineFunction &MF = DAG.getMachineFunction();

5413

const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

5414

5415

bool IsIndirect = false;

5416

Optional<MachineOperand> Op;

5417

// Some arguments' frame index is recorded during argument lowering.

5418

int FI = FuncInfo.getArgumentFrameIndex(Arg);

5419

if (FI != std::numeric_limits<int>::max())

5420

Op = MachineOperand::CreateFI(FI);

5421

5422

SmallVector<std::pair<unsigned, unsigned>, 8> ArgRegsAndSizes;

5423

if (!Op && N.getNode()) {

5424

getUnderlyingArgRegs(ArgRegsAndSizes, N);

5425

Register Reg;

5426

if (ArgRegsAndSizes.size() == 1)

5427

Reg = ArgRegsAndSizes.front().first;

5428

5429

if (Reg && Reg.isVirtual()) {

5430

MachineRegisterInfo &RegInfo = MF.getRegInfo();

5431

Register PR = RegInfo.getLiveInPhysReg(Reg);

5432

if (PR)

5433

Reg = PR;

5434

}

5435

if (Reg) {

5436

Op = MachineOperand::CreateReg(Reg, false);

5437

IsIndirect = IsDbgDeclare;

5438

}

5439

}

5440

5441

if (!Op && N.getNode()) {

5442

// Check if frame index is available.

5443

SDValue LCandidate = peekThroughBitcasts(N);

5444

if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))

5445

if (FrameIndexSDNode *FINode =

5446

dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))

5447

Op = MachineOperand::CreateFI(FINode->getIndex());

5448

}

5449

5450

if (!Op) {

5451

// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg

5452

auto splitMultiRegDbgValue

5453

= [&](ArrayRef<std::pair<unsigned, unsigned>> SplitRegs) {

5454

unsigned Offset = 0;

5455

for (auto RegAndSize : SplitRegs) {

5456

// If the expression is already a fragment, the current register

5457

// offset+size might extend beyond the fragment. In this case, only

5458

// the register bits that are inside the fragment are relevant.

5459

int RegFragmentSizeInBits = RegAndSize.second;

5460

if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {

5461

uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;

5462

// The register is entirely outside the expression fragment,

5463

// so is irrelevant for debug info.

5464

if (Offset >= ExprFragmentSizeInBits)

5465

break;

5466

// The register is partially outside the expression fragment, only

5467

// the low bits within the fragment are relevant for debug info.

5468

if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {

5469

RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;

5470

}

5471

}

5472

5473

auto FragmentExpr = DIExpression::createFragmentExpression(

5474

Expr, Offset, RegFragmentSizeInBits);

5475

Offset += RegAndSize.second;

5476

// If a valid fragment expression cannot be created, the variable's

5477

// correct value cannot be determined and so it is set as Undef.

5478

if (!FragmentExpr) {

5479

SDDbgValue *SDV = DAG.getConstantDbgValue(

5480

Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);

5481

DAG.AddDbgValue(SDV, nullptr, false);

5482

continue;

5483

}

5484

assert(!IsDbgDeclare && "DbgDeclare operand is not in memory?")((!IsDbgDeclare && "DbgDeclare operand is not in memory?"
) ? static_cast<void> (0) : __assert_fail ("!IsDbgDeclare && \"DbgDeclare operand is not in memory?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5484, __PRETTY_FUNCTION__));

5485

FuncInfo.ArgDbgValues.push_back(

5486

BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,

5487

RegAndSize.first, Variable, *FragmentExpr));

5488

}

5489

};

5490

5491

// Check if ValueMap has reg number.

5492

DenseMap<const Value *, Register>::const_iterator

5493

VMI = FuncInfo.ValueMap.find(V);

5494

if (VMI != FuncInfo.ValueMap.end()) {

5495

const auto &TLI = DAG.getTargetLoweringInfo();

5496

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,

5497

V->getType(), None);

5498

if (RFV.occupiesMultipleRegs()) {

5499

splitMultiRegDbgValue(RFV.getRegsAndSizes());

5500

return true;

5501

}

5502

5503

Op = MachineOperand::CreateReg(VMI->second, false);

5504

IsIndirect = IsDbgDeclare;

5505

} else if (ArgRegsAndSizes.size() > 1) {

5506

// This was split due to the calling convention, and no virtual register

5507

// mapping exists for the value.

5508

splitMultiRegDbgValue(ArgRegsAndSizes);

5509

return true;

5510

}

5511

}

5512

5513

if (!Op)

5514

return false;

5515

5516

assert(Variable->isValidLocationForIntrinsic(DL) &&((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5517, __PRETTY_FUNCTION__))

5517

"Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5517, __PRETTY_FUNCTION__));

5518

IsIndirect = (Op->isReg()) ? IsIndirect : true;

5519

FuncInfo.ArgDbgValues.push_back(

5520

BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,

5521

*Op, Variable, Expr));

5522

5523

return true;

5524

}

5525

5526

/// Return the appropriate SDDbgValue based on N.

5527

SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,

5528

DILocalVariable *Variable,

5529

DIExpression *Expr,

5530

const DebugLoc &dl,

5531

unsigned DbgSDNodeOrder) {

5532

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

5533

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe

5534

// stack slot locations.

5535

//

5536

// Consider "int x = 0; int *px = &x;". There are two kinds of interesting

5537

// debug values here after optimization:

5538

//

5539

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

5540

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

5541

//

5542

// Both describe the direct values of their associated variables.

5543

return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),

5544

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5545

}

5546

return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),

5547

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5548

}

5549

5550

static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {

5551

switch (Intrinsic) {

5552

case Intrinsic::smul_fix:

5553

return ISD::SMULFIX;

5554

case Intrinsic::umul_fix:

5555

return ISD::UMULFIX;

5556

case Intrinsic::smul_fix_sat:

5557

return ISD::SMULFIXSAT;

5558

case Intrinsic::umul_fix_sat:

5559

return ISD::UMULFIXSAT;

5560

case Intrinsic::sdiv_fix:

5561

return ISD::SDIVFIX;

5562

case Intrinsic::udiv_fix:

5563

return ISD::UDIVFIX;

5564

case Intrinsic::sdiv_fix_sat:

5565

return ISD::SDIVFIXSAT;

5566

case Intrinsic::udiv_fix_sat:

5567

return ISD::UDIVFIXSAT;

5568

default:

5569

llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5569);

5570

}

5571

}

5572

5573

void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,

5574

const char *FunctionName) {

5575

assert(FunctionName && "FunctionName must not be nullptr")((FunctionName && "FunctionName must not be nullptr")
? static_cast<void> (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5575, __PRETTY_FUNCTION__));

5576

SDValue Callee = DAG.getExternalSymbol(

5577

FunctionName,

5578

DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));

5579

LowerCallTo(I, Callee, I.isTailCall());

5580

}

5581

5582

/// Given a @llvm.call.preallocated.setup, return the corresponding

5583

/// preallocated call.

5584

static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {

5585

assert(cast<CallBase>(PreallocatedSetup)((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5588, __PRETTY_FUNCTION__))

5586

->getCalledFunction()((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5588, __PRETTY_FUNCTION__))

5587

->getIntrinsicID() == Intrinsic::call_preallocated_setup &&((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5588, __PRETTY_FUNCTION__))

5588

"expected call_preallocated_setup Value")((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5588, __PRETTY_FUNCTION__));

5589

for (auto *U : PreallocatedSetup->users()) {

5590

auto *UseCall = cast<CallBase>(U);

5591

const Function *Fn = UseCall->getCalledFunction();

5592

if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {

5593

return UseCall;

5594

}

5595

}

5596

llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5596);

5597

}

5598

5599

/// Lower the call to the specified intrinsic function.

5600

void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,

5601

unsigned Intrinsic) {

5602

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5603

SDLoc sdl = getCurSDLoc();

5604

DebugLoc dl = getCurDebugLoc();

5605

SDValue Res;

5606

5607

SDNodeFlags Flags;

5608

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

5609

Flags.copyFMF(*FPOp);

5610

5611

switch (Intrinsic) {

5612

default:

5613

// By default, turn this into a target intrinsic node.

5614

visitTargetIntrinsic(I, Intrinsic);

5615

return;

5616

case Intrinsic::vscale: {

5617

match(&I, m_VScale(DAG.getDataLayout()));

5618

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

5619

setValue(&I,

5620

DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)));

5621

return;

5622

}

5623

case Intrinsic::vastart: visitVAStart(I); return;

5624

case Intrinsic::vaend: visitVAEnd(I); return;

5625

case Intrinsic::vacopy: visitVACopy(I); return;

5626

case Intrinsic::returnaddress:

5627

setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,

5628

TLI.getPointerTy(DAG.getDataLayout()),

5629

getValue(I.getArgOperand(0))));

5630

return;

5631

case Intrinsic::addressofreturnaddress:

5632

setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,

5633

TLI.getPointerTy(DAG.getDataLayout())));

5634

return;

5635

case Intrinsic::sponentry:

5636

setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl,

5637

TLI.getFrameIndexTy(DAG.getDataLayout())));

5638

return;

5639

case Intrinsic::frameaddress:

5640

setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,

5641

TLI.getFrameIndexTy(DAG.getDataLayout()),

5642

getValue(I.getArgOperand(0))));

5643

return;

5644

case Intrinsic::read_volatile_register:

5645

case Intrinsic::read_register: {

5646

Value *Reg = I.getArgOperand(0);

5647

SDValue Chain = getRoot();

5648

SDValue RegName =

5649

DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));

5650

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

5651

Res = DAG.getNode(ISD::READ_REGISTER, sdl,

5652

DAG.getVTList(VT, MVT::Other), Chain, RegName);

5653

setValue(&I, Res);

5654

DAG.setRoot(Res.getValue(1));

5655

return;

5656

}

5657

case Intrinsic::write_register: {

5658

Value *Reg = I.getArgOperand(0);

5659

Value *RegValue = I.getArgOperand(1);

5660

SDValue Chain = getRoot();

5661

SDValue RegName =

5662

DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));

5663

DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,

5664

RegName, getValue(RegValue)));

5665

return;

5666

}

5667

case Intrinsic::memcpy: {

5668

const auto &MCI = cast<MemCpyInst>(I);

5669

SDValue Op1 = getValue(I.getArgOperand(0));

5670

SDValue Op2 = getValue(I.getArgOperand(1));

5671

SDValue Op3 = getValue(I.getArgOperand(2));

5672

// @llvm.memcpy defines 0 and 1 to both mean no alignment.

5673

Align DstAlign = MCI.getDestAlign().valueOrOne();

5674

Align SrcAlign = MCI.getSourceAlign().valueOrOne();

5675

Align Alignment = commonAlignment(DstAlign, SrcAlign);

5676

bool isVol = MCI.isVolatile();

5677

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5678

// FIXME: Support passing different dest/src alignments to the memcpy DAG

5679

// node.

5680

SDValue Root = isVol ? getRoot() : getMemoryRoot();

5681

SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,

5682

/* AlwaysInline */ false, isTC,

5683

MachinePointerInfo(I.getArgOperand(0)),

5684

MachinePointerInfo(I.getArgOperand(1)));

5685

updateDAGForMaybeTailCall(MC);

5686

return;

5687

}

5688

case Intrinsic::memcpy_inline: {

5689

const auto &MCI = cast<MemCpyInlineInst>(I);

5690

SDValue Dst = getValue(I.getArgOperand(0));

5691

SDValue Src = getValue(I.getArgOperand(1));

5692

SDValue Size = getValue(I.getArgOperand(2));

5693

assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")((isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(Size) && \"memcpy_inline needs constant size\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5693, __PRETTY_FUNCTION__));

5694

// @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.

5695

Align DstAlign = MCI.getDestAlign().valueOrOne();

5696

Align SrcAlign = MCI.getSourceAlign().valueOrOne();

5697

Align Alignment = commonAlignment(DstAlign, SrcAlign);

5698

bool isVol = MCI.isVolatile();

5699

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5700

// FIXME: Support passing different dest/src alignments to the memcpy DAG

5701

// node.

5702

SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,

5703

/* AlwaysInline */ true, isTC,

5704

MachinePointerInfo(I.getArgOperand(0)),

5705

MachinePointerInfo(I.getArgOperand(1)));

5706

updateDAGForMaybeTailCall(MC);

5707

return;

5708

}

5709

case Intrinsic::memset: {

5710

const auto &MSI = cast<MemSetInst>(I);

5711

SDValue Op1 = getValue(I.getArgOperand(0));

5712

SDValue Op2 = getValue(I.getArgOperand(1));

5713

SDValue Op3 = getValue(I.getArgOperand(2));

5714

// @llvm.memset defines 0 and 1 to both mean no alignment.

5715

Align Alignment = MSI.getDestAlign().valueOrOne();

5716

bool isVol = MSI.isVolatile();

5717

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5718

SDValue Root = isVol ? getRoot() : getMemoryRoot();

5719

SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC,

5720

MachinePointerInfo(I.getArgOperand(0)));

5721

updateDAGForMaybeTailCall(MS);

5722

return;

5723

}

5724

case Intrinsic::memmove: {

5725

const auto &MMI = cast<MemMoveInst>(I);

5726

SDValue Op1 = getValue(I.getArgOperand(0));

5727

SDValue Op2 = getValue(I.getArgOperand(1));

5728

SDValue Op3 = getValue(I.getArgOperand(2));

5729

// @llvm.memmove defines 0 and 1 to both mean no alignment.

5730

Align DstAlign = MMI.getDestAlign().valueOrOne();

5731

Align SrcAlign = MMI.getSourceAlign().valueOrOne();

5732

Align Alignment = commonAlignment(DstAlign, SrcAlign);

5733

bool isVol = MMI.isVolatile();

5734

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5735

// FIXME: Support passing different dest/src alignments to the memmove DAG

5736

// node.

5737

SDValue Root = isVol ? getRoot() : getMemoryRoot();

5738

SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,

5739

isTC, MachinePointerInfo(I.getArgOperand(0)),

5740

MachinePointerInfo(I.getArgOperand(1)));

5741

updateDAGForMaybeTailCall(MM);

5742

return;

5743

}

5744

case Intrinsic::memcpy_element_unordered_atomic: {

5745

const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);

5746

SDValue Dst = getValue(MI.getRawDest());

5747

SDValue Src = getValue(MI.getRawSource());

5748

SDValue Length = getValue(MI.getLength());

5749

5750

unsigned DstAlign = MI.getDestAlignment();

5751

unsigned SrcAlign = MI.getSourceAlignment();

5752

Type *LengthTy = MI.getLength()->getType();

5753

unsigned ElemSz = MI.getElementSizeInBytes();

5754

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5755

SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,

5756

SrcAlign, Length, LengthTy, ElemSz, isTC,

5757

MachinePointerInfo(MI.getRawDest()),

5758

MachinePointerInfo(MI.getRawSource()));

5759

updateDAGForMaybeTailCall(MC);

5760

return;

5761

}

5762

case Intrinsic::memmove_element_unordered_atomic: {

5763

auto &MI = cast<AtomicMemMoveInst>(I);

5764

SDValue Dst = getValue(MI.getRawDest());

5765

SDValue Src = getValue(MI.getRawSource());

5766

SDValue Length = getValue(MI.getLength());

5767

5768

unsigned DstAlign = MI.getDestAlignment();

5769

unsigned SrcAlign = MI.getSourceAlignment();

5770

Type *LengthTy = MI.getLength()->getType();

5771

unsigned ElemSz = MI.getElementSizeInBytes();

5772

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5773

SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,

5774

SrcAlign, Length, LengthTy, ElemSz, isTC,

5775

MachinePointerInfo(MI.getRawDest()),

5776

MachinePointerInfo(MI.getRawSource()));

5777

updateDAGForMaybeTailCall(MC);

5778

return;

5779

}

5780

case Intrinsic::memset_element_unordered_atomic: {

5781

auto &MI = cast<AtomicMemSetInst>(I);

5782

SDValue Dst = getValue(MI.getRawDest());

5783

SDValue Val = getValue(MI.getValue());

5784

SDValue Length = getValue(MI.getLength());

5785

5786

unsigned DstAlign = MI.getDestAlignment();

5787

Type *LengthTy = MI.getLength()->getType();

5788

unsigned ElemSz = MI.getElementSizeInBytes();

5789

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5790

SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,

5791

LengthTy, ElemSz, isTC,

5792

MachinePointerInfo(MI.getRawDest()));

5793

updateDAGForMaybeTailCall(MC);

5794

return;

5795

}

5796

case Intrinsic::call_preallocated_setup: {

5797

const CallBase *PreallocatedCall = FindPreallocatedCall(&I);

5798

SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);

5799

SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,

5800

getRoot(), SrcValue);

5801

setValue(&I, Res);

5802

DAG.setRoot(Res);

5803

return;

5804

}

5805

case Intrinsic::call_preallocated_arg: {

5806

const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));

5807

SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);

5808

SDValue Ops[3];

5809

Ops[0] = getRoot();

5810

Ops[1] = SrcValue;

5811

Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,

5812

MVT::i32); // arg index

5813

SDValue Res = DAG.getNode(

5814

ISD::PREALLOCATED_ARG, sdl,

5815

DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);

5816

setValue(&I, Res);

5817

DAG.setRoot(Res.getValue(1));

5818

return;

5819

}

5820

case Intrinsic::dbg_addr:

5821

case Intrinsic::dbg_declare: {

5822

const auto &DI = cast<DbgVariableIntrinsic>(I);

5823

DILocalVariable *Variable = DI.getVariable();

5824

DIExpression *Expression = DI.getExpression();

5825

dropDanglingDebugInfo(Variable, Expression);

5826

assert(Variable && "Missing variable")((Variable && "Missing variable") ? static_cast<void
> (0) : __assert_fail ("Variable && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5826, __PRETTY_FUNCTION__));

5827

LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: "
<< DI << "\n"; } } while (false)

5828

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: "
<< DI << "\n"; } } while (false);

5829

// Check if address has undef value.

5830

const Value *Address = DI.getVariableLocation();

5831

if (!Address || isa<UndefValue>(Address) ||

5832

(Address->use_empty() && !isa<Argument>(Address))) {

5833

LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << " (bad/undef/unused-arg address)\n"; } } while (
false)

5834

<< " (bad/undef/unused-arg address)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << " (bad/undef/unused-arg address)\n"; } } while (
false);

5835

return;

5836

}

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1101, column 10 Called C++ object pointer is null

Bug Summary

Annotated Source Code

/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp