/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1

//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements routines for translating from LLVM IR into SelectionDAG IR.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "SelectionDAGBuilder.h"

14

#include "SDNodeDbgValue.h"

15

#include "llvm/ADT/APFloat.h"

16

#include "llvm/ADT/APInt.h"

17

#include "llvm/ADT/BitVector.h"

18

#include "llvm/ADT/None.h"

19

#include "llvm/ADT/Optional.h"

20

#include "llvm/ADT/STLExtras.h"

21

#include "llvm/ADT/SmallPtrSet.h"

22

#include "llvm/ADT/SmallSet.h"

23

#include "llvm/ADT/StringRef.h"

24

#include "llvm/ADT/Triple.h"

25

#include "llvm/ADT/Twine.h"

26

#include "llvm/Analysis/AliasAnalysis.h"

27

#include "llvm/Analysis/BlockFrequencyInfo.h"

28

#include "llvm/Analysis/BranchProbabilityInfo.h"

29

#include "llvm/Analysis/ConstantFolding.h"

30

#include "llvm/Analysis/EHPersonalities.h"

31

#include "llvm/Analysis/Loads.h"

32

#include "llvm/Analysis/MemoryLocation.h"

33

#include "llvm/Analysis/ProfileSummaryInfo.h"

34

#include "llvm/Analysis/TargetLibraryInfo.h"

35

#include "llvm/Analysis/ValueTracking.h"

36

#include "llvm/Analysis/VectorUtils.h"

37

#include "llvm/CodeGen/Analysis.h"

38

#include "llvm/CodeGen/FunctionLoweringInfo.h"

39

#include "llvm/CodeGen/GCMetadata.h"

40

#include "llvm/CodeGen/MachineBasicBlock.h"

41

#include "llvm/CodeGen/MachineFrameInfo.h"

42

#include "llvm/CodeGen/MachineFunction.h"

43

#include "llvm/CodeGen/MachineInstr.h"

44

#include "llvm/CodeGen/MachineInstrBuilder.h"

45

#include "llvm/CodeGen/MachineJumpTableInfo.h"

46

#include "llvm/CodeGen/MachineMemOperand.h"

47

#include "llvm/CodeGen/MachineModuleInfo.h"

48

#include "llvm/CodeGen/MachineOperand.h"

49

#include "llvm/CodeGen/MachineRegisterInfo.h"

50

#include "llvm/CodeGen/RuntimeLibcalls.h"

51

#include "llvm/CodeGen/SelectionDAG.h"

52

#include "llvm/CodeGen/SelectionDAGTargetInfo.h"

53

#include "llvm/CodeGen/StackMaps.h"

54

#include "llvm/CodeGen/SwiftErrorValueTracking.h"

55

#include "llvm/CodeGen/TargetFrameLowering.h"

56

#include "llvm/CodeGen/TargetInstrInfo.h"

57

#include "llvm/CodeGen/TargetOpcodes.h"

58

#include "llvm/CodeGen/TargetRegisterInfo.h"

59

#include "llvm/CodeGen/TargetSubtargetInfo.h"

60

#include "llvm/CodeGen/WinEHFuncInfo.h"

61

#include "llvm/IR/Argument.h"

62

#include "llvm/IR/Attributes.h"

63

#include "llvm/IR/BasicBlock.h"

64

#include "llvm/IR/CFG.h"

65

#include "llvm/IR/CallingConv.h"

66

#include "llvm/IR/Constant.h"

67

#include "llvm/IR/ConstantRange.h"

68

#include "llvm/IR/Constants.h"

69

#include "llvm/IR/DataLayout.h"

70

#include "llvm/IR/DebugInfoMetadata.h"

71

#include "llvm/IR/DerivedTypes.h"

72

#include "llvm/IR/Function.h"

73

#include "llvm/IR/GetElementPtrTypeIterator.h"

74

#include "llvm/IR/InlineAsm.h"

75

#include "llvm/IR/InstrTypes.h"

76

#include "llvm/IR/Instructions.h"

77

#include "llvm/IR/IntrinsicInst.h"

78

#include "llvm/IR/Intrinsics.h"

79

#include "llvm/IR/IntrinsicsAArch64.h"

80

#include "llvm/IR/IntrinsicsWebAssembly.h"

81

#include "llvm/IR/LLVMContext.h"

82

#include "llvm/IR/Metadata.h"

83

#include "llvm/IR/Module.h"

84

#include "llvm/IR/Operator.h"

85

#include "llvm/IR/PatternMatch.h"

86

#include "llvm/IR/Statepoint.h"

87

#include "llvm/IR/Type.h"

88

#include "llvm/IR/User.h"

89

#include "llvm/IR/Value.h"

90

#include "llvm/MC/MCContext.h"

91

#include "llvm/MC/MCSymbol.h"

92

#include "llvm/Support/AtomicOrdering.h"

93

#include "llvm/Support/Casting.h"

94

#include "llvm/Support/CommandLine.h"

95

#include "llvm/Support/Compiler.h"

96

#include "llvm/Support/Debug.h"

97

#include "llvm/Support/MathExtras.h"

98

#include "llvm/Support/raw_ostream.h"

99

#include "llvm/Target/TargetIntrinsicInfo.h"

100

#include "llvm/Target/TargetMachine.h"

101

#include "llvm/Target/TargetOptions.h"

102

#include "llvm/Transforms/Utils/Local.h"

103

#include <cstddef>

104

#include <cstring>

105

#include <iterator>

106

#include <limits>

107

#include <numeric>

108

#include <tuple>

109

110

using namespace llvm;

111

using namespace PatternMatch;

112

using namespace SwitchCG;

113

114

#define DEBUG_TYPE"isel" "isel"

115

116

/// LimitFloatPrecision - Generate low-precision inline sequences for

117

/// some float libcalls (6, 8 or 12 bits).

118

static unsigned LimitFloatPrecision;

119

120

static cl::opt<bool>

121

InsertAssertAlign("insert-assert-align", cl::init(true),

122

cl::desc("Insert the experimental `assertalign` node."),

123

cl::ReallyHidden);

124

125

static cl::opt<unsigned, true>

126

LimitFPPrecision("limit-float-precision",

127

cl::desc("Generate low-precision inline sequences "

128

"for some float libcalls"),

129

cl::location(LimitFloatPrecision), cl::Hidden,

130

cl::init(0));

131

132

static cl::opt<unsigned> SwitchPeelThreshold(

133

"switch-peel-threshold", cl::Hidden, cl::init(66),

134

cl::desc("Set the case probability threshold for peeling the case from a "

135

"switch statement. A value greater than 100 will void this "

136

"optimization"));

137

138

// Limit the width of DAG chains. This is important in general to prevent

139

// DAG-based analysis from blowing up. For example, alias analysis and

140

// load clustering may not complete in reasonable time. It is difficult to

141

// recognize and avoid this situation within each individual analysis, and

142

// future analyses are likely to have the same behavior. Limiting DAG width is

143

// the safe approach and will be especially important with global DAGs.

144

//

145

// MaxParallelChains default is arbitrarily high to avoid affecting

146

// optimization, but could be lowered to improve compile time. Any ld-ld-st-st

147

// sequence over this should have been converted to llvm.memcpy by the

148

// frontend. It is easy to induce this behavior with .ll code such as:

149

// %buffer = alloca [4096 x i8]

150

// %data = load [4096 x i8]* %argPtr

151

// store [4096 x i8] %data, [4096 x i8]* %buffer

152

static const unsigned MaxParallelChains = 64;

153

154

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

155

const SDValue *Parts, unsigned NumParts,

156

MVT PartVT, EVT ValueVT, const Value *V,

157

Optional<CallingConv::ID> CC);

158

159

/// getCopyFromParts - Create a value that contains the specified legal parts

160

/// combined into the value they represent. If the parts combine to a type

161

/// larger than ValueVT then AssertOp can be used to specify whether the extra

162

/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT

163

/// (ISD::AssertSext).

164

static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,

165

const SDValue *Parts, unsigned NumParts,

166

MVT PartVT, EVT ValueVT, const Value *V,

167

Optional<CallingConv::ID> CC = None,

168

Optional<ISD::NodeType> AssertOp = None) {

169

// Let the target assemble the parts if it wants to

170

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

171

if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,

172

PartVT, ValueVT, CC))

173

return Val;

174

175

if (ValueVT.isVector())

176

return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,

177

CC);

178

179

assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 179, __PRETTY_FUNCTION__));

180

SDValue Val = Parts[0];

181

182

if (NumParts > 1) {

183

// Assemble the value from multiple parts.

184

if (ValueVT.isInteger()) {

185

unsigned PartBits = PartVT.getSizeInBits();

186

unsigned ValueBits = ValueVT.getSizeInBits();

187

188

// Assemble the power of 2 part.

189

unsigned RoundParts =

190

(NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;

191

unsigned RoundBits = PartBits * RoundParts;

192

EVT RoundVT = RoundBits == ValueBits ?

193

ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);

194

SDValue Lo, Hi;

195

196

EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

197

198

if (RoundParts > 2) {

199

Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,

200

PartVT, HalfVT, V);

201

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,

202

RoundParts / 2, PartVT, HalfVT, V);

203

} else {

204

Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);

205

Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);

206

}

207

208

if (DAG.getDataLayout().isBigEndian())

209

std::swap(Lo, Hi);

210

211

Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

212

213

if (RoundParts < NumParts) {

214

// Assemble the trailing non-power-of-2 part.

215

unsigned OddParts = NumParts - RoundParts;

216

EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);

217

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,

218

OddVT, V, CC);

219

220

// Combine the round and odd parts.

221

Lo = Val;

222

if (DAG.getDataLayout().isBigEndian())

223

std::swap(Lo, Hi);

224

EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

225

Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);

226

Hi =

227

DAG.getNode(ISD::SHL, DL, TotalVT, Hi,

228

DAG.getConstant(Lo.getValueSizeInBits(), DL,

229

TLI.getPointerTy(DAG.getDataLayout())));

230

Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);

231

Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);

232

}

233

} else if (PartVT.isFloatingPoint()) {

234

// FP split into multiple FP parts (for ppcf128)

235

assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __PRETTY_FUNCTION__))

236

"Unexpected split")((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 236, __PRETTY_FUNCTION__));

237

SDValue Lo, Hi;

238

Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);

239

Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);

240

if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

241

std::swap(Lo, Hi);

242

Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);

243

} else {

244

// FP split into integer parts (soft fp)

245

assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __PRETTY_FUNCTION__))

246

!PartVT.isVector() && "Unexpected split")((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 246, __PRETTY_FUNCTION__));

247

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

248

Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);

249

}

250

}

251

252

// There is now one part, held in Val. Correct it to match ValueVT.

253

// PartEVT is the type of the register class that holds the value.

254

// ValueVT is the type of the inline asm operation.

255

EVT PartEVT = Val.getValueType();

256

257

if (PartEVT == ValueVT)

258

return Val;

259

260

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&

261

ValueVT.bitsLT(PartEVT)) {

262

// For an FP value in an integer part, we need to truncate to the right

263

// width first.

264

PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

265

Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);

266

}

267

268

// Handle types that have the same size.

269

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())

270

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

271

272

// Handle types with different sizes.

273

if (PartEVT.isInteger() && ValueVT.isInteger()) {

274

if (ValueVT.bitsLT(PartEVT)) {

275

// For a truncate, see if we have any information to

276

// indicate whether the truncated bits will always be

277

// zero or sign-extension.

278

if (AssertOp.hasValue())

279

Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,

280

DAG.getValueType(ValueVT));

281

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

282

}

283

return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

284

}

285

286

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

287

// FP_ROUND's are always exact here.

288

if (ValueVT.bitsLT(Val.getValueType()))

289

return DAG.getNode(

290

ISD::FP_ROUND, DL, ValueVT, Val,

291

DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

292

293

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);

294

}

295

296

// Handle MMX to a narrower integer type by bitcasting MMX to integer and

297

// then truncating.

298

if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&

299

ValueVT.bitsLT(PartEVT)) {

300

Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);

301

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

302

}

303

304

report_fatal_error("Unknown mismatch in getCopyFromParts!");

305

}

306

307

static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,

308

const Twine &ErrMsg) {

309

const Instruction *I = dyn_cast_or_null<Instruction>(V);

310

if (!V)

311

return Ctx.emitError(ErrMsg);

312

313

const char *AsmError = ", possible invalid constraint for vector type";

314

if (const CallInst *CI = dyn_cast<CallInst>(I))

315

if (CI->isInlineAsm())

316

return Ctx.emitError(I, ErrMsg + AsmError);

317

318

return Ctx.emitError(I, ErrMsg);

319

}

320

321

/// getCopyFromPartsVector - Create a value that contains the specified legal

322

/// parts combined into the value they represent. If the parts combine to a

323

/// type larger than ValueVT then AssertOp can be used to specify whether the

324

/// extra bits are known to be zero (ISD::AssertZext) or sign extended from

325

/// ValueVT (ISD::AssertSext).

326

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

327

const SDValue *Parts, unsigned NumParts,

328

MVT PartVT, EVT ValueVT, const Value *V,

329

Optional<CallingConv::ID> CallConv) {

330

assert(ValueVT.isVector() && "Not a vector value")((ValueVT.isVector() && "Not a vector value") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 330, __PRETTY_FUNCTION__));

331

assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 331, __PRETTY_FUNCTION__));

332

const bool IsABIRegCopy = CallConv.hasValue();

333

334

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

335

SDValue Val = Parts[0];

336

337

// Handle a multi-element vector.

338

if (NumParts > 1) {

339

EVT IntermediateVT;

340

MVT RegisterVT;

341

unsigned NumIntermediates;

342

unsigned NumRegs;

343

344

if (IsABIRegCopy) {

345

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

346

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

347

NumIntermediates, RegisterVT);

348

} else {

349

NumRegs =

350

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

351

NumIntermediates, RegisterVT);

352

}

353

354

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 354, __PRETTY_FUNCTION__));

355

NumParts = NumRegs; // Silence a compiler warning.

356

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 356, __PRETTY_FUNCTION__));

357

assert(RegisterVT.getSizeInBits() ==((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__))

358

Parts[0].getSimpleValueType().getSizeInBits() &&((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__))

359

"Part type sizes don't match!")((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 359, __PRETTY_FUNCTION__));

360

361

// Assemble the parts into intermediate operands.

362

SmallVector<SDValue, 8> Ops(NumIntermediates);

363

if (NumIntermediates == NumParts) {

364

// If the register was not expanded, truncate or copy the value,

365

// as appropriate.

366

for (unsigned i = 0; i != NumParts; ++i)

367

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,

368

PartVT, IntermediateVT, V, CallConv);

369

} else if (NumParts > 0) {

370

// If the intermediate type was expanded, build the intermediate

371

// operands from the parts.

372

assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __PRETTY_FUNCTION__))

373

"Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __PRETTY_FUNCTION__));

374

unsigned Factor = NumParts / NumIntermediates;

375

for (unsigned i = 0; i != NumIntermediates; ++i)

376

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,

377

PartVT, IntermediateVT, V, CallConv);

378

}

379

380

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the

381

// intermediate operands.

382

EVT BuiltVectorTy =

383

IntermediateVT.isVector()

384

? EVT::getVectorVT(

385

*DAG.getContext(), IntermediateVT.getScalarType(),

386

IntermediateVT.getVectorElementCount() * NumParts)

387

: EVT::getVectorVT(*DAG.getContext(),

388

IntermediateVT.getScalarType(),

389

NumIntermediates);

390

Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS

391

: ISD::BUILD_VECTOR,

392

DL, BuiltVectorTy, Ops);

393

}

394

395

// There is now one part, held in Val. Correct it to match ValueVT.

396

EVT PartEVT = Val.getValueType();

397

398

if (PartEVT == ValueVT)

399

return Val;

400

401

if (PartEVT.isVector()) {

402

// If the element type of the source/dest vectors are the same, but the

403

// parts vector has more elements than the value vector, then we have a

404

// vector widening case (e.g. <2 x float> -> <4 x float>). Extract the

405

// elements we want.

406

if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {

407

assert((PartEVT.getVectorElementCount().getKnownMinValue() >(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

408

ValueVT.getVectorElementCount().getKnownMinValue()) &&(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

409

(PartEVT.getVectorElementCount().isScalable() ==(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

410

ValueVT.getVectorElementCount().isScalable()) &&(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__))

411

"Cannot narrow, it would be a lossy transformation")(((PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT
.getVectorElementCount().getKnownMinValue()) && (PartEVT
.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 411, __PRETTY_FUNCTION__));

412

return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,

413

DAG.getVectorIdxConstant(0, DL));

414

}

415

416

// Vector/Vector bitcast.

417

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())

418

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

419

420

assert(PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() &&((PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __PRETTY_FUNCTION__))

421

"Cannot handle this kind of promotion")((PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 421, __PRETTY_FUNCTION__));

422

// Promoted vector extract

423

return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);

424

425

}

426

427

// Trivial bitcast if the types are the same size and the destination

428

// vector type is legal.

429

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&

430

TLI.isTypeLegal(ValueVT))

431

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

432

433

if (ValueVT.getVectorNumElements() != 1) {

434

// Certain ABIs require that vectors are passed as integers. For vectors

435

// are the same size, this is an obvious bitcast.

436

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {

437

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

438

} else if (ValueVT.bitsLT(PartEVT)) {

439

const uint64_t ValueSize = ValueVT.getFixedSizeInBits();

440

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

441

// Drop the extra bits.

442

Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);

443

return DAG.getBitcast(ValueVT, Val);

444

}

445

446

diagnosePossiblyInvalidConstraint(

447

*DAG.getContext(), V, "non-trivial scalar-to-vector conversion");

448

return DAG.getUNDEF(ValueVT);

449

}

450

451

// Handle cases such as i8 -> <1 x i1>

452

EVT ValueSVT = ValueVT.getVectorElementType();

453

if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {

454

if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())

455

Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);

456

else

457

Val = ValueVT.isFloatingPoint()

458

? DAG.getFPExtendOrRound(Val, DL, ValueSVT)

459

: DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);

460

}

461

462

return DAG.getBuildVector(ValueVT, DL, Val);

463

}

464

465

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,

466

SDValue Val, SDValue *Parts, unsigned NumParts,

467

MVT PartVT, const Value *V,

468

Optional<CallingConv::ID> CallConv);

469

470

/// getCopyToParts - Create a series of nodes that contain the specified value

471

/// split into legal parts. If the parts contain more bits than Val, then, for

472

/// integers, ExtendKind can be used to specify how to generate the extra bits.

473

static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,

474

SDValue *Parts, unsigned NumParts, MVT PartVT,

475

const Value *V,

476

Optional<CallingConv::ID> CallConv = None,

477

ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {

478

// Let the target split the parts if it wants to

479

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

480

if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,

481

CallConv))

482

return;

483

EVT ValueVT = Val.getValueType();

484

485

// Handle the vector case separately.

486

if (ValueVT.isVector())

487

return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,

488

CallConv);

489

490

unsigned PartBits = PartVT.getSizeInBits();

491

unsigned OrigNumParts = NumParts;

492

assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __PRETTY_FUNCTION__))

493

"Copying to an illegal type!")((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 493, __PRETTY_FUNCTION__));

494

495

if (NumParts == 0)

496

return;

497

498

assert(!ValueVT.isVector() && "Vector case handled elsewhere")((!ValueVT.isVector() && "Vector case handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 498, __PRETTY_FUNCTION__));

499

EVT PartEVT = PartVT;

500

if (PartEVT == ValueVT) {

501

assert(NumParts == 1 && "No-op copy with multiple parts!")((NumParts == 1 && "No-op copy with multiple parts!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __PRETTY_FUNCTION__));

502

Parts[0] = Val;

503

return;

504

}

505

506

if (NumParts * PartBits > ValueVT.getSizeInBits()) {

507

// If the parts cover more bits than the value has, promote the value.

508

if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

509

assert(NumParts == 1 && "Do not know what to promote to!")((NumParts == 1 && "Do not know what to promote to!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 509, __PRETTY_FUNCTION__));

510

Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);

511

} else {

512

if (ValueVT.isFloatingPoint()) {

513

// FP values need to be bitcast, then extended if they are being put

514

// into a larger container.

515

ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

516

Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

517

}

518

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __PRETTY_FUNCTION__))

519

ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __PRETTY_FUNCTION__))

520

"Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 520, __PRETTY_FUNCTION__));

521

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

522

Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);

523

if (PartVT == MVT::x86mmx)

524

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

525

}

526

} else if (PartBits == ValueVT.getSizeInBits()) {

527

// Different types of the same size.

528

assert(NumParts == 1 && PartEVT != ValueVT)((NumParts == 1 && PartEVT != ValueVT) ? static_cast<
void> (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __PRETTY_FUNCTION__));

529

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

530

} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {

531

// If the parts cover less bits than value has, truncate the value.

532

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __PRETTY_FUNCTION__))

533

ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __PRETTY_FUNCTION__))

534

"Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 534, __PRETTY_FUNCTION__));

535

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

536

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

537

if (PartVT == MVT::x86mmx)

538

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

539

}

540

541

// The value may have changed - recompute ValueVT.

542

ValueVT = Val.getValueType();

543

assert(NumParts * PartBits == ValueVT.getSizeInBits() &&((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __PRETTY_FUNCTION__))

544

"Failed to tile the value with PartVT!")((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 544, __PRETTY_FUNCTION__));

545

546

if (NumParts == 1) {

547

if (PartEVT != ValueVT) {

548

diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,

549

"scalar-to-vector conversion failed");

550

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

551

}

552

553

Parts[0] = Val;

554

return;

555

}

556

557

// Expand the value into multiple parts.

558

if (NumParts & (NumParts - 1)) {

559

// The number of parts is not a power of 2. Split off and copy the tail.

560

assert(PartVT.isInteger() && ValueVT.isInteger() &&((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __PRETTY_FUNCTION__))

561

"Do not know what to expand to!")((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 561, __PRETTY_FUNCTION__));

562

unsigned RoundParts = 1 << Log2_32(NumParts);

563

unsigned RoundBits = RoundParts * PartBits;

564

unsigned OddParts = NumParts - RoundParts;

565

SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,

566

DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));

567

568

getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,

569

CallConv);

570

571

if (DAG.getDataLayout().isBigEndian())

572

// The odd parts were reversed by getCopyToParts - unreverse them.

573

std::reverse(Parts + RoundParts, Parts + NumParts);

574

575

NumParts = RoundParts;

576

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

577

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

578

}

579

580

// The number of parts is a power of 2. Repeatedly bisect the value using

581

// EXTRACT_ELEMENT.

582

Parts[0] = DAG.getNode(ISD::BITCAST, DL,

583

EVT::getIntegerVT(*DAG.getContext(),

584

ValueVT.getSizeInBits()),

585

Val);

586

587

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {

588

for (unsigned i = 0; i < NumParts; i += StepSize) {

589

unsigned ThisBits = StepSize * PartBits / 2;

590

EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);

591

SDValue &Part0 = Parts[i];

592

SDValue &Part1 = Parts[i+StepSize/2];

593

594

Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

595

ThisVT, Part0, DAG.getIntPtrConstant(1, DL));

596

Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

597

ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

598

599

if (ThisBits == PartBits && ThisVT != PartVT) {

600

Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);

601

Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);

602

}

603

}

604

}

605

606

if (DAG.getDataLayout().isBigEndian())

607

std::reverse(Parts, Parts + OrigNumParts);

608

}

609

610

static SDValue widenVectorToPartType(SelectionDAG &DAG,

611

SDValue Val, const SDLoc &DL, EVT PartVT) {

612

if (!PartVT.isFixedLengthVector())

613

return SDValue();

614

615

EVT ValueVT = Val.getValueType();

616

unsigned PartNumElts = PartVT.getVectorNumElements();

617

unsigned ValueNumElts = ValueVT.getVectorNumElements();

618

if (PartNumElts > ValueNumElts &&

619

PartVT.getVectorElementType() == ValueVT.getVectorElementType()) {

620

EVT ElementVT = PartVT.getVectorElementType();

621

// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in

622

// undef elements.

623

SmallVector<SDValue, 16> Ops;

624

DAG.ExtractVectorElements(Val, Ops);

625

SDValue EltUndef = DAG.getUNDEF(ElementVT);

626

for (unsigned i = ValueNumElts, e = PartNumElts; i != e; ++i)

627

Ops.push_back(EltUndef);

628

629

// FIXME: Use CONCAT for 2x -> 4x.

630

return DAG.getBuildVector(PartVT, DL, Ops);

631

}

632

633

return SDValue();

634

}

635

636

/// getCopyToPartsVector - Create a series of nodes that contain the specified

637

/// value split into legal parts.

638

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,

639

SDValue Val, SDValue *Parts, unsigned NumParts,

640

MVT PartVT, const Value *V,

641

Optional<CallingConv::ID> CallConv) {

642

EVT ValueVT = Val.getValueType();

643

assert(ValueVT.isVector() && "Not a vector")((ValueVT.isVector() && "Not a vector") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 643, __PRETTY_FUNCTION__));

644

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

645

const bool IsABIRegCopy = CallConv.hasValue();

646

647

if (NumParts == 1) {

648

EVT PartEVT = PartVT;

649

if (PartEVT == ValueVT) {

650

// Nothing to do.

651

} else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {

652

// Bitconvert vector->vector case.

653

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

654

} else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {

655

Val = Widened;

656

} else if (PartVT.isVector() &&

657

PartEVT.getVectorElementType().bitsGE(

658

ValueVT.getVectorElementType()) &&

659

PartEVT.getVectorElementCount() ==

660

ValueVT.getVectorElementCount()) {

661

662

// Promoted vector extract

663

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

664

} else {

665

if (ValueVT.getVectorElementCount().isScalar()) {

666

Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,

667

DAG.getVectorIdxConstant(0, DL));

668

} else {

669

uint64_t ValueSize = ValueVT.getFixedSizeInBits();

670

assert(PartVT.getFixedSizeInBits() > ValueSize &&((PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type"
) ? static_cast<void> (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 671, __PRETTY_FUNCTION__))

671

"lossy conversion of vector to scalar type")((PartVT.getFixedSizeInBits() > ValueSize && "lossy conversion of vector to scalar type"
) ? static_cast<void> (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 671, __PRETTY_FUNCTION__));

672

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

673

Val = DAG.getBitcast(IntermediateType, Val);

674

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

675

}

676

}

677

678

assert(Val.getValueType() == PartVT && "Unexpected vector part value type")((Val.getValueType() == PartVT && "Unexpected vector part value type"
) ? static_cast<void> (0) : __assert_fail ("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 678, __PRETTY_FUNCTION__));

679

Parts[0] = Val;

680

return;

681

}

682

683

// Handle a multi-element vector.

684

EVT IntermediateVT;

685

MVT RegisterVT;

686

unsigned NumIntermediates;

687

unsigned NumRegs;

688

if (IsABIRegCopy) {

689

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

690

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

691

NumIntermediates, RegisterVT);

692

} else {

693

NumRegs =

694

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

695

NumIntermediates, RegisterVT);

696

}

697

698

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 698, __PRETTY_FUNCTION__));

699

NumParts = NumRegs; // Silence a compiler warning.

700

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 700, __PRETTY_FUNCTION__));

701

702

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&((IntermediateVT.isScalableVector() == ValueVT.isScalableVector
() && "Mixing scalable and fixed vectors when copying in parts"
) ? static_cast<void> (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 703, __PRETTY_FUNCTION__))

703

"Mixing scalable and fixed vectors when copying in parts")((IntermediateVT.isScalableVector() == ValueVT.isScalableVector
() && "Mixing scalable and fixed vectors when copying in parts"
) ? static_cast<void> (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 703, __PRETTY_FUNCTION__));

704

705

Optional<ElementCount> DestEltCnt;

706

707

if (IntermediateVT.isVector())

708

DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;

709

else

710

DestEltCnt = ElementCount::getFixed(NumIntermediates);

711

712

EVT BuiltVectorTy = EVT::getVectorVT(

713

*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());

714

715

if (ValueVT == BuiltVectorTy) {

716

// Nothing to do.

717

} else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {

718

// Bitconvert vector->vector case.

719

Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);

720

} else if (SDValue Widened =

721

widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {

722

Val = Widened;

723

} else if (BuiltVectorTy.getVectorElementType().bitsGE(

724

ValueVT.getVectorElementType()) &&

725

BuiltVectorTy.getVectorElementCount() ==

726

ValueVT.getVectorElementCount()) {

727

// Promoted vector extract

728

Val = DAG.getAnyExtOrTrunc(Val, DL, BuiltVectorTy);

729

}

730

731

assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")((Val.getValueType() == BuiltVectorTy && "Unexpected vector value type"
) ? static_cast<void> (0) : __assert_fail ("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 731, __PRETTY_FUNCTION__));

732

733

// Split the vector into intermediate operands.

734

SmallVector<SDValue, 8> Ops(NumIntermediates);

735

for (unsigned i = 0; i != NumIntermediates; ++i) {

736

if (IntermediateVT.isVector()) {

737

// This does something sensible for scalable vectors - see the

738

// definition of EXTRACT_SUBVECTOR for further details.

739

unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();

740

Ops[i] =

741

DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,

742

DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));

743

} else {

744

Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,

745

DAG.getVectorIdxConstant(i, DL));

746

}

747

}

748

749

// Split the intermediate operands into legal parts.

750

if (NumParts == NumIntermediates) {

751

// If the register was not expanded, promote or copy the value,

752

// as appropriate.

753

for (unsigned i = 0; i != NumParts; ++i)

754

getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

755

} else if (NumParts > 0) {

756

// If the intermediate type was expanded, split each the value into

757

// legal parts.

758

assert(NumIntermediates != 0 && "division by zero")((NumIntermediates != 0 && "division by zero") ? static_cast
<void> (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 758, __PRETTY_FUNCTION__));

759

assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 760, __PRETTY_FUNCTION__))

760

"Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 760, __PRETTY_FUNCTION__));

761

unsigned Factor = NumParts / NumIntermediates;

762

for (unsigned i = 0; i != NumIntermediates; ++i)

763

getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,

764

CallConv);

765

}

766

}

767

768

RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,

769

EVT valuevt, Optional<CallingConv::ID> CC)

770

: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),

771

RegCount(1, regs.size()), CallConv(CC) {}

772

773

RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,

774

const DataLayout &DL, unsigned Reg, Type *Ty,

775

Optional<CallingConv::ID> CC) {

776

ComputeValueVTs(TLI, DL, Ty, ValueVTs);

777

778

CallConv = CC;

779

780

for (EVT ValueVT : ValueVTs) {

781

unsigned NumRegs =

782

isABIMangled()

783

? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)

784

: TLI.getNumRegisters(Context, ValueVT);

785

MVT RegisterVT =

786

isABIMangled()

787

? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)

788

: TLI.getRegisterType(Context, ValueVT);

789

for (unsigned i = 0; i != NumRegs; ++i)

790

Regs.push_back(Reg + i);

791

RegVTs.push_back(RegisterVT);

792

RegCount.push_back(NumRegs);

793

Reg += NumRegs;

794

}

795

}

796

797

SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,

798

FunctionLoweringInfo &FuncInfo,

799

const SDLoc &dl, SDValue &Chain,

800

SDValue *Flag, const Value *V) const {

801

// A Value with type {} or [0 x %t] needs no registers.

802

if (ValueVTs.empty())

803

return SDValue();

804

805

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

806

807

// Assemble the legal parts into the final values.

808

SmallVector<SDValue, 4> Values(ValueVTs.size());

809

SmallVector<SDValue, 8> Parts;

810

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

811

// Copy the legal parts from the registers.

812

EVT ValueVT = ValueVTs[Value];

813

unsigned NumRegs = RegCount[Value];

814

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

815

*DAG.getContext(),

816

CallConv.getValue(), RegVTs[Value])

817

: RegVTs[Value];

818

819

Parts.resize(NumRegs);

820

for (unsigned i = 0; i != NumRegs; ++i) {

821

SDValue P;

822

if (!Flag) {

823

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);

824

} else {

825

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);

826

*Flag = P.getValue(2);

827

}

828

829

Chain = P.getValue(1);

830

Parts[i] = P;

831

832

// If the source register was virtual and if we know something about it,

833

// add an assert node.

834

if (!Register::isVirtualRegister(Regs[Part + i]) ||

835

!RegisterVT.isInteger())

836

continue;

837

838

const FunctionLoweringInfo::LiveOutInfo *LOI =

839

FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);

840

if (!LOI)

841

continue;

842

843

unsigned RegSize = RegisterVT.getScalarSizeInBits();

844

unsigned NumSignBits = LOI->NumSignBits;

845

unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

846

847

if (NumZeroBits == RegSize) {

848

// The current value is a zero.

849

// Explicitly express that as it would be easier for

850

// optimizations to kick in.

851

Parts[i] = DAG.getConstant(0, dl, RegisterVT);

852

continue;

853

}

854

855

// FIXME: We capture more information than the dag can represent. For

856

// now, just use the tightest assertzext/assertsext possible.

857

bool isSExt;

858

EVT FromVT(MVT::Other);

859

if (NumZeroBits) {

860

FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);

861

isSExt = false;

862

} else if (NumSignBits > 1) {

863

FromVT =

864

EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);

865

isSExt = true;

866

} else {

867

continue;

868

}

869

// Add an assertion node.

870

assert(FromVT != MVT::Other)((FromVT != MVT::Other) ? static_cast<void> (0) : __assert_fail
("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 870, __PRETTY_FUNCTION__));

871

Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,

872

RegisterVT, P, DAG.getValueType(FromVT));

873

}

874

875

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,

876

RegisterVT, ValueVT, V, CallConv);

877

Part += NumRegs;

878

Parts.clear();

879

}

880

881

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);

882

}

883

884

void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,

885

const SDLoc &dl, SDValue &Chain, SDValue *Flag,

886

const Value *V,

887

ISD::NodeType PreferredExtendType) const {

888

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

889

ISD::NodeType ExtendKind = PreferredExtendType;

890

891

// Get the list of the values's legal parts.

892

unsigned NumRegs = Regs.size();

893

SmallVector<SDValue, 8> Parts(NumRegs);

894

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

895

unsigned NumParts = RegCount[Value];

896

897

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

898

*DAG.getContext(),

899

CallConv.getValue(), RegVTs[Value])

900

: RegVTs[Value];

901

902

if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))

903

ExtendKind = ISD::ZERO_EXTEND;

904

905

getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],

906

NumParts, RegisterVT, V, CallConv, ExtendKind);

907

Part += NumParts;

908

}

909

910

// Copy the parts into the registers.

911

SmallVector<SDValue, 8> Chains(NumRegs);

912

for (unsigned i = 0; i != NumRegs; ++i) {

913

SDValue Part;

914

if (!Flag) {

915

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);

916

} else {

917

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);

918

*Flag = Part.getValue(1);

919

}

920

921

Chains[i] = Part.getValue(0);

922

}

923

924

if (NumRegs == 1 || Flag)

925

// If NumRegs > 1 && Flag is used then the use of the last CopyToReg is

926

// flagged to it. That is the CopyToReg nodes and the user are considered

927

// a single scheduling unit. If we create a TokenFactor and return it as

928

// chain, then the TokenFactor is both a predecessor (operand) of the

929

// user as well as a successor (the TF operands are flagged to the user).

930

// c1, f1 = CopyToReg

931

// c2, f2 = CopyToReg

932

// c3 = TokenFactor c1, c2

933

// ...

934

// = op c3, ..., f2

935

Chain = Chains[NumRegs-1];

936

else

937

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);

938

}

939

940

void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,

941

unsigned MatchingIdx, const SDLoc &dl,

942

SelectionDAG &DAG,

943

std::vector<SDValue> &Ops) const {

944

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

945

946

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());

947

if (HasMatching)

948

Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);

949

else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {

950

// Put the register class of the virtual registers in the flag word. That

951

// way, later passes can recompute register class constraints for inline

952

// assembly as well as normal instructions.

953

// Don't do this for tied operands that can use the regclass information

954

// from the def.

955

const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

956

const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());

957

Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());

958

}

959

960

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);

961

Ops.push_back(Res);

962

963

if (Code == InlineAsm::Kind_Clobber) {

964

// Clobbers should always have a 1:1 mapping with registers, and may

965

// reference registers that have illegal (e.g. vector) types. Hence, we

966

// shouldn't try to apply any sort of splitting logic to them.

967

assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&((Regs.size() == RegVTs.size() && Regs.size() == ValueVTs
.size() && "No 1:1 mapping from clobbers to regs?") ?
static_cast<void> (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 968, __PRETTY_FUNCTION__))

968

"No 1:1 mapping from clobbers to regs?")((Regs.size() == RegVTs.size() && Regs.size() == ValueVTs
.size() && "No 1:1 mapping from clobbers to regs?") ?
static_cast<void> (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 968, __PRETTY_FUNCTION__));

969

Register SP = TLI.getStackPointerRegisterToSaveRestore();

970

(void)SP;

971

for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {

972

Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));

973

assert((((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 976, __PRETTY_FUNCTION__))

974

(Regs[I] != SP ||(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 976, __PRETTY_FUNCTION__))

975

DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 976, __PRETTY_FUNCTION__))

976

"If we clobbered the stack pointer, MFI should know about it.")(((Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) && "If we clobbered the stack pointer, MFI should know about it."
) ? static_cast<void> (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 976, __PRETTY_FUNCTION__));

977

}

978

return;

979

}

980

981

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {

982

unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);

983

MVT RegisterVT = RegVTs[Value];

984

for (unsigned i = 0; i != NumRegs; ++i) {

985

assert(Reg < Regs.size() && "Mismatch in # registers expected")((Reg < Regs.size() && "Mismatch in # registers expected"
) ? static_cast<void> (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 985, __PRETTY_FUNCTION__));

986

unsigned TheReg = Regs[Reg++];

987

Ops.push_back(DAG.getRegister(TheReg, RegisterVT));

988

}

989

}

990

}

991

992

SmallVector<std::pair<unsigned, TypeSize>, 4>

993

RegsForValue::getRegsAndSizes() const {

994

SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;

995

unsigned I = 0;

996

for (auto CountAndVT : zip_first(RegCount, RegVTs)) {

997

unsigned RegCount = std::get<0>(CountAndVT);

998

MVT RegisterVT = std::get<1>(CountAndVT);

999

TypeSize RegisterSize = RegisterVT.getSizeInBits();

1000

for (unsigned E = I + RegCount; I != E; ++I)

1001

OutVec.push_back(std::make_pair(Regs[I], RegisterSize));

1002

}

1003

return OutVec;

1004

}

1005

1006

void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,

1007

const TargetLibraryInfo *li) {

1008

AA = aa;

1009

GFI = gfi;

1010

LibInfo = li;

1011

DL = &DAG.getDataLayout();

1012

Context = DAG.getContext();

1013

LPadToCallSiteMap.clear();

1014

SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());

1015

}

1016

1017

void SelectionDAGBuilder::clear() {

1018

NodeMap.clear();

1019

UnusedArgNodeMap.clear();

1020

PendingLoads.clear();

1021

PendingExports.clear();

1022

PendingConstrainedFP.clear();

1023

PendingConstrainedFPStrict.clear();

1024

CurInst = nullptr;

1025

HasTailCall = false;

1026

SDNodeOrder = LowestSDNodeOrder;

1027

StatepointLowering.clear();

1028

}

1029

1030

void SelectionDAGBuilder::clearDanglingDebugInfo() {

1031

DanglingDebugInfoMap.clear();

1032

}

1033

1034

// Update DAG root to include dependencies on Pending chains.

1035

SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {

1036

SDValue Root = DAG.getRoot();

1037

1038

if (Pending.empty())

1039

return Root;

1040

1041

// Add current root to PendingChains, unless we already indirectly

1042

// depend on it.

1043

if (Root.getOpcode() != ISD::EntryToken) {

1044

unsigned i = 0, e = Pending.size();

1045

for (; i != e; ++i) {

1046

assert(Pending[i].getNode()->getNumOperands() > 1)((Pending[i].getNode()->getNumOperands() > 1) ? static_cast
<void> (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1046, __PRETTY_FUNCTION__));

1047

if (Pending[i].getNode()->getOperand(0) == Root)

1048

break; // Don't add the root if we already indirectly depend on it.

1049

}

1050

1051

if (i == e)

1052

Pending.push_back(Root);

1053

}

1054

1055

if (Pending.size() == 1)

1056

Root = Pending[0];

1057

else

1058

Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

1059

1060

DAG.setRoot(Root);

1061

Pending.clear();

1062

return Root;

1063

}

1064

1065

SDValue SelectionDAGBuilder::getMemoryRoot() {

1066

return updateRoot(PendingLoads);

1067

}

1068

1069

SDValue SelectionDAGBuilder::getRoot() {

1070

// Chain up all pending constrained intrinsics together with all

1071

// pending loads, by simply appending them to PendingLoads and

1072

// then calling getMemoryRoot().

1073

PendingLoads.reserve(PendingLoads.size() +

1074

PendingConstrainedFP.size() +

1075

PendingConstrainedFPStrict.size());

1076

PendingLoads.append(PendingConstrainedFP.begin(),

1077

PendingConstrainedFP.end());

1078

PendingLoads.append(PendingConstrainedFPStrict.begin(),

1079

PendingConstrainedFPStrict.end());

1080

PendingConstrainedFP.clear();

1081

PendingConstrainedFPStrict.clear();

1082

return getMemoryRoot();

1083

}

1084

1085

SDValue SelectionDAGBuilder::getControlRoot() {

1086

// We need to emit pending fpexcept.strict constrained intrinsics,

1087

// so append them to the PendingExports list.

1088

PendingExports.append(PendingConstrainedFPStrict.begin(),

1089

PendingConstrainedFPStrict.end());

1090

PendingConstrainedFPStrict.clear();

1091

return updateRoot(PendingExports);

1092

}

1093

1094

void SelectionDAGBuilder::visit(const Instruction &I) {

1095

// Set up outgoing PHI node register values before emitting the terminator.

1096

if (I.isTerminator()) {

1097

HandlePHINodesInSuccessorBlocks(I.getParent());

1098

}

1099

1100

// Increase the SDNodeOrder if dealing with a non-debug instruction.

1101

if (!isa<DbgInfoIntrinsic>(I))

1102

++SDNodeOrder;

1103

1104

CurInst = &I;

1105

1106

visit(I.getOpcode(), I);

1107

1108

if (!I.isTerminator() && !HasTailCall &&

1109

!isa<GCStatepointInst>(I)) // statepoints handle their exports internally

1110

CopyToExportRegsIfNeeded(&I);

1111

1112

CurInst = nullptr;

1113

}

1114

1115

void SelectionDAGBuilder::visitPHI(const PHINode &) {

1116

llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1116);

1117

}

1118

1119

void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {

1120

// Note: this doesn't use InstVisitor, because it has to work with

1121

// ConstantExpr's in addition to instructions.

1122

switch (Opcode) {

1123

default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1123);

1124

// Build the switch statement using the Instruction.def file.

1125

#define HANDLE_INST(NUM, OPCODE, CLASS) \

1126

case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;

1127

#include "llvm/IR/Instruction.def"

1128

}

1129

}

1130

1131

void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,

1132

DebugLoc DL, unsigned Order) {

1133

// We treat variadic dbg_values differently at this stage.

1134

if (DI->hasArgList()) {

1135

// For variadic dbg_values we will now insert an undef.

1136

// FIXME: We can potentially recover these!

1137

SmallVector<SDDbgOperand, 2> Locs;

1138

for (const Value *V : DI->getValues()) {

1139

auto Undef = UndefValue::get(V->getType());

1140

Locs.push_back(SDDbgOperand::fromConst(Undef));

1141

}

1142

SDDbgValue *SDV = DAG.getDbgValueList(

1143

DI->getVariable(), DI->getExpression(), Locs, {},

1144

/*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);

1145

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1146

} else {

1147

// TODO: Dangling debug info will eventually either be resolved or produce

1148

// an Undef DBG_VALUE. However in the resolution case, a gap may appear

1149

// between the original dbg.value location and its resolved DBG_VALUE,

1150

// which we should ideally fill with an extra Undef DBG_VALUE.

1151

assert(DI->getNumVariableLocationOps() == 1 &&((DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? static_cast<void> (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1153, __PRETTY_FUNCTION__))

1152

"DbgValueInst without an ArgList should have a single location "((DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? static_cast<void> (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1153, __PRETTY_FUNCTION__))

1153

"operand.")((DI->getNumVariableLocationOps() == 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? static_cast<void> (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1153, __PRETTY_FUNCTION__));

1154

DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);

1155

}

1156

}

1157

1158

void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,

1159

const DIExpression *Expr) {

1160

auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {

1161

const DbgValueInst *DI = DDI.getDI();

1162

DIVariable *DanglingVariable = DI->getVariable();

1163

DIExpression *DanglingExpr = DI->getExpression();

1164

if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {

1165

LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false);

1166

return true;

1167

}

1168

return false;

1169

};

1170

1171

for (auto &DDIMI : DanglingDebugInfoMap) {

1172

DanglingDebugInfoVector &DDIV = DDIMI.second;

1173

1174

// If debug info is to be dropped, run it through final checks to see

1175

// whether it can be salvaged.

1176

for (auto &DDI : DDIV)

1177

if (isMatchingDbgValue(DDI))

1178

salvageUnresolvedDbgValue(DDI);

1179

1180

erase_if(DDIV, isMatchingDbgValue);

1181

}

1182

}

1183

1184

// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,

1185

// generate the debug data structures now that we've seen its definition.

1186

void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,

1187

SDValue Val) {

1188

auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);

1189

if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())

1190

return;

1191

1192

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;

1193

for (auto &DDI : DDIV) {

1194

const DbgValueInst *DI = DDI.getDI();

1195

assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")((!DI->hasArgList() && "Not implemented for variadic dbg_values"
) ? static_cast<void> (0) : __assert_fail ("!DI->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1195, __PRETTY_FUNCTION__));

1196

assert(DI && "Ill-formed DanglingDebugInfo")((DI && "Ill-formed DanglingDebugInfo") ? static_cast
<void> (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1196, __PRETTY_FUNCTION__));

1197

DebugLoc dl = DDI.getdl();

1198

unsigned ValSDNodeOrder = Val.getNode()->getIROrder();

1199

unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();

1200

DILocalVariable *Variable = DI->getVariable();

1201

DIExpression *Expr = DI->getExpression();

1202

assert(Variable->isValidLocationForIntrinsic(dl) &&((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1203, __PRETTY_FUNCTION__))

1203

"Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1203, __PRETTY_FUNCTION__));

1204

SDDbgValue *SDV;

1205

if (Val.getNode()) {

1206

// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a

1207

// FuncArgumentDbgValue (it would be hoisted to the function entry, and if

1208

// we couldn't resolve it directly when examining the DbgValue intrinsic

1209

// in the first place we should not be more successful here). Unless we

1210

// have some test case that prove this to be correct we should avoid

1211

// calling EmitFuncArgumentDbgValue here.

1212

if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {

1213

LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)

1214

<< DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false);

1215

LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);

1216

// Increase the SDNodeOrder for the DbgValue here to make sure it is

1217

// inserted after the definition of Val when emitting the instructions

1218

// after ISel. An alternative could be to teach

1219

// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.

1220

LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1221

<< "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1222

<< ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);

1223

SDV = getDbgValue(Val, Variable, Expr, dl,

1224

std::max(DbgSDNodeOrder, ValSDNodeOrder));

1225

DAG.AddDbgValue(SDV, false);

1226

} else

1227

LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)

1228

<< "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false);

1229

} else {

1230

LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false);

1231

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1232

auto SDV =

1233

DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);

1234

DAG.AddDbgValue(SDV, false);

1235

}

1236

}

1237

DDIV.clear();

1238

}

1239

1240

void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {

1241

assert(!DDI.getDI()->hasArgList() &&((!DDI.getDI()->hasArgList() && "Not implemented for variadic dbg_values"
) ? static_cast<void> (0) : __assert_fail ("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1242, __PRETTY_FUNCTION__))

1242

"Not implemented for variadic dbg_values")((!DDI.getDI()->hasArgList() && "Not implemented for variadic dbg_values"
) ? static_cast<void> (0) : __assert_fail ("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1242, __PRETTY_FUNCTION__));

1243

Value *V = DDI.getDI()->getValue(0);

1244

DILocalVariable *Var = DDI.getDI()->getVariable();

1245

DIExpression *Expr = DDI.getDI()->getExpression();

1246

DebugLoc DL = DDI.getdl();

1247

DebugLoc InstDL = DDI.getDI()->getDebugLoc();

1248

unsigned SDOrder = DDI.getSDNodeOrder();

1249

// Currently we consider only dbg.value intrinsics -- we tell the salvager

1250

// that DW_OP_stack_value is desired.

1251

assert(isa<DbgValueInst>(DDI.getDI()))((isa<DbgValueInst>(DDI.getDI())) ? static_cast<void
> (0) : __assert_fail ("isa<DbgValueInst>(DDI.getDI())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1251, __PRETTY_FUNCTION__));

1252

bool StackValue = true;

1253

1254

// Can this Value can be encoded without any further work?

1255

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))

1256

return;

1257

1258

// Attempt to salvage back through as many instructions as possible. Bail if

1259

// a non-instruction is seen, such as a constant expression or global

1260

// variable. FIXME: Further work could recover those too.

1261

while (isa<Instruction>(V)) {

1262

Instruction &VAsInst = *cast<Instruction>(V);

1263

// Temporary "0", awaiting real implementation.

1264

DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue, 0);

1265

1266

// If we cannot salvage any further, and haven't yet found a suitable debug

1267

// expression, bail out.

1268

if (!NewExpr)

1269

break;

1270

1271

// New value and expr now represent this debuginfo.

1272

V = VAsInst.getOperand(0);

1273

Expr = NewExpr;

1274

1275

// Some kind of simplification occurred: check whether the operand of the

1276

// salvaged debug expression can be encoded in this DAG.

1277

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,

1278

/*IsVariadic=*/false)) {

1279

LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false)

1280

<< DDI.getDI() << "\nBy stripping back to:\n " << V)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false);

1281

return;

1282

}

1283

}

1284

1285

// This was the final opportunity to salvage this debug information, and it

1286

// couldn't be done. Place an undef DBG_VALUE at this location to terminate

1287

// any earlier variable location.

1288

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1289

auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);

1290

DAG.AddDbgValue(SDV, false);

1291

1292

LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false)

1293

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false);

1294

LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
)

1295

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
);

1296

}

1297

1298

bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,

1299

DILocalVariable *Var,

1300

DIExpression *Expr, DebugLoc dl,

1301

DebugLoc InstDL, unsigned Order,

1302

bool IsVariadic) {

1303

if (Values.empty())

1304

return true;

1305

SmallVector<SDDbgOperand> LocationOps;

1306

SmallVector<SDNode *> Dependencies;

1307

for (const Value *V : Values) {

1308

// Constant value.

1309

if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||

1310

isa<ConstantPointerNull>(V)) {

1311

LocationOps.emplace_back(SDDbgOperand::fromConst(V));

1312

continue;

1313

}

1314

1315

// If the Value is a frame index, we can create a FrameIndex debug value

1316

// without relying on the DAG at all.

1317

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1318

auto SI = FuncInfo.StaticAllocaMap.find(AI);

1319

if (SI != FuncInfo.StaticAllocaMap.end()) {

1320

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));

1321

continue;

1322

}

1323

}

1324

1325

// Do not use getValue() in here; we don't want to generate code at

1326

// this point if it hasn't been done yet.

1327

SDValue N = NodeMap[V];

1328

if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.

1329

N = UnusedArgNodeMap[V];

1330

if (N.getNode()) {

1331

// Only emit func arg dbg value for non-variadic dbg.values for now.

1332

if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))

1333

return true;

1334

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

1335

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can

1336

// describe stack slot locations.

1337

//

1338

// Consider "int x = 0; int *px = &x;". There are two kinds of

1339

// interesting debug values here after optimization:

1340

//

1341

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

1342

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

1343

//

1344

// Both describe the direct values of their associated variables.

1345

Dependencies.push_back(N.getNode());

1346

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));

1347

continue;

1348

}

1349

LocationOps.emplace_back(

1350

SDDbgOperand::fromNode(N.getNode(), N.getResNo()));

1351

continue;

1352

}

1353

1354

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1355

// Special rules apply for the first dbg.values of parameter variables in a

1356

// function. Identify them by the fact they reference Argument Values, that

1357

// they're parameters, and they are parameters of the current function. We

1358

// need to let them dangle until they get an SDNode.

1359

bool IsParamOfFunc =

1360

isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();

1361

if (IsParamOfFunc)

1362

return false;

1363

1364

// The value is not used in this block yet (or it would have an SDNode).

1365

// We still want the value to appear for the user if possible -- if it has

1366

// an associated VReg, we can refer to that instead.

1367

auto VMI = FuncInfo.ValueMap.find(V);

1368

if (VMI != FuncInfo.ValueMap.end()) {

1369

unsigned Reg = VMI->second;

1370

// If this is a PHI node, it may be split up into several MI PHI nodes

1371

// (in FunctionLoweringInfo::set).

1372

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,

1373

V->getType(), None);

1374

if (RFV.occupiesMultipleRegs()) {

1375

// FIXME: We could potentially support variadic dbg_values here.

1376

if (IsVariadic)

1377

return false;

1378

unsigned Offset = 0;

1379

unsigned BitsToDescribe = 0;

1380

if (auto VarSize = Var->getSizeInBits())

1381

BitsToDescribe = *VarSize;

1382

if (auto Fragment = Expr->getFragmentInfo())

1383

BitsToDescribe = Fragment->SizeInBits;

1384

for (auto RegAndSize : RFV.getRegsAndSizes()) {

1385

// Bail out if all bits are described already.

1386

if (Offset >= BitsToDescribe)

1387

break;

1388

// TODO: handle scalable vectors.

1389

unsigned RegisterSize = RegAndSize.second;

1390

unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)

1391

? BitsToDescribe - Offset

1392

: RegisterSize;

1393

auto FragmentExpr = DIExpression::createFragmentExpression(

1394

Expr, Offset, FragmentSize);

1395

if (!FragmentExpr)

1396

continue;

1397

SDDbgValue *SDV = DAG.getVRegDbgValue(

1398

Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);

1399

DAG.AddDbgValue(SDV, false);

1400

Offset += RegisterSize;

1401

}

1402

return true;

1403

}

1404

// We can use simple vreg locations for variadic dbg_values as well.

1405

LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));

1406

continue;

1407

}

1408

// We failed to create a SDDbgOperand for V.

1409

return false;

1410

}

1411

1412

// We have created a SDDbgOperand for each Value in Values.

1413

// Should use Order instead of SDNodeOrder?

1414

assert(!LocationOps.empty())((!LocationOps.empty()) ? static_cast<void> (0) : __assert_fail
("!LocationOps.empty()", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1414, __PRETTY_FUNCTION__));

1415

SDDbgValue *SDV =

1416

DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,

1417

/*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);

1418

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1419

return true;

1420

}

1421

1422

void SelectionDAGBuilder::resolveOrClearDbgInfo() {

1423

// Try to fixup any remaining dangling debug info -- and drop it if we can't.

1424

for (auto &Pair : DanglingDebugInfoMap)

1425

for (auto &DDI : Pair.second)

1426

salvageUnresolvedDbgValue(DDI);

1427

clearDanglingDebugInfo();

1428

}

1429

1430

/// getCopyFromRegs - If there was virtual register allocated for the value V

1431

/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.

1432

SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {

1433

DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);

1434

SDValue Result;

1435

1436

if (It != FuncInfo.ValueMap.end()) {

1437

Register InReg = It->second;

1438

1439

RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1440

DAG.getDataLayout(), InReg, Ty,

1441

None); // This is not an ABI copy.

1442

SDValue Chain = DAG.getEntryNode();

1443

Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,

1444

V);

1445

resolveDanglingDebugInfo(V, Result);

1446

}

1447

1448

return Result;

1449

}

1450

1451

/// getValue - Return an SDValue for the given Value.

1452

SDValue SelectionDAGBuilder::getValue(const Value *V) {

1453

// If we already have an SDValue for this value, use it. It's important

1454

// to do this first, so that we don't create a CopyFromReg if we already

1455

// have a regular SDValue.

1456

SDValue &N = NodeMap[V];

1457

if (N.getNode()) return N;

1458

1459

// If there's a virtual register allocated and initialized for this

1460

// value, use it.

1461

if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))

1462

return copyFromReg;

1463

1464

// Otherwise create a new SDValue and remember it.

1465

SDValue Val = getValueImpl(V);

1466

NodeMap[V] = Val;

1467

resolveDanglingDebugInfo(V, Val);

1468

return Val;

1469

}

1470

1471

/// getNonRegisterValue - Return an SDValue for the given Value, but

1472

/// don't look in FuncInfo.ValueMap for a virtual register.

1473

SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {

1474

// If we already have an SDValue for this value, use it.

1475

SDValue &N = NodeMap[V];

1476

if (N.getNode()) {

1477

if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {

1478

// Remove the debug location from the node as the node is about to be used

1479

// in a location which may differ from the original debug location. This

1480

// is relevant to Constant and ConstantFP nodes because they can appear

1481

// as constant expressions inside PHI nodes.

1482

N->setDebugLoc(DebugLoc());

1483

}

1484

return N;

1485

}

1486

1487

// Otherwise create a new SDValue and remember it.

1488

SDValue Val = getValueImpl(V);

1489

NodeMap[V] = Val;

1490

resolveDanglingDebugInfo(V, Val);

1491

return Val;

1492

}

1493

1494

/// getValueImpl - Helper function for getValue and getNonRegisterValue.

1495

/// Create an SDValue for the given value.

1496

SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {

1497

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1498

1499

if (const Constant *C = dyn_cast<Constant>(V)) {

1500

EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

1501

1502

if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))

1503

return DAG.getConstant(*CI, getCurSDLoc(), VT);

1504

1505

if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))

1506

return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

1507

1508

if (isa<ConstantPointerNull>(C)) {

1509

unsigned AS = V->getType()->getPointerAddressSpace();

1510

return DAG.getConstant(0, getCurSDLoc(),

1511

TLI.getPointerTy(DAG.getDataLayout(), AS));

1512

}

1513

1514

if (match(C, m_VScale(DAG.getDataLayout())))

1515

return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

1516

1517

if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))

1518

return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

1519

1520

if (isa<UndefValue>(C) && !V->getType()->isAggregateType())

1521

return DAG.getUNDEF(VT);

1522

1523

if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {

1524

visit(CE->getOpcode(), *CE);

1525

SDValue N1 = NodeMap[V];

1526

assert(N1.getNode() && "visit didn't populate the NodeMap!")((N1.getNode() && "visit didn't populate the NodeMap!"
) ? static_cast<void> (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1526, __PRETTY_FUNCTION__));

1527

return N1;

1528

}

1529

1530

if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {

1531

SmallVector<SDValue, 4> Constants;

1532

for (const Use &U : C->operands()) {

1533

SDNode *Val = getValue(U).getNode();

1534

// If the operand is an empty aggregate, there are no values.

1535

if (!Val) continue;

1536

// Add each leaf value from the operand to the Constants list

1537

// to form a flattened list of all the values.

1538

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1539

Constants.push_back(SDValue(Val, i));

1540

}

1541

1542

return DAG.getMergeValues(Constants, getCurSDLoc());

1543

}

1544

1545

if (const ConstantDataSequential *CDS =

1546

dyn_cast<ConstantDataSequential>(C)) {

1547

SmallVector<SDValue, 4> Ops;

1548

for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {

1549

SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();

1550

// Add each leaf value from the operand to the Constants list

1551

// to form a flattened list of all the values.

1552

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1553

Ops.push_back(SDValue(Val, i));

1554

}

1555

1556

if (isa<ArrayType>(CDS->getType()))

1557

return DAG.getMergeValues(Ops, getCurSDLoc());

1558

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1559

}

1560

1561

if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {

1562

assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1563, __PRETTY_FUNCTION__))

1563

"Unknown struct or array constant!")(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1563, __PRETTY_FUNCTION__));

1564

1565

SmallVector<EVT, 4> ValueVTs;

1566

ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);

1567

unsigned NumElts = ValueVTs.size();

1568

if (NumElts == 0)

1569

return SDValue(); // empty struct

1570

SmallVector<SDValue, 4> Constants(NumElts);

1571

for (unsigned i = 0; i != NumElts; ++i) {

1572

EVT EltVT = ValueVTs[i];

1573

if (isa<UndefValue>(C))

1574

Constants[i] = DAG.getUNDEF(EltVT);

1575

else if (EltVT.isFloatingPoint())

1576

Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1577

else

1578

Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);

1579

}

1580

1581

return DAG.getMergeValues(Constants, getCurSDLoc());

1582

}

1583

1584

if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))

1585

return DAG.getBlockAddress(BA, VT);

1586

1587

if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))

1588

return getValue(Equiv->getGlobalValue());

1589

1590

VectorType *VecTy = cast<VectorType>(V->getType());

1591

1592

// Now that we know the number and type of the elements, get that number of

1593

// elements into the Ops array based on what kind of constant it is.

1594

if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {

1595

SmallVector<SDValue, 16> Ops;

1596

unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();

1597

for (unsigned i = 0; i != NumElements; ++i)

1598

Ops.push_back(getValue(CV->getOperand(i)));

1599

1600

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1601

} else if (isa<ConstantAggregateZero>(C)) {

1602

EVT EltVT =

1603

TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

1604

1605

SDValue Op;

1606

if (EltVT.isFloatingPoint())

1607

Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1608

else

1609

Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

1610

1611

if (isa<ScalableVectorType>(VecTy))

1612

return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);

1613

else {

1614

SmallVector<SDValue, 16> Ops;

1615

Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);

1616

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1617

}

1618

}

1619

llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1619);

1620

}

1621

1622

// If this is a static alloca, generate it as the frameindex instead of

1623

// computation.

1624

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1625

DenseMap<const AllocaInst*, int>::iterator SI =

1626

FuncInfo.StaticAllocaMap.find(AI);

1627

if (SI != FuncInfo.StaticAllocaMap.end())

1628

return DAG.getFrameIndex(SI->second,

1629

TLI.getFrameIndexTy(DAG.getDataLayout()));

1630

}

1631

1632

// If this is an instruction which fast-isel has deferred, select it now.

1633

if (const Instruction *Inst = dyn_cast<Instruction>(V)) {

1634

unsigned InReg = FuncInfo.InitializeRegForValue(Inst);

1635

1636

RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,

1637

Inst->getType(), None);

1638

SDValue Chain = DAG.getEntryNode();

1639

return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);

1640

}

1641

1642

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {

1643

return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

1644

}

1645

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1645);

1646

}

1647

1648

void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {

1649

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1650

bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;

1651

bool IsCoreCLR = Pers == EHPersonality::CoreCLR;

1652

bool IsSEH = isAsynchronousEHPersonality(Pers);

1653

MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;

1654

if (!IsSEH)

1655

CatchPadMBB->setIsEHScopeEntry();

1656

// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.

1657

if (IsMSVCCXX || IsCoreCLR)

1658

CatchPadMBB->setIsEHFuncletEntry();

1659

}

1660

1661

void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {

1662

// Update machine-CFG edge.

1663

MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];

1664

FuncInfo.MBB->addSuccessor(TargetMBB);

1665

TargetMBB->setIsEHCatchretTarget(true);

1666

DAG.getMachineFunction().setHasEHCatchret(true);

1667

1668

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1669

bool IsSEH = isAsynchronousEHPersonality(Pers);

1670

if (IsSEH) {

1671

// If this is not a fall-through branch or optimizations are switched off,

1672

// emit the branch.

1673

if (TargetMBB != NextBlock(FuncInfo.MBB) ||

1674

TM.getOptLevel() == CodeGenOpt::None)

1675

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,

1676

getControlRoot(), DAG.getBasicBlock(TargetMBB)));

1677

return;

1678

}

1679

1680

// Figure out the funclet membership for the catchret's successor.

1681

// This will be used by the FuncletLayout pass to determine how to order the

1682

// BB's.

1683

// A 'catchret' returns to the outer scope's color.

1684

Value *ParentPad = I.getCatchSwitchParentPad();

1685

const BasicBlock *SuccessorColor;

1686

if (isa<ConstantTokenNone>(ParentPad))

1687

SuccessorColor = &FuncInfo.Fn->getEntryBlock();

1688

else

1689

SuccessorColor = cast<Instruction>(ParentPad)->getParent();

1690

assert(SuccessorColor && "No parent funclet for catchret!")((SuccessorColor && "No parent funclet for catchret!"
) ? static_cast<void> (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1690, __PRETTY_FUNCTION__));

1691

MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];

1692

assert(SuccessorColorMBB && "No MBB for SuccessorColor!")((SuccessorColorMBB && "No MBB for SuccessorColor!") ?
static_cast<void> (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1692, __PRETTY_FUNCTION__));

1693

1694

// Create the terminator node.

1695

SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,

1696

getControlRoot(), DAG.getBasicBlock(TargetMBB),

1697

DAG.getBasicBlock(SuccessorColorMBB));

1698

DAG.setRoot(Ret);

1699

}

1700

1701

void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {

1702

// Don't emit any special code for the cleanuppad instruction. It just marks

1703

// the start of an EH scope/funclet.

1704

FuncInfo.MBB->setIsEHScopeEntry();

1705

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1706

if (Pers != EHPersonality::Wasm_CXX) {

1707

FuncInfo.MBB->setIsEHFuncletEntry();

1708

FuncInfo.MBB->setIsCleanupFuncletEntry();

1709

}

1710

}

1711

1712

// In wasm EH, even though a catchpad may not catch an exception if a tag does

1713

// not match, it is OK to add only the first unwind destination catchpad to the

1714

// successors, because there will be at least one invoke instruction within the

1715

// catch scope that points to the next unwind destination, if one exists, so

1716

// CFGSort cannot mess up with BB sorting order.

1717

// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic

1718

// call within them, and catchpads only consisting of 'catch (...)' have a

1719

// '__cxa_end_catch' call within them, both of which generate invokes in case

1720

// the next unwind destination exists, i.e., the next unwind destination is not

1721

// the caller.)

1722

//

1723

// Having at most one EH pad successor is also simpler and helps later

1724

// transformations.

1725

//

1726

// For example,

1727

// current:

1728

// invoke void @foo to ... unwind label %catch.dispatch

1729

// catch.dispatch:

1730

// %0 = catchswitch within ... [label %catch.start] unwind label %next

1731

// catch.start:

1732

// ...

1733

// ... in this BB or some other child BB dominated by this BB there will be an

1734

// invoke that points to 'next' BB as an unwind destination

1735

//

1736

// next: ; We don't need to add this to 'current' BB's successor

1737

// ...

1738

static void findWasmUnwindDestinations(

1739

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1740

BranchProbability Prob,

1741

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1742

&UnwindDests) {

1743

while (EHPadBB) {

1744

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1745

if (isa<CleanupPadInst>(Pad)) {

1746

// Stop on cleanup pads.

1747

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1748

UnwindDests.back().first->setIsEHScopeEntry();

1749

break;

1750

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1751

// Add the catchpad handlers to the possible destinations. We don't

1752

// continue to the unwind destination of the catchswitch for wasm.

1753

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1754

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1755

UnwindDests.back().first->setIsEHScopeEntry();

1756

}

1757

break;

1758

} else {

1759

continue;

1760

}

1761

}

1762

}

1763

1764

/// When an invoke or a cleanupret unwinds to the next EH pad, there are

1765

/// many places it could ultimately go. In the IR, we have a single unwind

1766

/// destination, but in the machine CFG, we enumerate all the possible blocks.

1767

/// This function skips over imaginary basic blocks that hold catchswitch

1768

/// instructions, and finds all the "real" machine

1769

/// basic block destinations. As those destinations may not be successors of

1770

/// EHPadBB, here we also calculate the edge probability to those destinations.

1771

/// The passed-in Prob is the edge probability to EHPadBB.

1772

static void findUnwindDestinations(

1773

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1774

BranchProbability Prob,

1775

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1776

&UnwindDests) {

1777

EHPersonality Personality =

1778

classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1779

bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;

1780

bool IsCoreCLR = Personality == EHPersonality::CoreCLR;

1781

bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;

1782

bool IsSEH = isAsynchronousEHPersonality(Personality);

1783

1784

if (IsWasmCXX) {

1785

findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);

1786

assert(UnwindDests.size() <= 1 &&((UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm"
) ? static_cast<void> (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1787, __PRETTY_FUNCTION__))

1787

"There should be at most one unwind destination for wasm")((UnwindDests.size() <= 1 && "There should be at most one unwind destination for wasm"
) ? static_cast<void> (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1787, __PRETTY_FUNCTION__));

1788

return;

1789

}

1790

1791

while (EHPadBB) {

1792

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1793

BasicBlock *NewEHPadBB = nullptr;

1794

if (isa<LandingPadInst>(Pad)) {

1795

// Stop on landingpads. They are not funclets.

1796

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1797

break;

1798

} else if (isa<CleanupPadInst>(Pad)) {

1799

// Stop on cleanup pads. Cleanups are always funclet entries for all known

1800

// personalities.

1801

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1802

UnwindDests.back().first->setIsEHScopeEntry();

1803

UnwindDests.back().first->setIsEHFuncletEntry();

1804

break;

1805

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1806

// Add the catchpad handlers to the possible destinations.

1807

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1808

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1809

// For MSVC++ and the CLR, catchblocks are funclets and need prologues.

1810

if (IsMSVCCXX || IsCoreCLR)

1811

UnwindDests.back().first->setIsEHFuncletEntry();

1812

if (!IsSEH)

1813

UnwindDests.back().first->setIsEHScopeEntry();

1814

}

1815

NewEHPadBB = CatchSwitch->getUnwindDest();

1816

} else {

1817

continue;

1818

}

1819

1820

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1821

if (BPI && NewEHPadBB)

1822

Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);

1823

EHPadBB = NewEHPadBB;

1824

}

1825

}

1826

1827

void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {

1828

// Update successor info.

1829

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

1830

auto UnwindDest = I.getUnwindDest();

1831

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1832

BranchProbability UnwindDestProb =

1833

(BPI && UnwindDest)

1834

? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)

1835

: BranchProbability::getZero();

1836

findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);

1837

for (auto &UnwindDest : UnwindDests) {

1838

UnwindDest.first->setIsEHPad();

1839

addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);

1840

}

1841

FuncInfo.MBB->normalizeSuccProbs();

1842

1843

// Create the terminator node.

1844

SDValue Ret =

1845

DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());

1846

DAG.setRoot(Ret);

1847

}

1848

1849

void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {

1850

report_fatal_error("visitCatchSwitch not yet implemented!");

1851

}

1852

1853

void SelectionDAGBuilder::visitRet(const ReturnInst &I) {

1854

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1855

auto &DL = DAG.getDataLayout();

1856

SDValue Chain = getControlRoot();

1857

SmallVector<ISD::OutputArg, 8> Outs;

1858

SmallVector<SDValue, 8> OutVals;

1859

1860

// Calls to @llvm.experimental.deoptimize don't generate a return value, so

1861

// lower

1862

//

1863

// %val = call <ty> @llvm.experimental.deoptimize()

1864

// ret <ty> %val

1865

//

1866

// differently.

1867

if (I.getParent()->getTerminatingDeoptimizeCall()) {

1868

LowerDeoptimizingReturn();

1869

return;

1870

}

1871

1872

if (!FuncInfo.CanLowerReturn) {

1873

unsigned DemoteReg = FuncInfo.DemoteRegister;

1874

const Function *F = I.getParent()->getParent();

1875

1876

// Emit a store of the return value through the virtual register.

1877

// Leave Outs empty so that LowerReturn won't try to load return

1878

// registers the usual way.

1879

SmallVector<EVT, 1> PtrValueVTs;

1880

ComputeValueVTs(TLI, DL,

1881

F->getReturnType()->getPointerTo(

1882

DAG.getDataLayout().getAllocaAddrSpace()),

1883

PtrValueVTs);

1884

1885

SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),

1886

DemoteReg, PtrValueVTs[0]);

1887

SDValue RetOp = getValue(I.getOperand(0));

1888

1889

SmallVector<EVT, 4> ValueVTs, MemVTs;

1890

SmallVector<uint64_t, 4> Offsets;

1891

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,

1892

&Offsets);

1893

unsigned NumValues = ValueVTs.size();

1894

1895

SmallVector<SDValue, 4> Chains(NumValues);

1896

Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());

1897

for (unsigned i = 0; i != NumValues; ++i) {

1898

// An aggregate return value cannot wrap around the address space, so

1899

// offsets to its parts don't wrap either.

1900

SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,

1901

TypeSize::Fixed(Offsets[i]));

1902

1903

SDValue Val = RetOp.getValue(RetOp.getResNo() + i);

1904

if (MemVTs[i] != ValueVTs[i])

1905

Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);

1906

Chains[i] = DAG.getStore(

1907

Chain, getCurSDLoc(), Val,

1908

// FIXME: better loc info would be nice.

1909

Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),

1910

commonAlignment(BaseAlign, Offsets[i]));

1911

}

1912

1913

Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),

1914

MVT::Other, Chains);

1915

} else if (I.getNumOperands() != 0) {

1916

SmallVector<EVT, 4> ValueVTs;

1917

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);

1918

unsigned NumValues = ValueVTs.size();

1919

if (NumValues) {

1920

SDValue RetOp = getValue(I.getOperand(0));

1921

1922

const Function *F = I.getParent()->getParent();

1923

1924

bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(

1925

I.getOperand(0)->getType(), F->getCallingConv(),

1926

/*IsVarArg*/ false);

1927

1928

ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

1929

if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1930

Attribute::SExt))

1931

ExtendKind = ISD::SIGN_EXTEND;

1932

else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,

1933

Attribute::ZExt))

1934

ExtendKind = ISD::ZERO_EXTEND;

1935

1936

LLVMContext &Context = F->getContext();

1937

bool RetInReg = F->getAttributes().hasAttribute(

1938

AttributeList::ReturnIndex, Attribute::InReg);

1939

1940

for (unsigned j = 0; j != NumValues; ++j) {

1941

EVT VT = ValueVTs[j];

1942

1943

if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())

1944

VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

1945

1946

CallingConv::ID CC = F->getCallingConv();

1947

1948

unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);

1949

MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);

1950

SmallVector<SDValue, 4> Parts(NumParts);

1951

getCopyToParts(DAG, getCurSDLoc(),

1952

SDValue(RetOp.getNode(), RetOp.getResNo() + j),

1953

&Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

1954

1955

// 'inreg' on function refers to return value

1956

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1957

if (RetInReg)

1958

Flags.setInReg();

1959

1960

if (I.getOperand(0)->getType()->isPointerTy()) {

1961

Flags.setPointer();

1962

Flags.setPointerAddrSpace(

1963

cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());

1964

}

1965

1966

if (NeedsRegBlock) {

1967

Flags.setInConsecutiveRegs();

1968

if (j == NumValues - 1)

1969

Flags.setInConsecutiveRegsLast();

1970

}

1971

1972

// Propagate extension type if any

1973

if (ExtendKind == ISD::SIGN_EXTEND)

1974

Flags.setSExt();

1975

else if (ExtendKind == ISD::ZERO_EXTEND)

1976

Flags.setZExt();

1977

1978

for (unsigned i = 0; i < NumParts; ++i) {

1979

Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),

1980

VT, /*isfixed=*/true, 0, 0));

1981

OutVals.push_back(Parts[i]);

1982

}

1983

}

1984

}

1985

}

1986

1987

// Push in swifterror virtual register as the last element of Outs. This makes

1988

// sure swifterror virtual register will be returned in the swifterror

1989

// physical register.

1990

const Function *F = I.getParent()->getParent();

1991

if (TLI.supportSwiftError() &&

1992

F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {

1993

assert(SwiftError.getFunctionArg() && "Need a swift error argument")((SwiftError.getFunctionArg() && "Need a swift error argument"
) ? static_cast<void> (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1993, __PRETTY_FUNCTION__));

1994

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1995

Flags.setSwiftError();

1996

Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,

1997

EVT(TLI.getPointerTy(DL)) /*argvt*/,

1998

true /*isfixed*/, 1 /*origidx*/,

1999

0 /*partOffs*/));

2000

// Create SDNode for the swifterror virtual register.

2001

OutVals.push_back(

2002

DAG.getRegister(SwiftError.getOrCreateVRegUseAt(

2003

&I, FuncInfo.MBB, SwiftError.getFunctionArg()),

2004

EVT(TLI.getPointerTy(DL))));

2005

}

2006

2007

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();

2008

CallingConv::ID CallConv =

2009

DAG.getMachineFunction().getFunction().getCallingConv();

2010

Chain = DAG.getTargetLoweringInfo().LowerReturn(

2011

Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

2012

2013

// Verify that the target's LowerReturn behaved as expected.

2014

assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2015, __PRETTY_FUNCTION__))

2015

"LowerReturn didn't return a valid chain!")((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2015, __PRETTY_FUNCTION__));

2016

2017

// Update the DAG with the new chain value resulting from return lowering.

2018

DAG.setRoot(Chain);

2019

}

2020

2021

/// CopyToExportRegsIfNeeded - If the given value has virtual registers

2022

/// created for it, emit nodes to copy the value into the virtual

2023

/// registers.

2024

void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

2025

// Skip empty types

2026

if (V->getType()->isEmptyTy())

2027

return;

2028

2029

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);

2030

if (VMI != FuncInfo.ValueMap.end()) {

2031

assert(!V->use_empty() && "Unused value assigned virtual registers!")((!V->use_empty() && "Unused value assigned virtual registers!"
) ? static_cast<void> (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2031, __PRETTY_FUNCTION__));

2032

CopyValueToVirtualRegister(V, VMI->second);

2033

}

2034

}

2035

2036

/// ExportFromCurrentBlock - If this condition isn't known to be exported from

2037

/// the current basic block, add it to ValueMap now so that we'll get a

2038

/// CopyTo/FromReg.

2039

void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {

2040

// No need to export constants.

2041

if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

2042

2043

// Already exported?

2044

if (FuncInfo.isExportedInst(V)) return;

2045

2046

unsigned Reg = FuncInfo.InitializeRegForValue(V);

2047

CopyValueToVirtualRegister(V, Reg);

2048

}

2049

2050

bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,

2051

const BasicBlock *FromBB) {

2052

// The operands of the setcc have to be in this block. We don't know

2053

// how to export them from some other block.

2054

if (const Instruction *VI = dyn_cast<Instruction>(V)) {

2055

// Can export from current BB.

2056

if (VI->getParent() == FromBB)

2057

return true;

2058

2059

// Is already exported, noop.

2060

return FuncInfo.isExportedInst(V);

2061

}

2062

2063

// If this is an argument, we can export it if the BB is the entry block or

2064

// if it is already exported.

2065

if (isa<Argument>(V)) {

2066

if (FromBB == &FromBB->getParent()->getEntryBlock())

2067

return true;

2068

2069

// Otherwise, can only export this if it is already exported.

2070

return FuncInfo.isExportedInst(V);

2071

}

2072

2073

// Otherwise, constants can always be exported.

2074

return true;

2075

}

2076

2077

/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.

2078

BranchProbability

2079

SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,

2080

const MachineBasicBlock *Dst) const {

2081

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2082

const BasicBlock *SrcBB = Src->getBasicBlock();

2083

const BasicBlock *DstBB = Dst->getBasicBlock();

2084

if (!BPI) {

2085

// If BPI is not available, set the default probability as 1 / N, where N is

2086

// the number of successors.

2087

auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);

2088

return BranchProbability(1, SuccSize);

2089

}

2090

return BPI->getEdgeProbability(SrcBB, DstBB);

2091

}

2092

2093

void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,

2094

MachineBasicBlock *Dst,

2095

BranchProbability Prob) {

2096

if (!FuncInfo.BPI)

2097

Src->addSuccessorWithoutProb(Dst);

2098

else {

2099

if (Prob.isUnknown())

2100

Prob = getEdgeProbability(Src, Dst);

2101

Src->addSuccessor(Dst, Prob);

2102

}

2103

}

2104

2105

static bool InBlock(const Value *V, const BasicBlock *BB) {

2106

if (const Instruction *I = dyn_cast<Instruction>(V))

2107

return I->getParent() == BB;

2108

return true;

2109

}

2110

2111

/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.

2112

/// This function emits a branch and is used at the leaves of an OR or an

2113

/// AND operator tree.

2114

void

2115

SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,

2116

MachineBasicBlock *TBB,

2117

MachineBasicBlock *FBB,

2118

MachineBasicBlock *CurBB,

2119

MachineBasicBlock *SwitchBB,

2120

BranchProbability TProb,

2121

BranchProbability FProb,

2122

bool InvertCond) {

2123

const BasicBlock *BB = CurBB->getBasicBlock();

2124

2125

// If the leaf of the tree is a comparison, merge the condition into

2126

// the caseblock.

2127

if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {

2128

// The operands of the cmp have to be in this block. We don't know

2129

// how to export them from some other block. If this is the first block

2130

// of the sequence, no exporting is needed.

2131

if (CurBB == SwitchBB ||

2132

(isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&

2133

isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {

2134

ISD::CondCode Condition;

2135

if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {

2136

ICmpInst::Predicate Pred =

2137

InvertCond ? IC->getInversePredicate() : IC->getPredicate();

2138

Condition = getICmpCondCode(Pred);

2139

} else {

2140

const FCmpInst *FC = cast<FCmpInst>(Cond);

2141

FCmpInst::Predicate Pred =

2142

InvertCond ? FC->getInversePredicate() : FC->getPredicate();

2143

Condition = getFCmpCondCode(Pred);

2144

if (TM.Options.NoNaNsFPMath)

2145

Condition = getFCmpCodeWithoutNaN(Condition);

2146

}

2147

2148

CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,

2149

TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2150

SL->SwitchCases.push_back(CB);

2151

return;

2152

}

2153

}

2154

2155

// Create a CaseBlock record representing this branch.

2156

ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;

2157

CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),

2158

nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2159

SL->SwitchCases.push_back(CB);

2160

}

2161

2162

void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,

2163

MachineBasicBlock *TBB,

2164

MachineBasicBlock *FBB,

2165

MachineBasicBlock *CurBB,

2166

MachineBasicBlock *SwitchBB,

2167

Instruction::BinaryOps Opc,

2168

BranchProbability TProb,

2169

BranchProbability FProb,

2170

bool InvertCond) {

2171

// Skip over not part of the tree and remember to invert op and operands at

2172

// next level.

2173

Value *NotCond;

2174

if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&

2175

InBlock(NotCond, CurBB->getBasicBlock())) {

2176

FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,

2177

!InvertCond);

2178

return;

2179

}

2180

2181

const Instruction *BOp = dyn_cast<Instruction>(Cond);

2182

const Value *BOpOp0, *BOpOp1;

2183

// Compute the effective opcode for Cond, taking into account whether it needs

2184

// to be inverted, e.g.

2185

// and (not (or A, B)), C

2186

// gets lowered as

2187

// and (and (not A, not B), C)

2188

Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;

2189

if (BOp) {

2190

BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))

2191

? Instruction::And

2192

: (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))

2193

? Instruction::Or

2194

: (Instruction::BinaryOps)0);

2195

if (InvertCond) {

2196

if (BOpc == Instruction::And)

2197

BOpc = Instruction::Or;

2198

else if (BOpc == Instruction::Or)

2199

BOpc = Instruction::And;

2200

}

2201

}

2202

2203

// If this node is not part of the or/and tree, emit it as a branch.

2204

// Note that all nodes in the tree should have same opcode.

2205

bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();

2206

if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||

2207

!InBlock(BOpOp0, CurBB->getBasicBlock()) ||

2208

!InBlock(BOpOp1, CurBB->getBasicBlock())) {

2209

EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,

2210

TProb, FProb, InvertCond);

2211

return;

2212

}

2213

2214

// Create TmpBB after CurBB.

2215

MachineFunction::iterator BBI(CurBB);

2216

MachineFunction &MF = DAG.getMachineFunction();

2217

MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());

2218

CurBB->getParent()->insert(++BBI, TmpBB);

2219

2220

if (Opc == Instruction::Or) {

2221

// Codegen X | Y as:

2222

// BB1:

2223

// jmp_if_X TBB

2224

// jmp TmpBB

2225

// TmpBB:

2226

// jmp_if_Y TBB

2227

// jmp FBB

2228

//

2229

2230

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2231

// The requirement is that

2232

// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)

2233

// = TrueProb for original BB.

2234

// Assuming the original probabilities are A and B, one choice is to set

2235

// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to

2236

// A/(1+B) and 2B/(1+B). This choice assumes that

2237

// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.

2238

// Another choice is to assume TrueProb for BB1 equals to TrueProb for

2239

// TmpBB, but the math is more complicated.

2240

2241

auto NewTrueProb = TProb / 2;

2242

auto NewFalseProb = TProb / 2 + FProb;

2243

// Emit the LHS condition.

2244

FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,

2245

NewFalseProb, InvertCond);

2246

2247

// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).

2248

SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};

2249

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2250

// Emit the RHS condition into TmpBB.

2251

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2252

Probs[1], InvertCond);

2253

} else {

2254

assert(Opc == Instruction::And && "Unknown merge op!")((Opc == Instruction::And && "Unknown merge op!") ? static_cast
<void> (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2254, __PRETTY_FUNCTION__));

2255

// Codegen X & Y as:

2256

// BB1:

2257

// jmp_if_X TmpBB

2258

// jmp FBB

2259

// TmpBB:

2260

// jmp_if_Y TBB

2261

// jmp FBB

2262

//

2263

// This requires creation of TmpBB after CurBB.

2264

2265

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2266

// The requirement is that

2267

// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)

2268

// = FalseProb for original BB.

2269

// Assuming the original probabilities are A and B, one choice is to set

2270

// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to

2271

// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==

2272

// TrueProb for BB1 * FalseProb for TmpBB.

2273

2274

auto NewTrueProb = TProb + FProb / 2;

2275

auto NewFalseProb = FProb / 2;

2276

// Emit the LHS condition.

2277

FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,

2278

NewFalseProb, InvertCond);

2279

2280

// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).

2281

SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};

2282

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2283

// Emit the RHS condition into TmpBB.

2284

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2285

Probs[1], InvertCond);

2286

}

2287

}

2288

2289

/// If the set of cases should be emitted as a series of branches, return true.

2290

/// If we should emit this as a bunch of and/or'd together conditions, return

2291

/// false.

2292

bool

2293

SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {

2294

if (Cases.size() != 2) return true;

2295

2296

// If this is two comparisons of the same values or'd or and'd together, they

2297

// will get folded into a single comparison, so don't emit two blocks.

2298

if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&

2299

Cases[0].CmpRHS == Cases[1].CmpRHS) ||

2300

(Cases[0].CmpRHS == Cases[1].CmpLHS &&

2301

Cases[0].CmpLHS == Cases[1].CmpRHS)) {

2302

return false;

2303

}

2304

2305

// Handle: (X != null) | (Y != null) --> (X|Y) != 0

2306

// Handle: (X == null) & (Y == null) --> (X|Y) == 0

2307

if (Cases[0].CmpRHS == Cases[1].CmpRHS &&

2308

Cases[0].CC == Cases[1].CC &&

2309

isa<Constant>(Cases[0].CmpRHS) &&

2310

cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {

2311

if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)

2312

return false;

2313

if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)

2314

return false;

2315

}

2316

2317

return true;

2318

}

2319

2320

void SelectionDAGBuilder::visitBr(const BranchInst &I) {

2321

MachineBasicBlock *BrMBB = FuncInfo.MBB;

2322

2323

// Update machine-CFG edges.

2324

MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

2325

2326

if (I.isUnconditional()) {

2327

// Update machine-CFG edges.

2328

BrMBB->addSuccessor(Succ0MBB);

2329

2330

// If this is not a fall-through branch or optimizations are switched off,

2331

// emit the branch.

2332

if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)

2333

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2334

MVT::Other, getControlRoot(),

2335

DAG.getBasicBlock(Succ0MBB)));

2336

2337

return;

2338

}

2339

2340

// If this condition is one of the special cases we handle, do special stuff

2341

// now.

2342

const Value *CondVal = I.getCondition();

2343

MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

2344

2345

// If this is a series of conditions that are or'd or and'd together, emit

2346

// this as a sequence of branches instead of setcc's with and/or operations.

2347

// As long as jumps are not expensive (exceptions for multi-use logic ops,

2348

// unpredictable branches, and vector extracts because those jumps are likely

2349

// expensive for any target), this should improve performance.

2350

// For example, instead of something like:

2351

// cmp A, B

2352

// C = seteq

2353

// cmp D, E

2354

// F = setle

2355

// or C, F

2356

// jnz foo

2357

// Emit:

2358

// cmp A, B

2359

// je foo

2360

// cmp D, E

2361

// jle foo

2362

const Instruction *BOp = dyn_cast<Instruction>(CondVal);

2363

if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&

2364

BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {

2365

Value *Vec;

2366

const Value *BOp0, *BOp1;

2367

Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;

2368

if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))

2369

Opcode = Instruction::And;

2370

else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))

2371

Opcode = Instruction::Or;

2372

2373

if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&

2374

match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {

2375

FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,

2376

getEdgeProbability(BrMBB, Succ0MBB),

2377

getEdgeProbability(BrMBB, Succ1MBB),

2378

/*InvertCond=*/false);

2379

// If the compares in later blocks need to use values not currently

2380

// exported from this block, export them now. This block should always

2381

// be the first entry.

2382

assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")((SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"
) ? static_cast<void> (0) : __assert_fail ("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2382, __PRETTY_FUNCTION__));

2383

2384

// Allow some cases to be rejected.

2385

if (ShouldEmitAsBranches(SL->SwitchCases)) {

2386

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {

2387

ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);

2388

ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);

2389

}

2390

2391

// Emit the branch for this block.

2392

visitSwitchCase(SL->SwitchCases[0], BrMBB);

2393

SL->SwitchCases.erase(SL->SwitchCases.begin());

2394

return;

2395

}

2396

2397

// Okay, we decided not to do this, remove any inserted MBB's and clear

2398

// SwitchCases.

2399

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)

2400

FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

2401

2402

SL->SwitchCases.clear();

2403

}

2404

}

2405

2406

// Create a CaseBlock record representing this branch.

2407

CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),

2408

nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

2409

2410

// Use visitSwitchCase to actually insert the fast branch sequence for this

2411

// cond branch.

2412

visitSwitchCase(CB, BrMBB);

2413

}

2414

2415

/// visitSwitchCase - Emits the necessary code to represent a single node in

2416

/// the binary search tree resulting from lowering a switch instruction.

2417

void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,

2418

MachineBasicBlock *SwitchBB) {

2419

SDValue Cond;

2420

SDValue CondLHS = getValue(CB.CmpLHS);

2421

SDLoc dl = CB.DL;

2422

2423

if (CB.CC == ISD::SETTRUE) {

2424

// Branch or fall through to TrueBB.

2425

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2426

SwitchBB->normalizeSuccProbs();

2427

if (CB.TrueBB != NextBlock(SwitchBB)) {

2428

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),

2429

DAG.getBasicBlock(CB.TrueBB)));

2430

}

2431

return;

2432

}

2433

2434

auto &TLI = DAG.getTargetLoweringInfo();

2435

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

2436

2437

// Build the setcc now.

2438

if (!CB.CmpMHS) {

2439

// Fold "(X == true)" to X and "(X == false)" to !X to

2440

// handle common cases produced by branch lowering.

2441

if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&

2442

CB.CC == ISD::SETEQ)

2443

Cond = CondLHS;

2444

else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&

2445

CB.CC == ISD::SETEQ) {

2446

SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());

2447

Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);

2448

} else {

2449

SDValue CondRHS = getValue(CB.CmpRHS);

2450

2451

// If a pointer's DAG type is larger than its memory type then the DAG

2452

// values are zero-extended. This breaks signed comparisons so truncate

2453

// back to the underlying type before doing the compare.

2454

if (CondLHS.getValueType() != MemVT) {

2455

CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);

2456

CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);

2457

}

2458

Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);

2459

}

2460

} else {

2461

assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")((CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? static_cast<void> (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2461, __PRETTY_FUNCTION__));

2462

2463

const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();

2464

const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

2465

2466

SDValue CmpOp = getValue(CB.CmpMHS);

2467

EVT VT = CmpOp.getValueType();

2468

2469

if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {

2470

Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),

2471

ISD::SETLE);

2472

} else {

2473

SDValue SUB = DAG.getNode(ISD::SUB, dl,

2474

VT, CmpOp, DAG.getConstant(Low, dl, VT));

2475

Cond = DAG.getSetCC(dl, MVT::i1, SUB,

2476

DAG.getConstant(High-Low, dl, VT), ISD::SETULE);

2477

}

2478

}

2479

2480

// Update successor info

2481

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2482

// TrueBB and FalseBB are always different unless the incoming IR is

2483

// degenerate. This only happens when running llc on weird IR.

2484

if (CB.TrueBB != CB.FalseBB)

2485

addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);

2486

SwitchBB->normalizeSuccProbs();

2487

2488

// If the lhs block is the next block, invert the condition so that we can

2489

// fall through to the lhs instead of the rhs block.

2490

if (CB.TrueBB == NextBlock(SwitchBB)) {

2491

std::swap(CB.TrueBB, CB.FalseBB);

2492

SDValue True = DAG.getConstant(1, dl, Cond.getValueType());

2493

Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);

2494

}

2495

2496

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2497

MVT::Other, getControlRoot(), Cond,

2498

DAG.getBasicBlock(CB.TrueBB));

2499

2500

// Insert the false branch. Do this even if it's a fall through branch,

2501

// this makes it easier to do DAG optimizations which require inverting

2502

// the branch condition.

2503

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2504

DAG.getBasicBlock(CB.FalseBB));

2505

2506

DAG.setRoot(BrCond);

2507

}

2508

2509

/// visitJumpTable - Emit JumpTable node in the current MBB

2510

void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {

2511

// Emit the code for the jump table

2512

assert(JT.Reg != -1U && "Should lower JT Header first!")((JT.Reg != -1U && "Should lower JT Header first!") ?
static_cast<void> (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2512, __PRETTY_FUNCTION__));

2513

EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());

2514

SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),

2515

JT.Reg, PTy);

2516

SDValue Table = DAG.getJumpTable(JT.JTI, PTy);

2517

SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),

2518

MVT::Other, Index.getValue(1),

2519

Table, Index);

2520

DAG.setRoot(BrJumpTable);

2521

}

2522

2523

/// visitJumpTableHeader - This function emits necessary code to produce index

2524

/// in the JumpTable from switch case.

2525

void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,

2526

JumpTableHeader &JTH,

2527

MachineBasicBlock *SwitchBB) {

2528

SDLoc dl = getCurSDLoc();

2529

2530

// Subtract the lowest switch case value from the value being switched on.

2531

SDValue SwitchOp = getValue(JTH.SValue);

2532

EVT VT = SwitchOp.getValueType();

2533

SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,

2534

DAG.getConstant(JTH.First, dl, VT));

2535

2536

// The SDNode we just created, which holds the value being switched on minus

2537

// the smallest case value, needs to be copied to a virtual register so it

2538

// can be used as an index into the jump table in a subsequent basic block.

2539

// This value may be smaller or larger than the target's pointer type, and

2540

// therefore require extension or truncating.

2541

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2542

SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

2543

2544

unsigned JumpTableReg =

2545

FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));

2546

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,

2547

JumpTableReg, SwitchOp);

2548

JT.Reg = JumpTableReg;

2549

2550

if (!JTH.OmitRangeCheck) {

2551

// Emit the range check for the jump table, and branch to the default block

2552

// for the switch statement if the value being switched on exceeds the

2553

// largest case in the switch.

2554

SDValue CMP = DAG.getSetCC(

2555

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2556

Sub.getValueType()),

2557

Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

2558

2559

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2560

MVT::Other, CopyTo, CMP,

2561

DAG.getBasicBlock(JT.Default));

2562

2563

// Avoid emitting unnecessary branches to the next block.

2564

if (JT.MBB != NextBlock(SwitchBB))

2565

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2566

DAG.getBasicBlock(JT.MBB));

2567

2568

DAG.setRoot(BrCond);

2569

} else {

2570

// Avoid emitting unnecessary branches to the next block.

2571

if (JT.MBB != NextBlock(SwitchBB))

2572

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,

2573

DAG.getBasicBlock(JT.MBB)));

2574

else

2575

DAG.setRoot(CopyTo);

2576

}

2577

}

2578

2579

/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global

2580

/// variable if there exists one.

2581

static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,

2582

SDValue &Chain) {

2583

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2584

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2585

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2586

MachineFunction &MF = DAG.getMachineFunction();

2587

Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());

2588

MachineSDNode *Node =

2589

DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);

2590

if (Global) {

2591

MachinePointerInfo MPInfo(Global);

2592

auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |

2593

MachineMemOperand::MODereferenceable;

2594

MachineMemOperand *MemRef = MF.getMachineMemOperand(

2595

MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));

2596

DAG.setNodeMemRefs(Node, {MemRef});

2597

}

2598

if (PtrTy != PtrMemTy)

2599

return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);

2600

return SDValue(Node, 0);

2601

}

2602

2603

/// Codegen a new tail for a stack protector check ParentMBB which has had its

2604

/// tail spliced into a stack protector check success bb.

2605

///

2606

/// For a high level explanation of how this fits into the stack protector

2607

/// generation see the comment on the declaration of class

2608

/// StackProtectorDescriptor.

2609

void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,

2610

MachineBasicBlock *ParentBB) {

2611

2612

// First create the loads to the guard/stack slot for the comparison.

2613

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2614

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2615

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2616

2617

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();

2618

int FI = MFI.getStackProtectorIndex();

2619

2620

SDValue Guard;

2621

SDLoc dl = getCurSDLoc();

2622

SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);

2623

const Module &M = *ParentBB->getParent()->getFunction().getParent();

2624

Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

2625

2626

// Generate code to load the content of the guard slot.

2627

SDValue GuardVal = DAG.getLoad(

2628

PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,

2629

MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,

2630

MachineMemOperand::MOVolatile);

2631

2632

if (TLI.useStackGuardXorFP())

2633

GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

2634

2635

// Retrieve guard check function, nullptr if instrumentation is inlined.

2636

if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {

2637

// The target provides a guard check function to validate the guard value.

2638

// Generate a call to that function with the content of the guard slot as

2639

// argument.

2640

FunctionType *FnTy = GuardCheckFn->getFunctionType();

2641

assert(FnTy->getNumParams() == 1 && "Invalid function signature")((FnTy->getNumParams() == 1 && "Invalid function signature"
) ? static_cast<void> (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2641, __PRETTY_FUNCTION__));

2642

2643

TargetLowering::ArgListTy Args;

2644

TargetLowering::ArgListEntry Entry;

2645

Entry.Node = GuardVal;

2646

Entry.Ty = FnTy->getParamType(0);

2647

if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))

2648

Entry.IsInReg = true;

2649

Args.push_back(Entry);

2650

2651

TargetLowering::CallLoweringInfo CLI(DAG);

2652

CLI.setDebugLoc(getCurSDLoc())

2653

.setChain(DAG.getEntryNode())

2654

.setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),

2655

getValue(GuardCheckFn), std::move(Args));

2656

2657

std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

2658

DAG.setRoot(Result.second);

2659

return;

2660

}

2661

2662

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.

2663

// Otherwise, emit a volatile load to retrieve the stack guard value.

2664

SDValue Chain = DAG.getEntryNode();

2665

if (TLI.useLoadStackGuardNode()) {

2666

Guard = getLoadStackGuard(DAG, dl, Chain);

2667

} else {

2668

const Value *IRGuard = TLI.getSDagStackGuard(M);

2669

SDValue GuardPtr = getValue(IRGuard);

2670

2671

Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,

2672

MachinePointerInfo(IRGuard, 0), Align,

2673

MachineMemOperand::MOVolatile);

2674

}

2675

2676

// Perform the comparison via a getsetcc.

2677

SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),

2678

*DAG.getContext(),

2679

Guard.getValueType()),

2680

Guard, GuardVal, ISD::SETNE);

2681

2682

// If the guard/stackslot do not equal, branch to failure MBB.

2683

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2684

MVT::Other, GuardVal.getOperand(0),

2685

Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));

2686

// Otherwise branch to success MBB.

2687

SDValue Br = DAG.getNode(ISD::BR, dl,

2688

MVT::Other, BrCond,

2689

DAG.getBasicBlock(SPD.getSuccessMBB()));

2690

2691

DAG.setRoot(Br);

2692

}

2693

2694

/// Codegen the failure basic block for a stack protector check.

2695

///

2696

/// A failure stack protector machine basic block consists simply of a call to

2697

/// __stack_chk_fail().

2698

///

2699

/// For a high level explanation of how this fits into the stack protector

2700

/// generation see the comment on the declaration of class

2701

/// StackProtectorDescriptor.

2702

void

2703

SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {

2704

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2705

TargetLowering::MakeLibCallOptions CallOptions;

2706

CallOptions.setDiscardResult(true);

2707

SDValue Chain =

2708

TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,

2709

None, CallOptions, getCurSDLoc()).second;

2710

// On PS4, the "return address" must still be within the calling function,

2711

// even if it's at the very end, so emit an explicit TRAP here.

2712

// Passing 'true' for doesNotReturn above won't generate the trap for us.

2713

if (TM.getTargetTriple().isPS4CPU())

2714

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2715

// WebAssembly needs an unreachable instruction after a non-returning call,

2716

// because the function return type can be different from __stack_chk_fail's

2717

// return type (void).

2718

if (TM.getTargetTriple().isWasm())

2719

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2720

2721

DAG.setRoot(Chain);

2722

}

2723

2724

/// visitBitTestHeader - This function emits necessary code to produce value

2725

/// suitable for "bit tests"

2726

void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,

2727

MachineBasicBlock *SwitchBB) {

2728

SDLoc dl = getCurSDLoc();

2729

2730

// Subtract the minimum value.

2731

SDValue SwitchOp = getValue(B.SValue);

2732

EVT VT = SwitchOp.getValueType();

2733

SDValue RangeSub =

2734

DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

2735

2736

// Determine the type of the test operands.

2737

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2738

bool UsePtrType = false;

2739

if (!TLI.isTypeLegal(VT)) {

2740

UsePtrType = true;

2741

} else {

2742

for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)

2743

if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {

2744

// Switch table case range are encoded into series of masks.

2745

// Just use pointer type, it's guaranteed to fit.

2746

UsePtrType = true;

2747

break;

2748

}

2749

}

2750

SDValue Sub = RangeSub;

2751

if (UsePtrType) {

2752

VT = TLI.getPointerTy(DAG.getDataLayout());

2753

Sub = DAG.getZExtOrTrunc(Sub, dl, VT);

2754

}

2755

2756

B.RegVT = VT.getSimpleVT();

2757

B.Reg = FuncInfo.CreateReg(B.RegVT);

2758

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

2759

2760

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

2761

2762

if (!B.OmitRangeCheck)

2763

addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);

2764

addSuccessorWithProb(SwitchBB, MBB, B.Prob);

2765

SwitchBB->normalizeSuccProbs();

2766

2767

SDValue Root = CopyTo;

2768

if (!B.OmitRangeCheck) {

2769

// Conditional branch to the default block.

2770

SDValue RangeCmp = DAG.getSetCC(dl,

2771

TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2772

RangeSub.getValueType()),

2773

RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),

2774

ISD::SETUGT);

2775

2776

Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,

2777

DAG.getBasicBlock(B.Default));

2778

}

2779

2780

// Avoid emitting unnecessary branches to the next block.

2781

if (MBB != NextBlock(SwitchBB))

2782

Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

2783

2784

DAG.setRoot(Root);

2785

}

2786

2787

/// visitBitTestCase - this function produces one "bit test"

2788

void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,

2789

MachineBasicBlock* NextMBB,

2790

BranchProbability BranchProbToNext,

2791

unsigned Reg,

2792

BitTestCase &B,

2793

MachineBasicBlock *SwitchBB) {

2794

SDLoc dl = getCurSDLoc();

2795

MVT VT = BB.RegVT;

2796

SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);

2797

SDValue Cmp;

2798

unsigned PopCount = countPopulation(B.Mask);

2799

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2800

if (PopCount == 1) {

2801

// Testing for a single bit; just compare the shift count with what it

2802

// would need to be to shift a 1 bit in that position.

2803

Cmp = DAG.getSetCC(

2804

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2805

ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),

2806

ISD::SETEQ);

2807

} else if (PopCount == BB.Range) {

2808

// There is only one zero bit in the range, test for it directly.

2809

Cmp = DAG.getSetCC(

2810

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2811

ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),

2812

ISD::SETNE);

2813

} else {

2814

// Make desired shift

2815

SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,

2816

DAG.getConstant(1, dl, VT), ShiftOp);

2817

2818

// Emit bit tests and jumps

2819

SDValue AndOp = DAG.getNode(ISD::AND, dl,

2820

VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));

2821

Cmp = DAG.getSetCC(

2822

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2823

AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);

2824

}

2825

2826

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.

2827

addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);

2828

// The branch probability from SwitchBB to NextMBB is BranchProbToNext.

2829

addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);

2830

// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is

2831

// one as they are relative probabilities (and thus work more like weights),

2832

// and hence we need to normalize them to let the sum of them become one.

2833

SwitchBB->normalizeSuccProbs();

2834

2835

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,

2836

MVT::Other, getControlRoot(),

2837

Cmp, DAG.getBasicBlock(B.TargetBB));

2838

2839

// Avoid emitting unnecessary branches to the next block.

2840

if (NextMBB != NextBlock(SwitchBB))

2841

BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,

2842

DAG.getBasicBlock(NextMBB));

2843

2844

DAG.setRoot(BrAnd);

2845

}

2846

2847

void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {

2848

MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

2849

2850

// Retrieve successors. Look through artificial IR level blocks like

2851

// catchswitch for successors.

2852

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];

2853

const BasicBlock *EHPadBB = I.getSuccessor(1);

2854

2855

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2856

// have to do anything here to lower funclet bundles.

2857

assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__))

2858

{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__))

2859

LLVMContext::OB_gc_live, LLVMContext::OB_funclet,((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__))

2860

LLVMContext::OB_cfguardtarget,((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__))

2861

LLVMContext::OB_clang_arc_attachedcall}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__))

2862

"Cannot lower invokes with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet
, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall
}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2862, __PRETTY_FUNCTION__));

2863

2864

const Value *Callee(I.getCalledOperand());

2865

const Function *Fn = dyn_cast<Function>(Callee);

2866

if (isa<InlineAsm>(Callee))

2867

visitInlineAsm(I);

2868

else if (Fn && Fn->isIntrinsic()) {

2869

switch (Fn->getIntrinsicID()) {

2870

default:

2871

llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2871);

2872

case Intrinsic::donothing:

2873

// Ignore invokes to @llvm.donothing: jump directly to the next BB.

2874

break;

2875

case Intrinsic::experimental_patchpoint_void:

2876

case Intrinsic::experimental_patchpoint_i64:

2877

visitPatchpoint(I, EHPadBB);

2878

break;

2879

case Intrinsic::experimental_gc_statepoint:

2880

LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);

2881

break;

2882

case Intrinsic::wasm_rethrow: {

2883

// This is usually done in visitTargetIntrinsic, but this intrinsic is

2884

// special because it can be invoked, so we manually lower it to a DAG

2885

// node here.

2886

SmallVector<SDValue, 8> Ops;

2887

Ops.push_back(getRoot()); // inchain

2888

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2889

Ops.push_back(

2890

DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),

2891

TLI.getPointerTy(DAG.getDataLayout())));

2892

SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain

2893

DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));

2894

break;

2895

}

2896

}

2897

} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {

2898

// Currently we do not lower any intrinsic calls with deopt operand bundles.

2899

// Eventually we will support lowering the @llvm.experimental.deoptimize

2900

// intrinsic, and right now there are no plans to support other intrinsics

2901

// with deopt state.

2902

LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);

2903

} else {

2904

LowerCallTo(I, getValue(Callee), false, EHPadBB);

2905

}

2906

2907

// If the value of the invoke is used outside of its defining block, make it

2908

// available as a virtual register.

2909

// We already took care of the exported value for the statepoint instruction

2910

// during call to the LowerStatepoint.

2911

if (!isa<GCStatepointInst>(I)) {

2912

CopyToExportRegsIfNeeded(&I);

2913

}

2914

2915

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

2916

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2917

BranchProbability EHPadBBProb =

2918

BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)

2919

: BranchProbability::getZero();

2920

findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

2921

2922

// Update successor info.

2923

addSuccessorWithProb(InvokeMBB, Return);

2924

for (auto &UnwindDest : UnwindDests) {

2925

UnwindDest.first->setIsEHPad();

2926

addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);

2927

}

2928

InvokeMBB->normalizeSuccProbs();

2929

2930

// Drop into normal successor.

2931

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),

2932

DAG.getBasicBlock(Return)));

2933

}

2934

2935

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {

2936

MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

2937

2938

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2939

// have to do anything here to lower funclet bundles.

2940

assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2942, __PRETTY_FUNCTION__))

2941

{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2942, __PRETTY_FUNCTION__))

2942

"Cannot lower callbrs with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2942, __PRETTY_FUNCTION__));

2943

2944

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")((I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? static_cast<void> (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2944, __PRETTY_FUNCTION__));

2945

visitInlineAsm(I);

2946

CopyToExportRegsIfNeeded(&I);

2947

2948

// Retrieve successors.

2949

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

2950

2951

// Update successor info.

2952

addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());

2953

for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {

2954

MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];

2955

addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());

2956

Target->setIsInlineAsmBrIndirectTarget();

2957

}

2958

CallBrMBB->normalizeSuccProbs();

2959

2960

// Drop into default successor.

2961

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2962

MVT::Other, getControlRoot(),

2963

DAG.getBasicBlock(Return)));

2964

}

2965

2966

void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {

2967

llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2967);

2968

}

2969

2970

void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {

2971

assert(FuncInfo.MBB->isEHPad() &&((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2972, __PRETTY_FUNCTION__))

2972

"Call to landingpad not in landing pad!")((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2972, __PRETTY_FUNCTION__));

2973

2974

// If there aren't registers to copy the values into (e.g., during SjLj

2975

// exceptions), then don't bother to create these DAG nodes.

2976

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2977

const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();

2978

if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&

2979

TLI.getExceptionSelectorRegister(PersonalityFn) == 0)

2980

return;

2981

2982

// If landingpad's return type is token type, we don't create DAG nodes

2983

// for its exception pointer and selector value. The extraction of exception

2984

// pointer or selector value from token type landingpads is not currently

2985

// supported.

2986

if (LP.getType()->isTokenTy())

2987

return;

2988

2989

SmallVector<EVT, 2> ValueVTs;

2990

SDLoc dl = getCurSDLoc();

2991

ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);

2992

assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")((ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2992, __PRETTY_FUNCTION__));

2993

2994

// Get the two live-in registers as SDValues. The physregs have already been

2995

// copied into virtual registers.

2996

SDValue Ops[2];

2997

if (FuncInfo.ExceptionPointerVirtReg) {

2998

Ops[0] = DAG.getZExtOrTrunc(

2999

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3000

FuncInfo.ExceptionPointerVirtReg,

3001

TLI.getPointerTy(DAG.getDataLayout())),

3002

dl, ValueVTs[0]);

3003

} else {

3004

Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));

3005

}

3006

Ops[1] = DAG.getZExtOrTrunc(

3007

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3008

FuncInfo.ExceptionSelectorVirtReg,

3009

TLI.getPointerTy(DAG.getDataLayout())),

3010

dl, ValueVTs[1]);

3011

3012

// Merge into one.

3013

SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,

3014

DAG.getVTList(ValueVTs), Ops);

3015

setValue(&LP, Res);

3016

}

3017

3018

void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,

3019

MachineBasicBlock *Last) {

3020

// Update JTCases.

3021

for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)

3022

if (SL->JTCases[i].first.HeaderBB == First)

3023

SL->JTCases[i].first.HeaderBB = Last;

3024

3025

// Update BitTestCases.

3026

for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)

3027

if (SL->BitTestCases[i].Parent == First)

3028

SL->BitTestCases[i].Parent = Last;

3029

}

3030

3031

void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {

3032

MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

3033

3034

// Update machine-CFG edges with unique successors.

3035

SmallSet<BasicBlock*, 32> Done;

3036

for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {

3037

BasicBlock *BB = I.getSuccessor(i);

3038

bool Inserted = Done.insert(BB).second;

3039

if (!Inserted)

3040

continue;

3041

3042

MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];

3043

addSuccessorWithProb(IndirectBrMBB, Succ);

3044

}

3045

IndirectBrMBB->normalizeSuccProbs();

3046

3047

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),

3048

MVT::Other, getControlRoot(),

3049

getValue(I.getAddress())));

3050

}

3051

3052

void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {

3053

if (!DAG.getTarget().Options.TrapUnreachable)

3054

return;

3055

3056

// We may be able to ignore unreachable behind a noreturn call.

3057

if (DAG.getTarget().Options.NoTrapAfterNoreturn) {

3058

const BasicBlock &BB = *I.getParent();

3059

if (&I != &BB.front()) {

3060

BasicBlock::const_iterator PredI =

3061

std::prev(BasicBlock::const_iterator(&I));

3062

if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {

3063

if (Call->doesNotReturn())

3064

return;

3065

}

3066

}

3067

}

3068

3069

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

3070

}

3071

3072

void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {

3073

SDNodeFlags Flags;

3074

3075

SDValue Op = getValue(I.getOperand(0));

3076

SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),

3077

Op, Flags);

3078

setValue(&I, UnNodeValue);

3079

}

3080

3081

void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {

3082

SDNodeFlags Flags;

3083

if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {

3084

Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());

3085

Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());

3086

}

3087

if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))

3088

Flags.setExact(ExactOp->isExact());

3089

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3090

Flags.copyFMF(*FPOp);

3091

3092

SDValue Op1 = getValue(I.getOperand(0));

3093

SDValue Op2 = getValue(I.getOperand(1));

3094

SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),

3095

Op1, Op2, Flags);

3096

setValue(&I, BinNodeValue);

3097

}

3098

3099

void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {

3100

SDValue Op1 = getValue(I.getOperand(0));

3101

SDValue Op2 = getValue(I.getOperand(1));

3102

3103

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(

3104

Op1.getValueType(), DAG.getDataLayout());

3105

3106

// Coerce the shift amount to the right type if we can.

3107

if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {

3108

unsigned ShiftSize = ShiftTy.getSizeInBits();

3109

unsigned Op2Size = Op2.getValueSizeInBits();

3110

SDLoc DL = getCurSDLoc();

3111

3112

// If the operand is smaller than the shift count type, promote it.

3113

if (ShiftSize > Op2Size)

3114

Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);

3115

3116

// If the operand is larger than the shift count type but the shift

3117

// count type has enough bits to represent any shift value, truncate

3118

// it now. This is a common case and it exposes the truncate to

3119

// optimization early.

3120

else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))

3121

Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);

3122

// Otherwise we'll need to temporarily settle for some other convenient

3123

// type. Type legalization will make adjustments once the shiftee is split.

3124

else

3125

Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);

3126

}

3127

3128

bool nuw = false;

3129

bool nsw = false;

3130

bool exact = false;

3131

3132

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

3133

3134

if (const OverflowingBinaryOperator *OFBinOp =

3135

dyn_cast<const OverflowingBinaryOperator>(&I)) {

3136

nuw = OFBinOp->hasNoUnsignedWrap();

3137

nsw = OFBinOp->hasNoSignedWrap();

3138

}

3139

if (const PossiblyExactOperator *ExactOp =

3140

dyn_cast<const PossiblyExactOperator>(&I))

3141

exact = ExactOp->isExact();

3142

}

3143

SDNodeFlags Flags;

3144

Flags.setExact(exact);

3145

Flags.setNoSignedWrap(nsw);

3146

Flags.setNoUnsignedWrap(nuw);

3147

SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,

3148

Flags);

3149

setValue(&I, Res);

3150

}

3151

3152

void SelectionDAGBuilder::visitSDiv(const User &I) {

3153

SDValue Op1 = getValue(I.getOperand(0));

3154

SDValue Op2 = getValue(I.getOperand(1));

3155

3156

SDNodeFlags Flags;

3157

Flags.setExact(isa<PossiblyExactOperator>(&I) &&

3158

cast<PossiblyExactOperator>(&I)->isExact());

3159

setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,

3160

Op2, Flags));

3161

}

3162

3163

void SelectionDAGBuilder::visitICmp(const User &I) {

3164

ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;

3165

if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))

3166

predicate = IC->getPredicate();

3167

else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))

3168

predicate = ICmpInst::Predicate(IC->getPredicate());

3169

SDValue Op1 = getValue(I.getOperand(0));

3170

SDValue Op2 = getValue(I.getOperand(1));

3171

ISD::CondCode Opcode = getICmpCondCode(predicate);

3172

3173

auto &TLI = DAG.getTargetLoweringInfo();

3174

EVT MemVT =

3175

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3176

3177

// If a pointer's DAG type is larger than its memory type then the DAG values

3178

// are zero-extended. This breaks signed comparisons so truncate back to the

3179

// underlying type before doing the compare.

3180

if (Op1.getValueType() != MemVT) {

3181

Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);

3182

Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);

3183

}

3184

3185

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3186

I.getType());

3187

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));

3188

}

3189

3190

void SelectionDAGBuilder::visitFCmp(const User &I) {

3191

FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;

3192

if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))

3193

predicate = FC->getPredicate();

3194

else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))

3195

predicate = FCmpInst::Predicate(FC->getPredicate());

3196

SDValue Op1 = getValue(I.getOperand(0));

3197

SDValue Op2 = getValue(I.getOperand(1));

3198

3199

ISD::CondCode Condition = getFCmpCondCode(predicate);

3200

auto *FPMO = cast<FPMathOperator>(&I);

3201

if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)

3202

Condition = getFCmpCodeWithoutNaN(Condition);

3203

3204

SDNodeFlags Flags;

3205

Flags.copyFMF(*FPMO);

3206

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

3207

3208

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3209

I.getType());

3210

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));

3211

}

3212

3213

// Check if the condition of the select has one use or two users that are both

3214

// selects with the same condition.

3215

static bool hasOnlySelectUsers(const Value *Cond) {

3216

return llvm::all_of(Cond->users(), [](const Value *V) {

3217

return isa<SelectInst>(V);

3218

});

3219

}

3220

3221

void SelectionDAGBuilder::visitSelect(const User &I) {

3222

SmallVector<EVT, 4> ValueVTs;

3223

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),

3224

ValueVTs);

3225

unsigned NumValues = ValueVTs.size();

3226

if (NumValues == 0) return;

3227

3228

SmallVector<SDValue, 4> Values(NumValues);

3229

SDValue Cond = getValue(I.getOperand(0));

3230

SDValue LHSVal = getValue(I.getOperand(1));

3231

SDValue RHSVal = getValue(I.getOperand(2));

3232

SmallVector<SDValue, 1> BaseOps(1, Cond);

3233

ISD::NodeType OpCode =

3234

Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

3235

3236

bool IsUnaryAbs = false;

3237

bool Negate = false;

3238

3239

SDNodeFlags Flags;

3240

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3241

Flags.copyFMF(*FPOp);

3242

3243

// Min/max matching is only viable if all output VTs are the same.

3244

if (is_splat(ValueVTs)) {

3245

EVT VT = ValueVTs[0];

3246

LLVMContext &Ctx = *DAG.getContext();

3247

auto &TLI = DAG.getTargetLoweringInfo();

3248

3249

// We care about the legality of the operation after it has been type

3250

// legalized.

3251

while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)

3252

VT = TLI.getTypeToTransformTo(Ctx, VT);

3253

3254

// If the vselect is legal, assume we want to leave this as a vector setcc +

3255

// vselect. Otherwise, if this is going to be scalarized, we want to see if

3256

// min/max is legal on the scalar type.

3257

bool UseScalarMinMax = VT.isVector() &&

3258

!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

3259

3260

Value *LHS, *RHS;

3261

auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);

3262

ISD::NodeType Opc = ISD::DELETED_NODE;

3263

switch (SPR.Flavor) {

3264

case SPF_UMAX: Opc = ISD::UMAX; break;

3265

case SPF_UMIN: Opc = ISD::UMIN; break;

3266

case SPF_SMAX: Opc = ISD::SMAX; break;

3267

case SPF_SMIN: Opc = ISD::SMIN; break;

3268

case SPF_FMINNUM:

3269

switch (SPR.NaNBehavior) {

3270

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3270);

3271

case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;

3272

case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;

3273

case SPNB_RETURNS_ANY: {

3274

if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))

3275

Opc = ISD::FMINNUM;

3276

else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))

3277

Opc = ISD::FMINIMUM;

3278

else if (UseScalarMinMax)

3279

Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?

3280

ISD::FMINNUM : ISD::FMINIMUM;

3281

break;

3282

}

3283

}

3284

break;

3285

case SPF_FMAXNUM:

3286

switch (SPR.NaNBehavior) {

3287

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3287);

3288

case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;

3289

case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;

3290

case SPNB_RETURNS_ANY:

3291

3292

if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))

3293

Opc = ISD::FMAXNUM;

3294

else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))

3295

Opc = ISD::FMAXIMUM;

3296

else if (UseScalarMinMax)

3297

Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?

3298

ISD::FMAXNUM : ISD::FMAXIMUM;

3299

break;

3300

}

3301

break;

3302

case SPF_NABS:

3303

Negate = true;

3304

LLVM_FALLTHROUGH[[gnu::fallthrough]];

3305

case SPF_ABS:

3306

IsUnaryAbs = true;

3307

Opc = ISD::ABS;

3308

break;

3309

default: break;

3310

}

3311

3312

if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&

3313

(TLI.isOperationLegalOrCustom(Opc, VT) ||

3314

(UseScalarMinMax &&

3315

TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&

3316

// If the underlying comparison instruction is used by any other

3317

// instruction, the consumed instructions won't be destroyed, so it is

3318

// not profitable to convert to a min/max.

3319

hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {

3320

OpCode = Opc;

3321

LHSVal = getValue(LHS);

3322

RHSVal = getValue(RHS);

3323

BaseOps.clear();

3324

}

3325

3326

if (IsUnaryAbs) {

3327

OpCode = Opc;

3328

LHSVal = getValue(LHS);

3329

BaseOps.clear();

3330

}

3331

}

3332

3333

if (IsUnaryAbs) {

3334

for (unsigned i = 0; i != NumValues; ++i) {

3335

SDLoc dl = getCurSDLoc();

3336

EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);

3337

Values[i] =

3338

DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));

3339

if (Negate)

3340

Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),

3341

Values[i]);

3342

}

3343

} else {

3344

for (unsigned i = 0; i != NumValues; ++i) {

3345

SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());

3346

Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3347

Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));

3348

Values[i] = DAG.getNode(

3349

OpCode, getCurSDLoc(),

3350

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);

3351

}

3352

}

3353

3354

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3355

DAG.getVTList(ValueVTs), Values));

3356

}

3357

3358

void SelectionDAGBuilder::visitTrunc(const User &I) {

3359

// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).

3360

SDValue N = getValue(I.getOperand(0));

3361

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3362

I.getType());

3363

setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));

3364

}

3365

3366

void SelectionDAGBuilder::visitZExt(const User &I) {

3367

// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3368

// ZExt also can't be a cast to bool for same reason. So, nothing much to do

3369

SDValue N = getValue(I.getOperand(0));

3370

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3371

I.getType());

3372

setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));

3373

}

3374

3375

void SelectionDAGBuilder::visitSExt(const User &I) {

3376

// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3377

// SExt also can't be a cast to bool for same reason. So, nothing much to do

3378

SDValue N = getValue(I.getOperand(0));

3379

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3380

I.getType());

3381

setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));

3382

}

3383

3384

void SelectionDAGBuilder::visitFPTrunc(const User &I) {

3385

// FPTrunc is never a no-op cast, no need to check

3386

SDValue N = getValue(I.getOperand(0));

3387

SDLoc dl = getCurSDLoc();

3388

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3389

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3390

setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,

3391

DAG.getTargetConstant(

3392

0, dl, TLI.getPointerTy(DAG.getDataLayout()))));

3393

}

3394

3395

void SelectionDAGBuilder::visitFPExt(const User &I) {

3396

// FPExt is never a no-op cast, no need to check

3397

SDValue N = getValue(I.getOperand(0));

3398

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3399

I.getType());

3400

setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));

3401

}

3402

3403

void SelectionDAGBuilder::visitFPToUI(const User &I) {

3404

// FPToUI is never a no-op cast, no need to check

3405

SDValue N = getValue(I.getOperand(0));

3406

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3407

I.getType());

3408

setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));

3409

}

3410

3411

void SelectionDAGBuilder::visitFPToSI(const User &I) {

3412

// FPToSI is never a no-op cast, no need to check

3413

SDValue N = getValue(I.getOperand(0));

3414

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3415

I.getType());

3416

setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));

3417

}

3418

3419

void SelectionDAGBuilder::visitUIToFP(const User &I) {

3420

// UIToFP is never a no-op cast, no need to check

3421

SDValue N = getValue(I.getOperand(0));

3422

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3423

I.getType());

3424

setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));

3425

}

3426

3427

void SelectionDAGBuilder::visitSIToFP(const User &I) {

3428

// SIToFP is never a no-op cast, no need to check

3429

SDValue N = getValue(I.getOperand(0));

3430

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3431

I.getType());

3432

setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));

3433

}

3434

3435

void SelectionDAGBuilder::visitPtrToInt(const User &I) {

3436

// What to do depends on the size of the integer and the size of the pointer.

3437

// We can either truncate, zero extend, or no-op, accordingly.

3438

SDValue N = getValue(I.getOperand(0));

3439

auto &TLI = DAG.getTargetLoweringInfo();

3440

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3441

I.getType());

3442

EVT PtrMemVT =

3443

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3444

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3445

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);

3446

setValue(&I, N);

3447

}

3448

3449

void SelectionDAGBuilder::visitIntToPtr(const User &I) {

3450

// What to do depends on the size of the integer and the size of the pointer.

3451

// We can either truncate, zero extend, or no-op, accordingly.

3452

SDValue N = getValue(I.getOperand(0));

3453

auto &TLI = DAG.getTargetLoweringInfo();

3454

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3455

EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

3456

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3457

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);

3458

setValue(&I, N);

3459

}

3460

3461

void SelectionDAGBuilder::visitBitCast(const User &I) {

3462

SDValue N = getValue(I.getOperand(0));

3463

SDLoc dl = getCurSDLoc();

3464

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3465

I.getType());

3466

3467

// BitCast assures us that source and destination are the same size so this is

3468

// either a BITCAST or a no-op.

3469

if (DestVT != N.getValueType())

3470

setValue(&I, DAG.getNode(ISD::BITCAST, dl,

3471

DestVT, N)); // convert types.

3472

// Check if the original LLVM IR Operand was a ConstantInt, because getValue()

3473

// might fold any kind of constant expression to an integer constant and that

3474

// is not what we are looking for. Only recognize a bitcast of a genuine

3475

// constant integer as an opaque constant.

3476

else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))

3477

setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,

3478

/*isOpaque*/true));

3479

else

3480

setValue(&I, N); // noop cast.

3481

}

3482

3483

void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {

3484

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3485

const Value *SV = I.getOperand(0);

3486

SDValue N = getValue(SV);

3487

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3488

3489

unsigned SrcAS = SV->getType()->getPointerAddressSpace();

3490

unsigned DestAS = I.getType()->getPointerAddressSpace();

3491

3492

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))

3493

N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

3494

3495

setValue(&I, N);

3496

}

3497

3498

void SelectionDAGBuilder::visitInsertElement(const User &I) {

3499

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3500

SDValue InVec = getValue(I.getOperand(0));

3501

SDValue InVal = getValue(I.getOperand(1));

3502

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),

3503

TLI.getVectorIdxTy(DAG.getDataLayout()));

3504

setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),

3505

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3506

InVec, InVal, InIdx));

3507

}

3508

3509

void SelectionDAGBuilder::visitExtractElement(const User &I) {

3510

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3511

SDValue InVec = getValue(I.getOperand(0));

3512

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),

3513

TLI.getVectorIdxTy(DAG.getDataLayout()));

3514

setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),

3515

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3516

InVec, InIdx));

3517

}

3518

3519

void SelectionDAGBuilder::visitShuffleVector(const User &I) {

3520

SDValue Src1 = getValue(I.getOperand(0));

3521

SDValue Src2 = getValue(I.getOperand(1));

3522

ArrayRef<int> Mask;

3523

if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))

3524

Mask = SVI->getShuffleMask();

3525

else

3526

Mask = cast<ConstantExpr>(I).getShuffleMask();

3527

SDLoc DL = getCurSDLoc();

3528

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3529

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3530

EVT SrcVT = Src1.getValueType();

3531

3532

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&

3533

VT.isScalableVector()) {

3534

// Canonical splat form of first element of first input vector.

3535

SDValue FirstElt =

3536

DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,

3537

DAG.getVectorIdxConstant(0, DL));

3538

setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));

3539

return;

3540

}

3541

3542

// For now, we only handle splats for scalable vectors.

3543

// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation

3544

// for targets that support a SPLAT_VECTOR for non-scalable vector types.

3545

assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")((!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? static_cast<void> (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3545, __PRETTY_FUNCTION__));

3546

3547

unsigned SrcNumElts = SrcVT.getVectorNumElements();

3548

unsigned MaskNumElts = Mask.size();

3549

3550

if (SrcNumElts == MaskNumElts) {

3551

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));

3552

return;

3553

}

3554

3555

// Normalize the shuffle vector since mask and vector length don't match.

3556

if (SrcNumElts < MaskNumElts) {

3557

// Mask is longer than the source vectors. We can use concatenate vector to

3558

// make the mask and vectors lengths match.

3559

3560

if (MaskNumElts % SrcNumElts == 0) {

3561

// Mask length is a multiple of the source vector length.

3562

// Check if the shuffle is some kind of concatenation of the input

3563

// vectors.

3564

unsigned NumConcat = MaskNumElts / SrcNumElts;

3565

bool IsConcat = true;

3566

SmallVector<int, 8> ConcatSrcs(NumConcat, -1);

3567

for (unsigned i = 0; i != MaskNumElts; ++i) {

3568

int Idx = Mask[i];

3569

if (Idx < 0)

3570

continue;

3571

// Ensure the indices in each SrcVT sized piece are sequential and that

3572

// the same source is used for the whole piece.

3573

if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

3574

(ConcatSrcs[i / SrcNumElts] >= 0 &&

3575

ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {

3576

IsConcat = false;

3577

break;

3578

}

3579

// Remember which source this index came from.

3580

ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

3581

}

3582

3583

// The shuffle is concatenating multiple vectors together. Just emit

3584

// a CONCAT_VECTORS operation.

3585

if (IsConcat) {

3586

SmallVector<SDValue, 8> ConcatOps;

3587

for (auto Src : ConcatSrcs) {

3588

if (Src < 0)

3589

ConcatOps.push_back(DAG.getUNDEF(SrcVT));

3590

else if (Src == 0)

3591

ConcatOps.push_back(Src1);

3592

else

3593

ConcatOps.push_back(Src2);

3594

}

3595

setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));

3596

return;

3597

}

3598

}

3599

3600

unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);

3601

unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;

3602

EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),

3603

PaddedMaskNumElts);

3604

3605

// Pad both vectors with undefs to make them the same length as the mask.

3606

SDValue UndefVal = DAG.getUNDEF(SrcVT);

3607

3608

SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);

3609

SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);

3610

MOps1[0] = Src1;

3611

MOps2[0] = Src2;

3612

3613

Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);

3614

Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

3615

3616

// Readjust mask for new input vector length.

3617

SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);

3618

for (unsigned i = 0; i != MaskNumElts; ++i) {

3619

int Idx = Mask[i];

3620

if (Idx >= (int)SrcNumElts)

3621

Idx -= SrcNumElts - PaddedMaskNumElts;

3622

MappedOps[i] = Idx;

3623

}

3624

3625

SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

3626

3627

// If the concatenated vector was padded, extract a subvector with the

3628

// correct number of elements.

3629

if (MaskNumElts != PaddedMaskNumElts)

3630

Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,

3631

DAG.getVectorIdxConstant(0, DL));

3632

3633

setValue(&I, Result);

3634

return;

3635

}

3636

3637

if (SrcNumElts > MaskNumElts) {

3638

// Analyze the access pattern of the vector to see if we can extract

3639

// two subvectors and do the shuffle.

3640

int StartIdx[2] = { -1, -1 }; // StartIdx to extract from

3641

bool CanExtract = true;

3642

for (int Idx : Mask) {

3643

unsigned Input = 0;

3644

if (Idx < 0)

3645

continue;

3646

3647

if (Idx >= (int)SrcNumElts) {

3648

Input = 1;

3649

Idx -= SrcNumElts;

3650

}

3651

3652

// If all the indices come from the same MaskNumElts sized portion of

3653

// the sources we can use extract. Also make sure the extract wouldn't

3654

// extract past the end of the source.

3655

int NewStartIdx = alignDown(Idx, MaskNumElts);

3656

if (NewStartIdx + MaskNumElts > SrcNumElts ||

3657

(StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))

3658

CanExtract = false;

3659

// Make sure we always update StartIdx as we use it to track if all

3660

// elements are undef.

3661

StartIdx[Input] = NewStartIdx;

3662

}

3663

3664

if (StartIdx[0] < 0 && StartIdx[1] < 0) {

3665

setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.

3666

return;

3667

}

3668

if (CanExtract) {

3669

// Extract appropriate subvector and generate a vector shuffle

3670

for (unsigned Input = 0; Input < 2; ++Input) {

3671

SDValue &Src = Input == 0 ? Src1 : Src2;

3672

if (StartIdx[Input] < 0)

3673

Src = DAG.getUNDEF(VT);

3674

else {

3675

Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,

3676

DAG.getVectorIdxConstant(StartIdx[Input], DL));

3677

}

3678

}

3679

3680

// Calculate new mask.

3681

SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());

3682

for (int &Idx : MappedOps) {

3683

if (Idx >= (int)SrcNumElts)

3684

Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;

3685

else if (Idx >= 0)

3686

Idx -= StartIdx[0];

3687

}

3688

3689

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));

3690

return;

3691

}

3692

}

3693

3694

// We can't use either concat vectors or extract subvectors so fall back to

3695

// replacing the shuffle with extract and build vector.

3696

// to insert and build vector.

3697

EVT EltVT = VT.getVectorElementType();

3698

SmallVector<SDValue,8> Ops;

3699

for (int Idx : Mask) {

3700

SDValue Res;

3701

3702

if (Idx < 0) {

3703

Res = DAG.getUNDEF(EltVT);

3704

} else {

3705

SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;

3706

if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

3707

3708

Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,

3709

DAG.getVectorIdxConstant(Idx, DL));

3710

}

3711

3712

Ops.push_back(Res);

3713

}

3714

3715

setValue(&I, DAG.getBuildVector(VT, DL, Ops));

3716

}

3717

3718

void SelectionDAGBuilder::visitInsertValue(const User &I) {

3719

ArrayRef<unsigned> Indices;

3720

if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))

3721

Indices = IV->getIndices();

3722

else

3723

Indices = cast<ConstantExpr>(&I)->getIndices();

3724

3725

const Value *Op0 = I.getOperand(0);

3726

const Value *Op1 = I.getOperand(1);

3727

Type *AggTy = I.getType();

3728

Type *ValTy = Op1->getType();

3729

bool IntoUndef = isa<UndefValue>(Op0);

3730

bool FromUndef = isa<UndefValue>(Op1);

3731

3732

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3733

3734

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3735

SmallVector<EVT, 4> AggValueVTs;

3736

ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);

3737

SmallVector<EVT, 4> ValValueVTs;

3738

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3739

3740

unsigned NumAggValues = AggValueVTs.size();

3741

unsigned NumValValues = ValValueVTs.size();

3742

SmallVector<SDValue, 4> Values(NumAggValues);

3743

3744

// Ignore an insertvalue that produces an empty object

3745

if (!NumAggValues) {

3746

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3747

return;

3748

}

3749

3750

SDValue Agg = getValue(Op0);

3751

unsigned i = 0;

3752

// Copy the beginning value(s) from the original aggregate.

3753

for (; i != LinearIndex; ++i)

3754

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3755

SDValue(Agg.getNode(), Agg.getResNo() + i);

3756

// Copy values from the inserted value(s).

3757

if (NumValValues) {

3758

SDValue Val = getValue(Op1);

3759

for (; i != LinearIndex + NumValValues; ++i)

3760

Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3761

SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);

3762

}

3763

// Copy remaining value(s) from the original aggregate.

3764

for (; i != NumAggValues; ++i)

3765

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3766

SDValue(Agg.getNode(), Agg.getResNo() + i);

3767

3768

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3769

DAG.getVTList(AggValueVTs), Values));

3770

}

3771

3772

void SelectionDAGBuilder::visitExtractValue(const User &I) {

3773

ArrayRef<unsigned> Indices;

3774

if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))

3775

Indices = EV->getIndices();

3776

else

3777

Indices = cast<ConstantExpr>(&I)->getIndices();

3778

3779

const Value *Op0 = I.getOperand(0);

3780

Type *AggTy = Op0->getType();

3781

Type *ValTy = I.getType();

3782

bool OutOfUndef = isa<UndefValue>(Op0);

3783

3784

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3785

3786

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3787

SmallVector<EVT, 4> ValValueVTs;

3788

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3789

3790

unsigned NumValValues = ValValueVTs.size();

3791

3792

// Ignore a extractvalue that produces an empty object

3793

if (!NumValValues) {

3794

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3795

return;

3796

}

3797

3798

SmallVector<SDValue, 4> Values(NumValValues);

3799

3800

SDValue Agg = getValue(Op0);

3801

// Copy out the selected value(s).

3802

for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)

3803

Values[i - LinearIndex] =

3804

OutOfUndef ?

3805

DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :

3806

SDValue(Agg.getNode(), Agg.getResNo() + i);

3807

3808

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3809

DAG.getVTList(ValValueVTs), Values));

3810

}

3811

3812

void SelectionDAGBuilder::visitGetElementPtr(const User &I) {

3813

Value *Op0 = I.getOperand(0);

3814

// Note that the pointer operand may be a vector of pointers. Take the scalar

3815

// element which holds a pointer.

3816

unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();

3817

SDValue N = getValue(Op0);

3818

SDLoc dl = getCurSDLoc();

3819

auto &TLI = DAG.getTargetLoweringInfo();

3820

3821

// Normalize Vector GEP - all scalar operands should be converted to the

3822

// splat vector.

3823

bool IsVectorGEP = I.getType()->isVectorTy();

3824

ElementCount VectorElementCount =

3825

IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()

3826

: ElementCount::getFixed(0);

3827

3828

if (IsVectorGEP && !N.getValueType().isVector()) {

3829

LLVMContext &Context = *DAG.getContext();

3830

EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);

3831

if (VectorElementCount.isScalable())

3832

N = DAG.getSplatVector(VT, dl, N);

3833

else

3834

N = DAG.getSplatBuildVector(VT, dl, N);

3835

}

3836

3837

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);

3838

GTI != E; ++GTI) {

3839

const Value *Idx = GTI.getOperand();

3840

if (StructType *StTy = GTI.getStructTypeOrNull()) {

3841

unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();

3842

if (Field) {

3843

// N = N + Offset

3844

uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);

3845

3846

// In an inbounds GEP with an offset that is nonnegative even when

3847

// interpreted as signed, assume there is no unsigned overflow.

3848

SDNodeFlags Flags;

3849

if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())

3850

Flags.setNoUnsignedWrap(true);

3851

3852

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,

3853

DAG.getConstant(Offset, dl, N.getValueType()), Flags);

3854

}

3855

} else {

3856

// IdxSize is the width of the arithmetic according to IR semantics.

3857

// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth

3858

// (and fix up the result later).

3859

unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);

3860

MVT IdxTy = MVT::getIntegerVT(IdxSize);

3861

TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());

3862

// We intentionally mask away the high bits here; ElementSize may not

3863

// fit in IdxTy.

3864

APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());

3865

bool ElementScalable = ElementSize.isScalable();

3866

3867

// If this is a scalar constant or a splat vector of constants,

3868

// handle it quickly.

3869

const auto *C = dyn_cast<Constant>(Idx);

3870

if (C && isa<VectorType>(C->getType()))

3871

C = C->getSplatValue();

3872

3873

const auto *CI = dyn_cast_or_null<ConstantInt>(C);

3874

if (CI && CI->isZero())

3875

continue;

3876

if (CI && !ElementScalable) {

3877

APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);

3878

LLVMContext &Context = *DAG.getContext();

3879

SDValue OffsVal;

3880

if (IsVectorGEP)

3881

OffsVal = DAG.getConstant(

3882

Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));

3883

else

3884

OffsVal = DAG.getConstant(Offs, dl, IdxTy);

3885

3886

// In an inbounds GEP with an offset that is nonnegative even when

3887

// interpreted as signed, assume there is no unsigned overflow.

3888

SDNodeFlags Flags;

3889

if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())

3890

Flags.setNoUnsignedWrap(true);

3891

3892

OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

3893

3894

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);

3895

continue;

3896

}

3897

3898

// N = N + Idx * ElementMul;

3899

SDValue IdxN = getValue(Idx);

3900

3901

if (!IdxN.getValueType().isVector() && IsVectorGEP) {

3902

EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),

3903

VectorElementCount);

3904

if (VectorElementCount.isScalable())

3905

IdxN = DAG.getSplatVector(VT, dl, IdxN);

3906

else

3907

IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);

3908

}

3909

3910

// If the index is smaller or larger than intptr_t, truncate or extend

3911

// it.

3912

IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

3913

3914

if (ElementScalable) {

3915

EVT VScaleTy = N.getValueType().getScalarType();

3916

SDValue VScale = DAG.getNode(

3917

ISD::VSCALE, dl, VScaleTy,

3918

DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));

3919

if (IsVectorGEP)

3920

VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);

3921

IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);

3922

} else {

3923

// If this is a multiply by a power of two, turn it into a shl

3924

// immediately. This is a very common case.

3925

if (ElementMul != 1) {

3926

if (ElementMul.isPowerOf2()) {

3927

unsigned Amt = ElementMul.logBase2();

3928

IdxN = DAG.getNode(ISD::SHL, dl,

3929

N.getValueType(), IdxN,

3930

DAG.getConstant(Amt, dl, IdxN.getValueType()));

3931

} else {

3932

SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,

3933

IdxN.getValueType());

3934

IdxN = DAG.getNode(ISD::MUL, dl,

3935

N.getValueType(), IdxN, Scale);

3936

}

3937

}

3938

}

3939

3940

N = DAG.getNode(ISD::ADD, dl,

3941

N.getValueType(), N, IdxN);

3942

}

3943

}

3944

3945

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);

3946

MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);

3947

if (IsVectorGEP) {

3948

PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);

3949

PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);

3950

}

3951

3952

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())

3953

N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

3954

3955

setValue(&I, N);

3956

}

3957

3958

void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {

3959

// If this is a fixed sized alloca in the entry block of the function,

3960

// allocate it statically on the stack.

3961

if (FuncInfo.StaticAllocaMap.count(&I))

3962

return; // getValue will auto-populate this.

3963

3964

SDLoc dl = getCurSDLoc();

3965

Type *Ty = I.getAllocatedType();

3966

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3967

auto &DL = DAG.getDataLayout();

3968

uint64_t TySize = DL.getTypeAllocSize(Ty);

3969

MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

3970

3971

SDValue AllocSize = getValue(I.getArraySize());

3972

3973

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());

3974

if (AllocSize.getValueType() != IntPtr)

3975

AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

3976

3977

AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,

3978

AllocSize,

3979

DAG.getConstant(TySize, dl, IntPtr));

3980

3981

// Handle alignment. If the requested alignment is less than or equal to

3982

// the stack alignment, ignore it. If the size is greater than or equal to

3983

// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.

3984

Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();

3985

if (*Alignment <= StackAlign)

3986

Alignment = None;

3987

3988

const uint64_t StackAlignMask = StackAlign.value() - 1U;

3989

// Round the size of the allocation up to the stack alignment size

3990

// by add SA-1 to the size. This doesn't overflow because we're computing

3991

// an address inside an alloca.

3992

SDNodeFlags Flags;

3993

Flags.setNoUnsignedWrap(true);

3994

AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,

3995

DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

3996

3997

// Mask out the low bits for alignment purposes.

3998

AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,

3999

DAG.getConstant(~StackAlignMask, dl, IntPtr));

4000

4001

SDValue Ops[] = {

4002

getRoot(), AllocSize,

4003

DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};

4004

SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);

4005

SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);

4006

setValue(&I, DSA);

4007

DAG.setRoot(DSA.getValue(1));

4008

4009

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())((FuncInfo.MF->getFrameInfo().hasVarSizedObjects()) ? static_cast
<void> (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4009, __PRETTY_FUNCTION__));

4010

}

4011

4012

void SelectionDAGBuilder::visitLoad(const LoadInst &I) {

4013

if (I.isAtomic())

4014

return visitAtomicLoad(I);

4015

4016

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4017

const Value *SV = I.getOperand(0);

4018

if (TLI.supportSwiftError()) {

4019

// Swifterror values can come from either a function parameter with

4020

// swifterror attribute or an alloca with swifterror attribute.

4021

if (const Argument *Arg = dyn_cast<Argument>(SV)) {

4022

if (Arg->hasSwiftErrorAttr())

4023

return visitLoadFromSwiftError(I);

4024

}

4025

4026

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {

4027

if (Alloca->isSwiftError())

4028

return visitLoadFromSwiftError(I);

4029

}

4030

}

4031

4032

SDValue Ptr = getValue(SV);

4033

4034

Type *Ty = I.getType();

4035

Align Alignment = I.getAlign();

4036

4037

AAMDNodes AAInfo;

4038

I.getAAMetadata(AAInfo);

4039

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4040

4041

SmallVector<EVT, 4> ValueVTs, MemVTs;

4042

SmallVector<uint64_t, 4> Offsets;

4043

ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);

4044

unsigned NumValues = ValueVTs.size();

4045

if (NumValues == 0)

4046

return;

4047

4048

bool isVolatile = I.isVolatile();

4049

4050

SDValue Root;

4051

bool ConstantMemory = false;

4052

if (isVolatile)

4053

// Serialize volatile loads with other side effects.

4054

Root = getRoot();

4055

else if (NumValues > MaxParallelChains)

4056

Root = getMemoryRoot();

4057

else if (AA &&

4058

AA->pointsToConstantMemory(MemoryLocation(

4059

SV,

4060

LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),

4061

AAInfo))) {

4062

// Do not serialize (non-volatile) loads of constant memory with anything.

4063

Root = DAG.getEntryNode();

4064

ConstantMemory = true;

4065

} else {

4066

// Do not serialize non-volatile loads against each other.

4067

Root = DAG.getRoot();

4068

}

4069

4070

SDLoc dl = getCurSDLoc();

4071

4072

if (isVolatile)

4073

Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

4074

4075

// An aggregate load cannot wrap around the address space, so offsets to its

4076

// parts don't wrap either.

4077

SDNodeFlags Flags;

4078

Flags.setNoUnsignedWrap(true);

4079

4080

SmallVector<SDValue, 4> Values(NumValues);

4081

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4082

EVT PtrVT = Ptr.getValueType();

4083

4084

MachineMemOperand::Flags MMOFlags

4085

= TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4086

4087

unsigned ChainI = 0;

4088

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4089

// Serializing loads here may result in excessive register pressure, and

4090

// TokenFactor places arbitrary choke points on the scheduler. SD scheduling

4091

// could recover a bit by hoisting nodes upward in the chain by recognizing

4092

// they are side-effect free or do not alias. The optimizer should really

4093

// avoid this case by converting large object/array copies to llvm.memcpy

4094

// (MaxParallelChains should always remain as failsafe).

4095

if (ChainI == MaxParallelChains) {

4096

assert(PendingLoads.empty() && "PendingLoads must be serialized first")((PendingLoads.empty() && "PendingLoads must be serialized first"
) ? static_cast<void> (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4096, __PRETTY_FUNCTION__));

4097

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4098

makeArrayRef(Chains.data(), ChainI));

4099

Root = Chain;

4100

ChainI = 0;

4101

}

4102

SDValue A = DAG.getNode(ISD::ADD, dl,

4103

PtrVT, Ptr,

4104

DAG.getConstant(Offsets[i], dl, PtrVT),

4105

Flags);

4106

4107

SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,

4108

MachinePointerInfo(SV, Offsets[i]), Alignment,

4109

MMOFlags, AAInfo, Ranges);

4110

Chains[ChainI] = L.getValue(1);

4111

4112

if (MemVTs[i] != ValueVTs[i])

4113

L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

4114

4115

Values[i] = L;

4116

}

4117

4118

if (!ConstantMemory) {

4119

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4120

makeArrayRef(Chains.data(), ChainI));

4121

if (isVolatile)

4122

DAG.setRoot(Chain);

4123

else

4124

PendingLoads.push_back(Chain);

4125

}

4126

4127

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,

4128

DAG.getVTList(ValueVTs), Values));

4129

}

4130

4131

void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {

4132

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4133, __PRETTY_FUNCTION__))

4133

"call visitStoreToSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4133, __PRETTY_FUNCTION__));

4134

4135

SmallVector<EVT, 4> ValueVTs;

4136

SmallVector<uint64_t, 4> Offsets;

4137

const Value *SrcV = I.getOperand(0);

4138

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4139

SrcV->getType(), ValueVTs, &Offsets);

4140

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4141, __PRETTY_FUNCTION__))

4141

"expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4141, __PRETTY_FUNCTION__));

4142

4143

SDValue Src = getValue(SrcV);

4144

// Create a virtual register, then update the virtual register.

4145

Register VReg =

4146

SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());

4147

// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue

4148

// Chain can be getRoot or getControlRoot.

4149

SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,

4150

SDValue(Src.getNode(), Src.getResNo()));

4151

DAG.setRoot(CopyNode);

4152

}

4153

4154

void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {

4155

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4156, __PRETTY_FUNCTION__))

4156

"call visitLoadFromSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4156, __PRETTY_FUNCTION__));

4157

4158

assert(!I.isVolatile() &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4161, __PRETTY_FUNCTION__))

4159

!I.hasMetadata(LLVMContext::MD_nontemporal) &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4161, __PRETTY_FUNCTION__))

4160

!I.hasMetadata(LLVMContext::MD_invariant_load) &&((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4161, __PRETTY_FUNCTION__))

4161

"Support volatile, non temporal, invariant for load_from_swift_error")((!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal
) && !I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4161, __PRETTY_FUNCTION__));

4162

4163

const Value *SV = I.getOperand(0);

4164

Type *Ty = I.getType();

4165

AAMDNodes AAInfo;

4166

I.getAAMetadata(AAInfo);

4167

assert((((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__))

4168

(!AA ||(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__))

4169

!AA->pointsToConstantMemory(MemoryLocation((((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__))

4170

SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__))

4171

AAInfo))) &&(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__))

4172

"load_from_swift_error should not be constant memory")(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize
::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))
) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4172, __PRETTY_FUNCTION__));

4173

4174

SmallVector<EVT, 4> ValueVTs;

4175

SmallVector<uint64_t, 4> Offsets;

4176

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,

4177

ValueVTs, &Offsets);

4178

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4179, __PRETTY_FUNCTION__))

4179

"expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4179, __PRETTY_FUNCTION__));

4180

4181

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT

4182

SDValue L = DAG.getCopyFromReg(

4183

getRoot(), getCurSDLoc(),

4184

SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

4185

4186

setValue(&I, L);

4187

}

4188

4189

void SelectionDAGBuilder::visitStore(const StoreInst &I) {

4190

if (I.isAtomic())

4191

return visitAtomicStore(I);

4192

4193

const Value *SrcV = I.getOperand(0);

4194

const Value *PtrV = I.getOperand(1);

4195

4196

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4197

if (TLI.supportSwiftError()) {

4198

// Swifterror values can come from either a function parameter with

4199

// swifterror attribute or an alloca with swifterror attribute.

4200

if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {

4201

if (Arg->hasSwiftErrorAttr())

4202

return visitStoreToSwiftError(I);

4203

}

4204

4205

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {

4206

if (Alloca->isSwiftError())

4207

return visitStoreToSwiftError(I);

4208

}

4209

}

4210

4211

SmallVector<EVT, 4> ValueVTs, MemVTs;

4212

SmallVector<uint64_t, 4> Offsets;

4213

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4214

SrcV->getType(), ValueVTs, &MemVTs, &Offsets);

4215

unsigned NumValues = ValueVTs.size();

4216

if (NumValues == 0)

4217

return;

4218

4219

// Get the lowered operands. Note that we do this after

4220

// checking if NumResults is zero, because with zero results

4221

// the operands won't have values in the map.

4222

SDValue Src = getValue(SrcV);

4223

SDValue Ptr = getValue(PtrV);

4224

4225

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();

4226

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4227

SDLoc dl = getCurSDLoc();

4228

Align Alignment = I.getAlign();

4229

AAMDNodes AAInfo;

4230

I.getAAMetadata(AAInfo);

4231

4232

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4233

4234

// An aggregate load cannot wrap around the address space, so offsets to its

4235

// parts don't wrap either.

4236

SDNodeFlags Flags;

4237

Flags.setNoUnsignedWrap(true);

4238

4239

unsigned ChainI = 0;

4240

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4241

// See visitLoad comments.

4242

if (ChainI == MaxParallelChains) {

4243

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4244

makeArrayRef(Chains.data(), ChainI));

4245

Root = Chain;

4246

ChainI = 0;

4247

}

4248

SDValue Add =

4249

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);

4250

SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);

4251

if (MemVTs[i] != ValueVTs[i])

4252

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);

4253

SDValue St =

4254

DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),

4255

Alignment, MMOFlags, AAInfo);

4256

Chains[ChainI] = St;

4257

}

4258

4259

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4260

makeArrayRef(Chains.data(), ChainI));

4261

DAG.setRoot(StoreNode);

4262

}

4263

4264

void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,

4265

bool IsCompressing) {

4266

SDLoc sdl = getCurSDLoc();

4267

4268

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4269

MaybeAlign &Alignment) {

4270

// llvm.masked.store.*(Src0, Ptr, alignment, Mask)

4271

Src0 = I.getArgOperand(0);

4272

Ptr = I.getArgOperand(1);

4273

Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();

4274

Mask = I.getArgOperand(3);

4275

};

4276

auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4277

MaybeAlign &Alignment) {

4278

// llvm.masked.compressstore.*(Src0, Ptr, Mask)

4279

Src0 = I.getArgOperand(0);

4280

Ptr = I.getArgOperand(1);

4281

Mask = I.getArgOperand(2);

4282

Alignment = None;

4283

};

4284

4285

Value *PtrOperand, *MaskOperand, *Src0Operand;

4286

MaybeAlign Alignment;

4287

if (IsCompressing)

4288

getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4289

else

4290

getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4291

4292

SDValue Ptr = getValue(PtrOperand);

4293

SDValue Src0 = getValue(Src0Operand);

4294

SDValue Mask = getValue(MaskOperand);

4295

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4296

4297

EVT VT = Src0.getValueType();

4298

if (!Alignment)

4299

Alignment = DAG.getEVTAlign(VT);

4300

4301

AAMDNodes AAInfo;

4302

I.getAAMetadata(AAInfo);

4303

4304

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4305

MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,

4306

// TODO: Make MachineMemOperands aware of scalable

4307

// vectors.

4308

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);

4309

SDValue StoreNode =

4310

DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,

4311

ISD::UNINDEXED, false /* Truncating */, IsCompressing);

4312

DAG.setRoot(StoreNode);

4313

setValue(&I, StoreNode);

4314

}

4315

4316

// Get a uniform base for the Gather/Scatter intrinsic.

4317

// The first argument of the Gather/Scatter intrinsic is a vector of pointers.

4318

// We try to represent it as a base pointer + vector of indices.

4319

// Usually, the vector of pointers comes from a 'getelementptr' instruction.

4320

// The first operand of the GEP may be a single pointer or a vector of pointers

4321

// Example:

4322

// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind

4323

// or

4324

// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind

4325

// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..

4326

//

4327

// When the first GEP operand is a single pointer - it is the uniform base we

4328

// are looking for. If first operand of the GEP is a splat vector - we

4329

// extract the splat value and use it as a uniform base.

4330

// In all other cases the function returns 'false'.

4331

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,

4332

ISD::MemIndexType &IndexType, SDValue &Scale,

4333

SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {

4334

SelectionDAG& DAG = SDB->DAG;

4335

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4336

const DataLayout &DL = DAG.getDataLayout();

4337

4338

assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")((Ptr->getType()->isVectorTy() && "Uexpected pointer type"
) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4338, __PRETTY_FUNCTION__));

4339

4340

// Handle splat constant pointer.

4341

if (auto *C = dyn_cast<Constant>(Ptr)) {

4342

C = C->getSplatValue();

4343

if (!C)

4344

return false;

4345

4346

Base = SDB->getValue(C);

4347

4348

unsigned NumElts = cast<FixedVectorType>(Ptr->getType())->getNumElements();

4349

EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);

4350

Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);

4351

IndexType = ISD::SIGNED_SCALED;

4352

Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4353

return true;

4354

}

4355

4356

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

4357

if (!GEP || GEP->getParent() != CurBB)

4358

return false;

4359

4360

if (GEP->getNumOperands() != 2)

4361

return false;

4362

4363

const Value *BasePtr = GEP->getPointerOperand();

4364

const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

4365

4366

// Make sure the base is scalar and the index is a vector.

4367

if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())

4368

return false;

4369

4370

Base = SDB->getValue(BasePtr);

4371

Index = SDB->getValue(IndexVal);

4372

IndexType = ISD::SIGNED_SCALED;

4373

Scale = DAG.getTargetConstant(

4374

DL.getTypeAllocSize(GEP->getResultElementType()),

4375

SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4376

return true;

4377

}

4378

4379

void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {

4380

SDLoc sdl = getCurSDLoc();

4381

4382

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)

4383

const Value *Ptr = I.getArgOperand(1);

4384

SDValue Src0 = getValue(I.getArgOperand(0));

4385

SDValue Mask = getValue(I.getArgOperand(3));

4386

EVT VT = Src0.getValueType();

4387

Align Alignment = cast<ConstantInt>(I.getArgOperand(2))

4388

->getMaybeAlignValue()

4389

.getValueOr(DAG.getEVTAlign(VT));

4390

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4391

4392

AAMDNodes AAInfo;

4393

I.getAAMetadata(AAInfo);

4394

4395

SDValue Base;

4396

SDValue Index;

4397

ISD::MemIndexType IndexType;

4398

SDValue Scale;

4399

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4400

I.getParent());

4401

4402

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4403

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4404

MachinePointerInfo(AS), MachineMemOperand::MOStore,

4405

// TODO: Make MachineMemOperands aware of scalable

4406

// vectors.

4407

MemoryLocation::UnknownSize, Alignment, AAInfo);

4408

if (!UniformBase) {

4409

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4410

Index = getValue(Ptr);

4411

IndexType = ISD::SIGNED_UNSCALED;

4412

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4413

}

4414

4415

EVT IdxVT = Index.getValueType();

4416

EVT EltTy = IdxVT.getVectorElementType();

4417

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4418

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4419

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4420

}

4421

4422

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };

4423

SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,

4424

Ops, MMO, IndexType, false);

4425

DAG.setRoot(Scatter);

4426

setValue(&I, Scatter);

4427

}

4428

4429

void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {

4430

SDLoc sdl = getCurSDLoc();

4431

4432

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4433

MaybeAlign &Alignment) {

4434

// @llvm.masked.load.*(Ptr, alignment, Mask, Src0)

4435

Ptr = I.getArgOperand(0);

4436

Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();

4437

Mask = I.getArgOperand(2);

4438

Src0 = I.getArgOperand(3);

4439

};

4440

auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4441

MaybeAlign &Alignment) {

4442

// @llvm.masked.expandload.*(Ptr, Mask, Src0)

4443

Ptr = I.getArgOperand(0);

4444

Alignment = None;

4445

Mask = I.getArgOperand(1);

4446

Src0 = I.getArgOperand(2);

4447

};

4448

4449

Value *PtrOperand, *MaskOperand, *Src0Operand;

4450

MaybeAlign Alignment;

4451

if (IsExpanding)

4452

getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4453

else

4454

getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4455

4456

SDValue Ptr = getValue(PtrOperand);

4457

SDValue Src0 = getValue(Src0Operand);

4458

SDValue Mask = getValue(MaskOperand);

4459

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4460

4461

EVT VT = Src0.getValueType();

4462

if (!Alignment)

4463

Alignment = DAG.getEVTAlign(VT);

4464

4465

AAMDNodes AAInfo;

4466

I.getAAMetadata(AAInfo);

4467

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4468

4469

// Do not serialize masked loads of constant memory with anything.

4470

MemoryLocation ML;

4471

if (VT.isScalableVector())

4472

ML = MemoryLocation::getAfter(PtrOperand);

4473

else

4474

ML = MemoryLocation(PtrOperand, LocationSize::precise(

4475

DAG.getDataLayout().getTypeStoreSize(I.getType())),

4476

AAInfo);

4477

bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

4478

4479

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

4480

4481

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4482

MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,

4483

// TODO: Make MachineMemOperands aware of scalable

4484

// vectors.

4485

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);

4486

4487

SDValue Load =

4488

DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,

4489

ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);

4490

if (AddToChain)

4491

PendingLoads.push_back(Load.getValue(1));

4492

setValue(&I, Load);

4493

}

4494

4495

void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {

4496

SDLoc sdl = getCurSDLoc();

4497

4498

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)

4499

const Value *Ptr = I.getArgOperand(0);

4500

SDValue Src0 = getValue(I.getArgOperand(3));

4501

SDValue Mask = getValue(I.getArgOperand(2));

4502

4503

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4504

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4505

Align Alignment = cast<ConstantInt>(I.getArgOperand(1))

4506

->getMaybeAlignValue()

4507

.getValueOr(DAG.getEVTAlign(VT));

4508

4509

AAMDNodes AAInfo;

4510

I.getAAMetadata(AAInfo);

4511

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4512

4513

SDValue Root = DAG.getRoot();

4514

SDValue Base;

4515

SDValue Index;

4516

ISD::MemIndexType IndexType;

4517

SDValue Scale;

4518

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4519

I.getParent());

4520

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4521

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4522

MachinePointerInfo(AS), MachineMemOperand::MOLoad,

4523

// TODO: Make MachineMemOperands aware of scalable

4524

// vectors.

4525

MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);

4526

4527

if (!UniformBase) {

4528

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4529

Index = getValue(Ptr);

4530

IndexType = ISD::SIGNED_UNSCALED;

4531

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4532

}

4533

4534

EVT IdxVT = Index.getValueType();

4535

EVT EltTy = IdxVT.getVectorElementType();

4536

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4537

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4538

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4539

}

4540

4541

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };

4542

SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,

4543

Ops, MMO, IndexType, ISD::NON_EXTLOAD);

4544

4545

PendingLoads.push_back(Gather.getValue(1));

4546

setValue(&I, Gather);

4547

}

4548

4549

void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {

4550

SDLoc dl = getCurSDLoc();

4551

AtomicOrdering SuccessOrdering = I.getSuccessOrdering();

4552

AtomicOrdering FailureOrdering = I.getFailureOrdering();

4553

SyncScope::ID SSID = I.getSyncScopeID();

4554

4555

SDValue InChain = getRoot();

4556

4557

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();

4558

SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

4559

4560

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4561

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4562

4563

MachineFunction &MF = DAG.getMachineFunction();

4564

MachineMemOperand *MMO = MF.getMachineMemOperand(

4565

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4566

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,

4567

FailureOrdering);

4568

4569

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,

4570

dl, MemVT, VTs, InChain,

4571

getValue(I.getPointerOperand()),

4572

getValue(I.getCompareOperand()),

4573

getValue(I.getNewValOperand()), MMO);

4574

4575

SDValue OutChain = L.getValue(2);

4576

4577

setValue(&I, L);

4578

DAG.setRoot(OutChain);

4579

}

4580

4581

void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {

4582

SDLoc dl = getCurSDLoc();

4583

ISD::NodeType NT;

4584

switch (I.getOperation()) {

4585

default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4585);

4586

case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;

4587

case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;

4588

case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;

4589

case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;

4590

case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;

4591

case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;

4592

case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;

4593

case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;

4594

case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;

4595

case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;

4596

case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;

4597

case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;

4598

case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;

4599

}

4600

AtomicOrdering Ordering = I.getOrdering();

4601

SyncScope::ID SSID = I.getSyncScopeID();

4602

4603

SDValue InChain = getRoot();

4604

4605

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();

4606

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4607

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4608

4609

MachineFunction &MF = DAG.getMachineFunction();

4610

MachineMemOperand *MMO = MF.getMachineMemOperand(

4611

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4612

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

4613

4614

SDValue L =

4615

DAG.getAtomic(NT, dl, MemVT, InChain,

4616

getValue(I.getPointerOperand()), getValue(I.getValOperand()),

4617

MMO);

4618

4619

SDValue OutChain = L.getValue(1);

4620

4621

setValue(&I, L);

4622

DAG.setRoot(OutChain);

4623

}

4624

4625

void SelectionDAGBuilder::visitFence(const FenceInst &I) {

4626

SDLoc dl = getCurSDLoc();

4627

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4628

SDValue Ops[3];

4629

Ops[0] = getRoot();

4630

Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,

4631

TLI.getFenceOperandTy(DAG.getDataLayout()));

4632

Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,

4633

TLI.getFenceOperandTy(DAG.getDataLayout()));

4634

DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));

4635

}

4636

4637

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {

4638

SDLoc dl = getCurSDLoc();

4639

AtomicOrdering Order = I.getOrdering();

4640

SyncScope::ID SSID = I.getSyncScopeID();

4641

4642

SDValue InChain = getRoot();

4643

4644

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4645

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4646

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

4647

4648

if (!TLI.supportsUnalignedAtomics() &&

4649

I.getAlignment() < MemVT.getSizeInBits() / 8)

4650

report_fatal_error("Cannot generate unaligned atomic load");

4651

4652

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4653

4654

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4655

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4656

I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

4657

4658

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

4659

4660

SDValue Ptr = getValue(I.getPointerOperand());

4661

4662

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {

4663

// TODO: Once this is better exercised by tests, it should be merged with

4664

// the normal path for loads to prevent future divergence.

4665

SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);

4666

if (MemVT != VT)

4667

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4668

4669

setValue(&I, L);

4670

SDValue OutChain = L.getValue(1);

4671

if (!I.isUnordered())

4672

DAG.setRoot(OutChain);

4673

else

4674

PendingLoads.push_back(OutChain);

4675

return;

4676

}

4677

4678

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,

4679

Ptr, MMO);

4680

4681

SDValue OutChain = L.getValue(1);

4682

if (MemVT != VT)

4683

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4684

4685

setValue(&I, L);

4686

DAG.setRoot(OutChain);

4687

}

4688

4689

void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {

4690

SDLoc dl = getCurSDLoc();

4691

4692

AtomicOrdering Ordering = I.getOrdering();

4693

SyncScope::ID SSID = I.getSyncScopeID();

4694

4695

SDValue InChain = getRoot();

4696

4697

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4698

EVT MemVT =

4699

TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

4700

4701

if (I.getAlignment() < MemVT.getSizeInBits() / 8)

4702

report_fatal_error("Cannot generate unaligned atomic store");

4703

4704

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4705

4706

MachineFunction &MF = DAG.getMachineFunction();

4707

MachineMemOperand *MMO = MF.getMachineMemOperand(

4708

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4709

I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

4710

4711

SDValue Val = getValue(I.getValueOperand());

4712

if (Val.getValueType() != MemVT)

4713

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);

4714

SDValue Ptr = getValue(I.getPointerOperand());

4715

4716

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {

4717

// TODO: Once this is better exercised by tests, it should be merged with

4718

// the normal path for stores to prevent future divergence.

4719

SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);

4720

DAG.setRoot(S);

4721

return;

4722

}

4723

SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,

4724

Ptr, Val, MMO);

4725

4726

4727

DAG.setRoot(OutChain);

4728

}

4729

4730

/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC

4731

/// node.

4732

void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,

4733

unsigned Intrinsic) {

4734

// Ignore the callsite's attributes. A specific call site may be marked with

4735

// readnone, but the lowering code will expect the chain based on the

4736

// definition.

4737

const Function *F = I.getCalledFunction();

4738

bool HasChain = !F->doesNotAccessMemory();

4739

bool OnlyLoad = HasChain && F->onlyReadsMemory();

4740

4741

// Build the operand list.

4742

SmallVector<SDValue, 8> Ops;

4743

if (HasChain) { // If this intrinsic has side-effects, chainify it.

4744

if (OnlyLoad) {

4745

// We don't need to serialize loads against other loads.

4746

Ops.push_back(DAG.getRoot());

4747

} else {

4748

Ops.push_back(getRoot());

4749

}

4750

}

4751

4752

// Info is set by getTgtMemInstrinsic

4753

TargetLowering::IntrinsicInfo Info;

4754

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4755

bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,

4756

DAG.getMachineFunction(),

4757

Intrinsic);

4758

4759

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.

4760

if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||

4761

Info.opc == ISD::INTRINSIC_W_CHAIN)

4762

Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),

4763

TLI.getPointerTy(DAG.getDataLayout())));

4764

4765

// Add all operands of the call to the operand list.

4766

for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {

4767

const Value *Arg = I.getArgOperand(i);

4768

if (!I.paramHasAttr(i, Attribute::ImmArg)) {

4769

Ops.push_back(getValue(Arg));

4770

continue;

4771

}

4772

4773

// Use TargetConstant instead of a regular constant for immarg.

4774

EVT VT = TLI.getValueType(*DL, Arg->getType(), true);

4775

if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {

4776

assert(CI->getBitWidth() <= 64 &&((CI->getBitWidth() <= 64 && "large intrinsic immediates not handled"
) ? static_cast<void> (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4777, __PRETTY_FUNCTION__))

4777

"large intrinsic immediates not handled")((CI->getBitWidth() <= 64 && "large intrinsic immediates not handled"
) ? static_cast<void> (0) : __assert_fail ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4777, __PRETTY_FUNCTION__));

4778

Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));

4779

} else {

4780

Ops.push_back(

4781

DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));

4782

}

4783

}

4784

4785

SmallVector<EVT, 4> ValueVTs;

4786

ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

4787

4788

if (HasChain)

4789

ValueVTs.push_back(MVT::Other);

4790

4791

SDVTList VTs = DAG.getVTList(ValueVTs);

4792

4793

// Create the node.

4794

SDValue Result;

4795

if (IsTgtIntrinsic) {

4796

// This is target intrinsic that touches memory

4797

AAMDNodes AAInfo;

4798

I.getAAMetadata(AAInfo);

4799

Result =

4800

DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,

4801

MachinePointerInfo(Info.ptrVal, Info.offset),

4802

Info.align, Info.flags, Info.size, AAInfo);

4803

} else if (!HasChain) {

4804

Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);

4805

} else if (!I.getType()->isVoidTy()) {

4806

Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);

4807

} else {

4808

Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);

4809

}

4810

4811

if (HasChain) {

4812

SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);

4813

if (OnlyLoad)

4814

PendingLoads.push_back(Chain);

4815

else

4816

DAG.setRoot(Chain);

4817

}

4818

4819

if (!I.getType()->isVoidTy()) {

4820

if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {

4821

EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);

4822

Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);

4823

} else

4824

Result = lowerRangeToAssertZExt(DAG, I, Result);

4825

4826

MaybeAlign Alignment = I.getRetAlign();

4827

if (!Alignment)

4828

Alignment = F->getAttributes().getRetAlignment();

4829

// Insert `assertalign` node if there's an alignment.

4830

if (InsertAssertAlign && Alignment) {

4831

Result =

4832

DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());

4833

}

4834

4835

setValue(&I, Result);

4836

}

4837

}

4838

4839

/// GetSignificand - Get the significand and build it into a floating-point

4840

/// number with exponent of 1:

4841

///

4842

/// Op = (Op & 0x007fffff) | 0x3f800000;

4843

///

4844

/// where Op is the hexadecimal representation of floating point value.

4845

static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {

4846

SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4847

DAG.getConstant(0x007fffff, dl, MVT::i32));

4848

SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,

4849

DAG.getConstant(0x3f800000, dl, MVT::i32));

4850

return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);

4851

}

4852

4853

/// GetExponent - Get the exponent:

4854

///

4855

/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);

4856

///

4857

/// where Op is the hexadecimal representation of floating point value.

4858

static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,

4859

const TargetLowering &TLI, const SDLoc &dl) {

4860

SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4861

DAG.getConstant(0x7f800000, dl, MVT::i32));

4862

SDValue t1 = DAG.getNode(

4863

ISD::SRL, dl, MVT::i32, t0,

4864

DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));

4865

SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,

4866

DAG.getConstant(127, dl, MVT::i32));

4867

return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);

4868

}

4869

4870

/// getF32Constant - Get 32-bit floating point constant.

4871

static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,

4872

const SDLoc &dl) {

4873

return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,

4874

MVT::f32);

4875

}

4876

4877

static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,

4878

SelectionDAG &DAG) {

4879

// TODO: What fast-math-flags should be set on the floating-point nodes?

4880

4881

// IntegerPartOfX = ((int32_t)(t0);

4882

SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

4883

4884

// FractionalPartOfX = t0 - (float)IntegerPartOfX;

4885

SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);

4886

SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

4887

4888

// IntegerPartOfX <<= 23;

4889

IntegerPartOfX = DAG.getNode(

4890

ISD::SHL, dl, MVT::i32, IntegerPartOfX,

4891

DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(

4892

DAG.getDataLayout())));

4893

4894

SDValue TwoToFractionalPartOfX;

4895

if (LimitFloatPrecision <= 6) {

4896

// For floating-point precision of 6:

4897

//

4898

// TwoToFractionalPartOfX =

4899

// 0.997535578f +

4900

// (0.735607626f + 0.252464424f * x) * x;

4901

//

4902

// error 0.0144103317, which is 6 bits

4903

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4904

getF32Constant(DAG, 0x3e814304, dl));

4905

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4906

getF32Constant(DAG, 0x3f3c50c8, dl));

4907

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4908

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4909

getF32Constant(DAG, 0x3f7f5e7e, dl));

4910

} else if (LimitFloatPrecision <= 12) {

4911

// For floating-point precision of 12:

4912

//

4913

// TwoToFractionalPartOfX =

4914

// 0.999892986f +

4915

// (0.696457318f +

4916

// (0.224338339f + 0.792043434e-1f * x) * x) * x;

4917

//

4918

// error 0.000107046256, which is 13 to 14 bits

4919

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4920

getF32Constant(DAG, 0x3da235e3, dl));

4921

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4922

getF32Constant(DAG, 0x3e65b8f3, dl));

4923

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4924

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4925

getF32Constant(DAG, 0x3f324b07, dl));

4926

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4927

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4928

getF32Constant(DAG, 0x3f7ff8fd, dl));

4929

} else { // LimitFloatPrecision <= 18

4930

// For floating-point precision of 18:

4931

//

4932

// TwoToFractionalPartOfX =

4933

// 0.999999982f +

4934

// (0.693148872f +

4935

// (0.240227044f +

4936

// (0.554906021e-1f +

4937

// (0.961591928e-2f +

4938

// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;

4939

// error 2.47208000*10^(-7), which is better than 18 bits

4940

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4941

getF32Constant(DAG, 0x3924b03e, dl));

4942

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4943

getF32Constant(DAG, 0x3ab24b87, dl));

4944

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4945

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4946

getF32Constant(DAG, 0x3c1d8c17, dl));

4947

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4948

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4949

getF32Constant(DAG, 0x3d634a1d, dl));

4950

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

4951

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

4952

getF32Constant(DAG, 0x3e75fe14, dl));

4953

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

4954

SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,

4955

getF32Constant(DAG, 0x3f317234, dl));

4956

SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);

4957

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,

4958

getF32Constant(DAG, 0x3f800000, dl));

4959

}

4960

4961

// Add the exponent into the result in integer domain.

4962

SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);

4963

return DAG.getNode(ISD::BITCAST, dl, MVT::f32,

4964

DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));

4965

}

4966

4967

/// expandExp - Lower an exp intrinsic. Handles the special sequences for

4968

/// limited-precision mode.

4969

static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4970

const TargetLowering &TLI, SDNodeFlags Flags) {

4971

if (Op.getValueType() == MVT::f32 &&

4972

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

4973

4974

// Put the exponent in the right bit position for later addition to the

4975

// final result:

4976

//

4977

// t0 = Op * log2(e)

4978

4979

// TODO: What fast-math-flags should be set here?

4980

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,

4981

DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));

4982

return getLimitedPrecisionExp2(t0, dl, DAG);

4983

}

4984

4985

// No special expansion.

4986

return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);

4987

}

4988

4989

/// expandLog - Lower a log intrinsic. Handles the special sequences for

4990

/// limited-precision mode.

4991

static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

4992

const TargetLowering &TLI, SDNodeFlags Flags) {

4993

// TODO: What fast-math-flags should be set on the floating-point nodes?

4994

4995

if (Op.getValueType() == MVT::f32 &&

4996

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

4997

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

4998

4999

// Scale the exponent by log(2).

5000

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5001

SDValue LogOfExponent =

5002

DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5003

DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

5004

5005

// Get the significand and build it into a floating-point number with

5006

// exponent of 1.

5007

SDValue X = GetSignificand(DAG, Op1, dl);

5008

5009

SDValue LogOfMantissa;

5010

if (LimitFloatPrecision <= 6) {

5011

// For floating-point precision of 6:

5012

//

5013

// LogofMantissa =

5014

// -1.1609546f +

5015

// (1.4034025f - 0.23903021f * x) * x;

5016

//

5017

// error 0.0034276066, which is better than 8 bits

5018

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5019

getF32Constant(DAG, 0xbe74c456, dl));

5020

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5021

getF32Constant(DAG, 0x3fb3a2b1, dl));

5022

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5023

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5024

getF32Constant(DAG, 0x3f949a29, dl));

5025

} else if (LimitFloatPrecision <= 12) {

5026

// For floating-point precision of 12:

5027

//

5028

// LogOfMantissa =

5029

// -1.7417939f +

5030

// (2.8212026f +

5031

// (-1.4699568f +

5032

// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;

5033

//

5034

// error 0.000061011436, which is 14 bits

5035

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5036

getF32Constant(DAG, 0xbd67b6d6, dl));

5037

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5038

getF32Constant(DAG, 0x3ee4f4b8, dl));

5039

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5040

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5041

getF32Constant(DAG, 0x3fbc278b, dl));

5042

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5043

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5044

getF32Constant(DAG, 0x40348e95, dl));

5045

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5046

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5047

getF32Constant(DAG, 0x3fdef31a, dl));

5048

} else { // LimitFloatPrecision <= 18

5049

// For floating-point precision of 18:

5050

//

5051

// LogOfMantissa =

5052

// -2.1072184f +

5053

// (4.2372794f +

5054

// (-3.7029485f +

5055

// (2.2781945f +

5056

// (-0.87823314f +

5057

// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;

5058

//

5059

// error 0.0000023660568, which is better than 18 bits

5060

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5061

getF32Constant(DAG, 0xbc91e5ac, dl));

5062

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5063

getF32Constant(DAG, 0x3e4350aa, dl));

5064

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5065

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5066

getF32Constant(DAG, 0x3f60d3e3, dl));

5067

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5068

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5069

getF32Constant(DAG, 0x4011cdf0, dl));

5070

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5071

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5072

getF32Constant(DAG, 0x406cfd1c, dl));

5073

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5074

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5075

getF32Constant(DAG, 0x408797cb, dl));

5076

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5077

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5078

getF32Constant(DAG, 0x4006dcab, dl));

5079

}

5080

5081

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);

5082

}

5083

5084

// No special expansion.

5085

return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);

5086

}

5087

5088

/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for

5089

/// limited-precision mode.

5090

static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5091

const TargetLowering &TLI, SDNodeFlags Flags) {

5092

// TODO: What fast-math-flags should be set on the floating-point nodes?

5093

5094

if (Op.getValueType() == MVT::f32 &&

5095

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5096

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5097

5098

// Get the exponent.

5099

SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

5100

5101

// Get the significand and build it into a floating-point number with

5102

// exponent of 1.

5103

SDValue X = GetSignificand(DAG, Op1, dl);

5104

5105

// Different possible minimax approximations of significand in

5106

// floating-point for various degrees of accuracy over [1,2].

5107

SDValue Log2ofMantissa;

5108

if (LimitFloatPrecision <= 6) {

5109

// For floating-point precision of 6:

5110

//

5111

// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;

5112

//

5113

// error 0.0049451742, which is more than 7 bits

5114

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5115

getF32Constant(DAG, 0xbeb08fe0, dl));

5116

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5117

getF32Constant(DAG, 0x40019463, dl));

5118

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5119

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5120

getF32Constant(DAG, 0x3fd6633d, dl));

5121

} else if (LimitFloatPrecision <= 12) {

5122

// For floating-point precision of 12:

5123

//

5124

// Log2ofMantissa =

5125

// -2.51285454f +

5126

// (4.07009056f +

5127

// (-2.12067489f +

5128

// (.645142248f - 0.816157886e-1f * x) * x) * x) * x;

5129

//

5130

// error 0.0000876136000, which is better than 13 bits

5131

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5132

getF32Constant(DAG, 0xbda7262e, dl));

5133

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5134

getF32Constant(DAG, 0x3f25280b, dl));

5135

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5136

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5137

getF32Constant(DAG, 0x4007b923, dl));

5138

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5139

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5140

getF32Constant(DAG, 0x40823e2f, dl));

5141

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5142

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5143

getF32Constant(DAG, 0x4020d29c, dl));

5144

} else { // LimitFloatPrecision <= 18

5145

// For floating-point precision of 18:

5146

//

5147

// Log2ofMantissa =

5148

// -3.0400495f +

5149

// (6.1129976f +

5150

// (-5.3420409f +

5151

// (3.2865683f +

5152

// (-1.2669343f +

5153

// (0.27515199f -

5154

// 0.25691327e-1f * x) * x) * x) * x) * x) * x;

5155

//

5156

// error 0.0000018516, which is better than 18 bits

5157

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5158

getF32Constant(DAG, 0xbcd2769e, dl));

5159

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5160

getF32Constant(DAG, 0x3e8ce0b9, dl));

5161

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5162

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5163

getF32Constant(DAG, 0x3fa22ae7, dl));

5164

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5165

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5166

getF32Constant(DAG, 0x40525723, dl));

5167

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5168

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5169

getF32Constant(DAG, 0x40aaf200, dl));

5170

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5171

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5172

getF32Constant(DAG, 0x40c39dad, dl));

5173

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5174

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5175

getF32Constant(DAG, 0x4042902c, dl));

5176

}

5177

5178

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);

5179

}

5180

5181

// No special expansion.

5182

return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);

5183

}

5184

5185

/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for

5186

/// limited-precision mode.

5187

static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5188

const TargetLowering &TLI, SDNodeFlags Flags) {

5189

// TODO: What fast-math-flags should be set on the floating-point nodes?

5190

5191

if (Op.getValueType() == MVT::f32 &&

5192

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5193

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5194

5195

// Scale the exponent by log10(2) [0.30102999f].

5196

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5197

SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5198

getF32Constant(DAG, 0x3e9a209a, dl));

5199

5200

// Get the significand and build it into a floating-point number with

5201

// exponent of 1.

5202

SDValue X = GetSignificand(DAG, Op1, dl);

5203

5204

SDValue Log10ofMantissa;

5205

if (LimitFloatPrecision <= 6) {

5206

// For floating-point precision of 6:

5207

//

5208

// Log10ofMantissa =

5209

// -0.50419619f +

5210

// (0.60948995f - 0.10380950f * x) * x;

5211

//

5212

// error 0.0014886165, which is 6 bits

5213

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5214

getF32Constant(DAG, 0xbdd49a13, dl));

5215

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5216

getF32Constant(DAG, 0x3f1c0789, dl));

5217

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5218

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5219

getF32Constant(DAG, 0x3f011300, dl));

5220

} else if (LimitFloatPrecision <= 12) {

5221

// For floating-point precision of 12:

5222

//

5223

// Log10ofMantissa =

5224

// -0.64831180f +

5225

// (0.91751397f +

5226

// (-0.31664806f + 0.47637168e-1f * x) * x) * x;

5227

//

5228

// error 0.00019228036, which is better than 12 bits

5229

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5230

getF32Constant(DAG, 0x3d431f31, dl));

5231

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5232

getF32Constant(DAG, 0x3ea21fb2, dl));

5233

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5234

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5235

getF32Constant(DAG, 0x3f6ae232, dl));

5236

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5237

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5238

getF32Constant(DAG, 0x3f25f7c3, dl));

5239

} else { // LimitFloatPrecision <= 18

5240

// For floating-point precision of 18:

5241

//

5242

// Log10ofMantissa =

5243

// -0.84299375f +

5244

// (1.5327582f +

5245

// (-1.0688956f +

5246

// (0.49102474f +

5247

// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;

5248

//

5249

// error 0.0000037995730, which is better than 18 bits

5250

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5251

getF32Constant(DAG, 0x3c5d51ce, dl));

5252

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5253

getF32Constant(DAG, 0x3e00685a, dl));

5254

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5255

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5256

getF32Constant(DAG, 0x3efb6798, dl));

5257

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5258

SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5259

getF32Constant(DAG, 0x3f88d192, dl));

5260

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5261

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5262

getF32Constant(DAG, 0x3fc4316c, dl));

5263

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5264

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,

5265

getF32Constant(DAG, 0x3f57ce70, dl));

5266

}

5267

5268

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);

5269

}

5270

5271

// No special expansion.

5272

return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);

5273

}

5274

5275

/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for

5276

/// limited-precision mode.

5277

static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5278

const TargetLowering &TLI, SDNodeFlags Flags) {

5279

if (Op.getValueType() == MVT::f32 &&

5280

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)

5281

return getLimitedPrecisionExp2(Op, dl, DAG);

5282

5283

// No special expansion.

5284

return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);

5285

}

5286

5287

/// visitPow - Lower a pow intrinsic. Handles the special sequences for

5288

/// limited-precision mode with x == 10.0f.

5289

static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,

5290

SelectionDAG &DAG, const TargetLowering &TLI,

5291

SDNodeFlags Flags) {

5292

bool IsExp10 = false;

5293

if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&

5294

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5295

if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {

5296

APFloat Ten(10.0f);

5297

IsExp10 = LHSC->isExactlyValue(Ten);

5298

}

5299

}

5300

5301

// TODO: What fast-math-flags should be set on the FMUL node?

5302

if (IsExp10) {

5303

// Put the exponent in the right bit position for later addition to the

5304

// final result:

5305

//

5306

// #define LOG2OF10 3.3219281f

5307

// t0 = Op * LOG2OF10;

5308

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,

5309

getF32Constant(DAG, 0x40549a78, dl));

5310

return getLimitedPrecisionExp2(t0, dl, DAG);

5311

}

5312

5313

// No special expansion.

5314

return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);

5315

}

5316

5317

/// ExpandPowI - Expand a llvm.powi intrinsic.

5318

static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,

5319

SelectionDAG &DAG) {

5320

// If RHS is a constant, we can expand this out to a multiplication tree,

5321

// otherwise we end up lowering to a call to __powidf2 (for example). When

5322

// optimizing for size, we only want to do this if the expansion would produce

5323

// a small number of multiplies, otherwise we do the full expansion.

5324

if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {

5325

// Get the exponent as a positive value.

5326

unsigned Val = RHSC->getSExtValue();

5327

if ((int)Val < 0) Val = -Val;

5328

5329

// powi(x, 0) -> 1.0

5330

if (Val == 0)

5331

return DAG.getConstantFP(1.0, DL, LHS.getValueType());

5332

5333

bool OptForSize = DAG.shouldOptForSize();

5334

if (!OptForSize ||

5335

// If optimizing for size, don't insert too many multiplies.

5336

// This inserts up to 5 multiplies.

5337

countPopulation(Val) + Log2_32(Val) < 7) {

5338

// We use the simple binary decomposition method to generate the multiply

5339

// sequence. There are more optimal ways to do this (for example,

5340

// powi(x,15) generates one more multiply than it should), but this has

5341

// the benefit of being both really simple and much better than a libcall.

5342

SDValue Res; // Logically starts equal to 1.0

5343

SDValue CurSquare = LHS;

5344

// TODO: Intrinsics should have fast-math-flags that propagate to these

5345

// nodes.

5346

while (Val) {

5347

if (Val & 1) {

5348

if (Res.getNode())

5349

Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);

5350

else

5351

Res = CurSquare; // 1.0*CurSquare.

5352

}

5353

5354

CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),

5355

CurSquare, CurSquare);

5356

Val >>= 1;

5357

}

5358

5359

// If the original was negative, invert the result, producing 1/(x*x*x).

5360

if (RHSC->getSExtValue() < 0)

5361

Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),

5362

DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);

5363

return Res;

5364

}

5365

}

5366

5367

// Otherwise, expand to a libcall.

5368

return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);

5369

}

5370

5371

static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,

5372

SDValue LHS, SDValue RHS, SDValue Scale,

5373

SelectionDAG &DAG, const TargetLowering &TLI) {

5374

EVT VT = LHS.getValueType();

5375

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

5376

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

5377

LLVMContext &Ctx = *DAG.getContext();

5378

5379

// If the type is legal but the operation isn't, this node might survive all

5380

// the way to operation legalization. If we end up there and we do not have

5381

// the ability to widen the type (if VT*2 is not legal), we cannot expand the

5382

// node.

5383

5384

// Coax the legalizer into expanding the node during type legalization instead

5385

// by bumping the size by one bit. This will force it to Promote, enabling the

5386

// early expansion and avoiding the need to expand later.

5387

5388

// We don't have to do this if Scale is 0; that can always be expanded, unless

5389

// it's a saturating signed operation. Those can experience true integer

5390

// division overflow, a case which we must avoid.

5391

5392

// FIXME: We wouldn't have to do this (or any of the early

5393

// expansion/promotion) if it was possible to expand a libcall of an

5394

// illegal type during operation legalization. But it's not, so things

5395

// get a bit hacky.

5396

unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();

5397

if ((ScaleInt > 0 || (Saturating && Signed)) &&

5398

(TLI.isTypeLegal(VT) ||

5399

(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {

5400

TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(

5401

Opcode, VT, ScaleInt);

5402

if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {

5403

EVT PromVT;

5404

if (VT.isScalarInteger())

5405

PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);

5406

else if (VT.isVector()) {

5407

PromVT = VT.getVectorElementType();

5408

PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);

5409

PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());

5410

} else

5411

llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5411);

5412

if (Signed) {

5413

LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);

5414

RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);

5415

} else {

5416

LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);

5417

RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);

5418

}

5419

EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());

5420

// For saturating operations, we need to shift up the LHS to get the

5421

// proper saturation width, and then shift down again afterwards.

5422

if (Saturating)

5423

LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,

5424

DAG.getConstant(1, DL, ShiftTy));

5425

SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);

5426

if (Saturating)

5427

Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,

5428

DAG.getConstant(1, DL, ShiftTy));

5429

return DAG.getZExtOrTrunc(Res, DL, VT);

5430

}

5431

}

5432

5433

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);

5434

}

5435

5436

// getUnderlyingArgRegs - Find underlying registers used for a truncated,

5437

// bitcasted, or split argument. Returns a list of <Register, size in bits>

5438

static void

5439

getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,

5440

const SDValue &N) {

5441

switch (N.getOpcode()) {

5442

case ISD::CopyFromReg: {

5443

SDValue Op = N.getOperand(1);

5444

Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),

5445

Op.getValueType().getSizeInBits());

5446

return;

5447

}

5448

case ISD::BITCAST:

5449

case ISD::AssertZext:

5450

case ISD::AssertSext:

5451

case ISD::TRUNCATE:

5452

getUnderlyingArgRegs(Regs, N.getOperand(0));

5453

return;

5454

case ISD::BUILD_PAIR:

5455

case ISD::BUILD_VECTOR:

5456

case ISD::CONCAT_VECTORS:

5457

for (SDValue Op : N->op_values())

5458

getUnderlyingArgRegs(Regs, Op);

5459

return;

5460

default:

5461

return;

5462

}

5463

}

5464

5465

/// If the DbgValueInst is a dbg_value of a function argument, create the

5466

/// corresponding DBG_VALUE machine instruction for it now. At the end of

5467

/// instruction selection, they will be inserted to the entry BB.

5468

/// We don't currently support this for variadic dbg_values, as they shouldn't

5469

/// appear for function arguments or in the prologue.

5470

bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(

5471

const Value *V, DILocalVariable *Variable, DIExpression *Expr,

5472

DILocation *DL, bool IsDbgDeclare, const SDValue &N) {

5473

const Argument *Arg = dyn_cast<Argument>(V);

5474

if (!Arg)

5475

return false;

5476

5477

if (!IsDbgDeclare) {

5478

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5479

// should only emit as ArgDbgValue if the dbg.value intrinsic is found in

5480

// the entry block.

5481

bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();

5482

if (!IsInEntryBlock)

5483

return false;

5484

5485

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5486

// should only emit as ArgDbgValue if the dbg.value intrinsic describes a

5487

// variable that also is a param.

5488

//

5489

// Although, if we are at the top of the entry block already, we can still

5490

// emit using ArgDbgValue. This might catch some situations when the

5491

// dbg.value refers to an argument that isn't used in the entry block, so

5492

// any CopyToReg node would be optimized out and the only way to express

5493

// this DBG_VALUE is by using the physical reg (or FI) as done in this

5494

// method. ArgDbgValues are hoisted to the beginning of the entry block. So

5495

// we should only emit as ArgDbgValue if the Variable is an argument to the

5496

// current function, and the dbg.value intrinsic is found in the entry

5497

// block.

5498

bool VariableIsFunctionInputArg = Variable->isParameter() &&

5499

!DL->getInlinedAt();

5500

bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;

5501

if (!IsInPrologue && !VariableIsFunctionInputArg)

5502

return false;

5503

5504

// Here we assume that a function argument on IR level only can be used to

5505

// describe one input parameter on source level. If we for example have

5506

// source code like this

5507

//

5508

// struct A { long x, y; };

5509

// void foo(struct A a, long b) {

5510

// ...

5511

// b = a.x;

5512

// ...

5513

// }

5514

//

5515

// and IR like this

5516

//

5517

// define void @foo(i32 %a1, i32 %a2, i32 %b) {

5518

// entry:

5519

// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment

5520

// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment

5521

// call void @llvm.dbg.value(metadata i32 %b, "b",

5522

// ...

5523

// call void @llvm.dbg.value(metadata i32 %a1, "b"

5524

// ...

5525

//

5526

// then the last dbg.value is describing a parameter "b" using a value that

5527

// is an argument. But since we already has used %a1 to describe a parameter

5528

// we should not handle that last dbg.value here (that would result in an

5529

// incorrect hoisting of the DBG_VALUE to the function entry).

5530

// Notice that we allow one dbg.value per IR level argument, to accommodate

5531

// for the situation with fragments above.

5532

if (VariableIsFunctionInputArg) {

5533

unsigned ArgNo = Arg->getArgNo();

5534

if (ArgNo >= FuncInfo.DescribedArgs.size())

5535

FuncInfo.DescribedArgs.resize(ArgNo + 1, false);

5536

else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))

5537

return false;

5538

FuncInfo.DescribedArgs.set(ArgNo);

5539

}

5540

}

5541

5542

MachineFunction &MF = DAG.getMachineFunction();

5543

const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

5544

5545

bool IsIndirect = false;

5546

Optional<MachineOperand> Op;

5547

// Some arguments' frame index is recorded during argument lowering.

5548

int FI = FuncInfo.getArgumentFrameIndex(Arg);

5549

if (FI != std::numeric_limits<int>::max())

5550

Op = MachineOperand::CreateFI(FI);

5551

5552

SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;

5553

if (!Op && N.getNode()) {

5554

getUnderlyingArgRegs(ArgRegsAndSizes, N);

5555

Register Reg;

5556

if (ArgRegsAndSizes.size() == 1)

5557

Reg = ArgRegsAndSizes.front().first;

5558

5559

if (Reg && Reg.isVirtual()) {

5560

MachineRegisterInfo &RegInfo = MF.getRegInfo();

5561

Register PR = RegInfo.getLiveInPhysReg(Reg);

5562

if (PR)

5563

Reg = PR;

5564

}

5565

if (Reg) {

5566

Op = MachineOperand::CreateReg(Reg, false);

5567

IsIndirect = IsDbgDeclare;

5568

}

5569

}

5570

5571

if (!Op && N.getNode()) {

5572

// Check if frame index is available.

5573

SDValue LCandidate = peekThroughBitcasts(N);

5574

if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))

5575

if (FrameIndexSDNode *FINode =

5576

dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))

5577

Op = MachineOperand::CreateFI(FINode->getIndex());

5578

}

5579

5580

if (!Op) {

5581

// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg

5582

auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>

5583

SplitRegs) {

5584

unsigned Offset = 0;

5585

for (auto RegAndSize : SplitRegs) {

5586

// If the expression is already a fragment, the current register

5587

// offset+size might extend beyond the fragment. In this case, only

5588

// the register bits that are inside the fragment are relevant.

5589

int RegFragmentSizeInBits = RegAndSize.second;

5590

if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {

5591

uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;

5592

// The register is entirely outside the expression fragment,

5593

// so is irrelevant for debug info.

5594

if (Offset >= ExprFragmentSizeInBits)

5595

break;

5596

// The register is partially outside the expression fragment, only

5597

// the low bits within the fragment are relevant for debug info.

5598

if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {

5599

RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;

5600

}

5601

}

5602

5603

auto FragmentExpr = DIExpression::createFragmentExpression(

5604

Expr, Offset, RegFragmentSizeInBits);

5605

Offset += RegAndSize.second;

5606

// If a valid fragment expression cannot be created, the variable's

5607

// correct value cannot be determined and so it is set as Undef.

5608

if (!FragmentExpr) {

5609

SDDbgValue *SDV = DAG.getConstantDbgValue(

5610

Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);

5611

DAG.AddDbgValue(SDV, false);

5612

continue;

5613

}

5614

assert(!IsDbgDeclare && "DbgDeclare operand is not in memory?")((!IsDbgDeclare && "DbgDeclare operand is not in memory?"
) ? static_cast<void> (0) : __assert_fail ("!IsDbgDeclare && \"DbgDeclare operand is not in memory?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5614, __PRETTY_FUNCTION__));

5615

FuncInfo.ArgDbgValues.push_back(

5616

BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,

5617

RegAndSize.first, Variable, *FragmentExpr));

5618

}

5619

};

5620

5621

// Check if ValueMap has reg number.

5622

DenseMap<const Value *, Register>::const_iterator

5623

VMI = FuncInfo.ValueMap.find(V);

5624

if (VMI != FuncInfo.ValueMap.end()) {

5625

const auto &TLI = DAG.getTargetLoweringInfo();

5626

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,

5627

V->getType(), None);

5628

if (RFV.occupiesMultipleRegs()) {

5629

splitMultiRegDbgValue(RFV.getRegsAndSizes());

5630

return true;

5631

}

5632

5633

Op = MachineOperand::CreateReg(VMI->second, false);

5634

IsIndirect = IsDbgDeclare;

5635

} else if (ArgRegsAndSizes.size() > 1) {

5636

// This was split due to the calling convention, and no virtual register

5637

// mapping exists for the value.

5638

splitMultiRegDbgValue(ArgRegsAndSizes);

5639

return true;

5640

}

5641

}

5642

5643

if (!Op)

5644

return false;

5645

5646

assert(Variable->isValidLocationForIntrinsic(DL) &&((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5647, __PRETTY_FUNCTION__))

5647

"Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5647, __PRETTY_FUNCTION__));

5648

IsIndirect = (Op->isReg()) ? IsIndirect : true;

5649

FuncInfo.ArgDbgValues.push_back(

5650

BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,

5651

*Op, Variable, Expr));

5652

5653

return true;

5654

}

5655

5656

/// Return the appropriate SDDbgValue based on N.

5657

SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,

5658

DILocalVariable *Variable,

5659

DIExpression *Expr,

5660

const DebugLoc &dl,

5661

unsigned DbgSDNodeOrder) {

5662

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

5663

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe

5664

// stack slot locations.

5665

//

5666

// Consider "int x = 0; int *px = &x;". There are two kinds of interesting

5667

// debug values here after optimization:

5668

//

5669

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

5670

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

5671

//

5672

// Both describe the direct values of their associated variables.

5673

return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),

5674

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5675

}

5676

return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),

5677

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5678

}

5679

5680

static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {

5681

switch (Intrinsic) {

5682

case Intrinsic::smul_fix:

5683

return ISD::SMULFIX;

5684

case Intrinsic::umul_fix:

5685

return ISD::UMULFIX;

5686

case Intrinsic::smul_fix_sat:

5687

return ISD::SMULFIXSAT;

5688

case Intrinsic::umul_fix_sat:

5689

return ISD::UMULFIXSAT;

5690

case Intrinsic::sdiv_fix:

5691

return ISD::SDIVFIX;

5692

case Intrinsic::udiv_fix:

5693

return ISD::UDIVFIX;

5694

case Intrinsic::sdiv_fix_sat:

5695

return ISD::SDIVFIXSAT;

5696

case Intrinsic::udiv_fix_sat:

5697

return ISD::UDIVFIXSAT;

5698

default:

5699

llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5699);

5700

}

5701

}

5702

5703

void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,

5704

const char *FunctionName) {

5705

assert(FunctionName && "FunctionName must not be nullptr")((FunctionName && "FunctionName must not be nullptr")
? static_cast<void> (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5705, __PRETTY_FUNCTION__));

5706

SDValue Callee = DAG.getExternalSymbol(

5707

FunctionName,

5708

DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));

5709

LowerCallTo(I, Callee, I.isTailCall());

5710

}

5711

5712

/// Given a @llvm.call.preallocated.setup, return the corresponding

5713

/// preallocated call.

5714

static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {

5715

assert(cast<CallBase>(PreallocatedSetup)((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5718, __PRETTY_FUNCTION__))

5716

->getCalledFunction()((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5718, __PRETTY_FUNCTION__))

5717

->getIntrinsicID() == Intrinsic::call_preallocated_setup &&((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5718, __PRETTY_FUNCTION__))

5718

"expected call_preallocated_setup Value")((cast<CallBase>(PreallocatedSetup) ->getCalledFunction
() ->getIntrinsicID() == Intrinsic::call_preallocated_setup
&& "expected call_preallocated_setup Value") ? static_cast
<void> (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5718, __PRETTY_FUNCTION__));

5719

for (auto *U : PreallocatedSetup->users()) {

5720

auto *UseCall = cast<CallBase>(U);

5721

const Function *Fn = UseCall->getCalledFunction();

5722

if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {

5723

return UseCall;

5724

}

5725

}

5726

llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5726);

5727

}

5728

5729

/// Lower the call to the specified intrinsic function.

5730

void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,

5731

unsigned Intrinsic) {

5732

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5733

SDLoc sdl = getCurSDLoc();

5734

DebugLoc dl = getCurDebugLoc();

5735

SDValue Res;

5736

5737

SDNodeFlags Flags;

5738

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

5739

Flags.copyFMF(*FPOp);

5740

5741

switch (Intrinsic) {

5742

default:

5743

// By default, turn this into a target intrinsic node.

5744

visitTargetIntrinsic(I, Intrinsic);

5745

return;

5746

case Intrinsic::vscale: {

5747

match(&I, m_VScale(DAG.getDataLayout()));

5748

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

5749

setValue(&I,

5750

DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)));

5751

return;

5752

}

5753

case Intrinsic::vastart: visitVAStart(I); return;

5754

case Intrinsic::vaend: visitVAEnd(I); return;

5755

case Intrinsic::vacopy: visitVACopy(I); return;

5756

case Intrinsic::returnaddress:

5757

setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,

5758

TLI.getPointerTy(DAG.getDataLayout()),

5759

getValue(I.getArgOperand(0))));

5760

return;

5761

case Intrinsic::addressofreturnaddress:

5762

setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,

5763

TLI.getPointerTy(DAG.getDataLayout())));

5764

return;

5765

case Intrinsic::sponentry:

5766

setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl,

5767

TLI.getFrameIndexTy(DAG.getDataLayout())));

5768

return;

5769

case Intrinsic::frameaddress:

5770

setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,

5771

TLI.getFrameIndexTy(DAG.getDataLayout()),

5772

getValue(I.getArgOperand(0))));

5773

return;

5774

case Intrinsic::read_volatile_register:

5775

case Intrinsic::read_register: {

5776

Value *Reg = I.getArgOperand(0);

5777

SDValue Chain = getRoot();

5778

SDValue RegName =

5779

DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));

5780

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

5781

Res = DAG.getNode(ISD::READ_REGISTER, sdl,

5782

DAG.getVTList(VT, MVT::Other), Chain, RegName);

5783

setValue(&I, Res);

5784

DAG.setRoot(Res.getValue(1));

5785

return;

5786

}

5787

case Intrinsic::write_register: {

5788

Value *Reg = I.getArgOperand(0);

5789

Value *RegValue = I.getArgOperand(1);

5790

SDValue Chain = getRoot();

5791

SDValue RegName =

5792

DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));

5793

DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,

5794

RegName, getValue(RegValue)));

5795

return;

5796

}

5797

case Intrinsic::memcpy: {

5798

const auto &MCI = cast<MemCpyInst>(I);

5799

SDValue Op1 = getValue(I.getArgOperand(0));

5800

SDValue Op2 = getValue(I.getArgOperand(1));

5801

SDValue Op3 = getValue(I.getArgOperand(2));

5802

// @llvm.memcpy defines 0 and 1 to both mean no alignment.

5803

Align DstAlign = MCI.getDestAlign().valueOrOne();

5804

Align SrcAlign = MCI.getSourceAlign().valueOrOne();

5805

Align Alignment = commonAlignment(DstAlign, SrcAlign);

5806

bool isVol = MCI.isVolatile();

5807

bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());

5808

// FIXME: Support passing different dest/src alignments to the memcpy DAG

5809

// node.

5810

SDValue Root = isVol ? getRoot() : getMemoryRoot();

5811

SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,

5812

/* AlwaysInline */ false, isTC,

5813

MachinePointerInfo(I.getArgOperand(0)),

5814

MachinePointerInfo(I.getArgOperand(1)));

5815

updateDAGForMaybeTailCall(MC);

5816

return;

5817

}

5818

case Intrinsic::memcpy_inline: {

5819

const auto &MCI = cast<MemCpy

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1114, column 10 Called C++ object pointer is null

Bug Summary

Annotated Source Code

/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp