/build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1

//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements routines for translating from LLVM IR into SelectionDAG IR.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "SelectionDAGBuilder.h"

14

#include "SDNodeDbgValue.h"

15

#include "llvm/ADT/APFloat.h"

16

#include "llvm/ADT/APInt.h"

17

#include "llvm/ADT/BitVector.h"

18

#include "llvm/ADT/STLExtras.h"

19

#include "llvm/ADT/SmallPtrSet.h"

20

#include "llvm/ADT/SmallSet.h"

21

#include "llvm/ADT/StringRef.h"

22

#include "llvm/ADT/Twine.h"

23

#include "llvm/Analysis/AliasAnalysis.h"

24

#include "llvm/Analysis/BranchProbabilityInfo.h"

25

#include "llvm/Analysis/ConstantFolding.h"

26

#include "llvm/Analysis/Loads.h"

27

#include "llvm/Analysis/MemoryLocation.h"

28

#include "llvm/Analysis/TargetLibraryInfo.h"

29

#include "llvm/Analysis/ValueTracking.h"

30

#include "llvm/Analysis/VectorUtils.h"

31

#include "llvm/CodeGen/Analysis.h"

32

#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"

33

#include "llvm/CodeGen/CodeGenCommonISel.h"

34

#include "llvm/CodeGen/FunctionLoweringInfo.h"

35

#include "llvm/CodeGen/GCMetadata.h"

36

#include "llvm/CodeGen/MachineBasicBlock.h"

37

#include "llvm/CodeGen/MachineFrameInfo.h"

38

#include "llvm/CodeGen/MachineFunction.h"

39

#include "llvm/CodeGen/MachineInstrBuilder.h"

40

#include "llvm/CodeGen/MachineInstrBundleIterator.h"

41

#include "llvm/CodeGen/MachineMemOperand.h"

42

#include "llvm/CodeGen/MachineModuleInfo.h"

43

#include "llvm/CodeGen/MachineOperand.h"

44

#include "llvm/CodeGen/MachineRegisterInfo.h"

45

#include "llvm/CodeGen/RuntimeLibcalls.h"

46

#include "llvm/CodeGen/SelectionDAG.h"

47

#include "llvm/CodeGen/SelectionDAGTargetInfo.h"

48

#include "llvm/CodeGen/StackMaps.h"

49

#include "llvm/CodeGen/SwiftErrorValueTracking.h"

50

#include "llvm/CodeGen/TargetFrameLowering.h"

51

#include "llvm/CodeGen/TargetInstrInfo.h"

52

#include "llvm/CodeGen/TargetOpcodes.h"

53

#include "llvm/CodeGen/TargetRegisterInfo.h"

54

#include "llvm/CodeGen/TargetSubtargetInfo.h"

55

#include "llvm/CodeGen/WinEHFuncInfo.h"

56

#include "llvm/IR/Argument.h"

57

#include "llvm/IR/Attributes.h"

58

#include "llvm/IR/BasicBlock.h"

59

#include "llvm/IR/CFG.h"

60

#include "llvm/IR/CallingConv.h"

61

#include "llvm/IR/Constant.h"

62

#include "llvm/IR/ConstantRange.h"

63

#include "llvm/IR/Constants.h"

64

#include "llvm/IR/DataLayout.h"

65

#include "llvm/IR/DebugInfo.h"

66

#include "llvm/IR/DebugInfoMetadata.h"

67

#include "llvm/IR/DerivedTypes.h"

68

#include "llvm/IR/DiagnosticInfo.h"

69

#include "llvm/IR/EHPersonalities.h"

70

#include "llvm/IR/Function.h"

71

#include "llvm/IR/GetElementPtrTypeIterator.h"

72

#include "llvm/IR/InlineAsm.h"

73

#include "llvm/IR/InstrTypes.h"

74

#include "llvm/IR/Instructions.h"

75

#include "llvm/IR/IntrinsicInst.h"

76

#include "llvm/IR/Intrinsics.h"

77

#include "llvm/IR/IntrinsicsAArch64.h"

78

#include "llvm/IR/IntrinsicsWebAssembly.h"

79

#include "llvm/IR/LLVMContext.h"

80

#include "llvm/IR/Metadata.h"

81

#include "llvm/IR/Module.h"

82

#include "llvm/IR/Operator.h"

83

#include "llvm/IR/PatternMatch.h"

84

#include "llvm/IR/Statepoint.h"

85

#include "llvm/IR/Type.h"

86

#include "llvm/IR/User.h"

87

#include "llvm/IR/Value.h"

88

#include "llvm/MC/MCContext.h"

89

#include "llvm/Support/AtomicOrdering.h"

90

#include "llvm/Support/Casting.h"

91

#include "llvm/Support/CommandLine.h"

92

#include "llvm/Support/Compiler.h"

93

#include "llvm/Support/Debug.h"

94

#include "llvm/Support/MathExtras.h"

95

#include "llvm/Support/raw_ostream.h"

96

#include "llvm/Target/TargetIntrinsicInfo.h"

97

#include "llvm/Target/TargetMachine.h"

98

#include "llvm/Target/TargetOptions.h"

99

#include "llvm/TargetParser/Triple.h"

100

#include "llvm/Transforms/Utils/Local.h"

101

#include <cstddef>

102

#include <iterator>

103

#include <limits>

104

#include <optional>

105

#include <tuple>

106

107

using namespace llvm;

108

using namespace PatternMatch;

109

using namespace SwitchCG;

110

111

#define DEBUG_TYPE"isel" "isel"

112

113

/// LimitFloatPrecision - Generate low-precision inline sequences for

114

/// some float libcalls (6, 8 or 12 bits).

115

static unsigned LimitFloatPrecision;

116

117

static cl::opt<bool>

118

InsertAssertAlign("insert-assert-align", cl::init(true),

119

cl::desc("Insert the experimental `assertalign` node."),

120

cl::ReallyHidden);

121

122

static cl::opt<unsigned, true>

123

LimitFPPrecision("limit-float-precision",

124

cl::desc("Generate low-precision inline sequences "

125

"for some float libcalls"),

126

cl::location(LimitFloatPrecision), cl::Hidden,

127

cl::init(0));

128

129

static cl::opt<unsigned> SwitchPeelThreshold(

130

"switch-peel-threshold", cl::Hidden, cl::init(66),

131

cl::desc("Set the case probability threshold for peeling the case from a "

132

"switch statement. A value greater than 100 will void this "

133

"optimization"));

134

135

// Limit the width of DAG chains. This is important in general to prevent

136

// DAG-based analysis from blowing up. For example, alias analysis and

137

// load clustering may not complete in reasonable time. It is difficult to

138

// recognize and avoid this situation within each individual analysis, and

139

// future analyses are likely to have the same behavior. Limiting DAG width is

140

// the safe approach and will be especially important with global DAGs.

141

//

142

// MaxParallelChains default is arbitrarily high to avoid affecting

143

// optimization, but could be lowered to improve compile time. Any ld-ld-st-st

144

// sequence over this should have been converted to llvm.memcpy by the

145

// frontend. It is easy to induce this behavior with .ll code such as:

146

// %buffer = alloca [4096 x i8]

147

// %data = load [4096 x i8]* %argPtr

148

// store [4096 x i8] %data, [4096 x i8]* %buffer

149

static const unsigned MaxParallelChains = 64;

150

151

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

152

const SDValue *Parts, unsigned NumParts,

153

MVT PartVT, EVT ValueVT, const Value *V,

154

std::optional<CallingConv::ID> CC);

155

156

/// getCopyFromParts - Create a value that contains the specified legal parts

157

/// combined into the value they represent. If the parts combine to a type

158

/// larger than ValueVT then AssertOp can be used to specify whether the extra

159

/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT

160

/// (ISD::AssertSext).

161

static SDValue

162

getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,

163

unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,

164

std::optional<CallingConv::ID> CC = std::nullopt,

165

std::optional<ISD::NodeType> AssertOp = std::nullopt) {

166

// Let the target assemble the parts if it wants to

167

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

168

if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,

169

PartVT, ValueVT, CC))

170

return Val;

171

172

if (ValueVT.isVector())

173

return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,

174

CC);

175

176

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 176
, __extension__ __PRETTY_FUNCTION__));

177

SDValue Val = Parts[0];

178

179

if (NumParts > 1) {

180

// Assemble the value from multiple parts.

181

if (ValueVT.isInteger()) {

182

unsigned PartBits = PartVT.getSizeInBits();

183

unsigned ValueBits = ValueVT.getSizeInBits();

184

185

// Assemble the power of 2 part.

186

unsigned RoundParts = llvm::bit_floor(NumParts);

187

unsigned RoundBits = PartBits * RoundParts;

188

EVT RoundVT = RoundBits == ValueBits ?

189

ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);

190

SDValue Lo, Hi;

191

192

EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

193

194

if (RoundParts > 2) {

195

Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,

196

PartVT, HalfVT, V);

197

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,

198

RoundParts / 2, PartVT, HalfVT, V);

199

} else {

200

Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);

201

Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);

202

}

203

204

if (DAG.getDataLayout().isBigEndian())

205

std::swap(Lo, Hi);

206

207

Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

208

209

if (RoundParts < NumParts) {

210

// Assemble the trailing non-power-of-2 part.

211

unsigned OddParts = NumParts - RoundParts;

212

EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);

213

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,

214

OddVT, V, CC);

215

216

// Combine the round and odd parts.

217

Lo = Val;

218

if (DAG.getDataLayout().isBigEndian())

219

std::swap(Lo, Hi);

220

EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

221

Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);

222

Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,

223

DAG.getConstant(Lo.getValueSizeInBits(), DL,

224

TLI.getShiftAmountTy(

225

TotalVT, DAG.getDataLayout())));

226

Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);

227

Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);

228

}

229

} else if (PartVT.isFloatingPoint()) {

230

// FP split into multiple FP parts (for ppcf128)

231

assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232
, __extension__ __PRETTY_FUNCTION__))

232

"Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232
, __extension__ __PRETTY_FUNCTION__));

233

SDValue Lo, Hi;

234

Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);

235

Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);

236

if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

237

std::swap(Lo, Hi);

238

Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);

239

} else {

240

// FP split into integer parts (soft fp)

241

assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242
, __extension__ __PRETTY_FUNCTION__))

242

!PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242
, __extension__ __PRETTY_FUNCTION__));

243

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

244

Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);

245

}

246

}

247

248

// There is now one part, held in Val. Correct it to match ValueVT.

249

// PartEVT is the type of the register class that holds the value.

250

// ValueVT is the type of the inline asm operation.

251

EVT PartEVT = Val.getValueType();

252

253

if (PartEVT == ValueVT)

254

return Val;

255

256

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&

257

ValueVT.bitsLT(PartEVT)) {

258

// For an FP value in an integer part, we need to truncate to the right

259

// width first.

260

PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

261

Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);

262

}

263

264

// Handle types that have the same size.

265

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())

266

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

267

268

// Handle types with different sizes.

269

if (PartEVT.isInteger() && ValueVT.isInteger()) {

270

if (ValueVT.bitsLT(PartEVT)) {

271

// For a truncate, see if we have any information to

272

// indicate whether the truncated bits will always be

273

// zero or sign-extension.

274

if (AssertOp)

275

Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,

276

DAG.getValueType(ValueVT));

277

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

278

}

279

return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

280

}

281

282

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

283

// FP_ROUND's are always exact here.

284

if (ValueVT.bitsLT(Val.getValueType()))

285

return DAG.getNode(

286

ISD::FP_ROUND, DL, ValueVT, Val,

287

DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

288

289

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);

290

}

291

292

// Handle MMX to a narrower integer type by bitcasting MMX to integer and

293

// then truncating.

294

if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&

295

ValueVT.bitsLT(PartEVT)) {

296

Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);

297

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

298

}

299

300

report_fatal_error("Unknown mismatch in getCopyFromParts!");

301

}

302

303

static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,

304

const Twine &ErrMsg) {

305

const Instruction *I = dyn_cast_or_null<Instruction>(V);

306

if (!V)

307

return Ctx.emitError(ErrMsg);

308

309

const char *AsmError = ", possible invalid constraint for vector type";

310

if (const CallInst *CI = dyn_cast<CallInst>(I))

311

if (CI->isInlineAsm())

312

return Ctx.emitError(I, ErrMsg + AsmError);

313

314

return Ctx.emitError(I, ErrMsg);

315

}

316

317

/// getCopyFromPartsVector - Create a value that contains the specified legal

318

/// parts combined into the value they represent. If the parts combine to a

319

/// type larger than ValueVT then AssertOp can be used to specify whether the

320

/// extra bits are known to be zero (ISD::AssertZext) or sign extended from

321

/// ValueVT (ISD::AssertSext).

322

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

323

const SDValue *Parts, unsigned NumParts,

324

MVT PartVT, EVT ValueVT, const Value *V,

325

std::optional<CallingConv::ID> CallConv) {

326

assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 326
, __extension__ __PRETTY_FUNCTION__));

327

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 327
, __extension__ __PRETTY_FUNCTION__));

328

const bool IsABIRegCopy = CallConv.has_value();

329

330

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

331

SDValue Val = Parts[0];

332

333

// Handle a multi-element vector.

334

if (NumParts > 1) {

335

EVT IntermediateVT;

336

MVT RegisterVT;

337

unsigned NumIntermediates;

338

unsigned NumRegs;

339

340

if (IsABIRegCopy) {

341

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

342

*DAG.getContext(), *CallConv, ValueVT, IntermediateVT,

343

NumIntermediates, RegisterVT);

344

} else {

345

NumRegs =

346

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

347

NumIntermediates, RegisterVT);

348

}

349

350

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 350
, __extension__ __PRETTY_FUNCTION__));

351

NumParts = NumRegs; // Silence a compiler warning.

352

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 352
, __extension__ __PRETTY_FUNCTION__));

353

assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__))

354

Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__))

355

"Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__));

356

357

// Assemble the parts into intermediate operands.

358

SmallVector<SDValue, 8> Ops(NumIntermediates);

359

if (NumIntermediates == NumParts) {

360

// If the register was not expanded, truncate or copy the value,

361

// as appropriate.

362

for (unsigned i = 0; i != NumParts; ++i)

363

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,

364

PartVT, IntermediateVT, V, CallConv);

365

} else if (NumParts > 0) {

366

// If the intermediate type was expanded, build the intermediate

367

// operands from the parts.

368

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369
, __extension__ __PRETTY_FUNCTION__))

369

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369
, __extension__ __PRETTY_FUNCTION__));

370

unsigned Factor = NumParts / NumIntermediates;

371

for (unsigned i = 0; i != NumIntermediates; ++i)

372

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,

373

PartVT, IntermediateVT, V, CallConv);

374

}

375

376

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the

377

// intermediate operands.

378

EVT BuiltVectorTy =

379

IntermediateVT.isVector()

380

? EVT::getVectorVT(

381

*DAG.getContext(), IntermediateVT.getScalarType(),

382

IntermediateVT.getVectorElementCount() * NumParts)

383

: EVT::getVectorVT(*DAG.getContext(),

384

IntermediateVT.getScalarType(),

385

NumIntermediates);

386

Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS

387

: ISD::BUILD_VECTOR,

388

DL, BuiltVectorTy, Ops);

389

}

390

391

// There is now one part, held in Val. Correct it to match ValueVT.

392

EVT PartEVT = Val.getValueType();

393

394

if (PartEVT == ValueVT)

395

return Val;

396

397

if (PartEVT.isVector()) {

398

// Vector/Vector bitcast.

399

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())

400

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

401

402

// If the parts vector has more elements than the value vector, then we

403

// have a vector widening case (e.g. <2 x float> -> <4 x float>).

404

// Extract the elements we want.

405

if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {

406

assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))

407

ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))

408

(PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))

409

ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))

410

"Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__));

411

PartEVT =

412

EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),

413

ValueVT.getVectorElementCount());

414

Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,

415

DAG.getVectorIdxConstant(0, DL));

416

if (PartEVT == ValueVT)

417

return Val;

418

if (PartEVT.isInteger() && ValueVT.isFloatingPoint())

419

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

420

}

421

422

// Promoted vector extract

423

return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);

424

}

425

426

// Trivial bitcast if the types are the same size and the destination

427

// vector type is legal.

428

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&

429

TLI.isTypeLegal(ValueVT))

430

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

431

432

if (ValueVT.getVectorNumElements() != 1) {

433

// Certain ABIs require that vectors are passed as integers. For vectors

434

// are the same size, this is an obvious bitcast.

435

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {

436

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

437

} else if (ValueVT.bitsLT(PartEVT)) {

438

const uint64_t ValueSize = ValueVT.getFixedSizeInBits();

439

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

440

// Drop the extra bits.

441

Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);

442

return DAG.getBitcast(ValueVT, Val);

443

}

444

445

diagnosePossiblyInvalidConstraint(

446

*DAG.getContext(), V, "non-trivial scalar-to-vector conversion");

447

return DAG.getUNDEF(ValueVT);

448

}

449

450

// Handle cases such as i8 -> <1 x i1>

451

EVT ValueSVT = ValueVT.getVectorElementType();

452

if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {

453

unsigned ValueSize = ValueSVT.getSizeInBits();

454

if (ValueSize == PartEVT.getSizeInBits()) {

455

Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);

456

} else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {

457

// It's possible a scalar floating point type gets softened to integer and

458

// then promoted to a larger integer. If PartEVT is the larger integer

459

// we need to truncate it and then bitcast to the FP type.

460

assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types")(static_cast <bool> (ValueSVT.bitsLT(PartEVT) &&
"Unexpected types") ? void (0) : __assert_fail ("ValueSVT.bitsLT(PartEVT) && \"Unexpected types\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 460
, __extension__ __PRETTY_FUNCTION__));

461

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

462

Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);

463

Val = DAG.getBitcast(ValueSVT, Val);

464

} else {

465

Val = ValueVT.isFloatingPoint()

466

? DAG.getFPExtendOrRound(Val, DL, ValueSVT)

467

: DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);

468

}

469

}

470

471

return DAG.getBuildVector(ValueVT, DL, Val);

472

}

473

474

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,

475

SDValue Val, SDValue *Parts, unsigned NumParts,

476

MVT PartVT, const Value *V,

477

std::optional<CallingConv::ID> CallConv);

478

479

/// getCopyToParts - Create a series of nodes that contain the specified value

480

/// split into legal parts. If the parts contain more bits than Val, then, for

481

/// integers, ExtendKind can be used to specify how to generate the extra bits.

482

static void

483

getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,

484

unsigned NumParts, MVT PartVT, const Value *V,

485

std::optional<CallingConv::ID> CallConv = std::nullopt,

486

ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {

487

// Let the target split the parts if it wants to

488

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

489

if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,

490

CallConv))

491

return;

492

EVT ValueVT = Val.getValueType();

493

494

// Handle the vector case separately.

495

if (ValueVT.isVector())

496

return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,

497

CallConv);

498

499

unsigned OrigNumParts = NumParts;

500

assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501
, __extension__ __PRETTY_FUNCTION__))

501

"Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501
, __extension__ __PRETTY_FUNCTION__));

502

503

if (NumParts == 0)

504

return;

505

506

assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 506
, __extension__ __PRETTY_FUNCTION__));

507

EVT PartEVT = PartVT;

508

if (PartEVT == ValueVT) {

509

assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 509
, __extension__ __PRETTY_FUNCTION__));

510

Parts[0] = Val;

511

return;

512

}

513

514

unsigned PartBits = PartVT.getSizeInBits();

515

if (NumParts * PartBits > ValueVT.getSizeInBits()) {

516

// If the parts cover more bits than the value has, promote the value.

517

if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

518

assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 518
, __extension__ __PRETTY_FUNCTION__));

519

Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);

520

} else {

521

if (ValueVT.isFloatingPoint()) {

522

// FP values need to be bitcast, then extended if they are being put

523

// into a larger container.

524

ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

525

Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

526

}

527

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__))

528

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__))

529

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__));

530

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

531

Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);

532

if (PartVT == MVT::x86mmx)

533

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

534

}

535

} else if (PartBits == ValueVT.getSizeInBits()) {

536

// Different types of the same size.

537

assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 537
, __extension__ __PRETTY_FUNCTION__));

538

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

539

} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {

540

// If the parts cover less bits than value has, truncate the value.

541

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__))

542

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__))

543

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__));

544

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

545

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

546

if (PartVT == MVT::x86mmx)

547

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

548

}

549

550

// The value may have changed - recompute ValueVT.

551

ValueVT = Val.getValueType();

552

assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553
, __extension__ __PRETTY_FUNCTION__))

553

"Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553
, __extension__ __PRETTY_FUNCTION__));

554

555

if (NumParts == 1) {

556

if (PartEVT != ValueVT) {

557

diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,

558

"scalar-to-vector conversion failed");

559

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

560

}

561

562

Parts[0] = Val;

563

return;

564

}

565

566

// Expand the value into multiple parts.

567

if (NumParts & (NumParts - 1)) {

568

// The number of parts is not a power of 2. Split off and copy the tail.

569

assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570
, __extension__ __PRETTY_FUNCTION__))

570

"Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570
, __extension__ __PRETTY_FUNCTION__));

571

unsigned RoundParts = llvm::bit_floor(NumParts);

572

unsigned RoundBits = RoundParts * PartBits;

573

unsigned OddParts = NumParts - RoundParts;

574

SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,

575

DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));

576

577

getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,

578

CallConv);

579

580

if (DAG.getDataLayout().isBigEndian())

581

// The odd parts were reversed by getCopyToParts - unreverse them.

582

std::reverse(Parts + RoundParts, Parts + NumParts);

583

584

NumParts = RoundParts;

585

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

586

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

587

}

588

589

// The number of parts is a power of 2. Repeatedly bisect the value using

590

// EXTRACT_ELEMENT.

591

Parts[0] = DAG.getNode(ISD::BITCAST, DL,

592

EVT::getIntegerVT(*DAG.getContext(),

593

ValueVT.getSizeInBits()),

594

Val);

595

596

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {

597

for (unsigned i = 0; i < NumParts; i += StepSize) {

598

unsigned ThisBits = StepSize * PartBits / 2;

599

EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);

600

SDValue &Part0 = Parts[i];

601

SDValue &Part1 = Parts[i+StepSize/2];

602

603

Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

604

ThisVT, Part0, DAG.getIntPtrConstant(1, DL));

605

Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

606

ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

607

608

if (ThisBits == PartBits && ThisVT != PartVT) {

609

Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);

610

Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);

611

}

612

}

613

}

614

615

if (DAG.getDataLayout().isBigEndian())

616

std::reverse(Parts, Parts + OrigNumParts);

617

}

618

619

static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,

620

const SDLoc &DL, EVT PartVT) {

621

if (!PartVT.isVector())

622

return SDValue();

623

624

EVT ValueVT = Val.getValueType();

625

ElementCount PartNumElts = PartVT.getVectorElementCount();

626

ElementCount ValueNumElts = ValueVT.getVectorElementCount();

627

628

// We only support widening vectors with equivalent element types and

629

// fixed/scalable properties. If a target needs to widen a fixed-length type

630

// to a scalable one, it should be possible to use INSERT_SUBVECTOR below.

631

if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||

632

PartNumElts.isScalable() != ValueNumElts.isScalable() ||

633

PartVT.getVectorElementType() != ValueVT.getVectorElementType())

634

return SDValue();

635

636

// Widening a scalable vector to another scalable vector is done by inserting

637

// the vector into a larger undef one.

638

if (PartNumElts.isScalable())

639

return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),

640

Val, DAG.getVectorIdxConstant(0, DL));

641

642

EVT ElementVT = PartVT.getVectorElementType();

643

// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in

644

// undef elements.

645

SmallVector<SDValue, 16> Ops;

646

DAG.ExtractVectorElements(Val, Ops);

647

SDValue EltUndef = DAG.getUNDEF(ElementVT);

648

Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);

649

650

// FIXME: Use CONCAT for 2x -> 4x.

651

return DAG.getBuildVector(PartVT, DL, Ops);

652

}

653

654

/// getCopyToPartsVector - Create a series of nodes that contain the specified

655

/// value split into legal parts.

656

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,

657

SDValue Val, SDValue *Parts, unsigned NumParts,

658

MVT PartVT, const Value *V,

659

std::optional<CallingConv::ID> CallConv) {

660

EVT ValueVT = Val.getValueType();

661

assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 661
, __extension__ __PRETTY_FUNCTION__));

662

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

663

const bool IsABIRegCopy = CallConv.has_value();

664

665

if (NumParts == 1) {

666

EVT PartEVT = PartVT;

667

if (PartEVT == ValueVT) {

668

// Nothing to do.

669

} else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {

670

// Bitconvert vector->vector case.

671

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

672

} else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {

673

Val = Widened;

674

} else if (PartVT.isVector() &&

675

PartEVT.getVectorElementType().bitsGE(

676

ValueVT.getVectorElementType()) &&

677

PartEVT.getVectorElementCount() ==

678

ValueVT.getVectorElementCount()) {

679

680

// Promoted vector extract

681

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

682

} else if (PartEVT.isVector() &&

683

PartEVT.getVectorElementType() !=

684

ValueVT.getVectorElementType() &&

685

TLI.getTypeAction(*DAG.getContext(), ValueVT) ==

686

TargetLowering::TypeWidenVector) {

687

// Combination of widening and promotion.

688

EVT WidenVT =

689

EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),

690

PartVT.getVectorElementCount());

691

SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);

692

Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);

693

} else {

694

// Don't extract an integer from a float vector. This can happen if the

695

// FP type gets softened to integer and then promoted. The promotion

696

// prevents it from being picked up by the earlier bitcast case.

697

if (ValueVT.getVectorElementCount().isScalar() &&

698

(!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {

699

Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,

700

DAG.getVectorIdxConstant(0, DL));

701

} else {

702

uint64_t ValueSize = ValueVT.getFixedSizeInBits();

703

assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704
, __extension__ __PRETTY_FUNCTION__))

704

"lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704
, __extension__ __PRETTY_FUNCTION__));

705

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

706

Val = DAG.getBitcast(IntermediateType, Val);

707

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

708

}

709

}

710

711

assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
"Unexpected vector part value type") ? void (0) : __assert_fail
("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 711
, __extension__ __PRETTY_FUNCTION__));

712

Parts[0] = Val;

713

return;

714

}

715

716

// Handle a multi-element vector.

717

EVT IntermediateVT;

718

MVT RegisterVT;

719

unsigned NumIntermediates;

720

unsigned NumRegs;

721

if (IsABIRegCopy) {

722

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

723

*DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,

724

RegisterVT);

725

} else {

726

NumRegs =

727

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

728

NumIntermediates, RegisterVT);

729

}

730

731

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 731
, __extension__ __PRETTY_FUNCTION__));

732

NumParts = NumRegs; // Silence a compiler warning.

733

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 733
, __extension__ __PRETTY_FUNCTION__));

734

735

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736
, __extension__ __PRETTY_FUNCTION__))

736

"Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736
, __extension__ __PRETTY_FUNCTION__));

737

738

std::optional<ElementCount> DestEltCnt;

739

740

if (IntermediateVT.isVector())

741

DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;

742

else

743

DestEltCnt = ElementCount::getFixed(NumIntermediates);

744

745

EVT BuiltVectorTy = EVT::getVectorVT(

746

*DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);

747

748

if (ValueVT == BuiltVectorTy) {

749

// Nothing to do.

750

} else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {

751

// Bitconvert vector->vector case.

752

Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);

753

} else {

754

if (BuiltVectorTy.getVectorElementType().bitsGT(

755

ValueVT.getVectorElementType())) {

756

// Integer promotion.

757

ValueVT = EVT::getVectorVT(*DAG.getContext(),

758

BuiltVectorTy.getVectorElementType(),

759

ValueVT.getVectorElementCount());

760

Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

761

}

762

763

if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {

764

Val = Widened;

765

}

766

}

767

768

assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy
&& "Unexpected vector value type") ? void (0) : __assert_fail
("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 768
, __extension__ __PRETTY_FUNCTION__));

769

770

// Split the vector into intermediate operands.

771

SmallVector<SDValue, 8> Ops(NumIntermediates);

772

for (unsigned i = 0; i != NumIntermediates; ++i) {

773

if (IntermediateVT.isVector()) {

774

// This does something sensible for scalable vectors - see the

775

// definition of EXTRACT_SUBVECTOR for further details.

776

unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();

777

Ops[i] =

778

DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,

779

DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));

780

} else {

781

Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,

782

DAG.getVectorIdxConstant(i, DL));

783

}

784

}

785

786

// Split the intermediate operands into legal parts.

787

if (NumParts == NumIntermediates) {

788

// If the register was not expanded, promote or copy the value,

789

// as appropriate.

790

for (unsigned i = 0; i != NumParts; ++i)

791

getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

792

} else if (NumParts > 0) {

793

// If the intermediate type was expanded, split each the value into

794

// legal parts.

795

assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 795
, __extension__ __PRETTY_FUNCTION__));

796

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797
, __extension__ __PRETTY_FUNCTION__))

797

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797
, __extension__ __PRETTY_FUNCTION__));

798

unsigned Factor = NumParts / NumIntermediates;

799

for (unsigned i = 0; i != NumIntermediates; ++i)

800

getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,

801

CallConv);

802

}

803

}

804

805

RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,

806

EVT valuevt, std::optional<CallingConv::ID> CC)

807

: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),

808

RegCount(1, regs.size()), CallConv(CC) {}

809

810

RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,

811

const DataLayout &DL, unsigned Reg, Type *Ty,

812

std::optional<CallingConv::ID> CC) {

813

ComputeValueVTs(TLI, DL, Ty, ValueVTs);

814

815

CallConv = CC;

816

817

for (EVT ValueVT : ValueVTs) {

818

unsigned NumRegs =

819

isABIMangled()

820

? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)

821

: TLI.getNumRegisters(Context, ValueVT);

822

MVT RegisterVT =

823

isABIMangled()

824

? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)

825

: TLI.getRegisterType(Context, ValueVT);

826

for (unsigned i = 0; i != NumRegs; ++i)

827

Regs.push_back(Reg + i);

828

RegVTs.push_back(RegisterVT);

829

RegCount.push_back(NumRegs);

830

Reg += NumRegs;

831

}

832

}

833

834

SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,

835

FunctionLoweringInfo &FuncInfo,

836

const SDLoc &dl, SDValue &Chain,

837

SDValue *Flag, const Value *V) const {

838

// A Value with type {} or [0 x %t] needs no registers.

839

if (ValueVTs.empty())

840

return SDValue();

841

842

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

843

844

// Assemble the legal parts into the final values.

845

SmallVector<SDValue, 4> Values(ValueVTs.size());

846

SmallVector<SDValue, 8> Parts;

847

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

848

// Copy the legal parts from the registers.

849

EVT ValueVT = ValueVTs[Value];

850

unsigned NumRegs = RegCount[Value];

851

MVT RegisterVT = isABIMangled()

852

? TLI.getRegisterTypeForCallingConv(

853

*DAG.getContext(), *CallConv, RegVTs[Value])

854

: RegVTs[Value];

855

856

Parts.resize(NumRegs);

857

for (unsigned i = 0; i != NumRegs; ++i) {

858

SDValue P;

859

if (!Flag) {

860

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);

861

} else {

862

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);

863

*Flag = P.getValue(2);

864

}

865

866

Chain = P.getValue(1);

867

Parts[i] = P;

868

869

// If the source register was virtual and if we know something about it,

870

// add an assert node.

871

if (!Register::isVirtualRegister(Regs[Part + i]) ||

872

!RegisterVT.isInteger())

873

continue;

874

875

const FunctionLoweringInfo::LiveOutInfo *LOI =

876

FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);

877

if (!LOI)

878

continue;

879

880

unsigned RegSize = RegisterVT.getScalarSizeInBits();

881

unsigned NumSignBits = LOI->NumSignBits;

882

unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

883

884

if (NumZeroBits == RegSize) {

885

// The current value is a zero.

886

// Explicitly express that as it would be easier for

887

// optimizations to kick in.

888

Parts[i] = DAG.getConstant(0, dl, RegisterVT);

889

continue;

890

}

891

892

// FIXME: We capture more information than the dag can represent. For

893

// now, just use the tightest assertzext/assertsext possible.

894

bool isSExt;

895

EVT FromVT(MVT::Other);

896

if (NumZeroBits) {

897

FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);

898

isSExt = false;

899

} else if (NumSignBits > 1) {

900

FromVT =

901

EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);

902

isSExt = true;

903

} else {

904

continue;

905

}

906

// Add an assertion node.

907

assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
__assert_fail ("FromVT != MVT::Other", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 907, __extension__ __PRETTY_FUNCTION__));

908

Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,

909

RegisterVT, P, DAG.getValueType(FromVT));

910

}

911

912

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,

913

RegisterVT, ValueVT, V, CallConv);

914

Part += NumRegs;

915

Parts.clear();

916

}

917

918

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);

919

}

920

921

void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,

922

const SDLoc &dl, SDValue &Chain, SDValue *Flag,

923

const Value *V,

924

ISD::NodeType PreferredExtendType) const {

925

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

926

ISD::NodeType ExtendKind = PreferredExtendType;

927

928

// Get the list of the values's legal parts.

929

unsigned NumRegs = Regs.size();

930

SmallVector<SDValue, 8> Parts(NumRegs);

931

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

932

unsigned NumParts = RegCount[Value];

933

934

MVT RegisterVT = isABIMangled()

935

? TLI.getRegisterTypeForCallingConv(

936

*DAG.getContext(), *CallConv, RegVTs[Value])

937

: RegVTs[Value];

938

939

if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))

940

ExtendKind = ISD::ZERO_EXTEND;

941

942

getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],

943

NumParts, RegisterVT, V, CallConv, ExtendKind);

944

Part += NumParts;

945

}

946

947

// Copy the parts into the registers.

948

SmallVector<SDValue, 8> Chains(NumRegs);

949

for (unsigned i = 0; i != NumRegs; ++i) {

950

SDValue Part;

951

if (!Flag) {

952

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);

953

} else {

954

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);

955

*Flag = Part.getValue(1);

956

}

957

958

Chains[i] = Part.getValue(0);

959

}

960

961

if (NumRegs == 1 || Flag)

962

// If NumRegs > 1 && Flag is used then the use of the last CopyToReg is

963

// flagged to it. That is the CopyToReg nodes and the user are considered

964

// a single scheduling unit. If we create a TokenFactor and return it as

965

// chain, then the TokenFactor is both a predecessor (operand) of the

966

// user as well as a successor (the TF operands are flagged to the user).

967

// c1, f1 = CopyToReg

968

// c2, f2 = CopyToReg

969

// c3 = TokenFactor c1, c2

970

// ...

971

// = op c3, ..., f2

972

Chain = Chains[NumRegs-1];

973

else

974

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);

975

}

976

977

void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,

978

unsigned MatchingIdx, const SDLoc &dl,

979

SelectionDAG &DAG,

980

std::vector<SDValue> &Ops) const {

981

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

982

983

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());

984

if (HasMatching)

985

Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);

986

else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {

987

// Put the register class of the virtual registers in the flag word. That

988

// way, later passes can recompute register class constraints for inline

989

// assembly as well as normal instructions.

990

// Don't do this for tied operands that can use the regclass information

991

// from the def.

992

const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

993

const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());

994

Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());

995

}

996

997

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);

998

Ops.push_back(Res);

999

1000

if (Code == InlineAsm::Kind_Clobber) {

1001

// Clobbers should always have a 1:1 mapping with registers, and may

1002

// reference registers that have illegal (e.g. vector) types. Hence, we

1003

// shouldn't try to apply any sort of splitting logic to them.

1004

assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005
, __extension__ __PRETTY_FUNCTION__))

1005

"No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005
, __extension__ __PRETTY_FUNCTION__));

1006

Register SP = TLI.getStackPointerRegisterToSaveRestore();

1007

(void)SP;

1008

for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {

1009

Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));

1010

assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))

1011

(Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))

1012

DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))

1013

"If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__));

1014

}

1015

return;

1016

}

1017

1018

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {

1019

MVT RegisterVT = RegVTs[Value];

1020

unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],

1021

RegisterVT);

1022

for (unsigned i = 0; i != NumRegs; ++i) {

1023

assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1023
, __extension__ __PRETTY_FUNCTION__));

1024

unsigned TheReg = Regs[Reg++];

1025

Ops.push_back(DAG.getRegister(TheReg, RegisterVT));

1026

}

1027

}

1028

}

1029

1030

SmallVector<std::pair<unsigned, TypeSize>, 4>

1031

RegsForValue::getRegsAndSizes() const {

1032

SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;

1033

unsigned I = 0;

1034

for (auto CountAndVT : zip_first(RegCount, RegVTs)) {

1035

unsigned RegCount = std::get<0>(CountAndVT);

1036

MVT RegisterVT = std::get<1>(CountAndVT);

1037

TypeSize RegisterSize = RegisterVT.getSizeInBits();

1038

for (unsigned E = I + RegCount; I != E; ++I)

1039

OutVec.push_back(std::make_pair(Regs[I], RegisterSize));

1040

}

1041

return OutVec;

1042

}

1043

1044

void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,

1045

AssumptionCache *ac,

1046

const TargetLibraryInfo *li) {

1047

AA = aa;

1048

AC = ac;

1049

GFI = gfi;

1050

LibInfo = li;

1051

Context = DAG.getContext();

1052

LPadToCallSiteMap.clear();

1053

SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());

1054

}

1055

1056

void SelectionDAGBuilder::clear() {

1057

NodeMap.clear();

1058

UnusedArgNodeMap.clear();

1059

PendingLoads.clear();

1060

PendingExports.clear();

1061

PendingConstrainedFP.clear();

1062

PendingConstrainedFPStrict.clear();

1063

CurInst = nullptr;

1064

HasTailCall = false;

1065

SDNodeOrder = LowestSDNodeOrder;

1066

StatepointLowering.clear();

1067

}

1068

1069

void SelectionDAGBuilder::clearDanglingDebugInfo() {

1070

DanglingDebugInfoMap.clear();

1071

}

1072

1073

// Update DAG root to include dependencies on Pending chains.

1074

SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {

1075

SDValue Root = DAG.getRoot();

1076

1077

if (Pending.empty())

1078

return Root;

1079

1080

// Add current root to PendingChains, unless we already indirectly

1081

// depend on it.

1082

if (Root.getOpcode() != ISD::EntryToken) {

1083

unsigned i = 0, e = Pending.size();

1084

for (; i != e; ++i) {

1085

assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1085
, __extension__ __PRETTY_FUNCTION__));

1086

if (Pending[i].getNode()->getOperand(0) == Root)

1087

break; // Don't add the root if we already indirectly depend on it.

1088

}

1089

1090

if (i == e)

1091

Pending.push_back(Root);

1092

}

1093

1094

if (Pending.size() == 1)

1095

Root = Pending[0];

1096

else

1097

Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

1098

1099

DAG.setRoot(Root);

1100

Pending.clear();

1101

return Root;

1102

}

1103

1104

SDValue SelectionDAGBuilder::getMemoryRoot() {

1105

return updateRoot(PendingLoads);

1106

}

1107

1108

SDValue SelectionDAGBuilder::getRoot() {

1109

// Chain up all pending constrained intrinsics together with all

1110

// pending loads, by simply appending them to PendingLoads and

1111

// then calling getMemoryRoot().

1112

PendingLoads.reserve(PendingLoads.size() +

1113

PendingConstrainedFP.size() +

1114

PendingConstrainedFPStrict.size());

1115

PendingLoads.append(PendingConstrainedFP.begin(),

1116

PendingConstrainedFP.end());

1117

PendingLoads.append(PendingConstrainedFPStrict.begin(),

1118

PendingConstrainedFPStrict.end());

1119

PendingConstrainedFP.clear();

1120

PendingConstrainedFPStrict.clear();

1121

return getMemoryRoot();

1122

}

1123

1124

SDValue SelectionDAGBuilder::getControlRoot() {

1125

// We need to emit pending fpexcept.strict constrained intrinsics,

1126

// so append them to the PendingExports list.

1127

PendingExports.append(PendingConstrainedFPStrict.begin(),

1128

PendingConstrainedFPStrict.end());

1129

PendingConstrainedFPStrict.clear();

1130

return updateRoot(PendingExports);

1131

}

1132

1133

void SelectionDAGBuilder::visit(const Instruction &I) {

1134

// Set up outgoing PHI node register values before emitting the terminator.

1135

if (I.isTerminator()) {

1136

HandlePHINodesInSuccessorBlocks(I.getParent());

1137

}

1138

1139

// Add SDDbgValue nodes for any var locs here. Do so before updating

1140

// SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.

1141

if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {

1142

// Add SDDbgValue nodes for any var locs here. Do so before updating

1143

// SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.

1144

for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);

1145

It != End; ++It) {

1146

auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);

1147

dropDanglingDebugInfo(Var, It->Expr);

1148

SmallVector<Value *> Values(It->Values.location_ops());

1149

if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,

1150

It->Values.hasArgList()))

1151

addDanglingDebugInfo(It, SDNodeOrder);

1152

}

1153

}

1154

1155

// Increase the SDNodeOrder if dealing with a non-debug instruction.

1156

if (!isa<DbgInfoIntrinsic>(I))

1157

++SDNodeOrder;

1158

1159

CurInst = &I;

1160

1161

// Set inserted listener only if required.

1162

bool NodeInserted = false;

1163

std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;

1164

MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);

1165

if (PCSectionsMD) {

1166

InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(

1167

DAG, [&](SDNode *) { NodeInserted = true; });

1168

}

1169

1170

visit(I.getOpcode(), I);

1171

1172

if (!I.isTerminator() && !HasTailCall &&

1173

!isa<GCStatepointInst>(I)) // statepoints handle their exports internally

1174

CopyToExportRegsIfNeeded(&I);

1175

1176

// Handle metadata.

1177

if (PCSectionsMD) {

1178

auto It = NodeMap.find(&I);

1179

if (It != NodeMap.end()) {

1180

DAG.addPCSections(It->second.getNode(), PCSectionsMD);

1181

} else if (NodeInserted) {

1182

// This should not happen; if it does, don't let it go unnoticed so we can

1183

// fix it. Relevant visit*() function is probably missing a setValue().

1184

errs() << "warning: loosing !pcsections metadata ["

1185

<< I.getModule()->getName() << "]\n";

1186

LLVM_DEBUG(I.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { I.dump(); } } while (false);

1187

assert(false)(static_cast <bool> (false) ? void (0) : __assert_fail (
"false", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1187, __extension__ __PRETTY_FUNCTION__));

1188

}

1189

}

1190

1191

CurInst = nullptr;

1192

}

1193

1194

void SelectionDAGBuilder::visitPHI(const PHINode &) {

1195

llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1195
);

1196

}

1197

1198

void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {

1199

// Note: this doesn't use InstVisitor, because it has to work with

1200

// ConstantExpr's in addition to instructions.

1201

switch (Opcode) {

1202

default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1202
);

1203

// Build the switch statement using the Instruction.def file.

1204

#define HANDLE_INST(NUM, OPCODE, CLASS) \

1205

case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;

1206

#include "llvm/IR/Instruction.def"

1207

}

1208

}

1209

1210

static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,

1211

DILocalVariable *Variable,

1212

DebugLoc DL, unsigned Order,

1213

RawLocationWrapper Values,

1214

DIExpression *Expression) {

1215

if (!Values.hasArgList())

1216

return false;

1217

// For variadic dbg_values we will now insert an undef.

1218

// FIXME: We can potentially recover these!

1219

SmallVector<SDDbgOperand, 2> Locs;

1220

for (const Value *V : Values.location_ops()) {

1221

auto *Undef = UndefValue::get(V->getType());

1222

Locs.push_back(SDDbgOperand::fromConst(Undef));

1223

}

1224

SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},

1225

/*IsIndirect=*/false, DL, Order,

1226

/*IsVariadic=*/true);

1227

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1228

return true;

1229

}

1230

1231

void SelectionDAGBuilder::addDanglingDebugInfo(const VarLocInfo *VarLoc,

1232

unsigned Order) {

1233

if (!handleDanglingVariadicDebugInfo(

1234

DAG,

1235

const_cast<DILocalVariable *>(DAG.getFunctionVarLocs()

1236

->getVariable(VarLoc->VariableID)

1237

.getVariable()),

1238

VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) {

1239

DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back(

1240

VarLoc, Order);

1241

}

1242

}

1243

1244

void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,

1245

unsigned Order) {

1246

// We treat variadic dbg_values differently at this stage.

1247

if (!handleDanglingVariadicDebugInfo(

1248

DAG, DI->getVariable(), DI->getDebugLoc(), Order,

1249

DI->getWrappedLocation(), DI->getExpression())) {

1250

// TODO: Dangling debug info will eventually either be resolved or produce

1251

// an Undef DBG_VALUE. However in the resolution case, a gap may appear

1252

// between the original dbg.value location and its resolved DBG_VALUE,

1253

// which we should ideally fill with an extra Undef DBG_VALUE.

1254

assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1256
, __extension__ __PRETTY_FUNCTION__))

1255

"DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1256
, __extension__ __PRETTY_FUNCTION__))

1256

"operand.")(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1256
, __extension__ __PRETTY_FUNCTION__));

1257

DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);

1258

}

1259

}

1260

1261

void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,

1262

const DIExpression *Expr) {

1263

auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {

1264

DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs());

1265

DIExpression *DanglingExpr = DDI.getExpression();

1266

if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {

1267

LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< printDDI(DDI) << "\n"; } } while (false)

1268

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< printDDI(DDI) << "\n"; } } while (false);

1269

return true;

1270

}

1271

return false;

1272

};

1273

1274

for (auto &DDIMI : DanglingDebugInfoMap) {

1275

DanglingDebugInfoVector &DDIV = DDIMI.second;

1276

1277

// If debug info is to be dropped, run it through final checks to see

1278

// whether it can be salvaged.

1279

for (auto &DDI : DDIV)

1280

if (isMatchingDbgValue(DDI))

1281

salvageUnresolvedDbgValue(DDI);

1282

1283

erase_if(DDIV, isMatchingDbgValue);

1284

}

1285

}

1286

1287

// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,

1288

// generate the debug data structures now that we've seen its definition.

1289

void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,

1290

SDValue Val) {

1291

auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);

1292

if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())

1293

return;

1294

1295

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;

1296

for (auto &DDI : DDIV) {

1297

DebugLoc DL = DDI.getDebugLoc();

1298

unsigned ValSDNodeOrder = Val.getNode()->getIROrder();

1299

unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();

1300

DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs());

1301

DIExpression *Expr = DDI.getExpression();

1302

assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1303
, __extension__ __PRETTY_FUNCTION__))

1303

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1303
, __extension__ __PRETTY_FUNCTION__));

1304

SDDbgValue *SDV;

1305

if (Val.getNode()) {

1306

// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a

1307

// FuncArgumentDbgValue (it would be hoisted to the function entry, and if

1308

// we couldn't resolve it directly when examining the DbgValue intrinsic

1309

// in the first place we should not be more successful here). Unless we

1310

// have some test case that prove this to be correct we should avoid

1311

// calling EmitFuncArgumentDbgValue here.

1312

if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,

1313

FuncArgumentDbgValueKind::Value, Val)) {

1314

LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info for "
<< printDDI(DDI) << "\n"; } } while (false)

1315

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info for "
<< printDDI(DDI) << "\n"; } } while (false);

1316

LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);

1317

// Increase the SDNodeOrder for the DbgValue here to make sure it is

1318

// inserted after the definition of Val when emitting the instructions

1319

// after ISel. An alternative could be to teach

1320

// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.

1321

LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1322

<< "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1323

<< ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);

1324

SDV = getDbgValue(Val, Variable, Expr, DL,

1325

std::max(DbgSDNodeOrder, ValSDNodeOrder));

1326

DAG.AddDbgValue(SDV, false);

1327

} else

1328

LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< printDDI(DDI) << " in EmitFuncArgumentDbgValue\n"
; } } while (false)

1329

<< printDDI(DDI) << " in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< printDDI(DDI) << " in EmitFuncArgumentDbgValue\n"
; } } while (false);

1330

} else {

1331

LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
printDDI(DDI) << "\n"; } } while (false);

1332

auto Undef = UndefValue::get(V->getType());

1333

auto SDV =

1334

DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);

1335

DAG.AddDbgValue(SDV, false);

1336

}

1337

}

1338

DDIV.clear();

1339

}

1340

1341

void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {

1342

// TODO: For the variadic implementation, instead of only checking the fail

1343

// state of `handleDebugValue`, we need know specifically which values were

1344

// invalid, so that we attempt to salvage only those values when processing

1345

// a DIArgList.

1346

Value *V = DDI.getVariableLocationOp(0);

1347

Value *OrigV = V;

1348

DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs());

1349

DIExpression *Expr = DDI.getExpression();

1350

DebugLoc DL = DDI.getDebugLoc();

1351

unsigned SDOrder = DDI.getSDNodeOrder();

1352

1353

// Currently we consider only dbg.value intrinsics -- we tell the salvager

1354

// that DW_OP_stack_value is desired.

1355

bool StackValue = true;

1356

1357

// Can this Value can be encoded without any further work?

1358

if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))

1359

return;

1360

1361

// Attempt to salvage back through as many instructions as possible. Bail if

1362

// a non-instruction is seen, such as a constant expression or global

1363

// variable. FIXME: Further work could recover those too.

1364

while (isa<Instruction>(V)) {

1365

Instruction &VAsInst = *cast<Instruction>(V);

1366

// Temporary "0", awaiting real implementation.

1367

SmallVector<uint64_t, 16> Ops;

1368

SmallVector<Value *, 4> AdditionalValues;

1369

V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,

1370

AdditionalValues);

1371

// If we cannot salvage any further, and haven't yet found a suitable debug

1372

// expression, bail out.

1373

if (!V)

1374

break;

1375

1376

// TODO: If AdditionalValues isn't empty, then the salvage can only be

1377

// represented with a DBG_VALUE_LIST, so we give up. When we have support

1378

// here for variadic dbg_values, remove that condition.

1379

if (!AdditionalValues.empty())

1380

break;

1381

1382

// New value and expr now represent this debuginfo.

1383

Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);

1384

1385

// Some kind of simplification occurred: check whether the operand of the

1386

// salvaged debug expression can be encoded in this DAG.

1387

if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {

1388

LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< *Var << "\n" << *OrigV << "\nBy stripping back to:\n "
<< *V << "\n"; } } while (false)

1389

dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< *Var << "\n" << *OrigV << "\nBy stripping back to:\n "
<< *V << "\n"; } } while (false)

1390

<< *OrigV << "\nBy stripping back to:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< *Var << "\n" << *OrigV << "\nBy stripping back to:\n "
<< *V << "\n"; } } while (false);

1391

return;

1392

}

1393

}

1394

1395

// This was the final opportunity to salvage this debug information, and it

1396

// couldn't be done. Place an undef DBG_VALUE at this location to terminate

1397

// any earlier variable location.

1398

assert(OrigV && "V shouldn't be null")(static_cast <bool> (OrigV && "V shouldn't be null"
) ? void (0) : __assert_fail ("OrigV && \"V shouldn't be null\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1398
, __extension__ __PRETTY_FUNCTION__));

1399

auto *Undef = UndefValue::get(OrigV->getType());

1400

auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);

1401

DAG.AddDbgValue(SDV, false);

1402

LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< printDDI(DDI) << "\n"; } } while (false)

1403

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< printDDI(DDI) << "\n"; } } while (false);

1404

}

1405

1406

bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,

1407

DILocalVariable *Var,

1408

DIExpression *Expr, DebugLoc DbgLoc,

1409

unsigned Order, bool IsVariadic) {

1410

if (Values.empty())

1411

return true;

1412

SmallVector<SDDbgOperand> LocationOps;

1413

SmallVector<SDNode *> Dependencies;

1414

for (const Value *V : Values) {

1415

// Constant value.

1416

if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||

1417

isa<ConstantPointerNull>(V)) {

1418

LocationOps.emplace_back(SDDbgOperand::fromConst(V));

1419

continue;

1420

}

1421

1422

// Look through IntToPtr constants.

1423

if (auto *CE = dyn_cast<ConstantExpr>(V))

1424

if (CE->getOpcode() == Instruction::IntToPtr) {

1425

LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));

1426

continue;

1427

}

1428

1429

// If the Value is a frame index, we can create a FrameIndex debug value

1430

// without relying on the DAG at all.

1431

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1432

auto SI = FuncInfo.StaticAllocaMap.find(AI);

1433

if (SI != FuncInfo.StaticAllocaMap.end()) {

1434

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));

1435

continue;

1436

}

1437

}

1438

1439

// Do not use getValue() in here; we don't want to generate code at

1440

// this point if it hasn't been done yet.

1441

SDValue N = NodeMap[V];

1442

if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.

1443

N = UnusedArgNodeMap[V];

1444

if (N.getNode()) {

1445

// Only emit func arg dbg value for non-variadic dbg.values for now.

1446

if (!IsVariadic &&

1447

EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,

1448

FuncArgumentDbgValueKind::Value, N))

1449

return true;

1450

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

1451

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can

1452

// describe stack slot locations.

1453

//

1454

// Consider "int x = 0; int *px = &x;". There are two kinds of

1455

// interesting debug values here after optimization:

1456

//

1457

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

1458

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

1459

//

1460

// Both describe the direct values of their associated variables.

1461

Dependencies.push_back(N.getNode());

1462

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));

1463

continue;

1464

}

1465

LocationOps.emplace_back(

1466

SDDbgOperand::fromNode(N.getNode(), N.getResNo()));

1467

continue;

1468

}

1469

1470

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1471

// Special rules apply for the first dbg.values of parameter variables in a

1472

// function. Identify them by the fact they reference Argument Values, that

1473

// they're parameters, and they are parameters of the current function. We

1474

// need to let them dangle until they get an SDNode.

1475

bool IsParamOfFunc =

1476

isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();

1477

if (IsParamOfFunc)

1478

return false;

1479

1480

// The value is not used in this block yet (or it would have an SDNode).

1481

// We still want the value to appear for the user if possible -- if it has

1482

// an associated VReg, we can refer to that instead.

1483

auto VMI = FuncInfo.ValueMap.find(V);

1484

if (VMI != FuncInfo.ValueMap.end()) {

1485

unsigned Reg = VMI->second;

1486

// If this is a PHI node, it may be split up into several MI PHI nodes

1487

// (in FunctionLoweringInfo::set).

1488

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,

1489

V->getType(), std::nullopt);

1490

if (RFV.occupiesMultipleRegs()) {

1491

// FIXME: We could potentially support variadic dbg_values here.

1492

if (IsVariadic)

1493

return false;

1494

unsigned Offset = 0;

1495

unsigned BitsToDescribe = 0;

1496

if (auto VarSize = Var->getSizeInBits())

1497

BitsToDescribe = *VarSize;

1498

if (auto Fragment = Expr->getFragmentInfo())

1499

BitsToDescribe = Fragment->SizeInBits;

1500

for (const auto &RegAndSize : RFV.getRegsAndSizes()) {

1501

// Bail out if all bits are described already.

1502

if (Offset >= BitsToDescribe)

1503

break;

1504

// TODO: handle scalable vectors.

1505

unsigned RegisterSize = RegAndSize.second;

1506

unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)

1507

? BitsToDescribe - Offset

1508

: RegisterSize;

1509

auto FragmentExpr = DIExpression::createFragmentExpression(

1510

Expr, Offset, FragmentSize);

1511

if (!FragmentExpr)

1512

continue;

1513

SDDbgValue *SDV = DAG.getVRegDbgValue(

1514

Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);

1515

DAG.AddDbgValue(SDV, false);

1516

Offset += RegisterSize;

1517

}

1518

return true;

1519

}

1520

// We can use simple vreg locations for variadic dbg_values as well.

1521

LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));

1522

continue;

1523

}

1524

// We failed to create a SDDbgOperand for V.

1525

return false;

1526

}

1527

1528

// We have created a SDDbgOperand for each Value in Values.

1529

// Should use Order instead of SDNodeOrder?

1530

assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) :
__assert_fail ("!LocationOps.empty()", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1530, __extension__ __PRETTY_FUNCTION__));

1531

SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,

1532

/*IsIndirect=*/false, DbgLoc,

1533

SDNodeOrder, IsVariadic);

1534

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1535

return true;

1536

}

1537

1538

void SelectionDAGBuilder::resolveOrClearDbgInfo() {

1539

// Try to fixup any remaining dangling debug info -- and drop it if we can't.

1540

for (auto &Pair : DanglingDebugInfoMap)

1541

for (auto &DDI : Pair.second)

1542

salvageUnresolvedDbgValue(DDI);

1543

clearDanglingDebugInfo();

1544

}

1545

1546

/// getCopyFromRegs - If there was virtual register allocated for the value V

1547

/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.

1548

SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {

1549

DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);

1550

SDValue Result;

1551

1552

if (It != FuncInfo.ValueMap.end()) {

1553

Register InReg = It->second;

1554

1555

RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1556

DAG.getDataLayout(), InReg, Ty,

1557

std::nullopt); // This is not an ABI copy.

1558

SDValue Chain = DAG.getEntryNode();

1559

Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,

1560

V);

1561

resolveDanglingDebugInfo(V, Result);

1562

}

1563

1564

return Result;

1565

}

1566

1567

/// getValue - Return an SDValue for the given Value.

1568

SDValue SelectionDAGBuilder::getValue(const Value *V) {

1569

// If we already have an SDValue for this value, use it. It's important

1570

// to do this first, so that we don't create a CopyFromReg if we already

1571

// have a regular SDValue.

1572

SDValue &N = NodeMap[V];

1573

if (N.getNode()) return N;

1574

1575

// If there's a virtual register allocated and initialized for this

1576

// value, use it.

1577

if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))

1578

return copyFromReg;

1579

1580

// Otherwise create a new SDValue and remember it.

1581

SDValue Val = getValueImpl(V);

1582

NodeMap[V] = Val;

1583

resolveDanglingDebugInfo(V, Val);

1584

return Val;

1585

}

1586

1587

/// getNonRegisterValue - Return an SDValue for the given Value, but

1588

/// don't look in FuncInfo.ValueMap for a virtual register.

1589

SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {

1590

// If we already have an SDValue for this value, use it.

1591

SDValue &N = NodeMap[V];

1592

if (N.getNode()) {

1593

if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {

1594

// Remove the debug location from the node as the node is about to be used

1595

// in a location which may differ from the original debug location. This

1596

// is relevant to Constant and ConstantFP nodes because they can appear

1597

// as constant expressions inside PHI nodes.

1598

N->setDebugLoc(DebugLoc());

1599

}

1600

return N;

1601

}

1602

1603

// Otherwise create a new SDValue and remember it.

1604

SDValue Val = getValueImpl(V);

1605

NodeMap[V] = Val;

1606

resolveDanglingDebugInfo(V, Val);

1607

return Val;

1608

}

1609

1610

/// getValueImpl - Helper function for getValue and getNonRegisterValue.

1611

/// Create an SDValue for the given value.

1612

SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {

1613

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1614

1615

if (const Constant *C = dyn_cast<Constant>(V)) {

1616

EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

1617

1618

if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))

1619

return DAG.getConstant(*CI, getCurSDLoc(), VT);

1620

1621

if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))

1622

return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

1623

1624

if (isa<ConstantPointerNull>(C)) {

1625

unsigned AS = V->getType()->getPointerAddressSpace();

1626

return DAG.getConstant(0, getCurSDLoc(),

1627

TLI.getPointerTy(DAG.getDataLayout(), AS));

1628

}

1629

1630

if (match(C, m_VScale()))

1631

return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

1632

1633

if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))

1634

return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

1635

1636

if (isa<UndefValue>(C) && !V->getType()->isAggregateType())

1637

return DAG.getUNDEF(VT);

1638

1639

if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {

1640

visit(CE->getOpcode(), *CE);

1641

SDValue N1 = NodeMap[V];

1642

assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1642
, __extension__ __PRETTY_FUNCTION__));

1643

return N1;

1644

}

1645

1646

if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {

1647

SmallVector<SDValue, 4> Constants;

1648

for (const Use &U : C->operands()) {

1649

SDNode *Val = getValue(U).getNode();

1650

// If the operand is an empty aggregate, there are no values.

1651

if (!Val) continue;

1652

// Add each leaf value from the operand to the Constants list

1653

// to form a flattened list of all the values.

1654

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1655

Constants.push_back(SDValue(Val, i));

1656

}

1657

1658

return DAG.getMergeValues(Constants, getCurSDLoc());

1659

}

1660

1661

if (const ConstantDataSequential *CDS =

1662

dyn_cast<ConstantDataSequential>(C)) {

1663

SmallVector<SDValue, 4> Ops;

1664

for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {

1665

SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();

1666

// Add each leaf value from the operand to the Constants list

1667

// to form a flattened list of all the values.

1668

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1669

Ops.push_back(SDValue(Val, i));

1670

}

1671

1672

if (isa<ArrayType>(CDS->getType()))

1673

return DAG.getMergeValues(Ops, getCurSDLoc());

1674

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1675

}

1676

1677

if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {

1678

assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1679
, __extension__ __PRETTY_FUNCTION__))

1679

"Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1679
, __extension__ __PRETTY_FUNCTION__));

1680

1681

SmallVector<EVT, 4> ValueVTs;

1682

ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);

1683

unsigned NumElts = ValueVTs.size();

1684

if (NumElts == 0)

1685

return SDValue(); // empty struct

1686

SmallVector<SDValue, 4> Constants(NumElts);

1687

for (unsigned i = 0; i != NumElts; ++i) {

1688

EVT EltVT = ValueVTs[i];

1689

if (isa<UndefValue>(C))

1690

Constants[i] = DAG.getUNDEF(EltVT);

1691

else if (EltVT.isFloatingPoint())

1692

Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1693

else

1694

Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);

1695

}

1696

1697

return DAG.getMergeValues(Constants, getCurSDLoc());

1698

}

1699

1700

if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))

1701

return DAG.getBlockAddress(BA, VT);

1702

1703

if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))

1704

return getValue(Equiv->getGlobalValue());

1705

1706

if (const auto *NC = dyn_cast<NoCFIValue>(C))

1707

return getValue(NC->getGlobalValue());

1708

1709

VectorType *VecTy = cast<VectorType>(V->getType());

1710

1711

// Now that we know the number and type of the elements, get that number of

1712

// elements into the Ops array based on what kind of constant it is.

1713

if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {

1714

SmallVector<SDValue, 16> Ops;

1715

unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();

1716

for (unsigned i = 0; i != NumElements; ++i)

1717

Ops.push_back(getValue(CV->getOperand(i)));

1718

1719

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1720

}

1721

1722

if (isa<ConstantAggregateZero>(C)) {

1723

EVT EltVT =

1724

TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

1725

1726

SDValue Op;

1727

if (EltVT.isFloatingPoint())

1728

Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1729

else

1730

Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

1731

1732

return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);

1733

}

1734

1735

llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1735);

1736

}

1737

1738

// If this is a static alloca, generate it as the frameindex instead of

1739

// computation.

1740

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1741

DenseMap<const AllocaInst*, int>::iterator SI =

1742

FuncInfo.StaticAllocaMap.find(AI);

1743

if (SI != FuncInfo.StaticAllocaMap.end())

1744

return DAG.getFrameIndex(

1745

SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));

1746

}

1747

1748

// If this is an instruction which fast-isel has deferred, select it now.

1749

if (const Instruction *Inst = dyn_cast<Instruction>(V)) {

1750

Register InReg = FuncInfo.InitializeRegForValue(Inst);

1751

1752

RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,

1753

Inst->getType(), std::nullopt);

1754

SDValue Chain = DAG.getEntryNode();

1755

return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);

1756

}

1757

1758

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))

1759

return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

1760

1761

if (const auto *BB = dyn_cast<BasicBlock>(V))

1762

return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);

1763

1764

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1764
);

1765

}

1766

1767

void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {

1768

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1769

bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;

1770

bool IsCoreCLR = Pers == EHPersonality::CoreCLR;

1771

bool IsSEH = isAsynchronousEHPersonality(Pers);

1772

MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;

1773

if (!IsSEH)

1774

CatchPadMBB->setIsEHScopeEntry();

1775

// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.

1776

if (IsMSVCCXX || IsCoreCLR)

1777

CatchPadMBB->setIsEHFuncletEntry();

1778

}

1779

1780

void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {

1781

// Update machine-CFG edge.

1782

MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];

1783

FuncInfo.MBB->addSuccessor(TargetMBB);

1784

TargetMBB->setIsEHCatchretTarget(true);

1785

DAG.getMachineFunction().setHasEHCatchret(true);

1786

1787

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1788

bool IsSEH = isAsynchronousEHPersonality(Pers);

1789

if (IsSEH) {

1790

// If this is not a fall-through branch or optimizations are switched off,

1791

// emit the branch.

1792

if (TargetMBB != NextBlock(FuncInfo.MBB) ||

1793

TM.getOptLevel() == CodeGenOpt::None)

1794

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,

1795

getControlRoot(), DAG.getBasicBlock(TargetMBB)));

1796

return;

1797

}

1798

1799

// Figure out the funclet membership for the catchret's successor.

1800

// This will be used by the FuncletLayout pass to determine how to order the

1801

// BB's.

1802

// A 'catchret' returns to the outer scope's color.

1803

Value *ParentPad = I.getCatchSwitchParentPad();

1804

const BasicBlock *SuccessorColor;

1805

if (isa<ConstantTokenNone>(ParentPad))

1806

SuccessorColor = &FuncInfo.Fn->getEntryBlock();

1807

else

1808

SuccessorColor = cast<Instruction>(ParentPad)->getParent();

1809

assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1809
, __extension__ __PRETTY_FUNCTION__));

1810

MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];

1811

assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1811
, __extension__ __PRETTY_FUNCTION__));

1812

1813

// Create the terminator node.

1814

SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,

1815

getControlRoot(), DAG.getBasicBlock(TargetMBB),

1816

DAG.getBasicBlock(SuccessorColorMBB));

1817

DAG.setRoot(Ret);

1818

}

1819

1820

void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {

1821

// Don't emit any special code for the cleanuppad instruction. It just marks

1822

// the start of an EH scope/funclet.

1823

FuncInfo.MBB->setIsEHScopeEntry();

1824

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1825

if (Pers != EHPersonality::Wasm_CXX) {

1826

FuncInfo.MBB->setIsEHFuncletEntry();

1827

FuncInfo.MBB->setIsCleanupFuncletEntry();

1828

}

1829

}

1830

1831

// In wasm EH, even though a catchpad may not catch an exception if a tag does

1832

// not match, it is OK to add only the first unwind destination catchpad to the

1833

// successors, because there will be at least one invoke instruction within the

1834

// catch scope that points to the next unwind destination, if one exists, so

1835

// CFGSort cannot mess up with BB sorting order.

1836

// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic

1837

// call within them, and catchpads only consisting of 'catch (...)' have a

1838

// '__cxa_end_catch' call within them, both of which generate invokes in case

1839

// the next unwind destination exists, i.e., the next unwind destination is not

1840

// the caller.)

1841

//

1842

// Having at most one EH pad successor is also simpler and helps later

1843

// transformations.

1844

//

1845

// For example,

1846

// current:

1847

// invoke void @foo to ... unwind label %catch.dispatch

1848

// catch.dispatch:

1849

// %0 = catchswitch within ... [label %catch.start] unwind label %next

1850

// catch.start:

1851

// ...

1852

// ... in this BB or some other child BB dominated by this BB there will be an

1853

// invoke that points to 'next' BB as an unwind destination

1854

//

1855

// next: ; We don't need to add this to 'current' BB's successor

1856

// ...

1857

static void findWasmUnwindDestinations(

1858

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1859

BranchProbability Prob,

1860

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1861

&UnwindDests) {

1862

while (EHPadBB) {

1863

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1864

if (isa<CleanupPadInst>(Pad)) {

1865

// Stop on cleanup pads.

1866

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1867

UnwindDests.back().first->setIsEHScopeEntry();

1868

break;

1869

} else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1870

// Add the catchpad handlers to the possible destinations. We don't

1871

// continue to the unwind destination of the catchswitch for wasm.

1872

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1873

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1874

UnwindDests.back().first->setIsEHScopeEntry();

1875

}

1876

break;

1877

} else {

1878

continue;

1879

}

1880

}

1881

}

1882

1883

/// When an invoke or a cleanupret unwinds to the next EH pad, there are

1884

/// many places it could ultimately go. In the IR, we have a single unwind

1885

/// destination, but in the machine CFG, we enumerate all the possible blocks.

1886

/// This function skips over imaginary basic blocks that hold catchswitch

1887

/// instructions, and finds all the "real" machine

1888

/// basic block destinations. As those destinations may not be successors of

1889

/// EHPadBB, here we also calculate the edge probability to those destinations.

1890

/// The passed-in Prob is the edge probability to EHPadBB.

1891

static void findUnwindDestinations(

1892

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1893

BranchProbability Prob,

1894

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1895

&UnwindDests) {

1896

EHPersonality Personality =

1897

classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1898

bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;

1899

bool IsCoreCLR = Personality == EHPersonality::CoreCLR;

1900

bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;

1901

bool IsSEH = isAsynchronousEHPersonality(Personality);

1902

1903

if (IsWasmCXX) {

1904

findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);

1905

assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1906
, __extension__ __PRETTY_FUNCTION__))

1906

"There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1906
, __extension__ __PRETTY_FUNCTION__));

1907

return;

1908

}

1909

1910

while (EHPadBB) {

1911

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1912

BasicBlock *NewEHPadBB = nullptr;

1913

if (isa<LandingPadInst>(Pad)) {

1914

// Stop on landingpads. They are not funclets.

1915

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1916

break;

1917

} else if (isa<CleanupPadInst>(Pad)) {

1918

// Stop on cleanup pads. Cleanups are always funclet entries for all known

1919

// personalities.

1920

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1921

UnwindDests.back().first->setIsEHScopeEntry();

1922

UnwindDests.back().first->setIsEHFuncletEntry();

1923

break;

1924

} else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1925

// Add the catchpad handlers to the possible destinations.

1926

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1927

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1928

// For MSVC++ and the CLR, catchblocks are funclets and need prologues.

1929

if (IsMSVCCXX || IsCoreCLR)

1930

UnwindDests.back().first->setIsEHFuncletEntry();

1931

if (!IsSEH)

1932

UnwindDests.back().first->setIsEHScopeEntry();

1933

}

1934

NewEHPadBB = CatchSwitch->getUnwindDest();

1935

} else {

1936

continue;

1937

}

1938

1939

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1940

if (BPI && NewEHPadBB)

1941

Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);

1942

EHPadBB = NewEHPadBB;

1943

}

1944

}

1945

1946

void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {

1947

// Update successor info.

1948

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

1949

auto UnwindDest = I.getUnwindDest();

1950

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1951

BranchProbability UnwindDestProb =

1952

(BPI && UnwindDest)

1953

? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)

1954

: BranchProbability::getZero();

1955

findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);

1956

for (auto &UnwindDest : UnwindDests) {

1957

UnwindDest.first->setIsEHPad();

1958

addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);

1959

}

1960

FuncInfo.MBB->normalizeSuccProbs();

1961

1962

// Create the terminator node.

1963

SDValue Ret =

1964

DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());

1965

DAG.setRoot(Ret);

1966

}

1967

1968

void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {

1969

report_fatal_error("visitCatchSwitch not yet implemented!");

1970

}

1971

1972

void SelectionDAGBuilder::visitRet(const ReturnInst &I) {

1973

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1974

auto &DL = DAG.getDataLayout();

1975

SDValue Chain = getControlRoot();

1976

SmallVector<ISD::OutputArg, 8> Outs;

1977

SmallVector<SDValue, 8> OutVals;

1978

1979

// Calls to @llvm.experimental.deoptimize don't generate a return value, so

1980

// lower

1981

//

1982

// %val = call <ty> @llvm.experimental.deoptimize()

1983

// ret <ty> %val

1984

//

1985

// differently.

1986

if (I.getParent()->getTerminatingDeoptimizeCall()) {

1987

LowerDeoptimizingReturn();

1988

return;

1989

}

1990

1991

if (!FuncInfo.CanLowerReturn) {

1992

unsigned DemoteReg = FuncInfo.DemoteRegister;

1993

const Function *F = I.getParent()->getParent();

1994

1995

// Emit a store of the return value through the virtual register.

1996

// Leave Outs empty so that LowerReturn won't try to load return

1997

// registers the usual way.

1998

SmallVector<EVT, 1> PtrValueVTs;

1999

ComputeValueVTs(TLI, DL,

2000

F->getReturnType()->getPointerTo(

2001

DAG.getDataLayout().getAllocaAddrSpace()),

2002

PtrValueVTs);

2003

2004

SDValue RetPtr =

2005

DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);

2006

SDValue RetOp = getValue(I.getOperand(0));

2007

2008

SmallVector<EVT, 4> ValueVTs, MemVTs;

2009

SmallVector<uint64_t, 4> Offsets;

2010

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,

2011

&Offsets);

2012

unsigned NumValues = ValueVTs.size();

2013

2014

SmallVector<SDValue, 4> Chains(NumValues);

2015

Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());

2016

for (unsigned i = 0; i != NumValues; ++i) {

2017

// An aggregate return value cannot wrap around the address space, so

2018

// offsets to its parts don't wrap either.

2019

SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,

2020

TypeSize::Fixed(Offsets[i]));

2021

2022

SDValue Val = RetOp.getValue(RetOp.getResNo() + i);

2023

if (MemVTs[i] != ValueVTs[i])

2024

Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);

2025

Chains[i] = DAG.getStore(

2026

Chain, getCurSDLoc(), Val,

2027

// FIXME: better loc info would be nice.

2028

Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),

2029

commonAlignment(BaseAlign, Offsets[i]));

2030

}

2031

2032

Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),

2033

MVT::Other, Chains);

2034

} else if (I.getNumOperands() != 0) {

2035

SmallVector<EVT, 4> ValueVTs;

2036

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);

2037

unsigned NumValues = ValueVTs.size();

2038

if (NumValues) {

2039

SDValue RetOp = getValue(I.getOperand(0));

2040

2041

const Function *F = I.getParent()->getParent();

2042

2043

bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(

2044

I.getOperand(0)->getType(), F->getCallingConv(),

2045

/*IsVarArg*/ false, DL);

2046

2047

ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

2048

if (F->getAttributes().hasRetAttr(Attribute::SExt))

2049

ExtendKind = ISD::SIGN_EXTEND;

2050

else if (F->getAttributes().hasRetAttr(Attribute::ZExt))

2051

ExtendKind = ISD::ZERO_EXTEND;

2052

2053

LLVMContext &Context = F->getContext();

2054

bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);

2055

2056

for (unsigned j = 0; j != NumValues; ++j) {

2057

EVT VT = ValueVTs[j];

2058

2059

if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())

2060

VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

2061

2062

CallingConv::ID CC = F->getCallingConv();

2063

2064

unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);

2065

MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);

2066

SmallVector<SDValue, 4> Parts(NumParts);

2067

getCopyToParts(DAG, getCurSDLoc(),

2068

SDValue(RetOp.getNode(), RetOp.getResNo() + j),

2069

&Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

2070

2071

// 'inreg' on function refers to return value

2072

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

2073

if (RetInReg)

2074

Flags.setInReg();

2075

2076

if (I.getOperand(0)->getType()->isPointerTy()) {

2077

Flags.setPointer();

2078

Flags.setPointerAddrSpace(

2079

cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());

2080

}

2081

2082

if (NeedsRegBlock) {

2083

Flags.setInConsecutiveRegs();

2084

if (j == NumValues - 1)

2085

Flags.setInConsecutiveRegsLast();

2086

}

2087

2088

// Propagate extension type if any

2089

if (ExtendKind == ISD::SIGN_EXTEND)

2090

Flags.setSExt();

2091

else if (ExtendKind == ISD::ZERO_EXTEND)

2092

Flags.setZExt();

2093

2094

for (unsigned i = 0; i < NumParts; ++i) {

2095

Outs.push_back(ISD::OutputArg(Flags,

2096

Parts[i].getValueType().getSimpleVT(),

2097

VT, /*isfixed=*/true, 0, 0));

2098

OutVals.push_back(Parts[i]);

2099

}

2100

}

2101

}

2102

}

2103

2104

// Push in swifterror virtual register as the last element of Outs. This makes

2105

// sure swifterror virtual register will be returned in the swifterror

2106

// physical register.

2107

const Function *F = I.getParent()->getParent();

2108

if (TLI.supportSwiftError() &&

2109

F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {

2110

assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() &&
"Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2110
, __extension__ __PRETTY_FUNCTION__));

2111

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

2112

Flags.setSwiftError();

2113

Outs.push_back(ISD::OutputArg(

2114

Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),

2115

/*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));

2116

// Create SDNode for the swifterror virtual register.

2117

OutVals.push_back(

2118

DAG.getRegister(SwiftError.getOrCreateVRegUseAt(

2119

&I, FuncInfo.MBB, SwiftError.getFunctionArg()),

2120

EVT(TLI.getPointerTy(DL))));

2121

}

2122

2123

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();

2124

CallingConv::ID CallConv =

2125

DAG.getMachineFunction().getFunction().getCallingConv();

2126

Chain = DAG.getTargetLoweringInfo().LowerReturn(

2127

Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

2128

2129

// Verify that the target's LowerReturn behaved as expected.

2130

assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2131
, __extension__ __PRETTY_FUNCTION__))

2131

"LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2131
, __extension__ __PRETTY_FUNCTION__));

2132

2133

// Update the DAG with the new chain value resulting from return lowering.

2134

DAG.setRoot(Chain);

2135

}

2136

2137

/// CopyToExportRegsIfNeeded - If the given value has virtual registers

2138

/// created for it, emit nodes to copy the value into the virtual

2139

/// registers.

2140

void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

2141

// Skip empty types

2142

if (V->getType()->isEmptyTy())

2143

return;

2144

2145

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);

2146

if (VMI != FuncInfo.ValueMap.end()) {

2147

assert((!V->use_empty() || isa<CallBrInst>(V)) &&(static_cast <bool> ((!V->use_empty() || isa<CallBrInst
>(V)) && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2148
, __extension__ __PRETTY_FUNCTION__))

2148

"Unused value assigned virtual registers!")(static_cast <bool> ((!V->use_empty() || isa<CallBrInst
>(V)) && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2148
, __extension__ __PRETTY_FUNCTION__));

2149

CopyValueToVirtualRegister(V, VMI->second);

2150

}

2151

}

2152

2153

/// ExportFromCurrentBlock - If this condition isn't known to be exported from

2154

/// the current basic block, add it to ValueMap now so that we'll get a

2155

/// CopyTo/FromReg.

2156

void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {

2157

// No need to export constants.

2158

if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

2159

2160

// Already exported?

2161

if (FuncInfo.isExportedInst(V)) return;

2162

2163

Register Reg = FuncInfo.InitializeRegForValue(V);

2164

CopyValueToVirtualRegister(V, Reg);

2165

}

2166

2167

bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,

2168

const BasicBlock *FromBB) {

2169

// The operands of the setcc have to be in this block. We don't know

2170

// how to export them from some other block.

2171

if (const Instruction *VI = dyn_cast<Instruction>(V)) {

2172

// Can export from current BB.

2173

if (VI->getParent() == FromBB)

2174

return true;

2175

2176

// Is already exported, noop.

2177

return FuncInfo.isExportedInst(V);

2178

}

2179

2180

// If this is an argument, we can export it if the BB is the entry block or

2181

// if it is already exported.

2182

if (isa<Argument>(V)) {

2183

if (FromBB->isEntryBlock())

2184

return true;

2185

2186

// Otherwise, can only export this if it is already exported.

2187

return FuncInfo.isExportedInst(V);

2188

}

2189

2190

// Otherwise, constants can always be exported.

2191

return true;

2192

}

2193

2194

/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.

2195

BranchProbability

2196

SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,

2197

const MachineBasicBlock *Dst) const {

2198

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2199

const BasicBlock *SrcBB = Src->getBasicBlock();

2200

const BasicBlock *DstBB = Dst->getBasicBlock();

2201

if (!BPI) {

2202

// If BPI is not available, set the default probability as 1 / N, where N is

2203

// the number of successors.

2204

auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);

2205

return BranchProbability(1, SuccSize);

2206

}

2207

return BPI->getEdgeProbability(SrcBB, DstBB);

2208

}

2209

2210

void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,

2211

MachineBasicBlock *Dst,

2212

BranchProbability Prob) {

2213

if (!FuncInfo.BPI)

2214

Src->addSuccessorWithoutProb(Dst);

2215

else {

2216

if (Prob.isUnknown())

2217

Prob = getEdgeProbability(Src, Dst);

2218

Src->addSuccessor(Dst, Prob);

2219

}

2220

}

2221

2222

static bool InBlock(const Value *V, const BasicBlock *BB) {

2223

if (const Instruction *I = dyn_cast<Instruction>(V))

2224

return I->getParent() == BB;

2225

return true;

2226

}

2227

2228

/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.

2229

/// This function emits a branch and is used at the leaves of an OR or an

2230

/// AND operator tree.

2231

void

2232

SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,

2233

MachineBasicBlock *TBB,

2234

MachineBasicBlock *FBB,

2235

MachineBasicBlock *CurBB,

2236

MachineBasicBlock *SwitchBB,

2237

BranchProbability TProb,

2238

BranchProbability FProb,

2239

bool InvertCond) {

2240

const BasicBlock *BB = CurBB->getBasicBlock();

2241

2242

// If the leaf of the tree is a comparison, merge the condition into

2243

// the caseblock.

2244

if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {

2245

// The operands of the cmp have to be in this block. We don't know

2246

// how to export them from some other block. If this is the first block

2247

// of the sequence, no exporting is needed.

2248

if (CurBB == SwitchBB ||

2249

(isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&

2250

isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {

2251

ISD::CondCode Condition;

2252

if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {

2253

ICmpInst::Predicate Pred =

2254

InvertCond ? IC->getInversePredicate() : IC->getPredicate();

2255

Condition = getICmpCondCode(Pred);

2256

} else {

2257

const FCmpInst *FC = cast<FCmpInst>(Cond);

2258

FCmpInst::Predicate Pred =

2259

InvertCond ? FC->getInversePredicate() : FC->getPredicate();

2260

Condition = getFCmpCondCode(Pred);

2261

if (TM.Options.NoNaNsFPMath)

2262

Condition = getFCmpCodeWithoutNaN(Condition);

2263

}

2264

2265

CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,

2266

TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2267

SL->SwitchCases.push_back(CB);

2268

return;

2269

}

2270

}

2271

2272

// Create a CaseBlock record representing this branch.

2273

ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;

2274

CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),

2275

nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2276

SL->SwitchCases.push_back(CB);

2277

}

2278

2279

void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,

2280

MachineBasicBlock *TBB,

2281

MachineBasicBlock *FBB,

2282

MachineBasicBlock *CurBB,

2283

MachineBasicBlock *SwitchBB,

2284

Instruction::BinaryOps Opc,

2285

BranchProbability TProb,

2286

BranchProbability FProb,

2287

bool InvertCond) {

2288

// Skip over not part of the tree and remember to invert op and operands at

2289

// next level.

2290

Value *NotCond;

2291

if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&

2292

InBlock(NotCond, CurBB->getBasicBlock())) {

2293

FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,

2294

!InvertCond);

2295

return;

2296

}

2297

2298

const Instruction *BOp = dyn_cast<Instruction>(Cond);

2299

const Value *BOpOp0, *BOpOp1;

2300

// Compute the effective opcode for Cond, taking into account whether it needs

2301

// to be inverted, e.g.

2302

// and (not (or A, B)), C

2303

// gets lowered as

2304

// and (and (not A, not B), C)

2305

Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;

2306

if (BOp) {

2307

BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))

2308

? Instruction::And

2309

: (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))

2310

? Instruction::Or

2311

: (Instruction::BinaryOps)0);

2312

if (InvertCond) {

2313

if (BOpc == Instruction::And)

2314

BOpc = Instruction::Or;

2315

else if (BOpc == Instruction::Or)

2316

BOpc = Instruction::And;

2317

}

2318

}

2319

2320

// If this node is not part of the or/and tree, emit it as a branch.

2321

// Note that all nodes in the tree should have same opcode.

2322

bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();

2323

if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||

2324

!InBlock(BOpOp0, CurBB->getBasicBlock()) ||

2325

!InBlock(BOpOp1, CurBB->getBasicBlock())) {

2326

EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,

2327

TProb, FProb, InvertCond);

2328

return;

2329

}

2330

2331

// Create TmpBB after CurBB.

2332

MachineFunction::iterator BBI(CurBB);

2333

MachineFunction &MF = DAG.getMachineFunction();

2334

MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());

2335

CurBB->getParent()->insert(++BBI, TmpBB);

2336

2337

if (Opc == Instruction::Or) {

2338

// Codegen X | Y as:

2339

// BB1:

2340

// jmp_if_X TBB

2341

// jmp TmpBB

2342

// TmpBB:

2343

// jmp_if_Y TBB

2344

// jmp FBB

2345

//

2346

2347

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2348

// The requirement is that

2349

// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)

2350

// = TrueProb for original BB.

2351

// Assuming the original probabilities are A and B, one choice is to set

2352

// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to

2353

// A/(1+B) and 2B/(1+B). This choice assumes that

2354

// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.

2355

// Another choice is to assume TrueProb for BB1 equals to TrueProb for

2356

// TmpBB, but the math is more complicated.

2357

2358

auto NewTrueProb = TProb / 2;

2359

auto NewFalseProb = TProb / 2 + FProb;

2360

// Emit the LHS condition.

2361

FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,

2362

NewFalseProb, InvertCond);

2363

2364

// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).

2365

SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};

2366

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2367

// Emit the RHS condition into TmpBB.

2368

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2369

Probs[1], InvertCond);

2370

} else {

2371

assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
"Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2371
, __extension__ __PRETTY_FUNCTION__));

2372

// Codegen X & Y as:

2373

// BB1:

2374

// jmp_if_X TmpBB

2375

// jmp FBB

2376

// TmpBB:

2377

// jmp_if_Y TBB

2378

// jmp FBB

2379

//

2380

// This requires creation of TmpBB after CurBB.

2381

2382

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2383

// The requirement is that

2384

// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)

2385

// = FalseProb for original BB.

2386

// Assuming the original probabilities are A and B, one choice is to set

2387

// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to

2388

// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==

2389

// TrueProb for BB1 * FalseProb for TmpBB.

2390

2391

auto NewTrueProb = TProb + FProb / 2;

2392

auto NewFalseProb = FProb / 2;

2393

// Emit the LHS condition.

2394

FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,

2395

NewFalseProb, InvertCond);

2396

2397

// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).

2398

SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};

2399

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2400

// Emit the RHS condition into TmpBB.

2401

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2402

Probs[1], InvertCond);

2403

}

2404

}

2405

2406

/// If the set of cases should be emitted as a series of branches, return true.

2407

/// If we should emit this as a bunch of and/or'd together conditions, return

2408

/// false.

2409

bool

2410

SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {

2411

if (Cases.size() != 2) return true;

2412

2413

// If this is two comparisons of the same values or'd or and'd together, they

2414

// will get folded into a single comparison, so don't emit two blocks.

2415

if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&

2416

Cases[0].CmpRHS == Cases[1].CmpRHS) ||

2417

(Cases[0].CmpRHS == Cases[1].CmpLHS &&

2418

Cases[0].CmpLHS == Cases[1].CmpRHS)) {

2419

return false;

2420

}

2421

2422

// Handle: (X != null) | (Y != null) --> (X|Y) != 0

2423

// Handle: (X == null) & (Y == null) --> (X|Y) == 0

2424

if (Cases[0].CmpRHS == Cases[1].CmpRHS &&

2425

Cases[0].CC == Cases[1].CC &&

2426

isa<Constant>(Cases[0].CmpRHS) &&

2427

cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {

2428

if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)

2429

return false;

2430

if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)

2431

return false;

2432

}

2433

2434

return true;

2435

}

2436

2437

void SelectionDAGBuilder::visitBr(const BranchInst &I) {

2438

MachineBasicBlock *BrMBB = FuncInfo.MBB;

2439

2440

// Update machine-CFG edges.

2441

MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

2442

2443

if (I.isUnconditional()) {

2444

// Update machine-CFG edges.

2445

BrMBB->addSuccessor(Succ0MBB);

2446

2447

// If this is not a fall-through branch or optimizations are switched off,

2448

// emit the branch.

2449

if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)

2450

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2451

MVT::Other, getControlRoot(),

2452

DAG.getBasicBlock(Succ0MBB)));

2453

2454

return;

2455

}

2456

2457

// If this condition is one of the special cases we handle, do special stuff

2458

// now.

2459

const Value *CondVal = I.getCondition();

2460

MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

2461

2462

// If this is a series of conditions that are or'd or and'd together, emit

2463

// this as a sequence of branches instead of setcc's with and/or operations.

2464

// As long as jumps are not expensive (exceptions for multi-use logic ops,

2465

// unpredictable branches, and vector extracts because those jumps are likely

2466

// expensive for any target), this should improve performance.

2467

// For example, instead of something like:

2468

// cmp A, B

2469

// C = seteq

2470

// cmp D, E

2471

// F = setle

2472

// or C, F

2473

// jnz foo

2474

// Emit:

2475

// cmp A, B

2476

// je foo

2477

// cmp D, E

2478

// jle foo

2479

const Instruction *BOp = dyn_cast<Instruction>(CondVal);

2480

if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&

2481

BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {

2482

Value *Vec;

2483

const Value *BOp0, *BOp1;

2484

Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;

2485

if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))

2486

Opcode = Instruction::And;

2487

else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))

2488

Opcode = Instruction::Or;

2489

2490

if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&

2491

match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {

2492

FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,

2493

getEdgeProbability(BrMBB, Succ0MBB),

2494

getEdgeProbability(BrMBB, Succ1MBB),

2495

/*InvertCond=*/false);

2496

// If the compares in later blocks need to use values not currently

2497

// exported from this block, export them now. This block should always

2498

// be the first entry.

2499

assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB
&& "Unexpected lowering!") ? void (0) : __assert_fail
("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2499
, __extension__ __PRETTY_FUNCTION__));

2500

2501

// Allow some cases to be rejected.

2502

if (ShouldEmitAsBranches(SL->SwitchCases)) {

2503

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {

2504

ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);

2505

ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);

2506

}

2507

2508

// Emit the branch for this block.

2509

visitSwitchCase(SL->SwitchCases[0], BrMBB);

2510

SL->SwitchCases.erase(SL->SwitchCases.begin());

2511

return;

2512

}

2513

2514

// Okay, we decided not to do this, remove any inserted MBB's and clear

2515

// SwitchCases.

2516

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)

2517

FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

2518

2519

SL->SwitchCases.clear();

2520

}

2521

}

2522

2523

// Create a CaseBlock record representing this branch.

2524

CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),

2525

nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

2526

2527

// Use visitSwitchCase to actually insert the fast branch sequence for this

2528

// cond branch.

2529

visitSwitchCase(CB, BrMBB);

2530

}

2531

2532

/// visitSwitchCase - Emits the necessary code to represent a single node in

2533

/// the binary search tree resulting from lowering a switch instruction.

2534

void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,

2535

MachineBasicBlock *SwitchBB) {

2536

SDValue Cond;

2537

SDValue CondLHS = getValue(CB.CmpLHS);

2538

SDLoc dl = CB.DL;

2539

2540

if (CB.CC == ISD::SETTRUE) {

2541

// Branch or fall through to TrueBB.

2542

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2543

SwitchBB->normalizeSuccProbs();

2544

if (CB.TrueBB != NextBlock(SwitchBB)) {

2545

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),

2546

DAG.getBasicBlock(CB.TrueBB)));

2547

}

2548

return;

2549

}

2550

2551

auto &TLI = DAG.getTargetLoweringInfo();

2552

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

2553

2554

// Build the setcc now.

2555

if (!CB.CmpMHS) {

2556

// Fold "(X == true)" to X and "(X == false)" to !X to

2557

// handle common cases produced by branch lowering.

2558

if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&

2559

CB.CC == ISD::SETEQ)

2560

Cond = CondLHS;

2561

else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&

2562

CB.CC == ISD::SETEQ) {

2563

SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());

2564

Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);

2565

} else {

2566

SDValue CondRHS = getValue(CB.CmpRHS);

2567

2568

// If a pointer's DAG type is larger than its memory type then the DAG

2569

// values are zero-extended. This breaks signed comparisons so truncate

2570

// back to the underlying type before doing the compare.

2571

if (CondLHS.getValueType() != MemVT) {

2572

CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);

2573

CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);

2574

}

2575

Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);

2576

}

2577

} else {

2578

assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2578
, __extension__ __PRETTY_FUNCTION__));

2579

2580

const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();

2581

const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

2582

2583

SDValue CmpOp = getValue(CB.CmpMHS);

2584

EVT VT = CmpOp.getValueType();

2585

2586

if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {

2587

Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),

2588

ISD::SETLE);

2589

} else {

2590

SDValue SUB = DAG.getNode(ISD::SUB, dl,

2591

VT, CmpOp, DAG.getConstant(Low, dl, VT));

2592

Cond = DAG.getSetCC(dl, MVT::i1, SUB,

2593

DAG.getConstant(High-Low, dl, VT), ISD::SETULE);

2594

}

2595

}

2596

2597

// Update successor info

2598

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2599

// TrueBB and FalseBB are always different unless the incoming IR is

2600

// degenerate. This only happens when running llc on weird IR.

2601

if (CB.TrueBB != CB.FalseBB)

2602

addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);

2603

SwitchBB->normalizeSuccProbs();

2604

2605

// If the lhs block is the next block, invert the condition so that we can

2606

// fall through to the lhs instead of the rhs block.

2607

if (CB.TrueBB == NextBlock(SwitchBB)) {

2608

std::swap(CB.TrueBB, CB.FalseBB);

2609

SDValue True = DAG.getConstant(1, dl, Cond.getValueType());

2610

Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);

2611

}

2612

2613

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2614

MVT::Other, getControlRoot(), Cond,

2615

DAG.getBasicBlock(CB.TrueBB));

2616

2617

setValue(CurInst, BrCond);

2618

2619

// Insert the false branch. Do this even if it's a fall through branch,

2620

// this makes it easier to do DAG optimizations which require inverting

2621

// the branch condition.

2622

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2623

DAG.getBasicBlock(CB.FalseBB));

2624

2625

DAG.setRoot(BrCond);

2626

}

2627

2628

/// visitJumpTable - Emit JumpTable node in the current MBB

2629

void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {

2630

// Emit the code for the jump table

2631

assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2631
, __extension__ __PRETTY_FUNCTION__));

2632

EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());

2633

SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),

2634

JT.Reg, PTy);

2635

SDValue Table = DAG.getJumpTable(JT.JTI, PTy);

2636

SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),

2637

MVT::Other, Index.getValue(1),

2638

Table, Index);

2639

DAG.setRoot(BrJumpTable);

2640

}

2641

2642

/// visitJumpTableHeader - This function emits necessary code to produce index

2643

/// in the JumpTable from switch case.

2644

void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,

2645

JumpTableHeader &JTH,

2646

MachineBasicBlock *SwitchBB) {

2647

SDLoc dl = getCurSDLoc();

2648

2649

// Subtract the lowest switch case value from the value being switched on.

2650

SDValue SwitchOp = getValue(JTH.SValue);

2651

EVT VT = SwitchOp.getValueType();

2652

SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,

2653

DAG.getConstant(JTH.First, dl, VT));

2654

2655

// The SDNode we just created, which holds the value being switched on minus

2656

// the smallest case value, needs to be copied to a virtual register so it

2657

// can be used as an index into the jump table in a subsequent basic block.

2658

// This value may be smaller or larger than the target's pointer type, and

2659

// therefore require extension or truncating.

2660

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2661

SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

2662

2663

unsigned JumpTableReg =

2664

FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));

2665

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,

2666

JumpTableReg, SwitchOp);

2667

JT.Reg = JumpTableReg;

2668

2669

if (!JTH.FallthroughUnreachable) {

2670

// Emit the range check for the jump table, and branch to the default block

2671

// for the switch statement if the value being switched on exceeds the

2672

// largest case in the switch.

2673

SDValue CMP = DAG.getSetCC(

2674

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2675

Sub.getValueType()),

2676

Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

2677

2678

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2679

MVT::Other, CopyTo, CMP,

2680

DAG.getBasicBlock(JT.Default));

2681

2682

// Avoid emitting unnecessary branches to the next block.

2683

if (JT.MBB != NextBlock(SwitchBB))

2684

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2685

DAG.getBasicBlock(JT.MBB));

2686

2687

DAG.setRoot(BrCond);

2688

} else {

2689

// Avoid emitting unnecessary branches to the next block.

2690

if (JT.MBB != NextBlock(SwitchBB))

2691

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,

2692

DAG.getBasicBlock(JT.MBB)));

2693

else

2694

DAG.setRoot(CopyTo);

2695

}

2696

}

2697

2698

/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global

2699

/// variable if there exists one.

2700

static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,

2701

SDValue &Chain) {

2702

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2703

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2704

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2705

MachineFunction &MF = DAG.getMachineFunction();

2706

Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());

2707

MachineSDNode *Node =

2708

DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);

2709

if (Global) {

2710

MachinePointerInfo MPInfo(Global);

2711

auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |

2712

MachineMemOperand::MODereferenceable;

2713

MachineMemOperand *MemRef = MF.getMachineMemOperand(

2714

MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));

2715

DAG.setNodeMemRefs(Node, {MemRef});

2716

}

2717

if (PtrTy != PtrMemTy)

2718

return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);

2719

return SDValue(Node, 0);

2720

}

2721

2722

/// Codegen a new tail for a stack protector check ParentMBB which has had its

2723

/// tail spliced into a stack protector check success bb.

2724

///

2725

/// For a high level explanation of how this fits into the stack protector

2726

/// generation see the comment on the declaration of class

2727

/// StackProtectorDescriptor.

2728

void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,

2729

MachineBasicBlock *ParentBB) {

2730

2731

// First create the loads to the guard/stack slot for the comparison.

2732

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2733

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2734

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2735

2736

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();

2737

int FI = MFI.getStackProtectorIndex();

2738

2739

SDValue Guard;

2740

SDLoc dl = getCurSDLoc();

2741

SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);

2742

const Module &M = *ParentBB->getParent()->getFunction().getParent();

2743

Align Align =

2744

DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

2745

2746

// Generate code to load the content of the guard slot.

2747

SDValue GuardVal = DAG.getLoad(

2748

PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,

2749

MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,

2750

MachineMemOperand::MOVolatile);

2751

2752

if (TLI.useStackGuardXorFP())

2753

GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

2754

2755

// Retrieve guard check function, nullptr if instrumentation is inlined.

2756

if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {

2757

// The target provides a guard check function to validate the guard value.

2758

// Generate a call to that function with the content of the guard slot as

2759

// argument.

2760

FunctionType *FnTy = GuardCheckFn->getFunctionType();

2761

assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
"Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2761
, __extension__ __PRETTY_FUNCTION__));

2762

2763

TargetLowering::ArgListTy Args;

2764

TargetLowering::ArgListEntry Entry;

2765

Entry.Node = GuardVal;

2766

Entry.Ty = FnTy->getParamType(0);

2767

if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))

2768

Entry.IsInReg = true;

2769

Args.push_back(Entry);

2770

2771

TargetLowering::CallLoweringInfo CLI(DAG);

2772

CLI.setDebugLoc(getCurSDLoc())

2773

.setChain(DAG.getEntryNode())

2774

.setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),

2775

getValue(GuardCheckFn), std::move(Args));

2776

2777

std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

2778

DAG.setRoot(Result.second);

2779

return;

2780

}

2781

2782

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.

2783

// Otherwise, emit a volatile load to retrieve the stack guard value.

2784

SDValue Chain = DAG.getEntryNode();

2785

if (TLI.useLoadStackGuardNode()) {

2786

Guard = getLoadStackGuard(DAG, dl, Chain);

2787

} else {

2788

const Value *IRGuard = TLI.getSDagStackGuard(M);

2789

SDValue GuardPtr = getValue(IRGuard);

2790

2791

Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,

2792

MachinePointerInfo(IRGuard, 0), Align,

2793

MachineMemOperand::MOVolatile);

2794

}

2795

2796

// Perform the comparison via a getsetcc.

2797

SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),

2798

*DAG.getContext(),

2799

Guard.getValueType()),

2800

Guard, GuardVal, ISD::SETNE);

2801

2802

// If the guard/stackslot do not equal, branch to failure MBB.

2803

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2804

MVT::Other, GuardVal.getOperand(0),

2805

Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));

2806

// Otherwise branch to success MBB.

2807

SDValue Br = DAG.getNode(ISD::BR, dl,

2808

MVT::Other, BrCond,

2809

DAG.getBasicBlock(SPD.getSuccessMBB()));

2810

2811

DAG.setRoot(Br);

2812

}

2813

2814

/// Codegen the failure basic block for a stack protector check.

2815

///

2816

/// A failure stack protector machine basic block consists simply of a call to

2817

/// __stack_chk_fail().

2818

///

2819

/// For a high level explanation of how this fits into the stack protector

2820

/// generation see the comment on the declaration of class

2821

/// StackProtectorDescriptor.

2822

void

2823

SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {

2824

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2825

TargetLowering::MakeLibCallOptions CallOptions;

2826

CallOptions.setDiscardResult(true);

2827

SDValue Chain =

2828

TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,

2829

std::nullopt, CallOptions, getCurSDLoc())

2830

.second;

2831

// On PS4/PS5, the "return address" must still be within the calling

2832

// function, even if it's at the very end, so emit an explicit TRAP here.

2833

// Passing 'true' for doesNotReturn above won't generate the trap for us.

2834

if (TM.getTargetTriple().isPS())

2835

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2836

// WebAssembly needs an unreachable instruction after a non-returning call,

2837

// because the function return type can be different from __stack_chk_fail's

2838

// return type (void).

2839

if (TM.getTargetTriple().isWasm())

2840

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2841

2842

DAG.setRoot(Chain);

2843

}

2844

2845

/// visitBitTestHeader - This function emits necessary code to produce value

2846

/// suitable for "bit tests"

2847

void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,

2848

MachineBasicBlock *SwitchBB) {

2849

SDLoc dl = getCurSDLoc();

2850

2851

// Subtract the minimum value.

2852

SDValue SwitchOp = getValue(B.SValue);

2853

EVT VT = SwitchOp.getValueType();

2854

SDValue RangeSub =

2855

DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

2856

2857

// Determine the type of the test operands.

2858

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2859

bool UsePtrType = false;

2860

if (!TLI.isTypeLegal(VT)) {

2861

UsePtrType = true;

2862

} else {

2863

for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)

2864

if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {

2865

// Switch table case range are encoded into series of masks.

2866

// Just use pointer type, it's guaranteed to fit.

2867

UsePtrType = true;

2868

break;

2869

}

2870

}

2871

SDValue Sub = RangeSub;

2872

if (UsePtrType) {

2873

VT = TLI.getPointerTy(DAG.getDataLayout());

2874

Sub = DAG.getZExtOrTrunc(Sub, dl, VT);

2875

}

2876

2877

B.RegVT = VT.getSimpleVT();

2878

B.Reg = FuncInfo.CreateReg(B.RegVT);

2879

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

2880

2881

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

2882

2883

if (!B.FallthroughUnreachable)

2884

addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);

2885

addSuccessorWithProb(SwitchBB, MBB, B.Prob);

2886

SwitchBB->normalizeSuccProbs();

2887

2888

SDValue Root = CopyTo;

2889

if (!B.FallthroughUnreachable) {

2890

// Conditional branch to the default block.

2891

SDValue RangeCmp = DAG.getSetCC(dl,

2892

TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2893

RangeSub.getValueType()),

2894

RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),

2895

ISD::SETUGT);

2896

2897

Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,

2898

DAG.getBasicBlock(B.Default));

2899

}

2900

2901

// Avoid emitting unnecessary branches to the next block.

2902

if (MBB != NextBlock(SwitchBB))

2903

Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

2904

2905

DAG.setRoot(Root);

2906

}

2907

2908

/// visitBitTestCase - this function produces one "bit test"

2909

void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,

2910

MachineBasicBlock* NextMBB,

2911

BranchProbability BranchProbToNext,

2912

unsigned Reg,

2913

BitTestCase &B,

2914

MachineBasicBlock *SwitchBB) {

2915

SDLoc dl = getCurSDLoc();

2916

MVT VT = BB.RegVT;

2917

SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);

2918

SDValue Cmp;

2919

unsigned PopCount = llvm::popcount(B.Mask);

2920

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2921

if (PopCount == 1) {

2922

// Testing for a single bit; just compare the shift count with what it

2923

// would need to be to shift a 1 bit in that position.

2924

Cmp = DAG.getSetCC(

2925

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2926

ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),

2927

ISD::SETEQ);

2928

} else if (PopCount == BB.Range) {

2929

// There is only one zero bit in the range, test for it directly.

2930

Cmp = DAG.getSetCC(

2931

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2932

ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);

2933

} else {

2934

// Make desired shift

2935

SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,

2936

DAG.getConstant(1, dl, VT), ShiftOp);

2937

2938

// Emit bit tests and jumps

2939

SDValue AndOp = DAG.getNode(ISD::AND, dl,

2940

VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));

2941

Cmp = DAG.getSetCC(

2942

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2943

AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);

2944

}

2945

2946

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.

2947

addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);

2948

// The branch probability from SwitchBB to NextMBB is BranchProbToNext.

2949

addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);

2950

// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is

2951

// one as they are relative probabilities (and thus work more like weights),

2952

// and hence we need to normalize them to let the sum of them become one.

2953

SwitchBB->normalizeSuccProbs();

2954

2955

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,

2956

MVT::Other, getControlRoot(),

2957

Cmp, DAG.getBasicBlock(B.TargetBB));

2958

2959

// Avoid emitting unnecessary branches to the next block.

2960

if (NextMBB != NextBlock(SwitchBB))

2961

BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,

2962

DAG.getBasicBlock(NextMBB));

2963

2964

DAG.setRoot(BrAnd);

2965

}

2966

2967

void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {

2968

MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

2969

2970

// Retrieve successors. Look through artificial IR level blocks like

2971

// catchswitch for successors.

2972

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];

2973

const BasicBlock *EHPadBB = I.getSuccessor(1);

2974

2975

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2976

// have to do anything here to lower funclet bundles.

2977

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__))

2978

{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__))

2979

LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__))

2980

LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__))

2981

LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__))

2982

"Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2982
, __extension__ __PRETTY_FUNCTION__));

2983

2984

const Value *Callee(I.getCalledOperand());

2985

const Function *Fn = dyn_cast<Function>(Callee);

2986

if (isa<InlineAsm>(Callee))

2987

visitInlineAsm(I, EHPadBB);

2988

else if (Fn && Fn->isIntrinsic()) {

2989

switch (Fn->getIntrinsicID()) {

2990

default:

2991

llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2991
);

2992

case Intrinsic::donothing:

2993

// Ignore invokes to @llvm.donothing: jump directly to the next BB.

2994

case Intrinsic::seh_try_begin:

2995

case Intrinsic::seh_scope_begin:

2996

case Intrinsic::seh_try_end:

2997

case Intrinsic::seh_scope_end:

2998

break;

2999

case Intrinsic::experimental_patchpoint_void:

3000

case Intrinsic::experimental_patchpoint_i64:

3001

visitPatchpoint(I, EHPadBB);

3002

break;

3003

case Intrinsic::experimental_gc_statepoint:

3004

LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);

3005

break;

3006

case Intrinsic::wasm_rethrow: {

3007

// This is usually done in visitTargetIntrinsic, but this intrinsic is

3008

// special because it can be invoked, so we manually lower it to a DAG

3009

// node here.

3010

SmallVector<SDValue, 8> Ops;

3011

Ops.push_back(getRoot()); // inchain

3012

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3013

Ops.push_back(

3014

DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),

3015

TLI.getPointerTy(DAG.getDataLayout())));

3016

SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain

3017

DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));

3018

break;

3019

}

3020

}

3021

} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {

3022

// Currently we do not lower any intrinsic calls with deopt operand bundles.

3023

// Eventually we will support lowering the @llvm.experimental.deoptimize

3024

// intrinsic, and right now there are no plans to support other intrinsics

3025

// with deopt state.

3026

LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);

3027

} else {

3028

LowerCallTo(I, getValue(Callee), false, false, EHPadBB);

3029

}

3030

3031

// If the value of the invoke is used outside of its defining block, make it

3032

// available as a virtual register.

3033

// We already took care of the exported value for the statepoint instruction

3034

// during call to the LowerStatepoint.

3035

if (!isa<GCStatepointInst>(I)) {

3036

CopyToExportRegsIfNeeded(&I);

3037

}

3038

3039

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

3040

BranchProbabilityInfo *BPI = FuncInfo.BPI;

3041

BranchProbability EHPadBBProb =

3042

BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)

3043

: BranchProbability::getZero();

3044

findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

3045

3046

// Update successor info.

3047

addSuccessorWithProb(InvokeMBB, Return);

3048

for (auto &UnwindDest : UnwindDests) {

3049

UnwindDest.first->setIsEHPad();

3050

addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);

3051

}

3052

InvokeMBB->normalizeSuccProbs();

3053

3054

// Drop into normal successor.

3055

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),

3056

DAG.getBasicBlock(Return)));

3057

}

3058

3059

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {

3060

MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

3061

3062

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

3063

// have to do anything here to lower funclet bundles.

3064

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3066
, __extension__ __PRETTY_FUNCTION__))

3065

{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3066
, __extension__ __PRETTY_FUNCTION__))

3066

"Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3066
, __extension__ __PRETTY_FUNCTION__));

3067

3068

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3068
, __extension__ __PRETTY_FUNCTION__));

3069

visitInlineAsm(I);

3070

CopyToExportRegsIfNeeded(&I);

3071

3072

// Retrieve successors.

3073

SmallPtrSet<BasicBlock *, 8> Dests;

3074

Dests.insert(I.getDefaultDest());

3075

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

3076

3077

// Update successor info.

3078

addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());

3079

for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {

3080

BasicBlock *Dest = I.getIndirectDest(i);

3081

MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];

3082

Target->setIsInlineAsmBrIndirectTarget();

3083

Target->setMachineBlockAddressTaken();

3084

Target->setLabelMustBeEmitted();

3085

// Don't add duplicate machine successors.

3086

if (Dests.insert(Dest).second)

3087

addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());

3088

}

3089

CallBrMBB->normalizeSuccProbs();

3090

3091

// Drop into default successor.

3092

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

3093

MVT::Other, getControlRoot(),

3094

DAG.getBasicBlock(Return)));

3095

}

3096

3097

void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {

3098

llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3098
);

3099

}

3100

3101

void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {

3102

assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3103
, __extension__ __PRETTY_FUNCTION__))

3103

"Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3103
, __extension__ __PRETTY_FUNCTION__));

3104

3105

// If there aren't registers to copy the values into (e.g., during SjLj

3106

// exceptions), then don't bother to create these DAG nodes.

3107

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3108

const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();

3109

if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&

3110

TLI.getExceptionSelectorRegister(PersonalityFn) == 0)

3111

return;

3112

3113

// If landingpad's return type is token type, we don't create DAG nodes

3114

// for its exception pointer and selector value. The extraction of exception

3115

// pointer or selector value from token type landingpads is not currently

3116

// supported.

3117

if (LP.getType()->isTokenTy())

3118

return;

3119

3120

SmallVector<EVT, 2> ValueVTs;

3121

SDLoc dl = getCurSDLoc();

3122

ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);

3123

assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3123
, __extension__ __PRETTY_FUNCTION__));

3124

3125

// Get the two live-in registers as SDValues. The physregs have already been

3126

// copied into virtual registers.

3127

SDValue Ops[2];

3128

if (FuncInfo.ExceptionPointerVirtReg) {

3129

Ops[0] = DAG.getZExtOrTrunc(

3130

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3131

FuncInfo.ExceptionPointerVirtReg,

3132

TLI.getPointerTy(DAG.getDataLayout())),

3133

dl, ValueVTs[0]);

3134

} else {

3135

Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));

3136

}

3137

Ops[1] = DAG.getZExtOrTrunc(

3138

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3139

FuncInfo.ExceptionSelectorVirtReg,

3140

TLI.getPointerTy(DAG.getDataLayout())),

3141

dl, ValueVTs[1]);

3142

3143

// Merge into one.

3144

SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,

3145

DAG.getVTList(ValueVTs), Ops);

3146

setValue(&LP, Res);

3147

}

3148

3149

void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,

3150

MachineBasicBlock *Last) {

3151

// Update JTCases.

3152

for (JumpTableBlock &JTB : SL->JTCases)

3153

if (JTB.first.HeaderBB == First)

3154

JTB.first.HeaderBB = Last;

3155

3156

// Update BitTestCases.

3157

for (BitTestBlock &BTB : SL->BitTestCases)

3158

if (BTB.Parent == First)

3159

BTB.Parent = Last;

3160

}

3161

3162

void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {

3163

MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

3164

3165

// Update machine-CFG edges with unique successors.

3166

SmallSet<BasicBlock*, 32> Done;

3167

for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {

3168

BasicBlock *BB = I.getSuccessor(i);

3169

bool Inserted = Done.insert(BB).second;

3170

if (!Inserted)

3171

continue;

3172

3173

MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];

3174

addSuccessorWithProb(IndirectBrMBB, Succ);

3175

}

3176

IndirectBrMBB->normalizeSuccProbs();

3177

3178

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),

3179

MVT::Other, getControlRoot(),

3180

getValue(I.getAddress())));

3181

}

3182

3183

void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {

3184

if (!DAG.getTarget().Options.TrapUnreachable)

3185

return;

3186

3187

// We may be able to ignore unreachable behind a noreturn call.

3188

if (DAG.getTarget().Options.NoTrapAfterNoreturn) {

3189

const BasicBlock &BB = *I.getParent();

3190

if (&I != &BB.front()) {

3191

BasicBlock::const_iterator PredI =

3192

std::prev(BasicBlock::const_iterator(&I));

3193

if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {

3194

if (Call->doesNotReturn())

3195

return;

3196

}

3197

}

3198

}

3199

3200

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

3201

}

3202

3203

void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {

3204

SDNodeFlags Flags;

3205

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3206

Flags.copyFMF(*FPOp);

3207

3208

SDValue Op = getValue(I.getOperand(0));

3209

SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),

3210

Op, Flags);

3211

setValue(&I, UnNodeValue);

3212

}

3213

3214

void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {

3215

SDNodeFlags Flags;

3216

if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {

3217

Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());

3218

Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());

3219

}

3220

if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))

3221

Flags.setExact(ExactOp->isExact());

3222

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3223

Flags.copyFMF(*FPOp);

3224

3225

SDValue Op1 = getValue(I.getOperand(0));

3226

SDValue Op2 = getValue(I.getOperand(1));

3227

SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),

3228

Op1, Op2, Flags);

3229

setValue(&I, BinNodeValue);

3230

}

3231

3232

void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {

3233

SDValue Op1 = getValue(I.getOperand(0));

3234

SDValue Op2 = getValue(I.getOperand(1));

3235

3236

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(

3237

Op1.getValueType(), DAG.getDataLayout());

3238

3239

// Coerce the shift amount to the right type if we can. This exposes the

3240

// truncate or zext to optimization early.

3241

if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {

3242

assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil
(Op1.getValueSizeInBits()) && "Unexpected shift type"
) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3243
, __extension__ __PRETTY_FUNCTION__))

3243

"Unexpected shift type")(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil
(Op1.getValueSizeInBits()) && "Unexpected shift type"
) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3243
, __extension__ __PRETTY_FUNCTION__));

3244

Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);

3245

}

3246

3247

bool nuw = false;

3248

bool nsw = false;

3249

bool exact = false;

3250

3251

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

3252

3253

if (const OverflowingBinaryOperator *OFBinOp =

3254

dyn_cast<const OverflowingBinaryOperator>(&I)) {

3255

nuw = OFBinOp->hasNoUnsignedWrap();

3256

nsw = OFBinOp->hasNoSignedWrap();

3257

}

3258

if (const PossiblyExactOperator *ExactOp =

3259

dyn_cast<const PossiblyExactOperator>(&I))

3260

exact = ExactOp->isExact();

3261

}

3262

SDNodeFlags Flags;

3263

Flags.setExact(exact);

3264

Flags.setNoSignedWrap(nsw);

3265

Flags.setNoUnsignedWrap(nuw);

3266

SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,

3267

Flags);

3268

setValue(&I, Res);

3269

}

3270

3271

void SelectionDAGBuilder::visitSDiv(const User &I) {

3272

SDValue Op1 = getValue(I.getOperand(0));

3273

SDValue Op2 = getValue(I.getOperand(1));

3274

3275

SDNodeFlags Flags;

3276

Flags.setExact(isa<PossiblyExactOperator>(&I) &&

3277

cast<PossiblyExactOperator>(&I)->isExact());

3278

setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,

3279

Op2, Flags));

3280

}

3281

3282

void SelectionDAGBuilder::visitICmp(const User &I) {

3283

ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;

3284

if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))

3285

predicate = IC->getPredicate();

3286

else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))

3287

predicate = ICmpInst::Predicate(IC->getPredicate());

3288

SDValue Op1 = getValue(I.getOperand(0));

3289

SDValue Op2 = getValue(I.getOperand(1));

3290

ISD::CondCode Opcode = getICmpCondCode(predicate);

3291

3292

auto &TLI = DAG.getTargetLoweringInfo();

3293

EVT MemVT =

3294

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3295

3296

// If a pointer's DAG type is larger than its memory type then the DAG values

3297

// are zero-extended. This breaks signed comparisons so truncate back to the

3298

// underlying type before doing the compare.

3299

if (Op1.getValueType() != MemVT) {

3300

Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);

3301

Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);

3302

}

3303

3304

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3305

I.getType());

3306

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));

3307

}

3308

3309

void SelectionDAGBuilder::visitFCmp(const User &I) {

3310

FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;

3311

if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))

3312

predicate = FC->getPredicate();

3313

else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))

3314

predicate = FCmpInst::Predicate(FC->getPredicate());

3315

SDValue Op1 = getValue(I.getOperand(0));

3316

SDValue Op2 = getValue(I.getOperand(1));

3317

3318

ISD::CondCode Condition = getFCmpCondCode(predicate);

3319

auto *FPMO = cast<FPMathOperator>(&I);

3320

if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)

3321

Condition = getFCmpCodeWithoutNaN(Condition);

3322

3323

SDNodeFlags Flags;

3324

Flags.copyFMF(*FPMO);

3325

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

3326

3327

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3328

I.getType());

3329

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));

3330

}

3331

3332

// Check if the condition of the select has one use or two users that are both

3333

// selects with the same condition.

3334

static bool hasOnlySelectUsers(const Value *Cond) {

3335

return llvm::all_of(Cond->users(), [](const Value *V) {

3336

return isa<SelectInst>(V);

3337

});

3338

}

3339

3340

void SelectionDAGBuilder::visitSelect(const User &I) {

3341

SmallVector<EVT, 4> ValueVTs;

3342

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),

3343

ValueVTs);

3344

unsigned NumValues = ValueVTs.size();

3345

if (NumValues == 0) return;

3346

3347

SmallVector<SDValue, 4> Values(NumValues);

3348

SDValue Cond = getValue(I.getOperand(0));

3349

SDValue LHSVal = getValue(I.getOperand(1));

3350

SDValue RHSVal = getValue(I.getOperand(2));

3351

SmallVector<SDValue, 1> BaseOps(1, Cond);

3352

ISD::NodeType OpCode =

3353

Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

3354

3355

bool IsUnaryAbs = false;

3356

bool Negate = false;

3357

3358

SDNodeFlags Flags;

3359

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3360

Flags.copyFMF(*FPOp);

3361

3362

// Min/max matching is only viable if all output VTs are the same.

3363

if (all_equal(ValueVTs)) {

3364

EVT VT = ValueVTs[0];

3365

LLVMContext &Ctx = *DAG.getContext();

3366

auto &TLI = DAG.getTargetLoweringInfo();

3367

3368

// We care about the legality of the operation after it has been type

3369

// legalized.

3370

while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)

3371

VT = TLI.getTypeToTransformTo(Ctx, VT);

3372

3373

// If the vselect is legal, assume we want to leave this as a vector setcc +

3374

// vselect. Otherwise, if this is going to be scalarized, we want to see if

3375

// min/max is legal on the scalar type.

3376

bool UseScalarMinMax = VT.isVector() &&

3377

!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

3378

3379

// ValueTracking's select pattern matching does not account for -0.0,

3380

// so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that

3381

// -0.0 is less than +0.0.

3382

Value *LHS, *RHS;

3383

auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);

3384

ISD::NodeType Opc = ISD::DELETED_NODE;

3385

switch (SPR.Flavor) {

3386

case SPF_UMAX: Opc = ISD::UMAX; break;

3387

case SPF_UMIN: Opc = ISD::UMIN; break;

3388

case SPF_SMAX: Opc = ISD::SMAX; break;

3389

case SPF_SMIN: Opc = ISD::SMIN; break;

3390

case SPF_FMINNUM:

3391

switch (SPR.NaNBehavior) {

3392

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3392
);

3393

case SPNB_RETURNS_NAN: break;

3394

case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;

3395

case SPNB_RETURNS_ANY:

3396

if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ||

3397

(UseScalarMinMax &&

3398

TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType())))

3399

Opc = ISD::FMINNUM;

3400

break;

3401

}

3402

break;

3403

case SPF_FMAXNUM:

3404

switch (SPR.NaNBehavior) {

3405

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3405
);

3406

case SPNB_RETURNS_NAN: break;

3407

case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;

3408

case SPNB_RETURNS_ANY:

3409

if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ||

3410

(UseScalarMinMax &&

3411

TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType())))

3412

Opc = ISD::FMAXNUM;

3413

break;

3414

}

3415

break;

3416

case SPF_NABS:

3417

Negate = true;

3418

[[fallthrough]];

3419

case SPF_ABS:

3420

IsUnaryAbs = true;

3421

Opc = ISD::ABS;

3422

break;

3423

default: break;

3424

}

3425

3426

if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&

3427

(TLI.isOperationLegalOrCustom(Opc, VT) ||

3428

(UseScalarMinMax &&

3429

TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&

3430

// If the underlying comparison instruction is used by any other

3431

// instruction, the consumed instructions won't be destroyed, so it is

3432

// not profitable to convert to a min/max.

3433

hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {

3434

OpCode = Opc;

3435

LHSVal = getValue(LHS);

3436

RHSVal = getValue(RHS);

3437

BaseOps.clear();

3438

}

3439

3440

if (IsUnaryAbs) {

3441

OpCode = Opc;

3442

LHSVal = getValue(LHS);

3443

BaseOps.clear();

3444

}

3445

}

3446

3447

if (IsUnaryAbs) {

3448

for (unsigned i = 0; i != NumValues; ++i) {

3449

SDLoc dl = getCurSDLoc();

3450

EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);

3451

Values[i] =

3452

DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));

3453

if (Negate)

3454

Values[i] = DAG.getNegative(Values[i], dl, VT);

3455

}

3456

} else {

3457

for (unsigned i = 0; i != NumValues; ++i) {

3458

SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());

3459

Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3460

Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));

3461

Values[i] = DAG.getNode(

3462

OpCode, getCurSDLoc(),

3463

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);

3464

}

3465

}

3466

3467

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3468

DAG.getVTList(ValueVTs), Values));

3469

}

3470

3471

void SelectionDAGBuilder::visitTrunc(const User &I) {

3472

// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).

3473

SDValue N = getValue(I.getOperand(0));

3474

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3475

I.getType());

3476

setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));

3477

}

3478

3479

void SelectionDAGBuilder::visitZExt(const User &I) {

3480

// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3481

// ZExt also can't be a cast to bool for same reason. So, nothing much to do

3482

SDValue N = getValue(I.getOperand(0));

3483

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3484

I.getType());

3485

setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));

3486

}

3487

3488

void SelectionDAGBuilder::visitSExt(const User &I) {

3489

// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3490

// SExt also can't be a cast to bool for same reason. So, nothing much to do

3491

SDValue N = getValue(I.getOperand(0));

3492

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3493

I.getType());

3494

setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));

3495

}

3496

3497

void SelectionDAGBuilder::visitFPTrunc(const User &I) {

3498

// FPTrunc is never a no-op cast, no need to check

3499

SDValue N = getValue(I.getOperand(0));

3500

SDLoc dl = getCurSDLoc();

3501

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3502

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3503

setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,

3504

DAG.getTargetConstant(

3505

0, dl, TLI.getPointerTy(DAG.getDataLayout()))));

3506

}

3507

3508

void SelectionDAGBuilder::visitFPExt(const User &I) {

3509

// FPExt is never a no-op cast, no need to check

3510

SDValue N = getValue(I.getOperand(0));

3511

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3512

I.getType());

3513

setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));

3514

}

3515

3516

void SelectionDAGBuilder::visitFPToUI(const User &I) {

3517

// FPToUI is never a no-op cast, no need to check

3518

SDValue N = getValue(I.getOperand(0));

3519

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3520

I.getType());

3521

setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));

3522

}

3523

3524

void SelectionDAGBuilder::visitFPToSI(const User &I) {

3525

// FPToSI is never a no-op cast, no need to check

3526

SDValue N = getValue(I.getOperand(0));

3527

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3528

I.getType());

3529

setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));

3530

}

3531

3532

void SelectionDAGBuilder::visitUIToFP(const User &I) {

3533

// UIToFP is never a no-op cast, no need to check

3534

SDValue N = getValue(I.getOperand(0));

3535

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3536

I.getType());

3537

setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));

3538

}

3539

3540

void SelectionDAGBuilder::visitSIToFP(const User &I) {

3541

// SIToFP is never a no-op cast, no need to check

3542

SDValue N = getValue(I.getOperand(0));

3543

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3544

I.getType());

3545

setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));

3546

}

3547

3548

void SelectionDAGBuilder::visitPtrToInt(const User &I) {

3549

// What to do depends on the size of the integer and the size of the pointer.

3550

// We can either truncate, zero extend, or no-op, accordingly.

3551

SDValue N = getValue(I.getOperand(0));

3552

auto &TLI = DAG.getTargetLoweringInfo();

3553

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3554

I.getType());

3555

EVT PtrMemVT =

3556

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3557

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3558

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);

3559

setValue(&I, N);

3560

}

3561

3562

void SelectionDAGBuilder::visitIntToPtr(const User &I) {

3563

// What to do depends on the size of the integer and the size of the pointer.

3564

// We can either truncate, zero extend, or no-op, accordingly.

3565

SDValue N = getValue(I.getOperand(0));

3566

auto &TLI = DAG.getTargetLoweringInfo();

3567

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3568

EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

3569

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3570

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);

3571

setValue(&I, N);

3572

}

3573

3574

void SelectionDAGBuilder::visitBitCast(const User &I) {

3575

SDValue N = getValue(I.getOperand(0));

3576

SDLoc dl = getCurSDLoc();

3577

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3578

I.getType());

3579

3580

// BitCast assures us that source and destination are the same size so this is

3581

// either a BITCAST or a no-op.

3582

if (DestVT != N.getValueType())

3583

setValue(&I, DAG.getNode(ISD::BITCAST, dl,

3584

DestVT, N)); // convert types.

3585

// Check if the original LLVM IR Operand was a ConstantInt, because getValue()

3586

// might fold any kind of constant expression to an integer constant and that

3587

// is not what we are looking for. Only recognize a bitcast of a genuine

3588

// constant integer as an opaque constant.

3589

else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))

3590

setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,

3591

/*isOpaque*/true));

3592

else

3593

setValue(&I, N); // noop cast.

3594

}

3595

3596

void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {

3597

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3598

const Value *SV = I.getOperand(0);

3599

SDValue N = getValue(SV);

3600

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3601

3602

unsigned SrcAS = SV->getType()->getPointerAddressSpace();

3603

unsigned DestAS = I.getType()->getPointerAddressSpace();

3604

3605

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))

3606

N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

3607

3608

setValue(&I, N);

3609

}

3610

3611

void SelectionDAGBuilder::visitInsertElement(const User &I) {

3612

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3613

SDValue InVec = getValue(I.getOperand(0));

3614

SDValue InVal = getValue(I.getOperand(1));

3615

SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),

3616

TLI.getVectorIdxTy(DAG.getDataLayout()));

3617

setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),

3618

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3619

InVec, InVal, InIdx));

3620

}

3621

3622

void SelectionDAGBuilder::visitExtractElement(const User &I) {

3623

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3624

SDValue InVec = getValue(I.getOperand(0));

3625

SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),

3626

TLI.getVectorIdxTy(DAG.getDataLayout()));

3627

setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),

3628

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3629

InVec, InIdx));

3630

}

3631

3632

void SelectionDAGBuilder::visitShuffleVector(const User &I) {

3633

SDValue Src1 = getValue(I.getOperand(0));

3634

SDValue Src2 = getValue(I.getOperand(1));

3635

ArrayRef<int> Mask;

3636

if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))

3637

Mask = SVI->getShuffleMask();

3638

else

3639

Mask = cast<ConstantExpr>(I).getShuffleMask();

3640

SDLoc DL = getCurSDLoc();

3641

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3642

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3643

EVT SrcVT = Src1.getValueType();

3644

3645

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&

3646

VT.isScalableVector()) {

3647

// Canonical splat form of first element of first input vector.

3648

SDValue FirstElt =

3649

DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,

3650

DAG.getVectorIdxConstant(0, DL));

3651

setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));

3652

return;

3653

}

3654

3655

// For now, we only handle splats for scalable vectors.

3656

// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation

3657

// for targets that support a SPLAT_VECTOR for non-scalable vector types.

3658

assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3658
, __extension__ __PRETTY_FUNCTION__));

3659

3660

unsigned SrcNumElts = SrcVT.getVectorNumElements();

3661

unsigned MaskNumElts = Mask.size();

3662

3663

if (SrcNumElts == MaskNumElts) {

3664

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));

3665

return;

3666

}

3667

3668

// Normalize the shuffle vector since mask and vector length don't match.

3669

if (SrcNumElts < MaskNumElts) {

3670

// Mask is longer than the source vectors. We can use concatenate vector to

3671

// make the mask and vectors lengths match.

3672

3673

if (MaskNumElts % SrcNumElts == 0) {

3674

// Mask length is a multiple of the source vector length.

3675

// Check if the shuffle is some kind of concatenation of the input

3676

// vectors.

3677

unsigned NumConcat = MaskNumElts / SrcNumElts;

3678

bool IsConcat = true;

3679

SmallVector<int, 8> ConcatSrcs(NumConcat, -1);

3680

for (unsigned i = 0; i != MaskNumElts; ++i) {

3681

int Idx = Mask[i];

3682

if (Idx < 0)

3683

continue;

3684

// Ensure the indices in each SrcVT sized piece are sequential and that

3685

// the same source is used for the whole piece.

3686

if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

3687

(ConcatSrcs[i / SrcNumElts] >= 0 &&

3688

ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {

3689

IsConcat = false;

3690

break;

3691

}

3692

// Remember which source this index came from.

3693

ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

3694

}

3695

3696

// The shuffle is concatenating multiple vectors together. Just emit

3697

// a CONCAT_VECTORS operation.

3698

if (IsConcat) {

3699

SmallVector<SDValue, 8> ConcatOps;

3700

for (auto Src : ConcatSrcs) {

3701

if (Src < 0)

3702

ConcatOps.push_back(DAG.getUNDEF(SrcVT));

3703

else if (Src == 0)

3704

ConcatOps.push_back(Src1);

3705

else

3706

ConcatOps.push_back(Src2);

3707

}

3708

setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));

3709

return;

3710

}

3711

}

3712

3713

unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);

3714

unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;

3715

EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),

3716

PaddedMaskNumElts);

3717

3718

// Pad both vectors with undefs to make them the same length as the mask.

3719

SDValue UndefVal = DAG.getUNDEF(SrcVT);

3720

3721

SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);

3722

SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);

3723

MOps1[0] = Src1;

3724

MOps2[0] = Src2;

3725

3726

Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);

3727

Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

3728

3729

// Readjust mask for new input vector length.

3730

SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);

3731

for (unsigned i = 0; i != MaskNumElts; ++i) {

3732

int Idx = Mask[i];

3733

if (Idx >= (int)SrcNumElts)

3734

Idx -= SrcNumElts - PaddedMaskNumElts;

3735

MappedOps[i] = Idx;

3736

}

3737

3738

SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

3739

3740

// If the concatenated vector was padded, extract a subvector with the

3741

// correct number of elements.

3742

if (MaskNumElts != PaddedMaskNumElts)

3743

Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,

3744

DAG.getVectorIdxConstant(0, DL));

3745

3746

setValue(&I, Result);

3747

return;

3748

}

3749

3750

if (SrcNumElts > MaskNumElts) {

3751

// Analyze the access pattern of the vector to see if we can extract

3752

// two subvectors and do the shuffle.

3753

int StartIdx[2] = { -1, -1 }; // StartIdx to extract from

3754

bool CanExtract = true;

3755

for (int Idx : Mask) {

3756

unsigned Input = 0;

3757

if (Idx < 0)

3758

continue;

3759

3760

if (Idx >= (int)SrcNumElts) {

3761

Input = 1;

3762

Idx -= SrcNumElts;

3763

}

3764

3765

// If all the indices come from the same MaskNumElts sized portion of

3766

// the sources we can use extract. Also make sure the extract wouldn't

3767

// extract past the end of the source.

3768

int NewStartIdx = alignDown(Idx, MaskNumElts);

3769

if (NewStartIdx + MaskNumElts > SrcNumElts ||

3770

(StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))

3771

CanExtract = false;

3772

// Make sure we always update StartIdx as we use it to track if all

3773

// elements are undef.

3774

StartIdx[Input] = NewStartIdx;

3775

}

3776

3777

if (StartIdx[0] < 0 && StartIdx[1] < 0) {

3778

setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.

3779

return;

3780

}

3781

if (CanExtract) {

3782

// Extract appropriate subvector and generate a vector shuffle

3783

for (unsigned Input = 0; Input < 2; ++Input) {

3784

SDValue &Src = Input == 0 ? Src1 : Src2;

3785

if (StartIdx[Input] < 0)

3786

Src = DAG.getUNDEF(VT);

3787

else {

3788

Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,

3789

DAG.getVectorIdxConstant(StartIdx[Input], DL));

3790

}

3791

}

3792

3793

// Calculate new mask.

3794

SmallVector<int, 8> MappedOps(Mask);

3795

for (int &Idx : MappedOps) {

3796

if (Idx >= (int)SrcNumElts)

3797

Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;

3798

else if (Idx >= 0)

3799

Idx -= StartIdx[0];

3800

}

3801

3802

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));

3803

return;

3804

}

3805

}

3806

3807

// We can't use either concat vectors or extract subvectors so fall back to

3808

// replacing the shuffle with extract and build vector.

3809

// to insert and build vector.

3810

EVT EltVT = VT.getVectorElementType();

3811

SmallVector<SDValue,8> Ops;

3812

for (int Idx : Mask) {

3813

SDValue Res;

3814

3815

if (Idx < 0) {

3816

Res = DAG.getUNDEF(EltVT);

3817

} else {

3818

SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;

3819

if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

3820

3821

Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,

3822

DAG.getVectorIdxConstant(Idx, DL));

3823

}

3824

3825

Ops.push_back(Res);

3826

}

3827

3828

setValue(&I, DAG.getBuildVector(VT, DL, Ops));

3829

}

3830

3831

void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {

3832

ArrayRef<unsigned> Indices = I.getIndices();

3833

const Value *Op0 = I.getOperand(0);

3834

const Value *Op1 = I.getOperand(1);

3835

Type *AggTy = I.getType();

3836

Type *ValTy = Op1->getType();

3837

bool IntoUndef = isa<UndefValue>(Op0);

3838

bool FromUndef = isa<UndefValue>(Op1);

3839

3840

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3841

3842

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3843

SmallVector<EVT, 4> AggValueVTs;

3844

ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);

3845

SmallVector<EVT, 4> ValValueVTs;

3846

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3847

3848

unsigned NumAggValues = AggValueVTs.size();

3849

unsigned NumValValues = ValValueVTs.size();

3850

SmallVector<SDValue, 4> Values(NumAggValues);

3851

3852

// Ignore an insertvalue that produces an empty object

3853

if (!NumAggValues) {

3854

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3855

return;

3856

}

3857

3858

SDValue Agg = getValue(Op0);

3859

unsigned i = 0;

3860

// Copy the beginning value(s) from the original aggregate.

3861

for (; i != LinearIndex; ++i)

3862

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3863

SDValue(Agg.getNode(), Agg.getResNo() + i);

3864

// Copy values from the inserted value(s).

3865

if (NumValValues) {

3866

SDValue Val = getValue(Op1);

3867

for (; i != LinearIndex + NumValValues; ++i)

3868

Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3869

SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);

3870

}

3871

// Copy remaining value(s) from the original aggregate.

3872

for (; i != NumAggValues; ++i)

3873

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3874

SDValue(Agg.getNode(), Agg.getResNo() + i);

3875

3876

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3877

DAG.getVTList(AggValueVTs), Values));

3878

}

3879

3880

void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {

3881

ArrayRef<unsigned> Indices = I.getIndices();

3882

const Value *Op0 = I.getOperand(0);

3883

Type *AggTy = Op0->getType();

3884

Type *ValTy = I.getType();

3885

bool OutOfUndef = isa<UndefValue>(Op0);

3886

3887

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3888

3889

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3890

SmallVector<EVT, 4> ValValueVTs;

3891

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3892

3893

unsigned NumValValues = ValValueVTs.size();

3894

3895

// Ignore a extractvalue that produces an empty object

3896

if (!NumValValues) {

3897

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3898

return;

3899

}

3900

3901

SmallVector<SDValue, 4> Values(NumValValues);

3902

3903

SDValue Agg = getValue(Op0);

3904

// Copy out the selected value(s).

3905

for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)

3906

Values[i - LinearIndex] =

3907

OutOfUndef ?

3908

DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :

3909

SDValue(Agg.getNode(), Agg.getResNo() + i);

3910

3911

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3912

DAG.getVTList(ValValueVTs), Values));

3913

}

3914

3915

void SelectionDAGBuilder::visitGetElementPtr(const User &I) {

3916

Value *Op0 = I.getOperand(0);

3917

// Note that the pointer operand may be a vector of pointers. Take the scalar

3918

// element which holds a pointer.

3919

unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();

3920

SDValue N = getValue(Op0);

3921

SDLoc dl = getCurSDLoc();

3922

auto &TLI = DAG.getTargetLoweringInfo();

3923

3924

// Normalize Vector GEP - all scalar operands should be converted to the

3925

// splat vector.

3926

bool IsVectorGEP = I.getType()->isVectorTy();

3927

ElementCount VectorElementCount =

3928

IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()

3929

: ElementCount::getFixed(0);

3930

3931

if (IsVectorGEP && !N.getValueType().isVector()) {

3932

LLVMContext &Context = *DAG.getContext();

3933

EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);

3934

N = DAG.getSplat(VT, dl, N);

3935

}

3936

3937

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);

3938

GTI != E; ++GTI) {

3939

const Value *Idx = GTI.getOperand();

3940

if (StructType *StTy = GTI.getStructTypeOrNull()) {

3941

unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();

3942

if (Field) {

3943

// N = N + Offset

3944

uint64_t Offset =

3945

DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);

3946

3947

// In an inbounds GEP with an offset that is nonnegative even when

3948

// interpreted as signed, assume there is no unsigned overflow.

3949

SDNodeFlags Flags;

3950

if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())

3951

Flags.setNoUnsignedWrap(true);

3952

3953

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,

3954

DAG.getConstant(Offset, dl, N.getValueType()), Flags);

3955

}

3956

} else {

3957

// IdxSize is the width of the arithmetic according to IR semantics.

3958

// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth

3959

// (and fix up the result later).

3960

unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);

3961

MVT IdxTy = MVT::getIntegerVT(IdxSize);

3962

TypeSize ElementSize =

3963

DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());

3964

// We intentionally mask away the high bits here; ElementSize may not

3965

// fit in IdxTy.

3966

APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());

3967

bool ElementScalable = ElementSize.isScalable();

3968

3969

// If this is a scalar constant or a splat vector of constants,

3970

// handle it quickly.

3971

const auto *C = dyn_cast<Constant>(Idx);

3972

if (C && isa<VectorType>(C->getType()))

3973

C = C->getSplatValue();

3974

3975

const auto *CI = dyn_cast_or_null<ConstantInt>(C);

3976

if (CI && CI->isZero())

3977

continue;

3978

if (CI && !ElementScalable) {

3979

APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);

3980

LLVMContext &Context = *DAG.getContext();

3981

SDValue OffsVal;

3982

if (IsVectorGEP)

3983

OffsVal = DAG.getConstant(

3984

Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));

3985

else

3986

OffsVal = DAG.getConstant(Offs, dl, IdxTy);

3987

3988

// In an inbounds GEP with an offset that is nonnegative even when

3989

// interpreted as signed, assume there is no unsigned overflow.

3990

SDNodeFlags Flags;

3991

if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())

3992

Flags.setNoUnsignedWrap(true);

3993

3994

OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

3995

3996

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);

3997

continue;

3998

}

3999

4000

// N = N + Idx * ElementMul;

4001

SDValue IdxN = getValue(Idx);

4002

4003

if (!IdxN.getValueType().isVector() && IsVectorGEP) {

4004

EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),

4005

VectorElementCount);

4006

IdxN = DAG.getSplat(VT, dl, IdxN);

4007

}

4008

4009

// If the index is smaller or larger than intptr_t, truncate or extend

4010

// it.

4011

IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

4012

4013

if (ElementScalable) {

4014

EVT VScaleTy = N.getValueType().getScalarType();

4015

SDValue VScale = DAG.getNode(

4016

ISD::VSCALE, dl, VScaleTy,

4017

DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));

4018

if (IsVectorGEP)

4019

VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);

4020

IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);

4021

} else {

4022

// If this is a multiply by a power of two, turn it into a shl

4023

// immediately. This is a very common case.

4024

if (ElementMul != 1) {

4025

if (ElementMul.isPowerOf2()) {

4026

unsigned Amt = ElementMul.logBase2();

4027

IdxN = DAG.getNode(ISD::SHL, dl,

4028

N.getValueType(), IdxN,

4029

DAG.getConstant(Amt, dl, IdxN.getValueType()));

4030

} else {

4031

SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,

4032

IdxN.getValueType());

4033

IdxN = DAG.getNode(ISD::MUL, dl,

4034

N.getValueType(), IdxN, Scale);

4035

}

4036

}

4037

}

4038

4039

N = DAG.getNode(ISD::ADD, dl,

4040

N.getValueType(), N, IdxN);

4041

}

4042

}

4043

4044

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);

4045

MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);

4046

if (IsVectorGEP) {

4047

PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);

4048

PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);

4049

}

4050

4051

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())

4052

N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

4053

4054

setValue(&I, N);

4055

}

4056

4057

void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {

4058

// If this is a fixed sized alloca in the entry block of the function,

4059

// allocate it statically on the stack.

4060

if (FuncInfo.StaticAllocaMap.count(&I))

4061

return; // getValue will auto-populate this.

4062

4063

SDLoc dl = getCurSDLoc();

4064

Type *Ty = I.getAllocatedType();

4065

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4066

auto &DL = DAG.getDataLayout();

4067

TypeSize TySize = DL.getTypeAllocSize(Ty);

4068

MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

4069

4070

SDValue AllocSize = getValue(I.getArraySize());

4071

4072

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());

4073

if (AllocSize.getValueType() != IntPtr)

4074

AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

4075

4076

if (TySize.isScalable())

4077

AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,

4078

DAG.getVScale(dl, IntPtr,

4079

APInt(IntPtr.getScalarSizeInBits(),

4080

TySize.getKnownMinValue())));

4081

else

4082

AllocSize =

4083

DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,

4084

DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));

4085

4086

// Handle alignment. If the requested alignment is less than or equal to

4087

// the stack alignment, ignore it. If the size is greater than or equal to

4088

// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.

4089

Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();

4090

if (*Alignment <= StackAlign)

4091

Alignment = std::nullopt;

4092

4093

const uint64_t StackAlignMask = StackAlign.value() - 1U;

4094

// Round the size of the allocation up to the stack alignment size

4095

// by add SA-1 to the size. This doesn't overflow because we're computing

4096

// an address inside an alloca.

4097

SDNodeFlags Flags;

4098

Flags.setNoUnsignedWrap(true);

4099

AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,

4100

DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

4101

4102

// Mask out the low bits for alignment purposes.

4103

AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,

4104

DAG.getConstant(~StackAlignMask, dl, IntPtr));

4105

4106

SDValue Ops[] = {

4107

getRoot(), AllocSize,

4108

DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};

4109

SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);

4110

SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);

4111

setValue(&I, DSA);

4112

DAG.setRoot(DSA.getValue(1));

4113

4114

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4114
, __extension__ __PRETTY_FUNCTION__));

4115

}

4116

4117

void SelectionDAGBuilder::visitLoad(const LoadInst &I) {

4118

if (I.isAtomic())

4119

return visitAtomicLoad(I);

4120

4121

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4122

const Value *SV = I.getOperand(0);

4123

if (TLI.supportSwiftError()) {

4124

// Swifterror values can come from either a function parameter with

4125

// swifterror attribute or an alloca with swifterror attribute.

4126

if (const Argument *Arg = dyn_cast<Argument>(SV)) {

4127

if (Arg->hasSwiftErrorAttr())

4128

return visitLoadFromSwiftError(I);

4129

}

4130

4131

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {

4132

if (Alloca->isSwiftError())

4133

return visitLoadFromSwiftError(I);

4134

}

4135

}

4136

4137

SDValue Ptr = getValue(SV);

4138

4139

Type *Ty = I.getType();

4140

SmallVector<EVT, 4> ValueVTs, MemVTs;

4141

SmallVector<uint64_t, 4> Offsets;

4142

ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);

4143

unsigned NumValues = ValueVTs.size();

4144

if (NumValues == 0)

4145

return;

4146

4147

Align Alignment = I.getAlign();

4148

AAMDNodes AAInfo = I.getAAMetadata();

4149

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4150

bool isVolatile = I.isVolatile();

4151

MachineMemOperand::Flags MMOFlags =

4152

TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);

4153

4154

SDValue Root;

4155

bool ConstantMemory = false;

4156

if (isVolatile)

4157

// Serialize volatile loads with other side effects.

4158

Root = getRoot();

4159

else if (NumValues > MaxParallelChains)

4160

Root = getMemoryRoot();

4161

else if (AA &&

4162

AA->pointsToConstantMemory(MemoryLocation(

4163

SV,

4164

LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),

4165

AAInfo))) {

4166

// Do not serialize (non-volatile) loads of constant memory with anything.

4167

Root = DAG.getEntryNode();

4168

ConstantMemory = true;

4169

MMOFlags |= MachineMemOperand::MOInvariant;

4170

} else {

4171

// Do not serialize non-volatile loads against each other.

4172

Root = DAG.getRoot();

4173

}

4174

4175

SDLoc dl = getCurSDLoc();

4176

4177

if (isVolatile)

4178

Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

4179

4180

// An aggregate load cannot wrap around the address space, so offsets to its

4181

// parts don't wrap either.

4182

SDNodeFlags Flags;

4183

Flags.setNoUnsignedWrap(true);

4184

4185

SmallVector<SDValue, 4> Values(NumValues);

4186

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4187

EVT PtrVT = Ptr.getValueType();

4188

4189

unsigned ChainI = 0;

4190

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4191

// Serializing loads here may result in excessive register pressure, and

4192

// TokenFactor places arbitrary choke points on the scheduler. SD scheduling

4193

// could recover a bit by hoisting nodes upward in the chain by recognizing

4194

// they are side-effect free or do not alias. The optimizer should really

4195

// avoid this case by converting large object/array copies to llvm.memcpy

4196

// (MaxParallelChains should always remain as failsafe).

4197

if (ChainI == MaxParallelChains) {

4198

assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4198
, __extension__ __PRETTY_FUNCTION__));

4199

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4200

ArrayRef(Chains.data(), ChainI));

4201

Root = Chain;

4202

ChainI = 0;

4203

}

4204

SDValue A = DAG.getNode(ISD::ADD, dl,

4205

PtrVT, Ptr,

4206

DAG.getConstant(Offsets[i], dl, PtrVT),

4207

Flags);

4208

4209

SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,

4210

MachinePointerInfo(SV, Offsets[i]), Alignment,

4211

MMOFlags, AAInfo, Ranges);

4212

Chains[ChainI] = L.getValue(1);

4213

4214

if (MemVTs[i] != ValueVTs[i])

4215

L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

4216

4217

Values[i] = L;

4218

}

4219

4220

if (!ConstantMemory) {

4221

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4222

ArrayRef(Chains.data(), ChainI));

4223

if (isVolatile)

4224

DAG.setRoot(Chain);

4225

else

4226

PendingLoads.push_back(Chain);

4227

}

4228

4229

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,

4230

DAG.getVTList(ValueVTs), Values));

4231

}

4232

4233

void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {

4234

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4235
, __extension__ __PRETTY_FUNCTION__))

4235

"call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4235
, __extension__ __PRETTY_FUNCTION__));

4236

4237

SmallVector<EVT, 4> ValueVTs;

4238

SmallVector<uint64_t, 4> Offsets;

4239

const Value *SrcV = I.getOperand(0);

4240

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4241

SrcV->getType(), ValueVTs, &Offsets);

4242

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4243
, __extension__ __PRETTY_FUNCTION__))

4243

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4243
, __extension__ __PRETTY_FUNCTION__));

4244

4245

SDValue Src = getValue(SrcV);

4246

// Create a virtual register, then update the virtual register.

4247

Register VReg =

4248

SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());

4249

// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue

4250

// Chain can be getRoot or getControlRoot.

4251

SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,

4252

SDValue(Src.getNode(), Src.getResNo()));

4253

DAG.setRoot(CopyNode);

4254

}

4255

4256

void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {

4257

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4258
, __extension__ __PRETTY_FUNCTION__))

4258

"call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4258
, __extension__ __PRETTY_FUNCTION__));

4259

4260

assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4263
, __extension__ __PRETTY_FUNCTION__))

4261

!I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4263
, __extension__ __PRETTY_FUNCTION__))

4262

!I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4263
, __extension__ __PRETTY_FUNCTION__))

4263

"Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4263
, __extension__ __PRETTY_FUNCTION__));

4264

4265

const Value *SV = I.getOperand(0);

4266

Type *Ty = I.getType();

4267

assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__))

4268

(!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__))

4269

!AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__))

4270

SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__))

4271

I.getAAMetadata()))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__))

4272

"load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4272
, __extension__ __PRETTY_FUNCTION__));

4273

4274

SmallVector<EVT, 4> ValueVTs;

4275

SmallVector<uint64_t, 4> Offsets;

4276

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,

4277

ValueVTs, &Offsets);

4278

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4279
, __extension__ __PRETTY_FUNCTION__))

4279

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4279
, __extension__ __PRETTY_FUNCTION__));

4280

4281

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT

4282

SDValue L = DAG.getCopyFromReg(

4283

getRoot(), getCurSDLoc(),

4284

SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

4285

4286

setValue(&I, L);

4287

}

4288

4289

void SelectionDAGBuilder::visitStore(const StoreInst &I) {

4290

if (I.isAtomic())

4291

return visitAtomicStore(I);

4292

4293

const Value *SrcV = I.getOperand(0);

4294

const Value *PtrV = I.getOperand(1);

4295

4296

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4297

if (TLI.supportSwiftError()) {

4298

// Swifterror values can come from either a function parameter with

4299

// swifterror attribute or an alloca with swifterror attribute.

4300

if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {

4301

if (Arg->hasSwiftErrorAttr())

4302

return visitStoreToSwiftError(I);

4303

}

4304

4305

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {

4306

if (Alloca->isSwiftError())

4307

return visitStoreToSwiftError(I);

4308

}

4309

}

4310

4311

SmallVector<EVT, 4> ValueVTs, MemVTs;

4312

SmallVector<uint64_t, 4> Offsets;

4313

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4314

SrcV->getType(), ValueVTs, &MemVTs, &Offsets);

4315

unsigned NumValues = ValueVTs.size();

4316

if (NumValues == 0)

4317

return;

4318

4319

// Get the lowered operands. Note that we do this after

4320

// checking if NumResults is zero, because with zero results

4321

// the operands won't have values in the map.

4322

SDValue Src = getValue(SrcV);

4323

SDValue Ptr = getValue(PtrV);

4324

4325

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();

4326

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4327

SDLoc dl = getCurSDLoc();

4328

Align Alignment = I.getAlign();

4329

AAMDNodes AAInfo = I.getAAMetadata();

4330

4331

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4332

4333

// An aggregate load cannot wrap around the address space, so offsets to its

4334

// parts don't wrap either.

4335

SDNodeFlags Flags;

4336

Flags.setNoUnsignedWrap(true);

4337

4338

unsigned ChainI = 0;

4339

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4340

// See visitLoad comments.

4341

if (ChainI == MaxParallelChains) {

4342

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4343

ArrayRef(Chains.data(), ChainI));

4344

Root = Chain;

4345

ChainI = 0;

4346

}

4347

SDValue Add =

4348

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);

4349

SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);

4350

if (MemVTs[i] != ValueVTs[i])

4351

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);

4352

SDValue St =

4353

DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),

4354

Alignment, MMOFlags, AAInfo);

4355

Chains[ChainI] = St;

4356

}

4357

4358

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4359

ArrayRef(Chains.data(), ChainI));

4360

setValue(&I, StoreNode);

4361

DAG.setRoot(StoreNode);

4362

}

4363

4364

void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,

4365

bool IsCompressing) {

4366

SDLoc sdl = getCurSDLoc();

4367

4368

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4369

MaybeAlign &Alignment) {

4370

// llvm.masked.store.*(Src0, Ptr, alignment, Mask)

4371

Src0 = I.getArgOperand(0);

4372

Ptr = I.getArgOperand(1);

4373

Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();

4374

Mask = I.getArgOperand(3);

4375

};

4376

auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4377

MaybeAlign &Alignment) {

4378

// llvm.masked.compressstore.*(Src0, Ptr, Mask)

4379

Src0 = I.getArgOperand(0);

4380

Ptr = I.getArgOperand(1);

4381

Mask = I.getArgOperand(2);

4382

Alignment = std::nullopt;

4383

};

4384

4385

Value *PtrOperand, *MaskOperand, *Src0Operand;

4386

MaybeAlign Alignment;

4387

if (IsCompressing)

4388

getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4389

else

4390

getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4391

4392

SDValue Ptr = getValue(PtrOperand);

4393

SDValue Src0 = getValue(Src0Operand);

4394

SDValue Mask = getValue(MaskOperand);

4395

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4396

4397

EVT VT = Src0.getValueType();

4398

if (!Alignment)

4399

Alignment = DAG.getEVTAlign(VT);

4400

4401

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4402

MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,

4403

MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());

4404

SDValue StoreNode =

4405

DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,

4406

ISD::UNINDEXED, false /* Truncating */, IsCompressing);

4407

DAG.setRoot(StoreNode);

4408

setValue(&I, StoreNode);

4409

}

4410

4411

// Get a uniform base for the Gather/Scatter intrinsic.

4412

// The first argument of the Gather/Scatter intrinsic is a vector of pointers.

4413

// We try to represent it as a base pointer + vector of indices.

4414

// Usually, the vector of pointers comes from a 'getelementptr' instruction.

4415

// The first operand of the GEP may be a single pointer or a vector of pointers

4416

// Example:

4417

// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind

4418

// or

4419

// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind

4420

// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..

4421

//

4422

// When the first GEP operand is a single pointer - it is the uniform base we

4423

// are looking for. If first operand of the GEP is a splat vector - we

4424

// extract the splat value and use it as a uniform base.

4425

// In all other cases the function returns 'false'.

4426

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,

4427

ISD::MemIndexType &IndexType, SDValue &Scale,

4428

SelectionDAGBuilder *SDB, const BasicBlock *CurBB,

4429

uint64_t ElemSize) {

4430

SelectionDAG& DAG = SDB->DAG;

4431

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4432

const DataLayout &DL = DAG.getDataLayout();

4433

4434

assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Unexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Unexpected pointer type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4434
, __extension__ __PRETTY_FUNCTION__));

4435

4436

// Handle splat constant pointer.

4437

if (auto *C = dyn_cast<Constant>(Ptr)) {

4438

C = C->getSplatValue();

4439

if (!C)

4440

return false;

4441

4442

Base = SDB->getValue(C);

4443

4444

ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();

4445

EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);

4446

Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);

4447

IndexType = ISD::SIGNED_SCALED;

4448

Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4449

return true;

4450

}

4451

4452

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

4453

if (!GEP || GEP->getParent() != CurBB)

4454

return false;

4455

4456

if (GEP->getNumOperands() != 2)

4457

return false;

4458

4459

const Value *BasePtr = GEP->getPointerOperand();

4460

const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

4461

4462

// Make sure the base is scalar and the index is a vector.

4463

if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())

4464

return false;

4465

4466

uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());

4467

4468

// Target may not support the required addressing mode.

4469

if (ScaleVal != 1 &&

4470

!TLI.isLegalScaleForGatherScatter(ScaleVal, ElemSize))

4471

return false;

4472

4473

Base = SDB->getValue(BasePtr);

4474

Index = SDB->getValue(IndexVal);

4475

IndexType = ISD::SIGNED_SCALED;

4476

4477

Scale =

4478

DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4479

return true;

4480

}

4481

4482

void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {

4483

SDLoc sdl = getCurSDLoc();

4484

4485

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)

4486

const Value *Ptr = I.getArgOperand(1);

4487

SDValue Src0 = getValue(I.getArgOperand(0));

4488

SDValue Mask = getValue(I.getArgOperand(3));

4489

EVT VT = Src0.getValueType();

4490

Align Alignment = cast<ConstantInt>(I.getArgOperand(2))

4491

->getMaybeAlignValue()

4492

.value_or(DAG.getEVTAlign(VT.getScalarType()));

4493

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4494

4495

SDValue Base;

4496

SDValue Index;

4497

ISD::MemIndexType IndexType;

4498

SDValue Scale;

4499

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4500

I.getParent(), VT.getScalarStoreSize());

4501

4502

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4503

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4504

MachinePointerInfo(AS), MachineMemOperand::MOStore,

4505

// TODO: Make MachineMemOperands aware of scalable

4506

// vectors.

4507

MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());

4508

if (!UniformBase) {

4509

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4510

Index = getValue(Ptr);

4511

IndexType = ISD::SIGNED_SCALED;

4512

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4513

}

4514

4515

EVT IdxVT = Index.getValueType();

4516

EVT EltTy = IdxVT.getVectorElementType();

4517

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4518

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4519

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4520

}

4521

4522

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };

4523

SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,

4524

Ops, MMO, IndexType, false);

4525

DAG.setRoot(Scatter);

4526

setValue(&I, Scatter);

4527

}

4528

4529

void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {

4530

SDLoc sdl = getCurSDLoc();

4531

4532

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4533

MaybeAlign &Alignment) {

4534

// @llvm.masked.load.*(Ptr, alignment, Mask, Src0)

4535

Ptr = I.getArgOperand(0);

4536

Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();

4537

Mask = I.getArgOperand(2);

4538

Src0 = I.getArgOperand(3);

4539

};

4540

auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4541

MaybeAlign &Alignment) {

4542

// @llvm.masked.expandload.*(Ptr, Mask, Src0)

4543

Ptr = I.getArgOperand(0);

4544

Alignment = std::nullopt;

4545

Mask = I.getArgOperand(1);

4546

Src0 = I.getArgOperand(2);

4547

};

4548

4549

Value *PtrOperand, *MaskOperand, *Src0Operand;

4550

MaybeAlign Alignment;

4551

if (IsExpanding)

4552

getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4553

else

4554

getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4555

4556

SDValue Ptr = getValue(PtrOperand);

4557

SDValue Src0 = getValue(Src0Operand);

4558

SDValue Mask = getValue(MaskOperand);

4559

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4560

4561

EVT VT = Src0.getValueType();

4562

if (!Alignment)

4563

Alignment = DAG.getEVTAlign(VT);

4564

4565

AAMDNodes AAInfo = I.getAAMetadata();

4566

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4567

4568

// Do not serialize masked loads of constant memory with anything.

4569

MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);

4570

bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

4571

4572

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

4573

4574

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4575

MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,

4576

MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);

4577

4578

SDValue Load =

4579

DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,

4580

ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);

4581

if (AddToChain)

4582

PendingLoads.push_back(Load.getValue(1));

4583

setValue(&I, Load);

4584

}

4585

4586

void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {

4587

SDLoc sdl = getCurSDLoc();

4588

4589

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)

4590

const Value *Ptr = I.getArgOperand(0);

4591

SDValue Src0 = getValue(I.getArgOperand(3));

4592

SDValue Mask = getValue(I.getArgOperand(2));

4593

4594

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4595

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4596

Align Alignment = cast<ConstantInt>(I.getArgOperand(1))

4597

->getMaybeAlignValue()

4598

.value_or(DAG.getEVTAlign(VT.getScalarType()));

4599

4600

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4601

4602

SDValue Root = DAG.getRoot();

4603

SDValue Base;

4604

SDValue Index;

4605

ISD::MemIndexType IndexType;

4606

SDValue Scale;

4607

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4608

I.getParent(), VT.getScalarStoreSize());

4609

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4610

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4611

MachinePointerInfo(AS), MachineMemOperand::MOLoad,

4612

// TODO: Make MachineMemOperands aware of scalable

4613

// vectors.

4614

MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);

4615

4616

if (!UniformBase) {

4617

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4618

Index = getValue(Ptr);

4619

IndexType = ISD::SIGNED_SCALED;

4620

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4621

}

4622

4623

EVT IdxVT = Index.getValueType();

4624

EVT EltTy = IdxVT.getVectorElementType();

4625

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4626

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4627

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4628

}

4629

4630

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };

4631

SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,

4632

Ops, MMO, IndexType, ISD::NON_EXTLOAD);

4633

4634

PendingLoads.push_back(Gather.getValue(1));

4635

setValue(&I, Gather);

4636

}

4637

4638

void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {

4639

SDLoc dl = getCurSDLoc();

4640

AtomicOrdering SuccessOrdering = I.getSuccessOrdering();

4641

AtomicOrdering FailureOrdering = I.getFailureOrdering();

4642

SyncScope::ID SSID = I.getSyncScopeID();

4643

4644

SDValue InChain = getRoot();

4645

4646

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();

4647

SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

4648

4649

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4650

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4651

4652

MachineFunction &MF = DAG.getMachineFunction();

4653

MachineMemOperand *MMO = MF.getMachineMemOperand(

4654

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4655

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,

4656

FailureOrdering);

4657

4658

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,

4659

dl, MemVT, VTs, InChain,

4660

getValue(I.getPointerOperand()),

4661

getValue(I.getCompareOperand()),

4662

getValue(I.getNewValOperand()), MMO);

4663

4664

SDValue OutChain = L.getValue(2);

4665

4666

setValue(&I, L);

4667

DAG.setRoot(OutChain);

4668

}

4669

4670

void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {

4671

SDLoc dl = getCurSDLoc();

4672

ISD::NodeType NT;

4673

switch (I.getOperation()) {

4674

default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4674
);

4675

case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;

4676

case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;

4677

case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;

4678

case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;

4679

case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;

4680

case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;

4681

case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;

4682

case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;

4683

case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;

4684

case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;

4685

case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;

4686

case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;

4687

case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;

4688

case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;

4689

case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;

4690

case AtomicRMWInst::UIncWrap:

4691

NT = ISD::ATOMIC_LOAD_UINC_WRAP;

4692

break;

4693

case AtomicRMWInst::UDecWrap:

4694

NT = ISD::ATOMIC_LOAD_UDEC_WRAP;

4695

break;

4696

}

4697

AtomicOrdering Ordering = I.getOrdering();

4698

SyncScope::ID SSID = I.getSyncScopeID();

4699

4700

SDValue InChain = getRoot();

4701

4702

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();

4703

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4704

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4705

4706

MachineFunction &MF = DAG.getMachineFunction();

4707

MachineMemOperand *MMO = MF.getMachineMemOperand(

4708

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4709

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

4710

4711

SDValue L =

4712

DAG.getAtomic(NT, dl, MemVT, InChain,

4713

getValue(I.getPointerOperand()), getValue(I.getValOperand()),

4714

MMO);

4715

4716

SDValue OutChain = L.getValue(1);

4717

4718

setValue(&I, L);

4719

DAG.setRoot(OutChain);

4720

}

4721

4722

void SelectionDAGBuilder::visitFence(const FenceInst &I) {

4723

SDLoc dl = getCurSDLoc();

4724

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4725

SDValue Ops[3];

4726

Ops[0] = getRoot();

4727

Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,

4728

TLI.getFenceOperandTy(DAG.getDataLayout()));

4729

Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,

4730

TLI.getFenceOperandTy(DAG.getDataLayout()));

4731

SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);

4732

setValue(&I, N);

4733

DAG.setRoot(N);

4734

}

4735

4736

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {

4737

SDLoc dl = getCurSDLoc();

4738

AtomicOrdering Order = I.getOrdering();

4739

SyncScope::ID SSID = I.getSyncScopeID();

4740

4741

SDValue InChain = getRoot();

4742

4743

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4744

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4745

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

4746

4747

if (!TLI.supportsUnalignedAtomics() &&

4748

I.getAlign().value() < MemVT.getSizeInBits() / 8)

4749

report_fatal_error("Cannot generate unaligned atomic load");

4750

4751

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);

4752

4753

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4754

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4755

I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

4756

4757

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

4758

4759

SDValue Ptr = getValue(I.getPointerOperand());

4760

4761

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {

4762

// TODO: Once this is better exercised by tests, it should be merged with

4763

// the normal path for loads to prevent future divergence.

4764

SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);

4765

if (MemVT != VT)

4766

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4767

4768

setValue(&I, L);

4769

SDValue OutChain = L.getValue(1);

4770

if (!I.isUnordered())

4771

DAG.setRoot(OutChain);

4772

else

4773

PendingLoads.push_back(OutChain);

4774

return;

4775

}

4776

4777

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,

4778

Ptr, MMO);

4779

4780

SDValue OutChain = L.getValue(1);

4781

if (MemVT != VT)

4782

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4783

4784

setValue(&I, L);

4785

DAG.setRoot(OutChain);

4786

}

4787

4788

void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {

4789

SDLoc dl = getCurSDLoc();

4790

4791

AtomicOrdering Ordering = I.getOrdering();

4792

SyncScope::ID SSID = I.getSyncScopeID();

4793

4794

SDValue InChain = getRoot();

4795

4796

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4797

EVT MemVT =

4798

TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

4799

4800

if (!TLI.supportsUnalignedAtomics() &&

4801

I.getAlign().value() < MemVT.getSizeInBits() / 8)

4802

report_fatal_error("Cannot generate unaligned atomic store");

4803

4804

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4805

4806

MachineFunction &MF = DAG.getMachineFunction();

4807

MachineMemOperand *MMO = MF.getMachineMemOperand(

4808

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4809

I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

4810

4811

SDValue Val = getValue(I.getValueOperand());

4812

if (Val.getValueType() != MemVT)

4813

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);

4814

SDValue Ptr = getValue(I.getPointerOperand());

4815

4816

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {

4817

// TODO: Once this is better exercised by tests, it should be merged with

4818

// the normal path for stores to prevent future divergence.

4819

SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);

4820

setValue(&I, S);

4821

DAG.setRoot(S);

4822

return;

4823

}

4824

SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,

4825

Ptr, Val, MMO);

4826

4827

setValue(&I, OutChain);

4828

DAG.setRoot(OutChain);

4829

}

4830

4831

/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC

4832

/// node.

4833

void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,

4834

unsigned Intrinsic) {

4835

// Ignore the callsite's attributes. A specific call site may be marked with

4836

// readnone, but the lowering code will expect the chain based on the

4837

// definition.

4838

const Function *F = I.getCalledFunction();

4839

bool HasChain = !F->doesNotAccessMemory();

4840

bool OnlyLoad = HasChain && F->onlyReadsMemory();

4841

4842

// Build the operand list.

4843

SmallVector<SDValue, 8> Ops;

4844

if (HasChain) { // If this intrinsic has side-effects, chainify it.

4845

if (OnlyLoad) {

4846

// We don't need to serialize loads against other loads.

4847

Ops.push_back(DAG.getRoot());

4848

} else {

4849

Ops.push_back(getRoot());

4850

}

4851

}

4852

4853

// Info is set by getTgtMemIntrinsic

4854

TargetLowering::IntrinsicInfo Info;

4855

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4856

bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,

4857

DAG.getMachineFunction(),

4858

Intrinsic);

4859

4860

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.

4861

if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||

4862

Info.opc == ISD::INTRINSIC_W_CHAIN)

4863

Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),

4864

TLI.getPointerTy(DAG.getDataLayout())));

4865

4866

// Add all operands of the call to the operand list.

4867

for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {

4868

const Value *Arg = I.getArgOperand(i);

4869

if (!I.paramHasAttr(i, Attribute::ImmArg)) {

4870

Ops.push_back(getValue(Arg));

4871

continue;

4872

}

4873

4874

// Use TargetConstant instead of a regular constant for immarg.

4875

EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);

4876

if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {

4877

assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4878
, __extension__ __PRETTY_FUNCTION__))

4878

"large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4878
, __extension__ __PRETTY_FUNCTION__));

4879

Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));

4880

} else {

4881

Ops.push_back(

4882

DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));

4883

}

4884

}

4885

4886

SmallVector<EVT, 4> ValueVTs;

4887

ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

4888

4889

if (HasChain)

4890

ValueVTs.push_back(MVT::Other);

4891

4892

SDVTList VTs = DAG.getVTList(ValueVTs);

4893

4894

// Propagate fast-math-flags from IR to node(s).

4895

SDNodeFlags Flags;

4896

if (auto *FPMO = dyn_cast<FPMathOperator>(&I))

4897

Flags.copyFMF(*FPMO);

4898

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

4899

4900

// Create the node.

4901

SDValue Result;

4902

// In some cases, custom collection of operands from CallInst I may be needed.

4903

TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);

4904

if (IsTgtIntrinsic) {

4905

// This is target intrinsic that touches memory

4906

//

4907

// TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic

4908

// didn't yield anything useful.

4909

MachinePointerInfo MPI;

4910

if (Info.ptrVal)

4911

MPI = MachinePointerInfo(Info.ptrVal, Info.offset);

4912

else if (Info.fallbackAddressSpace)

4913

MPI = MachinePointerInfo(*Info.fallbackAddressSpace);

4914

Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,

4915

Info.memVT, MPI, Info.align, Info.flags,

4916

Info.size, I.getAAMetadata());

4917

} else if (!HasChain) {

4918

Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);

4919

} else if (!I.getType()->isVoidTy()) {

4920

Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);

4921

} else {

4922

Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);

4923

}

4924

4925

if (HasChain) {

4926

SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);

4927

if (OnlyLoad)

4928

PendingLoads.push_back(Chain);

4929

else

4930

DAG.setRoot(Chain);

4931

}

4932

4933

if (!I.getType()->isVoidTy()) {

4934

if (!isa<VectorType>(I.getType()))

4935

Result = lowerRangeToAssertZExt(DAG, I, Result);

4936

4937

MaybeAlign Alignment = I.getRetAlign();

4938

4939

// Insert `assertalign` node if there's an alignment.

4940

if (InsertAssertAlign && Alignment) {

4941

Result =

4942

DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());

4943

}

4944

4945

setValue(&I, Result);

4946

}

4947

}

4948

4949

/// GetSignificand - Get the significand and build it into a floating-point

4950

/// number with exponent of 1:

4951

///

4952

/// Op = (Op & 0x007fffff) | 0x3f800000;

4953

///

4954

/// where Op is the hexadecimal representation of floating point value.

4955

static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {

4956

SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4957

DAG.getConstant(0x007fffff, dl, MVT::i32));

4958

SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,

4959

DAG.getConstant(0x3f800000, dl, MVT::i32));

4960

return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);

4961

}

4962

4963

/// GetExponent - Get the exponent:

4964

///

4965

/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);

4966

///

4967

/// where Op is the hexadecimal representation of floating point value.

4968

static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,

4969

const TargetLowering &TLI, const SDLoc &dl) {

4970

SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4971

DAG.getConstant(0x7f800000, dl, MVT::i32));

4972

SDValue t1 = DAG.getNode(

4973

ISD::SRL, dl, MVT::i32, t0,

4974

DAG.getConstant(23, dl,

4975

TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));

4976

SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,

4977

DAG.getConstant(127, dl, MVT::i32));

4978

return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);

4979

}

4980

4981

/// getF32Constant - Get 32-bit floating point constant.

4982

static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,

4983

const SDLoc &dl) {

4984

return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,

4985

MVT::f32);

4986

}

4987

4988

static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,

4989

SelectionDAG &DAG) {

4990

// TODO: What fast-math-flags should be set on the floating-point nodes?

4991

4992

// IntegerPartOfX = ((int32_t)(t0);

4993

SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

4994

4995

// FractionalPartOfX = t0 - (float)IntegerPartOfX;

4996

SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);

4997

SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

4998

4999

// IntegerPartOfX <<= 23;

5000

IntegerPartOfX =

5001

DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,

5002

DAG.getConstant(23, dl,

5003

DAG.getTargetLoweringInfo().getShiftAmountTy(

5004

MVT::i32, DAG.getDataLayout())));

5005

5006

SDValue TwoToFractionalPartOfX;

5007

if (LimitFloatPrecision <= 6) {

5008

// For floating-point precision of 6:

5009

//

5010

// TwoToFractionalPartOfX =

5011

// 0.997535578f +

5012

// (0.735607626f + 0.252464424f * x) * x;

5013

//

5014

// error 0.0144103317, which is 6 bits

5015

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5016

getF32Constant(DAG, 0x3e814304, dl));

5017

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5018

getF32Constant(DAG, 0x3f3c50c8, dl));

5019

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5020

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5021

getF32Constant(DAG, 0x3f7f5e7e, dl));

5022

} else if (LimitFloatPrecision <= 12) {

5023

// For floating-point precision of 12:

5024

//

5025

// TwoToFractionalPartOfX =

5026

// 0.999892986f +

5027

// (0.696457318f +

5028

// (0.224338339f + 0.792043434e-1f * x) * x) * x;

5029

//

5030

// error 0.000107046256, which is 13 to 14 bits

5031

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5032

getF32Constant(DAG, 0x3da235e3, dl));

5033

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5034

getF32Constant(DAG, 0x3e65b8f3, dl));

5035

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5036

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5037

getF32Constant(DAG, 0x3f324b07, dl));

5038

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5039

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5040

getF32Constant(DAG, 0x3f7ff8fd, dl));

5041

} else { // LimitFloatPrecision <= 18

5042

// For floating-point precision of 18:

5043

//

5044

// TwoToFractionalPartOfX =

5045

// 0.999999982f +

5046

// (0.693148872f +

5047

// (0.240227044f +

5048

// (0.554906021e-1f +

5049

// (0.961591928e-2f +

5050

// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;

5051

// error 2.47208000*10^(-7), which is better than 18 bits

5052

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5053

getF32Constant(DAG, 0x3924b03e, dl));

5054

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5055

getF32Constant(DAG, 0x3ab24b87, dl));

5056

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5057

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5058

getF32Constant(DAG, 0x3c1d8c17, dl));

5059

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5060

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5061

getF32Constant(DAG, 0x3d634a1d, dl));

5062

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5063

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5064

getF32Constant(DAG, 0x3e75fe14, dl));

5065

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5066

SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,

5067

getF32Constant(DAG, 0x3f317234, dl));

5068

SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);

5069

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,

5070

getF32Constant(DAG, 0x3f800000, dl));

5071

}

5072

5073

// Add the exponent into the result in integer domain.

5074

SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);

5075

return DAG.getNode(ISD::BITCAST, dl, MVT::f32,

5076

DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));

5077

}

5078

5079

/// expandExp - Lower an exp intrinsic. Handles the special sequences for

5080

/// limited-precision mode.

5081

static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5082

const TargetLowering &TLI, SDNodeFlags Flags) {

5083

if (Op.getValueType() == MVT::f32 &&

5084

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5085

5086

// Put the exponent in the right bit position for later addition to the

5087

// final result:

5088

//

5089

// t0 = Op * log2(e)

5090

5091

// TODO: What fast-math-flags should be set here?

5092

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,

5093

DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));

5094

return getLimitedPrecisionExp2(t0, dl, DAG);

5095

}

5096

5097

// No special expansion.

5098

return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);

5099

}

5100

5101

/// expandLog - Lower a log intrinsic. Handles the special sequences for

5102

/// limited-precision mode.

5103

static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5104

const TargetLowering &TLI, SDNodeFlags Flags) {

5105

// TODO: What fast-math-flags should be set on the floating-point nodes?

5106

5107

if (Op.getValueType() == MVT::f32 &&

5108

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5109

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5110

5111

// Scale the exponent by log(2).

5112

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5113

SDValue LogOfExponent =

5114

DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5115

DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

5116

5117

// Get the significand and build it into a floating-point number with

5118

// exponent of 1.

5119

SDValue X = GetSignificand(DAG, Op1, dl);

5120

5121

SDValue LogOfMantissa;

5122

if (LimitFloatPrecision <= 6) {

5123

// For floating-point precision of 6:

5124

//

5125

// LogofMantissa =

5126

// -1.1609546f +

5127

// (1.4034025f - 0.23903021f * x) * x;

5128

//

5129

// error 0.0034276066, which is better than 8 bits

5130

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5131

getF32Constant(DAG, 0xbe74c456, dl));

5132

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5133

getF32Constant(DAG, 0x3fb3a2b1, dl));

5134

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5135

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5136

getF32Constant(DAG, 0x3f949a29, dl));

5137

} else if (LimitFloatPrecision <= 12) {

5138

// For floating-point precision of 12:

5139

//

5140

// LogOfMantissa =

5141

// -1.7417939f +

5142

// (2.8212026f +

5143

// (-1.4699568f +

5144

// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;

5145

//

5146

// error 0.000061011436, which is 14 bits

5147

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5148

getF32Constant(DAG, 0xbd67b6d6, dl));

5149

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5150

getF32Constant(DAG, 0x3ee4f4b8, dl));

5151

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5152

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5153

getF32Constant(DAG, 0x3fbc278b, dl));

5154

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5155

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5156

getF32Constant(DAG, 0x40348e95, dl));

5157

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5158

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5159

getF32Constant(DAG, 0x3fdef31a, dl));

5160

} else { // LimitFloatPrecision <= 18

5161

// For floating-point precision of 18:

5162

//

5163

// LogOfMantissa =

5164

// -2.1072184f +

5165

// (4.2372794f +

5166

// (-3.7029485f +

5167

// (2.2781945f +

5168

// (-0.87823314f +

5169

// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;

5170

//

5171

// error 0.0000023660568, which is better than 18 bits

5172

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5173

getF32Constant(DAG, 0xbc91e5ac, dl));

5174

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5175

getF32Constant(DAG, 0x3e4350aa, dl));

5176

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5177

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5178

getF32Constant(DAG, 0x3f60d3e3, dl));

5179

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5180

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5181

getF32Constant(DAG, 0x4011cdf0, dl));

5182

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5183

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5184

getF32Constant(DAG, 0x406cfd1c, dl));

5185

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5186

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5187

getF32Constant(DAG, 0x408797cb, dl));

5188

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5189

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5190

getF32Constant(DAG, 0x4006dcab, dl));

5191

}

5192

5193

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);

5194

}

5195

5196

// No special expansion.

5197

return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);

5198

}

5199

5200

/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for

5201

/// limited-precision mode.

5202

static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5203

const TargetLowering &TLI, SDNodeFlags Flags) {

5204

// TODO: What fast-math-flags should be set on the floating-point nodes?

5205

5206

if (Op.getValueType() == MVT::f32 &&

5207

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5208

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5209

5210

// Get the exponent.

5211

SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

5212

5213

// Get the significand and build it into a floating-point number with

5214

// exponent of 1.

5215

SDValue X = GetSignificand(DAG, Op1, dl);

5216

5217

// Different possible minimax approximations of significand in

5218

// floating-point for various degrees of accuracy over [1,2].

5219

SDValue Log2ofMantissa;

5220

if (LimitFloatPrecision <= 6) {

5221

// For floating-point precision of 6:

5222

//

5223

// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;

5224

//

5225

// error 0.0049451742, which is more than 7 bits

5226

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5227

getF32Constant(DAG, 0xbeb08fe0, dl));

5228

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5229

getF32Constant(DAG, 0x40019463, dl));

5230

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5231

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5232

getF32Constant(DAG, 0x3fd6633d, dl));

5233

} else if (LimitFloatPrecision <= 12) {

5234

// For floating-point precision of 12:

5235

//

5236

// Log2ofMantissa =

5237

// -2.51285454f +

5238

// (4.07009056f +

5239

// (-2.12067489f +

5240

// (.645142248f - 0.816157886e-1f * x) * x) * x) * x;

5241

//

5242

// error 0.0000876136000, which is better than 13 bits

5243

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5244

getF32Constant(DAG, 0xbda7262e, dl));

5245

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5246

getF32Constant(DAG, 0x3f25280b, dl));

5247

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5248

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5249

getF32Constant(DAG, 0x4007b923, dl));

5250

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5251

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5252

getF32Constant(DAG, 0x40823e2f, dl));

5253

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5254

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5255

getF32Constant(DAG, 0x4020d29c, dl));

5256

} else { // LimitFloatPrecision <= 18

5257

// For floating-point precision of 18:

5258

//

5259

// Log2ofMantissa =

5260

// -3.0400495f +

5261

// (6.1129976f +

5262

// (-5.3420409f +

5263

// (3.2865683f +

5264

// (-1.2669343f +

5265

// (0.27515199f -

5266

// 0.25691327e-1f * x) * x) * x) * x) * x) * x;

5267

//

5268

// error 0.0000018516, which is better than 18 bits

5269

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5270

getF32Constant(DAG, 0xbcd2769e, dl));

5271

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5272

getF32Constant(DAG, 0x3e8ce0b9, dl));

5273

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5274

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5275

getF32Constant(DAG, 0x3fa22ae7, dl));

5276

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5277

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5278

getF32Constant(DAG, 0x40525723, dl));

5279

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5280

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5281

getF32Constant(DAG, 0x40aaf200, dl));

5282

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5283

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5284

getF32Constant(DAG, 0x40c39dad, dl));

5285

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5286

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5287

getF32Constant(DAG, 0x4042902c, dl));

5288

}

5289

5290

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);

5291

}

5292

5293

// No special expansion.

5294

return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);

5295

}

5296

5297

/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for

5298

/// limited-precision mode.

5299

static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5300

const TargetLowering &TLI, SDNodeFlags Flags) {

5301

// TODO: What fast-math-flags should be set on the floating-point nodes?

5302

5303

if (Op.getValueType() == MVT::f32 &&

5304

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5305

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5306

5307

// Scale the exponent by log10(2) [0.30102999f].

5308

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5309

SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5310

getF32Constant(DAG, 0x3e9a209a, dl));

5311

5312

// Get the significand and build it into a floating-point number with

5313

// exponent of 1.

5314

SDValue X = GetSignificand(DAG, Op1, dl);

5315

5316

SDValue Log10ofMantissa;

5317

if (LimitFloatPrecision <= 6) {

5318

// For floating-point precision of 6:

5319

//

5320

// Log10ofMantissa =

5321

// -0.50419619f +

5322

// (0.60948995f - 0.10380950f * x) * x;

5323

//

5324

// error 0.0014886165, which is 6 bits

5325

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5326

getF32Constant(DAG, 0xbdd49a13, dl));

5327

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5328

getF32Constant(DAG, 0x3f1c0789, dl));

5329

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5330

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5331

getF32Constant(DAG, 0x3f011300, dl));

5332

} else if (LimitFloatPrecision <= 12) {

5333

// For floating-point precision of 12:

5334

//

5335

// Log10ofMantissa =

5336

// -0.64831180f +

5337

// (0.91751397f +

5338

// (-0.31664806f + 0.47637168e-1f * x) * x) * x;

5339

//

5340

// error 0.00019228036, which is better than 12 bits

5341

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5342

getF32Constant(DAG, 0x3d431f31, dl));

5343

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5344

getF32Constant(DAG, 0x3ea21fb2, dl));

5345

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5346

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5347

getF32Constant(DAG, 0x3f6ae232, dl));

5348

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5349

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5350

getF32Constant(DAG, 0x3f25f7c3, dl));

5351

} else { // LimitFloatPrecision <= 18

5352

// For floating-point precision of 18:

5353

//

5354

// Log10ofMantissa =

5355

// -0.84299375f +

5356

// (1.5327582f +

5357

// (-1.0688956f +

5358

// (0.49102474f +

5359

// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;

5360

//

5361

// error 0.0000037995730, which is better than 18 bits

5362

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5363

getF32Constant(DAG, 0x3c5d51ce, dl));

5364

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5365

getF32Constant(DAG, 0x3e00685a, dl));

5366

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5367

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5368

getF32Constant(DAG, 0x3efb6798, dl));

5369

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5370

SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5371

getF32Constant(DAG, 0x3f88d192, dl));

5372

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5373

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5374

getF32Constant(DAG, 0x3fc4316c, dl));

5375

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5376

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,

5377

getF32Constant(DAG, 0x3f57ce70, dl));

5378

}

5379

5380

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);

5381

}

5382

5383

// No special expansion.

5384

return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);

5385

}

5386

5387

/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for

5388

/// limited-precision mode.

5389

static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5390

const TargetLowering &TLI, SDNodeFlags Flags) {

5391

if (Op.getValueType() == MVT::f32 &&

5392

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)

5393

return getLimitedPrecisionExp2(Op, dl, DAG);

5394

5395

// No special expansion.

5396

return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);

5397

}

5398

5399

/// visitPow - Lower a pow intrinsic. Handles the special sequences for

5400

/// limited-precision mode with x == 10.0f.

5401

static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,

5402

SelectionDAG &DAG, const TargetLowering &TLI,

5403

SDNodeFlags Flags) {

5404

bool IsExp10 = false;

5405

if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&

5406

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5407

if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {

5408

APFloat Ten(10.0f);

5409

IsExp10 = LHSC->isExactlyValue(Ten);

5410

}

5411

}

5412

5413

// TODO: What fast-math-flags should be set on the FMUL node?

5414

if (IsExp10) {

5415

// Put the exponent in the right bit position for later addition to the

5416

// final result:

5417

//

5418

// #define LOG2OF10 3.3219281f

5419

// t0 = Op * LOG2OF10;

5420

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,

5421

getF32Constant(DAG, 0x40549a78, dl));

5422

return getLimitedPrecisionExp2(t0, dl, DAG);

5423

}

5424

5425

// No special expansion.

5426

return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);

5427

}

5428

5429

/// ExpandPowI - Expand a llvm.powi intrinsic.

5430

static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,

5431

SelectionDAG &DAG) {

5432

// If RHS is a constant, we can expand this out to a multiplication tree if

5433

// it's beneficial on the target, otherwise we end up lowering to a call to

5434

// __powidf2 (for example).

5435

if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {

5436

unsigned Val = RHSC->getSExtValue();

5437

5438

// powi(x, 0) -> 1.0

5439

if (Val == 0)

5440

return DAG.getConstantFP(1.0, DL, LHS.getValueType());

5441

5442

if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(

5443

Val, DAG.shouldOptForSize())) {

5444

// Get the exponent as a positive value.

5445

if ((int)Val < 0)

5446

Val = -Val;

5447

// We use the simple binary decomposition method to generate the multiply

5448

// sequence. There are more optimal ways to do this (for example,

5449

// powi(x,15) generates one more multiply than it should), but this has

5450

// the benefit of being both really simple and much better than a libcall.

5451

SDValue Res; // Logically starts equal to 1.0

5452

SDValue CurSquare = LHS;

5453

// TODO: Intrinsics should have fast-math-flags that propagate to these

5454

// nodes.

5455

while (Val) {

5456

if (Val & 1) {

5457

if (Res.getNode())

5458

Res =

5459

DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);

5460

else

5461

Res = CurSquare; // 1.0*CurSquare.

5462

}

5463

5464

CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),

5465

CurSquare, CurSquare);

5466

Val >>= 1;

5467

}

5468

5469

// If the original was negative, invert the result, producing 1/(x*x*x).

5470

if (RHSC->getSExtValue() < 0)

5471

Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),

5472

DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);

5473

return Res;

5474

}

5475

}

5476

5477

// Otherwise, expand to a libcall.

5478

return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);

5479

}

5480

5481

static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,

5482

SDValue LHS, SDValue RHS, SDValue Scale,

5483

SelectionDAG &DAG, const TargetLowering &TLI) {

5484

EVT VT = LHS.getValueType();

5485

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

5486

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

5487

LLVMContext &Ctx = *DAG.getContext();

5488

5489

// If the type is legal but the operation isn't, this node might survive all

5490

// the way to operation legalization. If we end up there and we do not have

5491

// the ability to widen the type (if VT*2 is not legal), we cannot expand the

5492

// node.

5493

5494

// Coax the legalizer into expanding the node during type legalization instead

5495

// by bumping the size by one bit. This will force it to Promote, enabling the

5496

// early expansion and avoiding the need to expand later.

5497

5498

// We don't have to do this if Scale is 0; that can always be expanded, unless

5499

// it's a saturating signed operation. Those can experience true integer

5500

// division overflow, a case which we must avoid.

5501

5502

// FIXME: We wouldn't have to do this (or any of the early

5503

// expansion/promotion) if it was possible to expand a libcall of an

5504

// illegal type during operation legalization. But it's not, so things

5505

// get a bit hacky.

5506

unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();

5507

if ((ScaleInt > 0 || (Saturating && Signed)) &&

5508

(TLI.isTypeLegal(VT) ||

5509

(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {

5510

TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(

5511

Opcode, VT, ScaleInt);

5512

if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {

5513

EVT PromVT;

5514

if (VT.isScalarInteger())

5515

PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);

5516

else if (VT.isVector()) {

5517

PromVT = VT.getVectorElementType();

5518

PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);

5519

PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());

5520

} else

5521

llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5521);

5522

if (Signed) {

5523

LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);

5524

RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);

5525

} else {

5526

LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);

5527

RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);

5528

}

5529

EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());

5530

// For saturating operations, we need to shift up the LHS to get the

5531

// proper saturation width, and then shift down again afterwards.

5532

if (Saturating)

5533

LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,

5534

DAG.getConstant(1, DL, ShiftTy));

5535

SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);

5536

if (Saturating)

5537

Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,

5538

DAG.getConstant(1, DL, ShiftTy));

5539

return DAG.getZExtOrTrunc(Res, DL, VT);

5540

}

5541

}

5542

5543

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);

5544

}

5545

5546

// getUnderlyingArgRegs - Find underlying registers used for a truncated,

5547

// bitcasted, or split argument. Returns a list of <Register, size in bits>

5548

static void

5549

getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,

5550

const SDValue &N) {

5551

switch (N.getOpcode()) {

5552

case ISD::CopyFromReg: {

5553

SDValue Op = N.getOperand(1);

5554

Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),

5555

Op.getValueType().getSizeInBits());

5556

return;

5557

}

5558

case ISD::BITCAST:

5559

case ISD::AssertZext:

5560

case ISD::AssertSext:

5561

case ISD::TRUNCATE:

5562

getUnderlyingArgRegs(Regs, N.getOperand(0));

5563

return;

5564

case ISD::BUILD_PAIR:

5565

case ISD::BUILD_VECTOR:

5566

case ISD::CONCAT_VECTORS:

5567

for (SDValue Op : N->op_values())

5568

getUnderlyingArgRegs(Regs, Op);

5569

return;

5570

default:

5571

return;

5572

}

5573

}

5574

5575

/// If the DbgValueInst is a dbg_value of a function argument, create the

5576

/// corresponding DBG_VALUE machine instruction for it now. At the end of

5577

/// instruction selection, they will be inserted to the entry BB.

5578

/// We don't currently support this for variadic dbg_values, as they shouldn't

5579

/// appear for function arguments or in the prologue.

5580

bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(

5581

const Value *V, DILocalVariable *Variable, DIExpression *Expr,

5582

DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {

5583

const Argument *Arg = dyn_cast<Argument>(V);

5584

if (!Arg)

5585

return false;

5586

5587

MachineFunction &MF = DAG.getMachineFunction();

5588

const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

5589

5590

// Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind

5591

// we've been asked to pursue.

5592

auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,

5593

bool Indirect) {

5594

if (Reg.isVirtual() && MF.useDebugInstrRef()) {

5595

// For VRegs, in instruction referencing mode, create a DBG_INSTR_REF

5596

// pointing at the VReg, which will be patched up later.

5597

auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);

5598

SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(

5599

/* Reg */ Reg, /* isDef */ false, /* isImp */ false,

5600

/* isKill */ false, /* isDead */ false,

5601

/* isUndef */ false, /* isEarlyClobber */ false,

5602

/* SubReg */ 0, /* isDebug */ true)});

5603

5604

auto *NewDIExpr = FragExpr;

5605

// We don't have an "Indirect" field in DBG_INSTR_REF, fold that into

5606

// the DIExpression.

5607

if (Indirect)

5608

NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);

5609

SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});

5610

NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);

5611

return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);

5612

} else {

5613

// Create a completely standard DBG_VALUE.

5614

auto &Inst = TII->get(TargetOpcode::DBG_VALUE);

5615

return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);

5616

}

5617

};

5618

5619

if (Kind == FuncArgumentDbgValueKind::Value) {

5620

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5621

// should only emit as ArgDbgValue if the dbg.value intrinsic is found in

5622

// the entry block.

5623

bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();

5624

if (!IsInEntryBlock)

5625

return false;

5626

5627

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5628

// should only emit as ArgDbgValue if the dbg.value intrinsic describes a

5629

// variable that also is a param.

5630

//

5631

// Although, if we are at the top of the entry block already, we can still

5632

// emit using ArgDbgValue. This might catch some situations when the

5633

// dbg.value refers to an argument that isn't used in the entry block, so

5634

// any CopyToReg node would be optimized out and the only way to express

5635

// this DBG_VALUE is by using the physical reg (or FI) as done in this

5636

// method. ArgDbgValues are hoisted to the beginning of the entry block. So

5637

// we should only emit as ArgDbgValue if the Variable is an argument to the

5638

// current function, and the dbg.value intrinsic is found in the entry

5639

// block.

5640

bool VariableIsFunctionInputArg = Variable->isParameter() &&

5641

!DL->getInlinedAt();

5642

bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;

5643

if (!IsInPrologue && !VariableIsFunctionInputArg)

5644

return false;

5645

5646

// Here we assume that a function argument on IR level only can be used to

5647

// describe one input parameter on source level. If we for example have

5648

// source code like this

5649

//

5650

// struct A { long x, y; };

5651

// void foo(struct A a, long b) {

5652

// ...

5653

// b = a.x;

5654

// ...

5655

// }

5656

//

5657

// and IR like this

5658

//

5659

// define void @foo(i32 %a1, i32 %a2, i32 %b) {

5660

// entry:

5661

// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment

5662

// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment

5663

// call void @llvm.dbg.value(metadata i32 %b, "b",

5664

// ...

5665

// call void @llvm.dbg.value(metadata i32 %a1, "b"

5666

// ...

5667

//

5668

// then the last dbg.value is describing a parameter "b" using a value that

5669

// is an argument. But since we already has used %a1 to describe a parameter

5670

// we should not handle that last dbg.value here (that would result in an

5671

// incorrect hoisting of the DBG_VALUE to the function entry).

5672

// Notice that we allow one dbg.value per IR level argument, to accommodate

5673

// for the situation with fragments above.

5674

if (VariableIsFunctionInputArg) {

5675

unsigned ArgNo = Arg->getArgNo();

5676

if (ArgNo >= FuncInfo.DescribedArgs.size())

5677

FuncInfo.DescribedArgs.resize(ArgNo + 1, false);

5678

else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))

5679

return false;

5680

FuncInfo.DescribedArgs.set(ArgNo);

5681

}

5682

}

5683

5684

bool IsIndirect = false;

5685

std::optional<MachineOperand> Op;

5686

// Some arguments' frame index is recorded during argument lowering.

5687

int FI = FuncInfo.getArgumentFrameIndex(Arg);

5688

if (FI != std::numeric_limits<int>::max())

5689

Op = MachineOperand::CreateFI(FI);

5690

5691

SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;

5692

if (!Op && N.getNode()) {

5693

getUnderlyingArgRegs(ArgRegsAndSizes, N);

5694

Register Reg;

5695

if (ArgRegsAndSizes.size() == 1)

5696

Reg = ArgRegsAndSizes.front().first;

5697

5698

if (Reg && Reg.isVirtual()) {

5699

MachineRegisterInfo &RegInfo = MF.getRegInfo();

5700

Register PR = RegInfo.getLiveInPhysReg(Reg);

5701

if (PR)

5702

Reg = PR;

5703

}

5704

if (Reg) {

5705

Op = MachineOperand::CreateReg(Reg, false);

5706

IsIndirect = Kind != FuncArgumentDbgValueKind::Value;

5707

}

5708

}

5709

5710

if (!Op && N.getNode()) {

5711

// Check if frame index is available.

5712

SDValue LCandidate = peekThroughBitcasts(N);

5713

if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))

5714

if (FrameIndexSDNode *FINode =

5715

dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))

5716

Op = MachineOperand::CreateFI(FINode->getIndex());

5717

}

5718

5719

if (!Op) {

5720

// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg

5721

auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>

5722

SplitRegs) {

5723

unsigned Offset = 0;

5724

for (const auto &RegAndSize : SplitRegs) {

5725

// If the expression is already a fragment, the current register

5726

// offset+size might extend beyond the fragment. In this case, only

5727

// the register bits that are inside the fragment are relevant.

5728

int RegFragmentSizeInBits = RegAndSize.second;

5729

if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {

5730

uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;

5731

// The register is entirely outside the expression fragment,

5732

// so is irrelevant for debug info.

5733

if (Offset >= ExprFragmentSizeInBits)

5734

break;

5735

// The register is partially outside the expression fragment, only

5736

// the low bits within the fragment are relevant for debug info.

5737

if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {

5738

RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;

5739

}

5740

}

5741

5742

auto FragmentExpr = DIExpression::createFragmentExpression(

5743

Expr, Offset, RegFragmentSizeInBits);

5744

Offset += RegAndSize.second;

5745

// If a valid fragment expression cannot be created, the variable's

5746

// correct value cannot be determined and so it is set as Undef.

5747

if (!FragmentExpr) {

5748

SDDbgValue *SDV = DAG.getConstantDbgValue(

5749

Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);

5750

DAG.AddDbgValue(SDV, false);

5751

continue;

5752

}

5753

MachineInstr *NewMI =

5754

MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,

5755

Kind != FuncArgumentDbgValueKind::Value);

5756

FuncInfo.ArgDbgValues.push_back(NewMI);

5757

}

5758

};

5759

5760

// Check if ValueMap has reg number.

5761

DenseMap<const Value *, Register>::const_iterator

5762

VMI = FuncInfo.ValueMap.find(V);

5763

if (VMI != FuncInfo.ValueMap.end()) {

5764

const auto &TLI = DAG.getTargetLoweringInfo();

5765

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,

5766

V->getType(), std::nullopt);

5767

if (RFV.occupiesMultipleRegs()) {

5768

splitMultiRegDbgValue(RFV.getRegsAndSizes());

5769

return true;

5770

}

5771

5772

Op = MachineOperand::CreateReg(VMI->second, false);

5773

IsIndirect = Kind != FuncArgumentDbgValueKind::Value;

5774

} else if (ArgRegsAndSizes.size() > 1) {

5775

// This was split due to the calling convention, and no virtual register

5776

// mapping exists for the value.

5777

splitMultiRegDbgValue(ArgRegsAndSizes);

5778

return true;

5779

}

5780

}

5781

5782

if (!Op)

5783

return false;

5784

5785

assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5786
, __extension__ __PRETTY_FUNCTION__))

5786

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5786
, __extension__ __PRETTY_FUNCTION__));

5787

MachineInstr *NewMI = nullptr;

5788

5789

if (Op->isReg())

5790

NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);

5791

else

5792

NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,

5793

Variable, Expr);

5794

5795

// Otherwise, use ArgDbgValues.

5796

FuncInfo.ArgDbgValues.push_back(NewMI);

5797

return true;

5798

}

5799

5800

/// Return the appropriate SDDbgValue based on N.

5801

SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,

5802

DILocalVariable *Variable,

5803

DIExpression *Expr,

File:	build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1138, column 10 Called C++ object pointer is null

Bug Summary

Annotated Source Code

/build/source/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp