/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1

//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements routines for translating from LLVM IR into SelectionDAG IR.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "SelectionDAGBuilder.h"

14

#include "SDNodeDbgValue.h"

15

#include "llvm/ADT/APFloat.h"

16

#include "llvm/ADT/APInt.h"

17

#include "llvm/ADT/BitVector.h"

18

#include "llvm/ADT/None.h"

19

#include "llvm/ADT/Optional.h"

20

#include "llvm/ADT/STLExtras.h"

21

#include "llvm/ADT/SmallPtrSet.h"

22

#include "llvm/ADT/SmallSet.h"

23

#include "llvm/ADT/StringRef.h"

24

#include "llvm/ADT/Triple.h"

25

#include "llvm/ADT/Twine.h"

26

#include "llvm/Analysis/AliasAnalysis.h"

27

#include "llvm/Analysis/BlockFrequencyInfo.h"

28

#include "llvm/Analysis/BranchProbabilityInfo.h"

29

#include "llvm/Analysis/ConstantFolding.h"

30

#include "llvm/Analysis/EHPersonalities.h"

31

#include "llvm/Analysis/Loads.h"

32

#include "llvm/Analysis/MemoryLocation.h"

33

#include "llvm/Analysis/ProfileSummaryInfo.h"

34

#include "llvm/Analysis/TargetLibraryInfo.h"

35

#include "llvm/Analysis/ValueTracking.h"

36

#include "llvm/Analysis/VectorUtils.h"

37

#include "llvm/CodeGen/Analysis.h"

38

#include "llvm/CodeGen/FunctionLoweringInfo.h"

39

#include "llvm/CodeGen/GCMetadata.h"

40

#include "llvm/CodeGen/MachineBasicBlock.h"

41

#include "llvm/CodeGen/MachineFrameInfo.h"

42

#include "llvm/CodeGen/MachineFunction.h"

43

#include "llvm/CodeGen/MachineInstr.h"

44

#include "llvm/CodeGen/MachineInstrBuilder.h"

45

#include "llvm/CodeGen/MachineJumpTableInfo.h"

46

#include "llvm/CodeGen/MachineMemOperand.h"

47

#include "llvm/CodeGen/MachineModuleInfo.h"

48

#include "llvm/CodeGen/MachineOperand.h"

49

#include "llvm/CodeGen/MachineRegisterInfo.h"

50

#include "llvm/CodeGen/RuntimeLibcalls.h"

51

#include "llvm/CodeGen/SelectionDAG.h"

52

#include "llvm/CodeGen/SelectionDAGTargetInfo.h"

53

#include "llvm/CodeGen/StackMaps.h"

54

#include "llvm/CodeGen/SwiftErrorValueTracking.h"

55

#include "llvm/CodeGen/TargetFrameLowering.h"

56

#include "llvm/CodeGen/TargetInstrInfo.h"

57

#include "llvm/CodeGen/TargetOpcodes.h"

58

#include "llvm/CodeGen/TargetRegisterInfo.h"

59

#include "llvm/CodeGen/TargetSubtargetInfo.h"

60

#include "llvm/CodeGen/WinEHFuncInfo.h"

61

#include "llvm/IR/Argument.h"

62

#include "llvm/IR/Attributes.h"

63

#include "llvm/IR/BasicBlock.h"

64

#include "llvm/IR/CFG.h"

65

#include "llvm/IR/CallingConv.h"

66

#include "llvm/IR/Constant.h"

67

#include "llvm/IR/ConstantRange.h"

68

#include "llvm/IR/Constants.h"

69

#include "llvm/IR/DataLayout.h"

70

#include "llvm/IR/DebugInfoMetadata.h"

71

#include "llvm/IR/DerivedTypes.h"

72

#include "llvm/IR/DiagnosticInfo.h"

73

#include "llvm/IR/Function.h"

74

#include "llvm/IR/GetElementPtrTypeIterator.h"

75

#include "llvm/IR/InlineAsm.h"

76

#include "llvm/IR/InstrTypes.h"

77

#include "llvm/IR/Instructions.h"

78

#include "llvm/IR/IntrinsicInst.h"

79

#include "llvm/IR/Intrinsics.h"

80

#include "llvm/IR/IntrinsicsAArch64.h"

81

#include "llvm/IR/IntrinsicsWebAssembly.h"

82

#include "llvm/IR/LLVMContext.h"

83

#include "llvm/IR/Metadata.h"

84

#include "llvm/IR/Module.h"

85

#include "llvm/IR/Operator.h"

86

#include "llvm/IR/PatternMatch.h"

87

#include "llvm/IR/Statepoint.h"

88

#include "llvm/IR/Type.h"

89

#include "llvm/IR/User.h"

90

#include "llvm/IR/Value.h"

91

#include "llvm/MC/MCContext.h"

92

#include "llvm/MC/MCSymbol.h"

93

#include "llvm/Support/AtomicOrdering.h"

94

#include "llvm/Support/Casting.h"

95

#include "llvm/Support/CommandLine.h"

96

#include "llvm/Support/Compiler.h"

97

#include "llvm/Support/Debug.h"

98

#include "llvm/Support/MathExtras.h"

99

#include "llvm/Support/raw_ostream.h"

100

#include "llvm/Target/TargetIntrinsicInfo.h"

101

#include "llvm/Target/TargetMachine.h"

102

#include "llvm/Target/TargetOptions.h"

103

#include "llvm/Transforms/Utils/Local.h"

104

#include <cstddef>

105

#include <cstring>

106

#include <iterator>

107

#include <limits>

108

#include <numeric>

109

#include <tuple>

110

111

using namespace llvm;

112

using namespace PatternMatch;

113

using namespace SwitchCG;

114

115

#define DEBUG_TYPE"isel" "isel"

116

117

/// LimitFloatPrecision - Generate low-precision inline sequences for

118

/// some float libcalls (6, 8 or 12 bits).

119

static unsigned LimitFloatPrecision;

120

121

static cl::opt<bool>

122

InsertAssertAlign("insert-assert-align", cl::init(true),

123

cl::desc("Insert the experimental `assertalign` node."),

124

cl::ReallyHidden);

125

126

static cl::opt<unsigned, true>

127

LimitFPPrecision("limit-float-precision",

128

cl::desc("Generate low-precision inline sequences "

129

"for some float libcalls"),

130

cl::location(LimitFloatPrecision), cl::Hidden,

131

cl::init(0));

132

133

static cl::opt<unsigned> SwitchPeelThreshold(

134

"switch-peel-threshold", cl::Hidden, cl::init(66),

135

cl::desc("Set the case probability threshold for peeling the case from a "

136

"switch statement. A value greater than 100 will void this "

137

"optimization"));

138

139

// Limit the width of DAG chains. This is important in general to prevent

140

// DAG-based analysis from blowing up. For example, alias analysis and

141

// load clustering may not complete in reasonable time. It is difficult to

142

// recognize and avoid this situation within each individual analysis, and

143

// future analyses are likely to have the same behavior. Limiting DAG width is

144

// the safe approach and will be especially important with global DAGs.

145

//

146

// MaxParallelChains default is arbitrarily high to avoid affecting

147

// optimization, but could be lowered to improve compile time. Any ld-ld-st-st

148

// sequence over this should have been converted to llvm.memcpy by the

149

// frontend. It is easy to induce this behavior with .ll code such as:

150

// %buffer = alloca [4096 x i8]

151

// %data = load [4096 x i8]* %argPtr

152

// store [4096 x i8] %data, [4096 x i8]* %buffer

153

static const unsigned MaxParallelChains = 64;

154

155

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

156

const SDValue *Parts, unsigned NumParts,

157

MVT PartVT, EVT ValueVT, const Value *V,

158

Optional<CallingConv::ID> CC);

159

160

/// getCopyFromParts - Create a value that contains the specified legal parts

161

/// combined into the value they represent. If the parts combine to a type

162

/// larger than ValueVT then AssertOp can be used to specify whether the extra

163

/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT

164

/// (ISD::AssertSext).

165

static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,

166

const SDValue *Parts, unsigned NumParts,

167

MVT PartVT, EVT ValueVT, const Value *V,

168

Optional<CallingConv::ID> CC = None,

169

Optional<ISD::NodeType> AssertOp = None) {

170

// Let the target assemble the parts if it wants to

171

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

172

if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,

173

PartVT, ValueVT, CC))

174

return Val;

175

176

if (ValueVT.isVector())

177

return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,

178

CC);

179

180

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 180, __extension__ __PRETTY_FUNCTION__));

181

SDValue Val = Parts[0];

182

183

if (NumParts > 1) {

184

// Assemble the value from multiple parts.

185

if (ValueVT.isInteger()) {

186

unsigned PartBits = PartVT.getSizeInBits();

187

unsigned ValueBits = ValueVT.getSizeInBits();

188

189

// Assemble the power of 2 part.

190

unsigned RoundParts =

191

(NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;

192

unsigned RoundBits = PartBits * RoundParts;

193

EVT RoundVT = RoundBits == ValueBits ?

194

ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);

195

SDValue Lo, Hi;

196

197

EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

198

199

if (RoundParts > 2) {

200

Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,

201

PartVT, HalfVT, V);

202

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,

203

RoundParts / 2, PartVT, HalfVT, V);

204

} else {

205

Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);

206

Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);

207

}

208

209

if (DAG.getDataLayout().isBigEndian())

210

std::swap(Lo, Hi);

211

212

Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

213

214

if (RoundParts < NumParts) {

215

// Assemble the trailing non-power-of-2 part.

216

unsigned OddParts = NumParts - RoundParts;

217

EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);

218

Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,

219

OddVT, V, CC);

220

221

// Combine the round and odd parts.

222

Lo = Val;

223

if (DAG.getDataLayout().isBigEndian())

224

std::swap(Lo, Hi);

225

EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

226

Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);

227

Hi =

228

DAG.getNode(ISD::SHL, DL, TotalVT, Hi,

229

DAG.getConstant(Lo.getValueSizeInBits(), DL,

230

TLI.getPointerTy(DAG.getDataLayout())));

231

Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);

232

Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);

233

}

234

} else if (PartVT.isFloatingPoint()) {

235

// FP split into multiple FP parts (for ppcf128)

236

assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 237, __extension__ __PRETTY_FUNCTION__))

237

"Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 237, __extension__ __PRETTY_FUNCTION__));

238

SDValue Lo, Hi;

239

Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);

240

Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);

241

if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

242

std::swap(Lo, Hi);

243

Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);

244

} else {

245

// FP split into integer parts (soft fp)

246

assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 247, __extension__ __PRETTY_FUNCTION__))

247

!PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 247, __extension__ __PRETTY_FUNCTION__));

248

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

249

Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);

250

}

251

}

252

253

// There is now one part, held in Val. Correct it to match ValueVT.

254

// PartEVT is the type of the register class that holds the value.

255

// ValueVT is the type of the inline asm operation.

256

EVT PartEVT = Val.getValueType();

257

258

if (PartEVT == ValueVT)

259

return Val;

260

261

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&

262

ValueVT.bitsLT(PartEVT)) {

263

// For an FP value in an integer part, we need to truncate to the right

264

// width first.

265

PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

266

Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);

267

}

268

269

// Handle types that have the same size.

270

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())

271

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

272

273

// Handle types with different sizes.

274

if (PartEVT.isInteger() && ValueVT.isInteger()) {

275

if (ValueVT.bitsLT(PartEVT)) {

276

// For a truncate, see if we have any information to

277

// indicate whether the truncated bits will always be

278

// zero or sign-extension.

279

if (AssertOp.hasValue())

280

Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,

281

DAG.getValueType(ValueVT));

282

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

283

}

284

return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

285

}

286

287

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

288

// FP_ROUND's are always exact here.

289

if (ValueVT.bitsLT(Val.getValueType()))

290

return DAG.getNode(

291

ISD::FP_ROUND, DL, ValueVT, Val,

292

DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

293

294

return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);

295

}

296

297

// Handle MMX to a narrower integer type by bitcasting MMX to integer and

298

// then truncating.

299

if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&

300

ValueVT.bitsLT(PartEVT)) {

301

Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);

302

return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

303

}

304

305

report_fatal_error("Unknown mismatch in getCopyFromParts!");

306

}

307

308

static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,

309

const Twine &ErrMsg) {

310

const Instruction *I = dyn_cast_or_null<Instruction>(V);

311

if (!V)

312

return Ctx.emitError(ErrMsg);

313

314

const char *AsmError = ", possible invalid constraint for vector type";

315

if (const CallInst *CI = dyn_cast<CallInst>(I))

316

if (CI->isInlineAsm())

317

return Ctx.emitError(I, ErrMsg + AsmError);

318

319

return Ctx.emitError(I, ErrMsg);

320

}

321

322

/// getCopyFromPartsVector - Create a value that contains the specified legal

323

/// parts combined into the value they represent. If the parts combine to a

324

/// type larger than ValueVT then AssertOp can be used to specify whether the

325

/// extra bits are known to be zero (ISD::AssertZext) or sign extended from

326

/// ValueVT (ISD::AssertSext).

327

static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,

328

const SDValue *Parts, unsigned NumParts,

329

MVT PartVT, EVT ValueVT, const Value *V,

330

Optional<CallingConv::ID> CallConv) {

331

assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 331, __extension__ __PRETTY_FUNCTION__));

332

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 332, __extension__ __PRETTY_FUNCTION__));

333

const bool IsABIRegCopy = CallConv.hasValue();

334

335

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

336

SDValue Val = Parts[0];

337

338

// Handle a multi-element vector.

339

if (NumParts > 1) {

340

EVT IntermediateVT;

341

MVT RegisterVT;

342

unsigned NumIntermediates;

343

unsigned NumRegs;

344

345

if (IsABIRegCopy) {

346

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

347

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

348

NumIntermediates, RegisterVT);

349

} else {

350

NumRegs =

351

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

352

NumIntermediates, RegisterVT);

353

}

354

355

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 355, __extension__ __PRETTY_FUNCTION__));

356

NumParts = NumRegs; // Silence a compiler warning.

357

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 357, __extension__ __PRETTY_FUNCTION__));

358

assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 360, __extension__ __PRETTY_FUNCTION__))

359

Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 360, __extension__ __PRETTY_FUNCTION__))

360

"Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 360, __extension__ __PRETTY_FUNCTION__));

361

362

// Assemble the parts into intermediate operands.

363

SmallVector<SDValue, 8> Ops(NumIntermediates);

364

if (NumIntermediates == NumParts) {

365

// If the register was not expanded, truncate or copy the value,

366

// as appropriate.

367

for (unsigned i = 0; i != NumParts; ++i)

368

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,

369

PartVT, IntermediateVT, V, CallConv);

370

} else if (NumParts > 0) {

371

// If the intermediate type was expanded, build the intermediate

372

// operands from the parts.

373

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 374, __extension__ __PRETTY_FUNCTION__))

374

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 374, __extension__ __PRETTY_FUNCTION__));

375

unsigned Factor = NumParts / NumIntermediates;

376

for (unsigned i = 0; i != NumIntermediates; ++i)

377

Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,

378

PartVT, IntermediateVT, V, CallConv);

379

}

380

381

// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the

382

// intermediate operands.

383

EVT BuiltVectorTy =

384

IntermediateVT.isVector()

385

? EVT::getVectorVT(

386

*DAG.getContext(), IntermediateVT.getScalarType(),

387

IntermediateVT.getVectorElementCount() * NumParts)

388

: EVT::getVectorVT(*DAG.getContext(),

389

IntermediateVT.getScalarType(),

390

NumIntermediates);

391

Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS

392

: ISD::BUILD_VECTOR,

393

DL, BuiltVectorTy, Ops);

394

}

395

396

// There is now one part, held in Val. Correct it to match ValueVT.

397

EVT PartEVT = Val.getValueType();

398

399

if (PartEVT == ValueVT)

400

return Val;

401

402

if (PartEVT.isVector()) {

403

// Vector/Vector bitcast.

404

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())

405

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

406

407

// If the element type of the source/dest vectors are the same, but the

408

// parts vector has more elements than the value vector, then we have a

409

// vector widening case (e.g. <2 x float> -> <4 x float>). Extract the

410

// elements we want.

411

if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {

412

assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 416, __extension__ __PRETTY_FUNCTION__))

413

ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 416, __extension__ __PRETTY_FUNCTION__))

414

(PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 416, __extension__ __PRETTY_FUNCTION__))

415

ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 416, __extension__ __PRETTY_FUNCTION__))

416

"Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
(PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 416, __extension__ __PRETTY_FUNCTION__));

417

PartEVT =

418

EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),

419

ValueVT.getVectorElementCount());

420

Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,

421

DAG.getVectorIdxConstant(0, DL));

422

if (PartEVT == ValueVT)

423

return Val;

424

}

425

426

// Promoted vector extract

427

return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);

428

}

429

430

// Trivial bitcast if the types are the same size and the destination

431

// vector type is legal.

432

if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&

433

TLI.isTypeLegal(ValueVT))

434

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

435

436

if (ValueVT.getVectorNumElements() != 1) {

437

// Certain ABIs require that vectors are passed as integers. For vectors

438

// are the same size, this is an obvious bitcast.

439

if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {

440

return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

441

} else if (ValueVT.bitsLT(PartEVT)) {

442

const uint64_t ValueSize = ValueVT.getFixedSizeInBits();

443

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

444

// Drop the extra bits.

445

Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);

446

return DAG.getBitcast(ValueVT, Val);

447

}

448

449

diagnosePossiblyInvalidConstraint(

450

*DAG.getContext(), V, "non-trivial scalar-to-vector conversion");

451

return DAG.getUNDEF(ValueVT);

452

}

453

454

// Handle cases such as i8 -> <1 x i1>

455

EVT ValueSVT = ValueVT.getVectorElementType();

456

if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {

457

if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())

458

Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);

459

else

460

Val = ValueVT.isFloatingPoint()

461

? DAG.getFPExtendOrRound(Val, DL, ValueSVT)

462

: DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);

463

}

464

465

return DAG.getBuildVector(ValueVT, DL, Val);

466

}

467

468

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,

469

SDValue Val, SDValue *Parts, unsigned NumParts,

470

MVT PartVT, const Value *V,

471

Optional<CallingConv::ID> CallConv);

472

473

/// getCopyToParts - Create a series of nodes that contain the specified value

474

/// split into legal parts. If the parts contain more bits than Val, then, for

475

/// integers, ExtendKind can be used to specify how to generate the extra bits.

476

static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,

477

SDValue *Parts, unsigned NumParts, MVT PartVT,

478

const Value *V,

479

Optional<CallingConv::ID> CallConv = None,

480

ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {

481

// Let the target split the parts if it wants to

482

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

483

if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,

484

CallConv))

485

return;

486

EVT ValueVT = Val.getValueType();

487

488

// Handle the vector case separately.

489

if (ValueVT.isVector())

490

return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,

491

CallConv);

492

493

unsigned PartBits = PartVT.getSizeInBits();

494

unsigned OrigNumParts = NumParts;

495

assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 496, __extension__ __PRETTY_FUNCTION__))

496

"Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 496, __extension__ __PRETTY_FUNCTION__));

497

498

if (NumParts == 0)

499

return;

500

501

assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __extension__ __PRETTY_FUNCTION__));

502

EVT PartEVT = PartVT;

503

if (PartEVT == ValueVT) {

504

assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 504, __extension__ __PRETTY_FUNCTION__));

505

Parts[0] = Val;

506

return;

507

}

508

509

if (NumParts * PartBits > ValueVT.getSizeInBits()) {

510

// If the parts cover more bits than the value has, promote the value.

511

if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {

512

assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 512, __extension__ __PRETTY_FUNCTION__));

513

Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);

514

} else {

515

if (ValueVT.isFloatingPoint()) {

516

// FP values need to be bitcast, then extended if they are being put

517

// into a larger container.

518

ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());

519

Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

520

}

521

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __extension__ __PRETTY_FUNCTION__))

522

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __extension__ __PRETTY_FUNCTION__))

523

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 523, __extension__ __PRETTY_FUNCTION__));

524

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

525

Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);

526

if (PartVT == MVT::x86mmx)

527

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

528

}

529

} else if (PartBits == ValueVT.getSizeInBits()) {

530

// Different types of the same size.

531

assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 531, __extension__ __PRETTY_FUNCTION__));

532

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

533

} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {

534

// If the parts cover less bits than value has, truncate the value.

535

assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __extension__ __PRETTY_FUNCTION__))

536

ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __extension__ __PRETTY_FUNCTION__))

537

"Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 537, __extension__ __PRETTY_FUNCTION__));

538

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

539

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

540

if (PartVT == MVT::x86mmx)

541

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

542

}

543

544

// The value may have changed - recompute ValueVT.

545

ValueVT = Val.getValueType();

546

assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 547, __extension__ __PRETTY_FUNCTION__))

547

"Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 547, __extension__ __PRETTY_FUNCTION__));

548

549

if (NumParts == 1) {

550

if (PartEVT != ValueVT) {

551

diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,

552

"scalar-to-vector conversion failed");

553

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

554

}

555

556

Parts[0] = Val;

557

return;

558

}

559

560

// Expand the value into multiple parts.

561

if (NumParts & (NumParts - 1)) {

562

// The number of parts is not a power of 2. Split off and copy the tail.

563

assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 564, __extension__ __PRETTY_FUNCTION__))

564

"Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 564, __extension__ __PRETTY_FUNCTION__));

565

unsigned RoundParts = 1 << Log2_32(NumParts);

566

unsigned RoundBits = RoundParts * PartBits;

567

unsigned OddParts = NumParts - RoundParts;

568

SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,

569

DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));

570

571

getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,

572

CallConv);

573

574

if (DAG.getDataLayout().isBigEndian())

575

// The odd parts were reversed by getCopyToParts - unreverse them.

576

std::reverse(Parts + RoundParts, Parts + NumParts);

577

578

NumParts = RoundParts;

579

ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);

580

Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);

581

}

582

583

// The number of parts is a power of 2. Repeatedly bisect the value using

584

// EXTRACT_ELEMENT.

585

Parts[0] = DAG.getNode(ISD::BITCAST, DL,

586

EVT::getIntegerVT(*DAG.getContext(),

587

ValueVT.getSizeInBits()),

588

Val);

589

590

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {

591

for (unsigned i = 0; i < NumParts; i += StepSize) {

592

unsigned ThisBits = StepSize * PartBits / 2;

593

EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);

594

SDValue &Part0 = Parts[i];

595

SDValue &Part1 = Parts[i+StepSize/2];

596

597

Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

598

ThisVT, Part0, DAG.getIntPtrConstant(1, DL));

599

Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,

600

ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

601

602

if (ThisBits == PartBits && ThisVT != PartVT) {

603

Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);

604

Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);

605

}

606

}

607

}

608

609

if (DAG.getDataLayout().isBigEndian())

610

std::reverse(Parts, Parts + OrigNumParts);

611

}

612

613

static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,

614

const SDLoc &DL, EVT PartVT) {

615

if (!PartVT.isVector())

616

return SDValue();

617

618

EVT ValueVT = Val.getValueType();

619

ElementCount PartNumElts = PartVT.getVectorElementCount();

620

ElementCount ValueNumElts = ValueVT.getVectorElementCount();

621

622

// We only support widening vectors with equivalent element types and

623

// fixed/scalable properties. If a target needs to widen a fixed-length type

624

// to a scalable one, it should be possible to use INSERT_SUBVECTOR below.

625

if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||

626

PartNumElts.isScalable() != ValueNumElts.isScalable() ||

627

PartVT.getVectorElementType() != ValueVT.getVectorElementType())

628

return SDValue();

629

630

// Widening a scalable vector to another scalable vector is done by inserting

631

// the vector into a larger undef one.

632

if (PartNumElts.isScalable())

633

return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),

634

Val, DAG.getVectorIdxConstant(0, DL));

635

636

EVT ElementVT = PartVT.getVectorElementType();

637

// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in

638

// undef elements.

639

SmallVector<SDValue, 16> Ops;

640

DAG.ExtractVectorElements(Val, Ops);

641

SDValue EltUndef = DAG.getUNDEF(ElementVT);

642

Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);

643

644

// FIXME: Use CONCAT for 2x -> 4x.

645

return DAG.getBuildVector(PartVT, DL, Ops);

646

}

647

648

/// getCopyToPartsVector - Create a series of nodes that contain the specified

649

/// value split into legal parts.

650

static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,

651

SDValue Val, SDValue *Parts, unsigned NumParts,

652

MVT PartVT, const Value *V,

653

Optional<CallingConv::ID> CallConv) {

654

EVT ValueVT = Val.getValueType();

655

assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 655, __extension__ __PRETTY_FUNCTION__));

656

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

657

const bool IsABIRegCopy = CallConv.hasValue();

658

659

if (NumParts == 1) {

660

EVT PartEVT = PartVT;

661

if (PartEVT == ValueVT) {

662

// Nothing to do.

663

} else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {

664

// Bitconvert vector->vector case.

665

Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);

666

} else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {

667

Val = Widened;

668

} else if (PartVT.isVector() &&

669

PartEVT.getVectorElementType().bitsGE(

670

ValueVT.getVectorElementType()) &&

671

PartEVT.getVectorElementCount() ==

672

ValueVT.getVectorElementCount()) {

673

674

// Promoted vector extract

675

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

676

} else {

677

if (ValueVT.getVectorElementCount().isScalar()) {

678

Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,

679

DAG.getVectorIdxConstant(0, DL));

680

} else {

681

uint64_t ValueSize = ValueVT.getFixedSizeInBits();

682

assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 683, __extension__ __PRETTY_FUNCTION__))

683

"lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
&& "lossy conversion of vector to scalar type") ? void
(0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 683, __extension__ __PRETTY_FUNCTION__));

684

EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);

685

Val = DAG.getBitcast(IntermediateType, Val);

686

Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);

687

}

688

}

689

690

assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
"Unexpected vector part value type") ? void (0) : __assert_fail
("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 690, __extension__ __PRETTY_FUNCTION__));

691

Parts[0] = Val;

692

return;

693

}

694

695

// Handle a multi-element vector.

696

EVT IntermediateVT;

697

MVT RegisterVT;

698

unsigned NumIntermediates;

699

unsigned NumRegs;

700

if (IsABIRegCopy) {

701

NumRegs = TLI.getVectorTypeBreakdownForCallingConv(

702

*DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,

703

NumIntermediates, RegisterVT);

704

} else {

705

NumRegs =

706

TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,

707

NumIntermediates, RegisterVT);

708

}

709

710

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 710, __extension__ __PRETTY_FUNCTION__));

711

NumParts = NumRegs; // Silence a compiler warning.

712

assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 712, __extension__ __PRETTY_FUNCTION__));

713

714

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 715, __extension__ __PRETTY_FUNCTION__))

715

"Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() ==
ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 715, __extension__ __PRETTY_FUNCTION__));

716

717

Optional<ElementCount> DestEltCnt;

718

719

if (IntermediateVT.isVector())

720

DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;

721

else

722

DestEltCnt = ElementCount::getFixed(NumIntermediates);

723

724

EVT BuiltVectorTy = EVT::getVectorVT(

725

*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());

726

727

if (ValueVT == BuiltVectorTy) {

728

// Nothing to do.

729

} else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {

730

// Bitconvert vector->vector case.

731

Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);

732

} else {

733

if (BuiltVectorTy.getVectorElementType().bitsGT(

734

ValueVT.getVectorElementType())) {

735

// Integer promotion.

736

ValueVT = EVT::getVectorVT(*DAG.getContext(),

737

BuiltVectorTy.getVectorElementType(),

738

ValueVT.getVectorElementCount());

739

Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);

740

}

741

742

if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {

743

Val = Widened;

744

}

745

}

746

747

assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy
&& "Unexpected vector value type") ? void (0) : __assert_fail
("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 747, __extension__ __PRETTY_FUNCTION__));

748

749

// Split the vector into intermediate operands.

750

SmallVector<SDValue, 8> Ops(NumIntermediates);

751

for (unsigned i = 0; i != NumIntermediates; ++i) {

752

if (IntermediateVT.isVector()) {

753

// This does something sensible for scalable vectors - see the

754

// definition of EXTRACT_SUBVECTOR for further details.

755

unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();

756

Ops[i] =

757

DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,

758

DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));

759

} else {

760

Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,

761

DAG.getVectorIdxConstant(i, DL));

762

}

763

}

764

765

// Split the intermediate operands into legal parts.

766

if (NumParts == NumIntermediates) {

767

// If the register was not expanded, promote or copy the value,

768

// as appropriate.

769

for (unsigned i = 0; i != NumParts; ++i)

770

getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);

771

} else if (NumParts > 0) {

772

// If the intermediate type was expanded, split each the value into

773

// legal parts.

774

assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 774, __extension__ __PRETTY_FUNCTION__));

775

assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 776, __extension__ __PRETTY_FUNCTION__))

776

"Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 776, __extension__ __PRETTY_FUNCTION__));

777

unsigned Factor = NumParts / NumIntermediates;

778

for (unsigned i = 0; i != NumIntermediates; ++i)

779

getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,

780

CallConv);

781

}

782

}

783

784

RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,

785

EVT valuevt, Optional<CallingConv::ID> CC)

786

: ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),

787

RegCount(1, regs.size()), CallConv(CC) {}

788

789

RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,

790

const DataLayout &DL, unsigned Reg, Type *Ty,

791

Optional<CallingConv::ID> CC) {

792

ComputeValueVTs(TLI, DL, Ty, ValueVTs);

793

794

CallConv = CC;

795

796

for (EVT ValueVT : ValueVTs) {

797

unsigned NumRegs =

798

isABIMangled()

799

? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)

800

: TLI.getNumRegisters(Context, ValueVT);

801

MVT RegisterVT =

802

isABIMangled()

803

? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)

804

: TLI.getRegisterType(Context, ValueVT);

805

for (unsigned i = 0; i != NumRegs; ++i)

806

Regs.push_back(Reg + i);

807

RegVTs.push_back(RegisterVT);

808

RegCount.push_back(NumRegs);

809

Reg += NumRegs;

810

}

811

}

812

813

SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,

814

FunctionLoweringInfo &FuncInfo,

815

const SDLoc &dl, SDValue &Chain,

816

SDValue *Flag, const Value *V) const {

817

// A Value with type {} or [0 x %t] needs no registers.

818

if (ValueVTs.empty())

819

return SDValue();

820

821

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

822

823

// Assemble the legal parts into the final values.

824

SmallVector<SDValue, 4> Values(ValueVTs.size());

825

SmallVector<SDValue, 8> Parts;

826

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

827

// Copy the legal parts from the registers.

828

EVT ValueVT = ValueVTs[Value];

829

unsigned NumRegs = RegCount[Value];

830

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

831

*DAG.getContext(),

832

CallConv.getValue(), RegVTs[Value])

833

: RegVTs[Value];

834

835

Parts.resize(NumRegs);

836

for (unsigned i = 0; i != NumRegs; ++i) {

837

SDValue P;

838

if (!Flag) {

839

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);

840

} else {

841

P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);

842

*Flag = P.getValue(2);

843

}

844

845

Chain = P.getValue(1);

846

Parts[i] = P;

847

848

// If the source register was virtual and if we know something about it,

849

// add an assert node.

850

if (!Register::isVirtualRegister(Regs[Part + i]) ||

851

!RegisterVT.isInteger())

852

continue;

853

854

const FunctionLoweringInfo::LiveOutInfo *LOI =

855

FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);

856

if (!LOI)

857

continue;

858

859

unsigned RegSize = RegisterVT.getScalarSizeInBits();

860

unsigned NumSignBits = LOI->NumSignBits;

861

unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

862

863

if (NumZeroBits == RegSize) {

864

// The current value is a zero.

865

// Explicitly express that as it would be easier for

866

// optimizations to kick in.

867

Parts[i] = DAG.getConstant(0, dl, RegisterVT);

868

continue;

869

}

870

871

// FIXME: We capture more information than the dag can represent. For

872

// now, just use the tightest assertzext/assertsext possible.

873

bool isSExt;

874

EVT FromVT(MVT::Other);

875

if (NumZeroBits) {

876

FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);

877

isSExt = false;

878

} else if (NumSignBits > 1) {

879

FromVT =

880

EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);

881

isSExt = true;

882

} else {

883

continue;

884

}

885

// Add an assertion node.

886

assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
__assert_fail ("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 886, __extension__ __PRETTY_FUNCTION__));

887

Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,

888

RegisterVT, P, DAG.getValueType(FromVT));

889

}

890

891

Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,

892

RegisterVT, ValueVT, V, CallConv);

893

Part += NumRegs;

894

Parts.clear();

895

}

896

897

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);

898

}

899

900

void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,

901

const SDLoc &dl, SDValue &Chain, SDValue *Flag,

902

const Value *V,

903

ISD::NodeType PreferredExtendType) const {

904

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

905

ISD::NodeType ExtendKind = PreferredExtendType;

906

907

// Get the list of the values's legal parts.

908

unsigned NumRegs = Regs.size();

909

SmallVector<SDValue, 8> Parts(NumRegs);

910

for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {

911

unsigned NumParts = RegCount[Value];

912

913

MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(

914

*DAG.getContext(),

915

CallConv.getValue(), RegVTs[Value])

916

: RegVTs[Value];

917

918

if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))

919

ExtendKind = ISD::ZERO_EXTEND;

920

921

getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],

922

NumParts, RegisterVT, V, CallConv, ExtendKind);

923

Part += NumParts;

924

}

925

926

// Copy the parts into the registers.

927

SmallVector<SDValue, 8> Chains(NumRegs);

928

for (unsigned i = 0; i != NumRegs; ++i) {

929

SDValue Part;

930

if (!Flag) {

931

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);

932

} else {

933

Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);

934

*Flag = Part.getValue(1);

935

}

936

937

Chains[i] = Part.getValue(0);

938

}

939

940

if (NumRegs == 1 || Flag)

941

// If NumRegs > 1 && Flag is used then the use of the last CopyToReg is

942

// flagged to it. That is the CopyToReg nodes and the user are considered

943

// a single scheduling unit. If we create a TokenFactor and return it as

944

// chain, then the TokenFactor is both a predecessor (operand) of the

945

// user as well as a successor (the TF operands are flagged to the user).

946

// c1, f1 = CopyToReg

947

// c2, f2 = CopyToReg

948

// c3 = TokenFactor c1, c2

949

// ...

950

// = op c3, ..., f2

951

Chain = Chains[NumRegs-1];

952

else

953

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);

954

}

955

956

void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,

957

unsigned MatchingIdx, const SDLoc &dl,

958

SelectionDAG &DAG,

959

std::vector<SDValue> &Ops) const {

960

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

961

962

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());

963

if (HasMatching)

964

Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);

965

else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {

966

// Put the register class of the virtual registers in the flag word. That

967

// way, later passes can recompute register class constraints for inline

968

// assembly as well as normal instructions.

969

// Don't do this for tied operands that can use the regclass information

970

// from the def.

971

const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

972

const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());

973

Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());

974

}

975

976

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);

977

Ops.push_back(Res);

978

979

if (Code == InlineAsm::Kind_Clobber) {

980

// Clobbers should always have a 1:1 mapping with registers, and may

981

// reference registers that have illegal (e.g. vector) types. Hence, we

982

// shouldn't try to apply any sort of splitting logic to them.

983

assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))

984

"No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 984, __extension__ __PRETTY_FUNCTION__));

985

Register SP = TLI.getStackPointerRegisterToSaveRestore();

986

(void)SP;

987

for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {

988

Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));

989

assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))

990

(Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))

991

DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 992, __extension__ __PRETTY_FUNCTION__))

992

"If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 992, __extension__ __PRETTY_FUNCTION__));

993

}

994

return;

995

}

996

997

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {

998

MVT RegisterVT = RegVTs[Value];

999

unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],

1000

RegisterVT);

1001

for (unsigned i = 0; i != NumRegs; ++i) {

1002

assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1002, __extension__ __PRETTY_FUNCTION__));

1003

unsigned TheReg = Regs[Reg++];

1004

Ops.push_back(DAG.getRegister(TheReg, RegisterVT));

1005

}

1006

}

1007

}

1008

1009

SmallVector<std::pair<unsigned, TypeSize>, 4>

1010

RegsForValue::getRegsAndSizes() const {

1011

SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;

1012

unsigned I = 0;

1013

for (auto CountAndVT : zip_first(RegCount, RegVTs)) {

1014

unsigned RegCount = std::get<0>(CountAndVT);

1015

MVT RegisterVT = std::get<1>(CountAndVT);

1016

TypeSize RegisterSize = RegisterVT.getSizeInBits();

1017

for (unsigned E = I + RegCount; I != E; ++I)

1018

OutVec.push_back(std::make_pair(Regs[I], RegisterSize));

1019

}

1020

return OutVec;

1021

}

1022

1023

void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,

1024

const TargetLibraryInfo *li) {

1025

AA = aa;

1026

GFI = gfi;

1027

LibInfo = li;

1028

DL = &DAG.getDataLayout();

1029

Context = DAG.getContext();

1030

LPadToCallSiteMap.clear();

1031

SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());

1032

}

1033

1034

void SelectionDAGBuilder::clear() {

1035

NodeMap.clear();

1036

UnusedArgNodeMap.clear();

1037

PendingLoads.clear();

1038

PendingExports.clear();

1039

PendingConstrainedFP.clear();

1040

PendingConstrainedFPStrict.clear();

1041

CurInst = nullptr;

1042

HasTailCall = false;

1043

SDNodeOrder = LowestSDNodeOrder;

1044

StatepointLowering.clear();

1045

}

1046

1047

void SelectionDAGBuilder::clearDanglingDebugInfo() {

1048

DanglingDebugInfoMap.clear();

1049

}

1050

1051

// Update DAG root to include dependencies on Pending chains.

1052

SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {

1053

SDValue Root = DAG.getRoot();

1054

1055

if (Pending.empty())

1056

return Root;

1057

1058

// Add current root to PendingChains, unless we already indirectly

1059

// depend on it.

1060

if (Root.getOpcode() != ISD::EntryToken) {

1061

unsigned i = 0, e = Pending.size();

1062

for (; i != e; ++i) {

1063

assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1063, __extension__ __PRETTY_FUNCTION__));

1064

if (Pending[i].getNode()->getOperand(0) == Root)

1065

break; // Don't add the root if we already indirectly depend on it.

1066

}

1067

1068

if (i == e)

1069

Pending.push_back(Root);

1070

}

1071

1072

if (Pending.size() == 1)

1073

Root = Pending[0];

1074

else

1075

Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

1076

1077

DAG.setRoot(Root);

1078

Pending.clear();

1079

return Root;

1080

}

1081

1082

SDValue SelectionDAGBuilder::getMemoryRoot() {

1083

return updateRoot(PendingLoads);

1084

}

1085

1086

SDValue SelectionDAGBuilder::getRoot() {

1087

// Chain up all pending constrained intrinsics together with all

1088

// pending loads, by simply appending them to PendingLoads and

1089

// then calling getMemoryRoot().

1090

PendingLoads.reserve(PendingLoads.size() +

1091

PendingConstrainedFP.size() +

1092

PendingConstrainedFPStrict.size());

1093

PendingLoads.append(PendingConstrainedFP.begin(),

1094

PendingConstrainedFP.end());

1095

PendingLoads.append(PendingConstrainedFPStrict.begin(),

1096

PendingConstrainedFPStrict.end());

1097

PendingConstrainedFP.clear();

1098

PendingConstrainedFPStrict.clear();

1099

return getMemoryRoot();

1100

}

1101

1102

SDValue SelectionDAGBuilder::getControlRoot() {

1103

// We need to emit pending fpexcept.strict constrained intrinsics,

1104

// so append them to the PendingExports list.

1105

PendingExports.append(PendingConstrainedFPStrict.begin(),

1106

PendingConstrainedFPStrict.end());

1107

PendingConstrainedFPStrict.clear();

1108

return updateRoot(PendingExports);

1109

}

1110

1111

void SelectionDAGBuilder::visit(const Instruction &I) {

1112

// Set up outgoing PHI node register values before emitting the terminator.

1113

if (I.isTerminator()) {

1114

HandlePHINodesInSuccessorBlocks(I.getParent());

1115

}

1116

1117

// Increase the SDNodeOrder if dealing with a non-debug instruction.

1118

if (!isa<DbgInfoIntrinsic>(I))

1119

++SDNodeOrder;

1120

1121

CurInst = &I;

1122

1123

visit(I.getOpcode(), I);

1124

1125

if (!I.isTerminator() && !HasTailCall &&

1126

!isa<GCStatepointInst>(I)) // statepoints handle their exports internally

1127

CopyToExportRegsIfNeeded(&I);

1128

1129

CurInst = nullptr;

1130

}

1131

1132

void SelectionDAGBuilder::visitPHI(const PHINode &) {

1133

llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1133);

1134

}

1135

1136

void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {

1137

// Note: this doesn't use InstVisitor, because it has to work with

1138

// ConstantExpr's in addition to instructions.

1139

switch (Opcode) {

1140

default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1140);

1141

// Build the switch statement using the Instruction.def file.

1142

#define HANDLE_INST(NUM, OPCODE, CLASS) \

1143

case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;

1144

#include "llvm/IR/Instruction.def"

1145

}

1146

}

1147

1148

void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,

1149

DebugLoc DL, unsigned Order) {

1150

// We treat variadic dbg_values differently at this stage.

1151

if (DI->hasArgList()) {

1152

// For variadic dbg_values we will now insert an undef.

1153

// FIXME: We can potentially recover these!

1154

SmallVector<SDDbgOperand, 2> Locs;

1155

for (const Value *V : DI->getValues()) {

1156

auto Undef = UndefValue::get(V->getType());

1157

Locs.push_back(SDDbgOperand::fromConst(Undef));

1158

}

1159

SDDbgValue *SDV = DAG.getDbgValueList(

1160

DI->getVariable(), DI->getExpression(), Locs, {},

1161

/*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);

1162

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1163

} else {

1164

// TODO: Dangling debug info will eventually either be resolved or produce

1165

// an Undef DBG_VALUE. However in the resolution case, a gap may appear

1166

// between the original dbg.value location and its resolved DBG_VALUE,

1167

// which we should ideally fill with an extra Undef DBG_VALUE.

1168

assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__))

1169

"DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__))

1170

"operand.")(static_cast <bool> (DI->getNumVariableLocationOps()
== 1 && "DbgValueInst without an ArgList should have a single location "
"operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__));

1171

DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);

1172

}

1173

}

1174

1175

void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,

1176

const DIExpression *Expr) {

1177

auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {

1178

const DbgValueInst *DI = DDI.getDI();

1179

DIVariable *DanglingVariable = DI->getVariable();

1180

DIExpression *DanglingExpr = DI->getExpression();

1181

if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {

1182

LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false);

1183

return true;

1184

}

1185

return false;

1186

};

1187

1188

for (auto &DDIMI : DanglingDebugInfoMap) {

1189

DanglingDebugInfoVector &DDIV = DDIMI.second;

1190

1191

// If debug info is to be dropped, run it through final checks to see

1192

// whether it can be salvaged.

1193

for (auto &DDI : DDIV)

1194

if (isMatchingDbgValue(DDI))

1195

salvageUnresolvedDbgValue(DDI);

1196

1197

erase_if(DDIV, isMatchingDbgValue);

1198

}

1199

}

1200

1201

// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,

1202

// generate the debug data structures now that we've seen its definition.

1203

void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,

1204

SDValue Val) {

1205

auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);

1206

if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())

1207

return;

1208

1209

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;

1210

for (auto &DDI : DDIV) {

1211

const DbgValueInst *DI = DDI.getDI();

1212

assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")(static_cast <bool> (!DI->hasArgList() && "Not implemented for variadic dbg_values"
) ? void (0) : __assert_fail ("!DI->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1212, __extension__ __PRETTY_FUNCTION__));

1213

assert(DI && "Ill-formed DanglingDebugInfo")(static_cast <bool> (DI && "Ill-formed DanglingDebugInfo"
) ? void (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1213, __extension__ __PRETTY_FUNCTION__));

1214

DebugLoc dl = DDI.getdl();

1215

unsigned ValSDNodeOrder = Val.getNode()->getIROrder();

1216

unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();

1217

DILocalVariable *Variable = DI->getVariable();

1218

DIExpression *Expr = DI->getExpression();

1219

assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1220, __extension__ __PRETTY_FUNCTION__))

1220

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1220, __extension__ __PRETTY_FUNCTION__));

1221

SDDbgValue *SDV;

1222

if (Val.getNode()) {

1223

// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a

1224

// FuncArgumentDbgValue (it would be hoisted to the function entry, and if

1225

// we couldn't resolve it directly when examining the DbgValue intrinsic

1226

// in the first place we should not be more successful here). Unless we

1227

// have some test case that prove this to be correct we should avoid

1228

// calling EmitFuncArgumentDbgValue here.

1229

if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {

1230

LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)

1231

<< DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false);

1232

LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);

1233

// Increase the SDNodeOrder for the DbgValue here to make sure it is

1234

// inserted after the definition of Val when emitting the instructions

1235

// after ISel. An alternative could be to teach

1236

// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.

1237

LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1238

<< "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)

1239

<< ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);

1240

SDV = getDbgValue(Val, Variable, Expr, dl,

1241

std::max(DbgSDNodeOrder, ValSDNodeOrder));

1242

DAG.AddDbgValue(SDV, false);

1243

} else

1244

LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)

1245

<< "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false);

1246

} else {

1247

LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false);

1248

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1249

auto SDV =

1250

DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);

1251

DAG.AddDbgValue(SDV, false);

1252

}

1253

}

1254

DDIV.clear();

1255

}

1256

1257

void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {

1258

// TODO: For the variadic implementation, instead of only checking the fail

1259

// state of `handleDebugValue`, we need know specifically which values were

1260

// invalid, so that we attempt to salvage only those values when processing

1261

// a DIArgList.

1262

assert(!DDI.getDI()->hasArgList() &&(static_cast <bool> (!DDI.getDI()->hasArgList() &&
"Not implemented for variadic dbg_values") ? void (0) : __assert_fail
("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1263, __extension__ __PRETTY_FUNCTION__))

1263

"Not implemented for variadic dbg_values")(static_cast <bool> (!DDI.getDI()->hasArgList() &&
"Not implemented for variadic dbg_values") ? void (0) : __assert_fail
("!DDI.getDI()->hasArgList() && \"Not implemented for variadic dbg_values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1263, __extension__ __PRETTY_FUNCTION__));

1264

Value *V = DDI.getDI()->getValue(0);

1265

DILocalVariable *Var = DDI.getDI()->getVariable();

1266

DIExpression *Expr = DDI.getDI()->getExpression();

1267

DebugLoc DL = DDI.getdl();

1268

DebugLoc InstDL = DDI.getDI()->getDebugLoc();

1269

unsigned SDOrder = DDI.getSDNodeOrder();

1270

// Currently we consider only dbg.value intrinsics -- we tell the salvager

1271

// that DW_OP_stack_value is desired.

1272

assert(isa<DbgValueInst>(DDI.getDI()))(static_cast <bool> (isa<DbgValueInst>(DDI.getDI(
))) ? void (0) : __assert_fail ("isa<DbgValueInst>(DDI.getDI())"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1272, __extension__ __PRETTY_FUNCTION__));

1273

bool StackValue = true;

1274

1275

// Can this Value can be encoded without any further work?

1276

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))

1277

return;

1278

1279

// Attempt to salvage back through as many instructions as possible. Bail if

1280

// a non-instruction is seen, such as a constant expression or global

1281

// variable. FIXME: Further work could recover those too.

1282

while (isa<Instruction>(V)) {

1283

Instruction &VAsInst = *cast<Instruction>(V);

1284

// Temporary "0", awaiting real implementation.

1285

SmallVector<uint64_t, 16> Ops;

1286

SmallVector<Value *, 4> AdditionalValues;

1287

V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,

1288

AdditionalValues);

1289

// If we cannot salvage any further, and haven't yet found a suitable debug

1290

// expression, bail out.

1291

if (!V)

1292

break;

1293

1294

// TODO: If AdditionalValues isn't empty, then the salvage can only be

1295

// represented with a DBG_VALUE_LIST, so we give up. When we have support

1296

// here for variadic dbg_values, remove that condition.

1297

if (!AdditionalValues.empty())

1298

break;

1299

1300

// New value and expr now represent this debuginfo.

1301

Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);

1302

1303

// Some kind of simplification occurred: check whether the operand of the

1304

// salvaged debug expression can be encoded in this DAG.

1305

if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,

1306

/*IsVariadic=*/false)) {

1307

LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false)

1308

<< DDI.getDI() << "\nBy stripping back to:\n " << V)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n "
<< DDI.getDI() << "\nBy stripping back to:\n " <<
V; } } while (false);

1309

return;

1310

}

1311

}

1312

1313

// This was the final opportunity to salvage this debug information, and it

1314

// couldn't be done. Place an undef DBG_VALUE at this location to terminate

1315

// any earlier variable location.

1316

auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());

1317

auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);

1318

DAG.AddDbgValue(SDV, false);

1319

1320

LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false)

1321

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n "
<< DDI.getDI() << "\n"; } } while (false);

1322

LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
)

1323

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
DDI.getDI()->getOperand(0) << "\n"; } } while (false
);

1324

}

1325

1326

bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,

1327

DILocalVariable *Var,

1328

DIExpression *Expr, DebugLoc dl,

1329

DebugLoc InstDL, unsigned Order,

1330

bool IsVariadic) {

1331

if (Values.empty())

1332

return true;

1333

SmallVector<SDDbgOperand> LocationOps;

1334

SmallVector<SDNode *> Dependencies;

1335

for (const Value *V : Values) {

1336

// Constant value.

1337

if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||

1338

isa<ConstantPointerNull>(V)) {

1339

LocationOps.emplace_back(SDDbgOperand::fromConst(V));

1340

continue;

1341

}

1342

1343

// If the Value is a frame index, we can create a FrameIndex debug value

1344

// without relying on the DAG at all.

1345

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1346

auto SI = FuncInfo.StaticAllocaMap.find(AI);

1347

if (SI != FuncInfo.StaticAllocaMap.end()) {

1348

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));

1349

continue;

1350

}

1351

}

1352

1353

// Do not use getValue() in here; we don't want to generate code at

1354

// this point if it hasn't been done yet.

1355

SDValue N = NodeMap[V];

1356

if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.

1357

N = UnusedArgNodeMap[V];

1358

if (N.getNode()) {

1359

// Only emit func arg dbg value for non-variadic dbg.values for now.

1360

if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))

1361

return true;

1362

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

1363

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can

1364

// describe stack slot locations.

1365

//

1366

// Consider "int x = 0; int *px = &x;". There are two kinds of

1367

// interesting debug values here after optimization:

1368

//

1369

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

1370

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

1371

//

1372

// Both describe the direct values of their associated variables.

1373

Dependencies.push_back(N.getNode());

1374

LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));

1375

continue;

1376

}

1377

LocationOps.emplace_back(

1378

SDDbgOperand::fromNode(N.getNode(), N.getResNo()));

1379

continue;

1380

}

1381

1382

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1383

// Special rules apply for the first dbg.values of parameter variables in a

1384

// function. Identify them by the fact they reference Argument Values, that

1385

// they're parameters, and they are parameters of the current function. We

1386

// need to let them dangle until they get an SDNode.

1387

bool IsParamOfFunc =

1388

isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();

1389

if (IsParamOfFunc)

1390

return false;

1391

1392

// The value is not used in this block yet (or it would have an SDNode).

1393

// We still want the value to appear for the user if possible -- if it has

1394

// an associated VReg, we can refer to that instead.

1395

auto VMI = FuncInfo.ValueMap.find(V);

1396

if (VMI != FuncInfo.ValueMap.end()) {

1397

unsigned Reg = VMI->second;

1398

// If this is a PHI node, it may be split up into several MI PHI nodes

1399

// (in FunctionLoweringInfo::set).

1400

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,

1401

V->getType(), None);

1402

if (RFV.occupiesMultipleRegs()) {

1403

// FIXME: We could potentially support variadic dbg_values here.

1404

if (IsVariadic)

1405

return false;

1406

unsigned Offset = 0;

1407

unsigned BitsToDescribe = 0;

1408

if (auto VarSize = Var->getSizeInBits())

1409

BitsToDescribe = *VarSize;

1410

if (auto Fragment = Expr->getFragmentInfo())

1411

BitsToDescribe = Fragment->SizeInBits;

1412

for (auto RegAndSize : RFV.getRegsAndSizes()) {

1413

// Bail out if all bits are described already.

1414

if (Offset >= BitsToDescribe)

1415

break;

1416

// TODO: handle scalable vectors.

1417

unsigned RegisterSize = RegAndSize.second;

1418

unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)

1419

? BitsToDescribe - Offset

1420

: RegisterSize;

1421

auto FragmentExpr = DIExpression::createFragmentExpression(

1422

Expr, Offset, FragmentSize);

1423

if (!FragmentExpr)

1424

continue;

1425

SDDbgValue *SDV = DAG.getVRegDbgValue(

1426

Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);

1427

DAG.AddDbgValue(SDV, false);

1428

Offset += RegisterSize;

1429

}

1430

return true;

1431

}

1432

// We can use simple vreg locations for variadic dbg_values as well.

1433

LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));

1434

continue;

1435

}

1436

// We failed to create a SDDbgOperand for V.

1437

return false;

1438

}

1439

1440

// We have created a SDDbgOperand for each Value in Values.

1441

// Should use Order instead of SDNodeOrder?

1442

assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) :
__assert_fail ("!LocationOps.empty()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1442, __extension__ __PRETTY_FUNCTION__));

1443

SDDbgValue *SDV =

1444

DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,

1445

/*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);

1446

DAG.AddDbgValue(SDV, /*isParameter=*/false);

1447

return true;

1448

}

1449

1450

void SelectionDAGBuilder::resolveOrClearDbgInfo() {

1451

// Try to fixup any remaining dangling debug info -- and drop it if we can't.

1452

for (auto &Pair : DanglingDebugInfoMap)

1453

for (auto &DDI : Pair.second)

1454

salvageUnresolvedDbgValue(DDI);

1455

clearDanglingDebugInfo();

1456

}

1457

1458

/// getCopyFromRegs - If there was virtual register allocated for the value V

1459

/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.

1460

SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {

1461

DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);

1462

SDValue Result;

1463

1464

if (It != FuncInfo.ValueMap.end()) {

1465

Register InReg = It->second;

1466

1467

RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1468

DAG.getDataLayout(), InReg, Ty,

1469

None); // This is not an ABI copy.

1470

SDValue Chain = DAG.getEntryNode();

1471

Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,

1472

V);

1473

resolveDanglingDebugInfo(V, Result);

1474

}

1475

1476

return Result;

1477

}

1478

1479

/// getValue - Return an SDValue for the given Value.

1480

SDValue SelectionDAGBuilder::getValue(const Value *V) {

1481

// If we already have an SDValue for this value, use it. It's important

1482

// to do this first, so that we don't create a CopyFromReg if we already

1483

// have a regular SDValue.

1484

SDValue &N = NodeMap[V];

1485

if (N.getNode()) return N;

1486

1487

// If there's a virtual register allocated and initialized for this

1488

// value, use it.

1489

if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))

1490

return copyFromReg;

1491

1492

// Otherwise create a new SDValue and remember it.

1493

SDValue Val = getValueImpl(V);

1494

NodeMap[V] = Val;

1495

resolveDanglingDebugInfo(V, Val);

1496

return Val;

1497

}

1498

1499

/// getNonRegisterValue - Return an SDValue for the given Value, but

1500

/// don't look in FuncInfo.ValueMap for a virtual register.

1501

SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {

1502

// If we already have an SDValue for this value, use it.

1503

SDValue &N = NodeMap[V];

1504

if (N.getNode()) {

1505

if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {

1506

// Remove the debug location from the node as the node is about to be used

1507

// in a location which may differ from the original debug location. This

1508

// is relevant to Constant and ConstantFP nodes because they can appear

1509

// as constant expressions inside PHI nodes.

1510

N->setDebugLoc(DebugLoc());

1511

}

1512

return N;

1513

}

1514

1515

// Otherwise create a new SDValue and remember it.

1516

SDValue Val = getValueImpl(V);

1517

NodeMap[V] = Val;

1518

resolveDanglingDebugInfo(V, Val);

1519

return Val;

1520

}

1521

1522

/// getValueImpl - Helper function for getValue and getNonRegisterValue.

1523

/// Create an SDValue for the given value.

1524

SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {

1525

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1526

1527

if (const Constant *C = dyn_cast<Constant>(V)) {

1528

EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

1529

1530

if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))

1531

return DAG.getConstant(*CI, getCurSDLoc(), VT);

1532

1533

if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))

1534

return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

1535

1536

if (isa<ConstantPointerNull>(C)) {

1537

unsigned AS = V->getType()->getPointerAddressSpace();

1538

return DAG.getConstant(0, getCurSDLoc(),

1539

TLI.getPointerTy(DAG.getDataLayout(), AS));

1540

}

1541

1542

if (match(C, m_VScale(DAG.getDataLayout())))

1543

return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

1544

1545

if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))

1546

return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

1547

1548

if (isa<UndefValue>(C) && !V->getType()->isAggregateType())

1549

return DAG.getUNDEF(VT);

1550

1551

if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {

1552

visit(CE->getOpcode(), *CE);

1553

SDValue N1 = NodeMap[V];

1554

assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1554, __extension__ __PRETTY_FUNCTION__));

1555

return N1;

1556

}

1557

1558

if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {

1559

SmallVector<SDValue, 4> Constants;

1560

for (const Use &U : C->operands()) {

1561

SDNode *Val = getValue(U).getNode();

1562

// If the operand is an empty aggregate, there are no values.

1563

if (!Val) continue;

1564

// Add each leaf value from the operand to the Constants list

1565

// to form a flattened list of all the values.

1566

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1567

Constants.push_back(SDValue(Val, i));

1568

}

1569

1570

return DAG.getMergeValues(Constants, getCurSDLoc());

1571

}

1572

1573

if (const ConstantDataSequential *CDS =

1574

dyn_cast<ConstantDataSequential>(C)) {

1575

SmallVector<SDValue, 4> Ops;

1576

for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {

1577

SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();

1578

// Add each leaf value from the operand to the Constants list

1579

// to form a flattened list of all the values.

1580

for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)

1581

Ops.push_back(SDValue(Val, i));

1582

}

1583

1584

if (isa<ArrayType>(CDS->getType()))

1585

return DAG.getMergeValues(Ops, getCurSDLoc());

1586

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1587

}

1588

1589

if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {

1590

assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1591, __extension__ __PRETTY_FUNCTION__))

1591

"Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1591, __extension__ __PRETTY_FUNCTION__));

1592

1593

SmallVector<EVT, 4> ValueVTs;

1594

ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);

1595

unsigned NumElts = ValueVTs.size();

1596

if (NumElts == 0)

1597

return SDValue(); // empty struct

1598

SmallVector<SDValue, 4> Constants(NumElts);

1599

for (unsigned i = 0; i != NumElts; ++i) {

1600

EVT EltVT = ValueVTs[i];

1601

if (isa<UndefValue>(C))

1602

Constants[i] = DAG.getUNDEF(EltVT);

1603

else if (EltVT.isFloatingPoint())

1604

Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1605

else

1606

Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);

1607

}

1608

1609

return DAG.getMergeValues(Constants, getCurSDLoc());

1610

}

1611

1612

if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))

1613

return DAG.getBlockAddress(BA, VT);

1614

1615

if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))

1616

return getValue(Equiv->getGlobalValue());

1617

1618

VectorType *VecTy = cast<VectorType>(V->getType());

1619

1620

// Now that we know the number and type of the elements, get that number of

1621

// elements into the Ops array based on what kind of constant it is.

1622

if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {

1623

SmallVector<SDValue, 16> Ops;

1624

unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();

1625

for (unsigned i = 0; i != NumElements; ++i)

1626

Ops.push_back(getValue(CV->getOperand(i)));

1627

1628

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1629

} else if (isa<ConstantAggregateZero>(C)) {

1630

EVT EltVT =

1631

TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

1632

1633

SDValue Op;

1634

if (EltVT.isFloatingPoint())

1635

Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);

1636

else

1637

Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

1638

1639

if (isa<ScalableVectorType>(VecTy))

1640

return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);

1641

else {

1642

SmallVector<SDValue, 16> Ops;

1643

Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);

1644

return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);

1645

}

1646

}

1647

llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1647);

1648

}

1649

1650

// If this is a static alloca, generate it as the frameindex instead of

1651

// computation.

1652

if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {

1653

DenseMap<const AllocaInst*, int>::iterator SI =

1654

FuncInfo.StaticAllocaMap.find(AI);

1655

if (SI != FuncInfo.StaticAllocaMap.end())

1656

return DAG.getFrameIndex(SI->second,

1657

TLI.getFrameIndexTy(DAG.getDataLayout()));

1658

}

1659

1660

// If this is an instruction which fast-isel has deferred, select it now.

1661

if (const Instruction *Inst = dyn_cast<Instruction>(V)) {

1662

unsigned InReg = FuncInfo.InitializeRegForValue(Inst);

1663

1664

RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,

1665

Inst->getType(), None);

1666

SDValue Chain = DAG.getEntryNode();

1667

return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);

1668

}

1669

1670

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {

1671

return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

1672

}

1673

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1673);

1674

}

1675

1676

void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {

1677

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1678

bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;

1679

bool IsCoreCLR = Pers == EHPersonality::CoreCLR;

1680

bool IsSEH = isAsynchronousEHPersonality(Pers);

1681

MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;

1682

if (!IsSEH)

1683

CatchPadMBB->setIsEHScopeEntry();

1684

// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.

1685

if (IsMSVCCXX || IsCoreCLR)

1686

CatchPadMBB->setIsEHFuncletEntry();

1687

}

1688

1689

void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {

1690

// Update machine-CFG edge.

1691

MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];

1692

FuncInfo.MBB->addSuccessor(TargetMBB);

1693

TargetMBB->setIsEHCatchretTarget(true);

1694

DAG.getMachineFunction().setHasEHCatchret(true);

1695

1696

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1697

bool IsSEH = isAsynchronousEHPersonality(Pers);

1698

if (IsSEH) {

1699

// If this is not a fall-through branch or optimizations are switched off,

1700

// emit the branch.

1701

if (TargetMBB != NextBlock(FuncInfo.MBB) ||

1702

TM.getOptLevel() == CodeGenOpt::None)

1703

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,

1704

getControlRoot(), DAG.getBasicBlock(TargetMBB)));

1705

return;

1706

}

1707

1708

// Figure out the funclet membership for the catchret's successor.

1709

// This will be used by the FuncletLayout pass to determine how to order the

1710

// BB's.

1711

// A 'catchret' returns to the outer scope's color.

1712

Value *ParentPad = I.getCatchSwitchParentPad();

1713

const BasicBlock *SuccessorColor;

1714

if (isa<ConstantTokenNone>(ParentPad))

1715

SuccessorColor = &FuncInfo.Fn->getEntryBlock();

1716

else

1717

SuccessorColor = cast<Instruction>(ParentPad)->getParent();

1718

assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1718, __extension__ __PRETTY_FUNCTION__));

1719

MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];

1720

assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1720, __extension__ __PRETTY_FUNCTION__));

1721

1722

// Create the terminator node.

1723

SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,

1724

getControlRoot(), DAG.getBasicBlock(TargetMBB),

1725

DAG.getBasicBlock(SuccessorColorMBB));

1726

DAG.setRoot(Ret);

1727

}

1728

1729

void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {

1730

// Don't emit any special code for the cleanuppad instruction. It just marks

1731

// the start of an EH scope/funclet.

1732

FuncInfo.MBB->setIsEHScopeEntry();

1733

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1734

if (Pers != EHPersonality::Wasm_CXX) {

1735

FuncInfo.MBB->setIsEHFuncletEntry();

1736

FuncInfo.MBB->setIsCleanupFuncletEntry();

1737

}

1738

}

1739

1740

// In wasm EH, even though a catchpad may not catch an exception if a tag does

1741

// not match, it is OK to add only the first unwind destination catchpad to the

1742

// successors, because there will be at least one invoke instruction within the

1743

// catch scope that points to the next unwind destination, if one exists, so

1744

// CFGSort cannot mess up with BB sorting order.

1745

// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic

1746

// call within them, and catchpads only consisting of 'catch (...)' have a

1747

// '__cxa_end_catch' call within them, both of which generate invokes in case

1748

// the next unwind destination exists, i.e., the next unwind destination is not

1749

// the caller.)

1750

//

1751

// Having at most one EH pad successor is also simpler and helps later

1752

// transformations.

1753

//

1754

// For example,

1755

// current:

1756

// invoke void @foo to ... unwind label %catch.dispatch

1757

// catch.dispatch:

1758

// %0 = catchswitch within ... [label %catch.start] unwind label %next

1759

// catch.start:

1760

// ...

1761

// ... in this BB or some other child BB dominated by this BB there will be an

1762

// invoke that points to 'next' BB as an unwind destination

1763

//

1764

// next: ; We don't need to add this to 'current' BB's successor

1765

// ...

1766

static void findWasmUnwindDestinations(

1767

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1768

BranchProbability Prob,

1769

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1770

&UnwindDests) {

1771

while (EHPadBB) {

1772

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1773

if (isa<CleanupPadInst>(Pad)) {

1774

// Stop on cleanup pads.

1775

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1776

UnwindDests.back().first->setIsEHScopeEntry();

1777

break;

1778

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1779

// Add the catchpad handlers to the possible destinations. We don't

1780

// continue to the unwind destination of the catchswitch for wasm.

1781

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1782

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1783

UnwindDests.back().first->setIsEHScopeEntry();

1784

}

1785

break;

1786

} else {

1787

continue;

1788

}

1789

}

1790

}

1791

1792

/// When an invoke or a cleanupret unwinds to the next EH pad, there are

1793

/// many places it could ultimately go. In the IR, we have a single unwind

1794

/// destination, but in the machine CFG, we enumerate all the possible blocks.

1795

/// This function skips over imaginary basic blocks that hold catchswitch

1796

/// instructions, and finds all the "real" machine

1797

/// basic block destinations. As those destinations may not be successors of

1798

/// EHPadBB, here we also calculate the edge probability to those destinations.

1799

/// The passed-in Prob is the edge probability to EHPadBB.

1800

static void findUnwindDestinations(

1801

FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,

1802

BranchProbability Prob,

1803

SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>

1804

&UnwindDests) {

1805

EHPersonality Personality =

1806

classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());

1807

bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;

1808

bool IsCoreCLR = Personality == EHPersonality::CoreCLR;

1809

bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;

1810

bool IsSEH = isAsynchronousEHPersonality(Personality);

1811

1812

if (IsWasmCXX) {

1813

findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);

1814

assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1815, __extension__ __PRETTY_FUNCTION__))

1815

"There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 &&
"There should be at most one unwind destination for wasm") ?
void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1815, __extension__ __PRETTY_FUNCTION__));

1816

return;

1817

}

1818

1819

while (EHPadBB) {

1820

const Instruction *Pad = EHPadBB->getFirstNonPHI();

1821

BasicBlock *NewEHPadBB = nullptr;

1822

if (isa<LandingPadInst>(Pad)) {

1823

// Stop on landingpads. They are not funclets.

1824

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1825

break;

1826

} else if (isa<CleanupPadInst>(Pad)) {

1827

// Stop on cleanup pads. Cleanups are always funclet entries for all known

1828

// personalities.

1829

UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);

1830

UnwindDests.back().first->setIsEHScopeEntry();

1831

UnwindDests.back().first->setIsEHFuncletEntry();

1832

break;

1833

} else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {

1834

// Add the catchpad handlers to the possible destinations.

1835

for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {

1836

UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);

1837

// For MSVC++ and the CLR, catchblocks are funclets and need prologues.

1838

if (IsMSVCCXX || IsCoreCLR)

1839

UnwindDests.back().first->setIsEHFuncletEntry();

1840

if (!IsSEH)

1841

UnwindDests.back().first->setIsEHScopeEntry();

1842

}

1843

NewEHPadBB = CatchSwitch->getUnwindDest();

1844

} else {

1845

continue;

1846

}

1847

1848

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1849

if (BPI && NewEHPadBB)

1850

Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);

1851

EHPadBB = NewEHPadBB;

1852

}

1853

}

1854

1855

void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {

1856

// Update successor info.

1857

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

1858

auto UnwindDest = I.getUnwindDest();

1859

BranchProbabilityInfo *BPI = FuncInfo.BPI;

1860

BranchProbability UnwindDestProb =

1861

(BPI && UnwindDest)

1862

? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)

1863

: BranchProbability::getZero();

1864

findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);

1865

for (auto &UnwindDest : UnwindDests) {

1866

UnwindDest.first->setIsEHPad();

1867

addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);

1868

}

1869

FuncInfo.MBB->normalizeSuccProbs();

1870

1871

// Create the terminator node.

1872

SDValue Ret =

1873

DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());

1874

DAG.setRoot(Ret);

1875

}

1876

1877

void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {

1878

report_fatal_error("visitCatchSwitch not yet implemented!");

1879

}

1880

1881

void SelectionDAGBuilder::visitRet(const ReturnInst &I) {

1882

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

1883

auto &DL = DAG.getDataLayout();

1884

SDValue Chain = getControlRoot();

1885

SmallVector<ISD::OutputArg, 8> Outs;

1886

SmallVector<SDValue, 8> OutVals;

1887

1888

// Calls to @llvm.experimental.deoptimize don't generate a return value, so

1889

// lower

1890

//

1891

// %val = call <ty> @llvm.experimental.deoptimize()

1892

// ret <ty> %val

1893

//

1894

// differently.

1895

if (I.getParent()->getTerminatingDeoptimizeCall()) {

1896

LowerDeoptimizingReturn();

1897

return;

1898

}

1899

1900

if (!FuncInfo.CanLowerReturn) {

1901

unsigned DemoteReg = FuncInfo.DemoteRegister;

1902

const Function *F = I.getParent()->getParent();

1903

1904

// Emit a store of the return value through the virtual register.

1905

// Leave Outs empty so that LowerReturn won't try to load return

1906

// registers the usual way.

1907

SmallVector<EVT, 1> PtrValueVTs;

1908

ComputeValueVTs(TLI, DL,

1909

F->getReturnType()->getPointerTo(

1910

DAG.getDataLayout().getAllocaAddrSpace()),

1911

PtrValueVTs);

1912

1913

SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),

1914

DemoteReg, PtrValueVTs[0]);

1915

SDValue RetOp = getValue(I.getOperand(0));

1916

1917

SmallVector<EVT, 4> ValueVTs, MemVTs;

1918

SmallVector<uint64_t, 4> Offsets;

1919

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,

1920

&Offsets);

1921

unsigned NumValues = ValueVTs.size();

1922

1923

SmallVector<SDValue, 4> Chains(NumValues);

1924

Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());

1925

for (unsigned i = 0; i != NumValues; ++i) {

1926

// An aggregate return value cannot wrap around the address space, so

1927

// offsets to its parts don't wrap either.

1928

SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,

1929

TypeSize::Fixed(Offsets[i]));

1930

1931

SDValue Val = RetOp.getValue(RetOp.getResNo() + i);

1932

if (MemVTs[i] != ValueVTs[i])

1933

Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);

1934

Chains[i] = DAG.getStore(

1935

Chain, getCurSDLoc(), Val,

1936

// FIXME: better loc info would be nice.

1937

Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),

1938

commonAlignment(BaseAlign, Offsets[i]));

1939

}

1940

1941

Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),

1942

MVT::Other, Chains);

1943

} else if (I.getNumOperands() != 0) {

1944

SmallVector<EVT, 4> ValueVTs;

1945

ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);

1946

unsigned NumValues = ValueVTs.size();

1947

if (NumValues) {

1948

SDValue RetOp = getValue(I.getOperand(0));

1949

1950

const Function *F = I.getParent()->getParent();

1951

1952

bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(

1953

I.getOperand(0)->getType(), F->getCallingConv(),

1954

/*IsVarArg*/ false, DL);

1955

1956

ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

1957

if (F->getAttributes().hasRetAttr(Attribute::SExt))

1958

ExtendKind = ISD::SIGN_EXTEND;

1959

else if (F->getAttributes().hasRetAttr(Attribute::ZExt))

1960

ExtendKind = ISD::ZERO_EXTEND;

1961

1962

LLVMContext &Context = F->getContext();

1963

bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);

1964

1965

for (unsigned j = 0; j != NumValues; ++j) {

1966

EVT VT = ValueVTs[j];

1967

1968

if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())

1969

VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

1970

1971

CallingConv::ID CC = F->getCallingConv();

1972

1973

unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);

1974

MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);

1975

SmallVector<SDValue, 4> Parts(NumParts);

1976

getCopyToParts(DAG, getCurSDLoc(),

1977

SDValue(RetOp.getNode(), RetOp.getResNo() + j),

1978

&Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

1979

1980

// 'inreg' on function refers to return value

1981

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

1982

if (RetInReg)

1983

Flags.setInReg();

1984

1985

if (I.getOperand(0)->getType()->isPointerTy()) {

1986

Flags.setPointer();

1987

Flags.setPointerAddrSpace(

1988

cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());

1989

}

1990

1991

if (NeedsRegBlock) {

1992

Flags.setInConsecutiveRegs();

1993

if (j == NumValues - 1)

1994

Flags.setInConsecutiveRegsLast();

1995

}

1996

1997

// Propagate extension type if any

1998

if (ExtendKind == ISD::SIGN_EXTEND)

1999

Flags.setSExt();

2000

else if (ExtendKind == ISD::ZERO_EXTEND)

2001

Flags.setZExt();

2002

2003

for (unsigned i = 0; i < NumParts; ++i) {

2004

Outs.push_back(ISD::OutputArg(Flags,

2005

Parts[i].getValueType().getSimpleVT(),

2006

VT, /*isfixed=*/true, 0, 0));

2007

OutVals.push_back(Parts[i]);

2008

}

2009

}

2010

}

2011

}

2012

2013

// Push in swifterror virtual register as the last element of Outs. This makes

2014

// sure swifterror virtual register will be returned in the swifterror

2015

// physical register.

2016

const Function *F = I.getParent()->getParent();

2017

if (TLI.supportSwiftError() &&

2018

F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {

2019

assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() &&
"Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2019, __extension__ __PRETTY_FUNCTION__));

2020

ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();

2021

Flags.setSwiftError();

2022

Outs.push_back(ISD::OutputArg(

2023

Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),

2024

/*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));

2025

// Create SDNode for the swifterror virtual register.

2026

OutVals.push_back(

2027

DAG.getRegister(SwiftError.getOrCreateVRegUseAt(

2028

&I, FuncInfo.MBB, SwiftError.getFunctionArg()),

2029

EVT(TLI.getPointerTy(DL))));

2030

}

2031

2032

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();

2033

CallingConv::ID CallConv =

2034

DAG.getMachineFunction().getFunction().getCallingConv();

2035

Chain = DAG.getTargetLoweringInfo().LowerReturn(

2036

Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

2037

2038

// Verify that the target's LowerReturn behaved as expected.

2039

assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2040, __extension__ __PRETTY_FUNCTION__))

2040

"LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2040, __extension__ __PRETTY_FUNCTION__));

2041

2042

// Update the DAG with the new chain value resulting from return lowering.

2043

DAG.setRoot(Chain);

2044

}

2045

2046

/// CopyToExportRegsIfNeeded - If the given value has virtual registers

2047

/// created for it, emit nodes to copy the value into the virtual

2048

/// registers.

2049

void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {

2050

// Skip empty types

2051

if (V->getType()->isEmptyTy())

2052

return;

2053

2054

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);

2055

if (VMI != FuncInfo.ValueMap.end()) {

2056

assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast <bool> (!V->use_empty() && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2056, __extension__ __PRETTY_FUNCTION__));

2057

CopyValueToVirtualRegister(V, VMI->second);

2058

}

2059

}

2060

2061

/// ExportFromCurrentBlock - If this condition isn't known to be exported from

2062

/// the current basic block, add it to ValueMap now so that we'll get a

2063

/// CopyTo/FromReg.

2064

void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {

2065

// No need to export constants.

2066

if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

2067

2068

// Already exported?

2069

if (FuncInfo.isExportedInst(V)) return;

2070

2071

unsigned Reg = FuncInfo.InitializeRegForValue(V);

2072

CopyValueToVirtualRegister(V, Reg);

2073

}

2074

2075

bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,

2076

const BasicBlock *FromBB) {

2077

// The operands of the setcc have to be in this block. We don't know

2078

// how to export them from some other block.

2079

if (const Instruction *VI = dyn_cast<Instruction>(V)) {

2080

// Can export from current BB.

2081

if (VI->getParent() == FromBB)

2082

return true;

2083

2084

// Is already exported, noop.

2085

return FuncInfo.isExportedInst(V);

2086

}

2087

2088

// If this is an argument, we can export it if the BB is the entry block or

2089

// if it is already exported.

2090

if (isa<Argument>(V)) {

2091

if (FromBB->isEntryBlock())

2092

return true;

2093

2094

// Otherwise, can only export this if it is already exported.

2095

return FuncInfo.isExportedInst(V);

2096

}

2097

2098

// Otherwise, constants can always be exported.

2099

return true;

2100

}

2101

2102

/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.

2103

BranchProbability

2104

SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,

2105

const MachineBasicBlock *Dst) const {

2106

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2107

const BasicBlock *SrcBB = Src->getBasicBlock();

2108

const BasicBlock *DstBB = Dst->getBasicBlock();

2109

if (!BPI) {

2110

// If BPI is not available, set the default probability as 1 / N, where N is

2111

// the number of successors.

2112

auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);

2113

return BranchProbability(1, SuccSize);

2114

}

2115

return BPI->getEdgeProbability(SrcBB, DstBB);

2116

}

2117

2118

void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,

2119

MachineBasicBlock *Dst,

2120

BranchProbability Prob) {

2121

if (!FuncInfo.BPI)

2122

Src->addSuccessorWithoutProb(Dst);

2123

else {

2124

if (Prob.isUnknown())

2125

Prob = getEdgeProbability(Src, Dst);

2126

Src->addSuccessor(Dst, Prob);

2127

}

2128

}

2129

2130

static bool InBlock(const Value *V, const BasicBlock *BB) {

2131

if (const Instruction *I = dyn_cast<Instruction>(V))

2132

return I->getParent() == BB;

2133

return true;

2134

}

2135

2136

/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.

2137

/// This function emits a branch and is used at the leaves of an OR or an

2138

/// AND operator tree.

2139

void

2140

SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,

2141

MachineBasicBlock *TBB,

2142

MachineBasicBlock *FBB,

2143

MachineBasicBlock *CurBB,

2144

MachineBasicBlock *SwitchBB,

2145

BranchProbability TProb,

2146

BranchProbability FProb,

2147

bool InvertCond) {

2148

const BasicBlock *BB = CurBB->getBasicBlock();

2149

2150

// If the leaf of the tree is a comparison, merge the condition into

2151

// the caseblock.

2152

if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {

2153

// The operands of the cmp have to be in this block. We don't know

2154

// how to export them from some other block. If this is the first block

2155

// of the sequence, no exporting is needed.

2156

if (CurBB == SwitchBB ||

2157

(isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&

2158

isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {

2159

ISD::CondCode Condition;

2160

if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {

2161

ICmpInst::Predicate Pred =

2162

InvertCond ? IC->getInversePredicate() : IC->getPredicate();

2163

Condition = getICmpCondCode(Pred);

2164

} else {

2165

const FCmpInst *FC = cast<FCmpInst>(Cond);

2166

FCmpInst::Predicate Pred =

2167

InvertCond ? FC->getInversePredicate() : FC->getPredicate();

2168

Condition = getFCmpCondCode(Pred);

2169

if (TM.Options.NoNaNsFPMath)

2170

Condition = getFCmpCodeWithoutNaN(Condition);

2171

}

2172

2173

CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,

2174

TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2175

SL->SwitchCases.push_back(CB);

2176

return;

2177

}

2178

}

2179

2180

// Create a CaseBlock record representing this branch.

2181

ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;

2182

CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),

2183

nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);

2184

SL->SwitchCases.push_back(CB);

2185

}

2186

2187

void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,

2188

MachineBasicBlock *TBB,

2189

MachineBasicBlock *FBB,

2190

MachineBasicBlock *CurBB,

2191

MachineBasicBlock *SwitchBB,

2192

Instruction::BinaryOps Opc,

2193

BranchProbability TProb,

2194

BranchProbability FProb,

2195

bool InvertCond) {

2196

// Skip over not part of the tree and remember to invert op and operands at

2197

// next level.

2198

Value *NotCond;

2199

if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&

2200

InBlock(NotCond, CurBB->getBasicBlock())) {

2201

FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,

2202

!InvertCond);

2203

return;

2204

}

2205

2206

const Instruction *BOp = dyn_cast<Instruction>(Cond);

2207

const Value *BOpOp0, *BOpOp1;

2208

// Compute the effective opcode for Cond, taking into account whether it needs

2209

// to be inverted, e.g.

2210

// and (not (or A, B)), C

2211

// gets lowered as

2212

// and (and (not A, not B), C)

2213

Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;

2214

if (BOp) {

2215

BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))

2216

? Instruction::And

2217

: (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))

2218

? Instruction::Or

2219

: (Instruction::BinaryOps)0);

2220

if (InvertCond) {

2221

if (BOpc == Instruction::And)

2222

BOpc = Instruction::Or;

2223

else if (BOpc == Instruction::Or)

2224

BOpc = Instruction::And;

2225

}

2226

}

2227

2228

// If this node is not part of the or/and tree, emit it as a branch.

2229

// Note that all nodes in the tree should have same opcode.

2230

bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();

2231

if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||

2232

!InBlock(BOpOp0, CurBB->getBasicBlock()) ||

2233

!InBlock(BOpOp1, CurBB->getBasicBlock())) {

2234

EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,

2235

TProb, FProb, InvertCond);

2236

return;

2237

}

2238

2239

// Create TmpBB after CurBB.

2240

MachineFunction::iterator BBI(CurBB);

2241

MachineFunction &MF = DAG.getMachineFunction();

2242

MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());

2243

CurBB->getParent()->insert(++BBI, TmpBB);

2244

2245

if (Opc == Instruction::Or) {

2246

// Codegen X | Y as:

2247

// BB1:

2248

// jmp_if_X TBB

2249

// jmp TmpBB

2250

// TmpBB:

2251

// jmp_if_Y TBB

2252

// jmp FBB

2253

//

2254

2255

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2256

// The requirement is that

2257

// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)

2258

// = TrueProb for original BB.

2259

// Assuming the original probabilities are A and B, one choice is to set

2260

// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to

2261

// A/(1+B) and 2B/(1+B). This choice assumes that

2262

// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.

2263

// Another choice is to assume TrueProb for BB1 equals to TrueProb for

2264

// TmpBB, but the math is more complicated.

2265

2266

auto NewTrueProb = TProb / 2;

2267

auto NewFalseProb = TProb / 2 + FProb;

2268

// Emit the LHS condition.

2269

FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,

2270

NewFalseProb, InvertCond);

2271

2272

// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).

2273

SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};

2274

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2275

// Emit the RHS condition into TmpBB.

2276

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2277

Probs[1], InvertCond);

2278

} else {

2279

assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
"Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2279, __extension__ __PRETTY_FUNCTION__));

2280

// Codegen X & Y as:

2281

// BB1:

2282

// jmp_if_X TmpBB

2283

// jmp FBB

2284

// TmpBB:

2285

// jmp_if_Y TBB

2286

// jmp FBB

2287

//

2288

// This requires creation of TmpBB after CurBB.

2289

2290

// We have flexibility in setting Prob for BB1 and Prob for TmpBB.

2291

// The requirement is that

2292

// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)

2293

// = FalseProb for original BB.

2294

// Assuming the original probabilities are A and B, one choice is to set

2295

// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to

2296

// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==

2297

// TrueProb for BB1 * FalseProb for TmpBB.

2298

2299

auto NewTrueProb = TProb + FProb / 2;

2300

auto NewFalseProb = FProb / 2;

2301

// Emit the LHS condition.

2302

FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,

2303

NewFalseProb, InvertCond);

2304

2305

// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).

2306

SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};

2307

BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());

2308

// Emit the RHS condition into TmpBB.

2309

FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],

2310

Probs[1], InvertCond);

2311

}

2312

}

2313

2314

/// If the set of cases should be emitted as a series of branches, return true.

2315

/// If we should emit this as a bunch of and/or'd together conditions, return

2316

/// false.

2317

bool

2318

SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {

2319

if (Cases.size() != 2) return true;

2320

2321

// If this is two comparisons of the same values or'd or and'd together, they

2322

// will get folded into a single comparison, so don't emit two blocks.

2323

if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&

2324

Cases[0].CmpRHS == Cases[1].CmpRHS) ||

2325

(Cases[0].CmpRHS == Cases[1].CmpLHS &&

2326

Cases[0].CmpLHS == Cases[1].CmpRHS)) {

2327

return false;

2328

}

2329

2330

// Handle: (X != null) | (Y != null) --> (X|Y) != 0

2331

// Handle: (X == null) & (Y == null) --> (X|Y) == 0

2332

if (Cases[0].CmpRHS == Cases[1].CmpRHS &&

2333

Cases[0].CC == Cases[1].CC &&

2334

isa<Constant>(Cases[0].CmpRHS) &&

2335

cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {

2336

if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)

2337

return false;

2338

if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)

2339

return false;

2340

}

2341

2342

return true;

2343

}

2344

2345

void SelectionDAGBuilder::visitBr(const BranchInst &I) {

2346

MachineBasicBlock *BrMBB = FuncInfo.MBB;

2347

2348

// Update machine-CFG edges.

2349

MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

2350

2351

if (I.isUnconditional()) {

2352

// Update machine-CFG edges.

2353

BrMBB->addSuccessor(Succ0MBB);

2354

2355

// If this is not a fall-through branch or optimizations are switched off,

2356

// emit the branch.

2357

if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)

2358

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2359

MVT::Other, getControlRoot(),

2360

DAG.getBasicBlock(Succ0MBB)));

2361

2362

return;

2363

}

2364

2365

// If this condition is one of the special cases we handle, do special stuff

2366

// now.

2367

const Value *CondVal = I.getCondition();

2368

MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

2369

2370

// If this is a series of conditions that are or'd or and'd together, emit

2371

// this as a sequence of branches instead of setcc's with and/or operations.

2372

// As long as jumps are not expensive (exceptions for multi-use logic ops,

2373

// unpredictable branches, and vector extracts because those jumps are likely

2374

// expensive for any target), this should improve performance.

2375

// For example, instead of something like:

2376

// cmp A, B

2377

// C = seteq

2378

// cmp D, E

2379

// F = setle

2380

// or C, F

2381

// jnz foo

2382

// Emit:

2383

// cmp A, B

2384

// je foo

2385

// cmp D, E

2386

// jle foo

2387

const Instruction *BOp = dyn_cast<Instruction>(CondVal);

2388

if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&

2389

BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {

2390

Value *Vec;

2391

const Value *BOp0, *BOp1;

2392

Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;

2393

if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))

2394

Opcode = Instruction::And;

2395

else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))

2396

Opcode = Instruction::Or;

2397

2398

if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&

2399

match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {

2400

FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,

2401

getEdgeProbability(BrMBB, Succ0MBB),

2402

getEdgeProbability(BrMBB, Succ1MBB),

2403

/*InvertCond=*/false);

2404

// If the compares in later blocks need to use values not currently

2405

// exported from this block, export them now. This block should always

2406

// be the first entry.

2407

assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB
&& "Unexpected lowering!") ? void (0) : __assert_fail
("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2407, __extension__ __PRETTY_FUNCTION__));

2408

2409

// Allow some cases to be rejected.

2410

if (ShouldEmitAsBranches(SL->SwitchCases)) {

2411

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {

2412

ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);

2413

ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);

2414

}

2415

2416

// Emit the branch for this block.

2417

visitSwitchCase(SL->SwitchCases[0], BrMBB);

2418

SL->SwitchCases.erase(SL->SwitchCases.begin());

2419

return;

2420

}

2421

2422

// Okay, we decided not to do this, remove any inserted MBB's and clear

2423

// SwitchCases.

2424

for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)

2425

FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

2426

2427

SL->SwitchCases.clear();

2428

}

2429

}

2430

2431

// Create a CaseBlock record representing this branch.

2432

CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),

2433

nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

2434

2435

// Use visitSwitchCase to actually insert the fast branch sequence for this

2436

// cond branch.

2437

visitSwitchCase(CB, BrMBB);

2438

}

2439

2440

/// visitSwitchCase - Emits the necessary code to represent a single node in

2441

/// the binary search tree resulting from lowering a switch instruction.

2442

void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,

2443

MachineBasicBlock *SwitchBB) {

2444

SDValue Cond;

2445

SDValue CondLHS = getValue(CB.CmpLHS);

2446

SDLoc dl = CB.DL;

2447

2448

if (CB.CC == ISD::SETTRUE) {

2449

// Branch or fall through to TrueBB.

2450

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2451

SwitchBB->normalizeSuccProbs();

2452

if (CB.TrueBB != NextBlock(SwitchBB)) {

2453

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),

2454

DAG.getBasicBlock(CB.TrueBB)));

2455

}

2456

return;

2457

}

2458

2459

auto &TLI = DAG.getTargetLoweringInfo();

2460

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

2461

2462

// Build the setcc now.

2463

if (!CB.CmpMHS) {

2464

// Fold "(X == true)" to X and "(X == false)" to !X to

2465

// handle common cases produced by branch lowering.

2466

if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&

2467

CB.CC == ISD::SETEQ)

2468

Cond = CondLHS;

2469

else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&

2470

CB.CC == ISD::SETEQ) {

2471

SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());

2472

Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);

2473

} else {

2474

SDValue CondRHS = getValue(CB.CmpRHS);

2475

2476

// If a pointer's DAG type is larger than its memory type then the DAG

2477

// values are zero-extended. This breaks signed comparisons so truncate

2478

// back to the underlying type before doing the compare.

2479

if (CondLHS.getValueType() != MemVT) {

2480

CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);

2481

CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);

2482

}

2483

Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);

2484

}

2485

} else {

2486

assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2486, __extension__ __PRETTY_FUNCTION__));

2487

2488

const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();

2489

const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

2490

2491

SDValue CmpOp = getValue(CB.CmpMHS);

2492

EVT VT = CmpOp.getValueType();

2493

2494

if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {

2495

Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),

2496

ISD::SETLE);

2497

} else {

2498

SDValue SUB = DAG.getNode(ISD::SUB, dl,

2499

VT, CmpOp, DAG.getConstant(Low, dl, VT));

2500

Cond = DAG.getSetCC(dl, MVT::i1, SUB,

2501

DAG.getConstant(High-Low, dl, VT), ISD::SETULE);

2502

}

2503

}

2504

2505

// Update successor info

2506

addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);

2507

// TrueBB and FalseBB are always different unless the incoming IR is

2508

// degenerate. This only happens when running llc on weird IR.

2509

if (CB.TrueBB != CB.FalseBB)

2510

addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);

2511

SwitchBB->normalizeSuccProbs();

2512

2513

// If the lhs block is the next block, invert the condition so that we can

2514

// fall through to the lhs instead of the rhs block.

2515

if (CB.TrueBB == NextBlock(SwitchBB)) {

2516

std::swap(CB.TrueBB, CB.FalseBB);

2517

SDValue True = DAG.getConstant(1, dl, Cond.getValueType());

2518

Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);

2519

}

2520

2521

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2522

MVT::Other, getControlRoot(), Cond,

2523

DAG.getBasicBlock(CB.TrueBB));

2524

2525

// Insert the false branch. Do this even if it's a fall through branch,

2526

// this makes it easier to do DAG optimizations which require inverting

2527

// the branch condition.

2528

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2529

DAG.getBasicBlock(CB.FalseBB));

2530

2531

DAG.setRoot(BrCond);

2532

}

2533

2534

/// visitJumpTable - Emit JumpTable node in the current MBB

2535

void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {

2536

// Emit the code for the jump table

2537

assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2537, __extension__ __PRETTY_FUNCTION__));

2538

EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());

2539

SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),

2540

JT.Reg, PTy);

2541

SDValue Table = DAG.getJumpTable(JT.JTI, PTy);

2542

SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),

2543

MVT::Other, Index.getValue(1),

2544

Table, Index);

2545

DAG.setRoot(BrJumpTable);

2546

}

2547

2548

/// visitJumpTableHeader - This function emits necessary code to produce index

2549

/// in the JumpTable from switch case.

2550

void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,

2551

JumpTableHeader &JTH,

2552

MachineBasicBlock *SwitchBB) {

2553

SDLoc dl = getCurSDLoc();

2554

2555

// Subtract the lowest switch case value from the value being switched on.

2556

SDValue SwitchOp = getValue(JTH.SValue);

2557

EVT VT = SwitchOp.getValueType();

2558

SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,

2559

DAG.getConstant(JTH.First, dl, VT));

2560

2561

// The SDNode we just created, which holds the value being switched on minus

2562

// the smallest case value, needs to be copied to a virtual register so it

2563

// can be used as an index into the jump table in a subsequent basic block.

2564

// This value may be smaller or larger than the target's pointer type, and

2565

// therefore require extension or truncating.

2566

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2567

SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

2568

2569

unsigned JumpTableReg =

2570

FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));

2571

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,

2572

JumpTableReg, SwitchOp);

2573

JT.Reg = JumpTableReg;

2574

2575

if (!JTH.OmitRangeCheck) {

2576

// Emit the range check for the jump table, and branch to the default block

2577

// for the switch statement if the value being switched on exceeds the

2578

// largest case in the switch.

2579

SDValue CMP = DAG.getSetCC(

2580

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2581

Sub.getValueType()),

2582

Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

2583

2584

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2585

MVT::Other, CopyTo, CMP,

2586

DAG.getBasicBlock(JT.Default));

2587

2588

// Avoid emitting unnecessary branches to the next block.

2589

if (JT.MBB != NextBlock(SwitchBB))

2590

BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,

2591

DAG.getBasicBlock(JT.MBB));

2592

2593

DAG.setRoot(BrCond);

2594

} else {

2595

// Avoid emitting unnecessary branches to the next block.

2596

if (JT.MBB != NextBlock(SwitchBB))

2597

DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,

2598

DAG.getBasicBlock(JT.MBB)));

2599

else

2600

DAG.setRoot(CopyTo);

2601

}

2602

}

2603

2604

/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global

2605

/// variable if there exists one.

2606

static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,

2607

SDValue &Chain) {

2608

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2609

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2610

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2611

MachineFunction &MF = DAG.getMachineFunction();

2612

Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());

2613

MachineSDNode *Node =

2614

DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);

2615

if (Global) {

2616

MachinePointerInfo MPInfo(Global);

2617

auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |

2618

MachineMemOperand::MODereferenceable;

2619

MachineMemOperand *MemRef = MF.getMachineMemOperand(

2620

MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));

2621

DAG.setNodeMemRefs(Node, {MemRef});

2622

}

2623

if (PtrTy != PtrMemTy)

2624

return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);

2625

return SDValue(Node, 0);

2626

}

2627

2628

/// Codegen a new tail for a stack protector check ParentMBB which has had its

2629

/// tail spliced into a stack protector check success bb.

2630

///

2631

/// For a high level explanation of how this fits into the stack protector

2632

/// generation see the comment on the declaration of class

2633

/// StackProtectorDescriptor.

2634

void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,

2635

MachineBasicBlock *ParentBB) {

2636

2637

// First create the loads to the guard/stack slot for the comparison.

2638

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2639

EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());

2640

EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

2641

2642

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();

2643

int FI = MFI.getStackProtectorIndex();

2644

2645

SDValue Guard;

2646

SDLoc dl = getCurSDLoc();

2647

SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);

2648

const Module &M = *ParentBB->getParent()->getFunction().getParent();

2649

Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

2650

2651

// Generate code to load the content of the guard slot.

2652

SDValue GuardVal = DAG.getLoad(

2653

PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,

2654

MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,

2655

MachineMemOperand::MOVolatile);

2656

2657

if (TLI.useStackGuardXorFP())

2658

GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

2659

2660

// Retrieve guard check function, nullptr if instrumentation is inlined.

2661

if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {

2662

// The target provides a guard check function to validate the guard value.

2663

// Generate a call to that function with the content of the guard slot as

2664

// argument.

2665

FunctionType *FnTy = GuardCheckFn->getFunctionType();

2666

assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
"Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2666, __extension__ __PRETTY_FUNCTION__));

2667

2668

TargetLowering::ArgListTy Args;

2669

TargetLowering::ArgListEntry Entry;

2670

Entry.Node = GuardVal;

2671

Entry.Ty = FnTy->getParamType(0);

2672

if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))

2673

Entry.IsInReg = true;

2674

Args.push_back(Entry);

2675

2676

TargetLowering::CallLoweringInfo CLI(DAG);

2677

CLI.setDebugLoc(getCurSDLoc())

2678

.setChain(DAG.getEntryNode())

2679

.setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),

2680

getValue(GuardCheckFn), std::move(Args));

2681

2682

std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

2683

DAG.setRoot(Result.second);

2684

return;

2685

}

2686

2687

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.

2688

// Otherwise, emit a volatile load to retrieve the stack guard value.

2689

SDValue Chain = DAG.getEntryNode();

2690

if (TLI.useLoadStackGuardNode()) {

2691

Guard = getLoadStackGuard(DAG, dl, Chain);

2692

} else {

2693

const Value *IRGuard = TLI.getSDagStackGuard(M);

2694

SDValue GuardPtr = getValue(IRGuard);

2695

2696

Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,

2697

MachinePointerInfo(IRGuard, 0), Align,

2698

MachineMemOperand::MOVolatile);

2699

}

2700

2701

// Perform the comparison via a getsetcc.

2702

SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),

2703

*DAG.getContext(),

2704

Guard.getValueType()),

2705

Guard, GuardVal, ISD::SETNE);

2706

2707

// If the guard/stackslot do not equal, branch to failure MBB.

2708

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,

2709

MVT::Other, GuardVal.getOperand(0),

2710

Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));

2711

// Otherwise branch to success MBB.

2712

SDValue Br = DAG.getNode(ISD::BR, dl,

2713

MVT::Other, BrCond,

2714

DAG.getBasicBlock(SPD.getSuccessMBB()));

2715

2716

DAG.setRoot(Br);

2717

}

2718

2719

/// Codegen the failure basic block for a stack protector check.

2720

///

2721

/// A failure stack protector machine basic block consists simply of a call to

2722

/// __stack_chk_fail().

2723

///

2724

/// For a high level explanation of how this fits into the stack protector

2725

/// generation see the comment on the declaration of class

2726

/// StackProtectorDescriptor.

2727

void

2728

SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {

2729

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2730

TargetLowering::MakeLibCallOptions CallOptions;

2731

CallOptions.setDiscardResult(true);

2732

SDValue Chain =

2733

TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,

2734

None, CallOptions, getCurSDLoc()).second;

2735

// On PS4, the "return address" must still be within the calling function,

2736

// even if it's at the very end, so emit an explicit TRAP here.

2737

// Passing 'true' for doesNotReturn above won't generate the trap for us.

2738

if (TM.getTargetTriple().isPS4CPU())

2739

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2740

// WebAssembly needs an unreachable instruction after a non-returning call,

2741

// because the function return type can be different from __stack_chk_fail's

2742

// return type (void).

2743

if (TM.getTargetTriple().isWasm())

2744

Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

2745

2746

DAG.setRoot(Chain);

2747

}

2748

2749

/// visitBitTestHeader - This function emits necessary code to produce value

2750

/// suitable for "bit tests"

2751

void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,

2752

MachineBasicBlock *SwitchBB) {

2753

SDLoc dl = getCurSDLoc();

2754

2755

// Subtract the minimum value.

2756

SDValue SwitchOp = getValue(B.SValue);

2757

EVT VT = SwitchOp.getValueType();

2758

SDValue RangeSub =

2759

DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

2760

2761

// Determine the type of the test operands.

2762

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2763

bool UsePtrType = false;

2764

if (!TLI.isTypeLegal(VT)) {

2765

UsePtrType = true;

2766

} else {

2767

for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)

2768

if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {

2769

// Switch table case range are encoded into series of masks.

2770

// Just use pointer type, it's guaranteed to fit.

2771

UsePtrType = true;

2772

break;

2773

}

2774

}

2775

SDValue Sub = RangeSub;

2776

if (UsePtrType) {

2777

VT = TLI.getPointerTy(DAG.getDataLayout());

2778

Sub = DAG.getZExtOrTrunc(Sub, dl, VT);

2779

}

2780

2781

B.RegVT = VT.getSimpleVT();

2782

B.Reg = FuncInfo.CreateReg(B.RegVT);

2783

SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

2784

2785

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

2786

2787

if (!B.OmitRangeCheck)

2788

addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);

2789

addSuccessorWithProb(SwitchBB, MBB, B.Prob);

2790

SwitchBB->normalizeSuccProbs();

2791

2792

SDValue Root = CopyTo;

2793

if (!B.OmitRangeCheck) {

2794

// Conditional branch to the default block.

2795

SDValue RangeCmp = DAG.getSetCC(dl,

2796

TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),

2797

RangeSub.getValueType()),

2798

RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),

2799

ISD::SETUGT);

2800

2801

Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,

2802

DAG.getBasicBlock(B.Default));

2803

}

2804

2805

// Avoid emitting unnecessary branches to the next block.

2806

if (MBB != NextBlock(SwitchBB))

2807

Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

2808

2809

DAG.setRoot(Root);

2810

}

2811

2812

/// visitBitTestCase - this function produces one "bit test"

2813

void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,

2814

MachineBasicBlock* NextMBB,

2815

BranchProbability BranchProbToNext,

2816

unsigned Reg,

2817

BitTestCase &B,

2818

MachineBasicBlock *SwitchBB) {

2819

SDLoc dl = getCurSDLoc();

2820

MVT VT = BB.RegVT;

2821

SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);

2822

SDValue Cmp;

2823

unsigned PopCount = countPopulation(B.Mask);

2824

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2825

if (PopCount == 1) {

2826

// Testing for a single bit; just compare the shift count with what it

2827

// would need to be to shift a 1 bit in that position.

2828

Cmp = DAG.getSetCC(

2829

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2830

ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),

2831

ISD::SETEQ);

2832

} else if (PopCount == BB.Range) {

2833

// There is only one zero bit in the range, test for it directly.

2834

Cmp = DAG.getSetCC(

2835

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2836

ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),

2837

ISD::SETNE);

2838

} else {

2839

// Make desired shift

2840

SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,

2841

DAG.getConstant(1, dl, VT), ShiftOp);

2842

2843

// Emit bit tests and jumps

2844

SDValue AndOp = DAG.getNode(ISD::AND, dl,

2845

VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));

2846

Cmp = DAG.getSetCC(

2847

dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),

2848

AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);

2849

}

2850

2851

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.

2852

addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);

2853

// The branch probability from SwitchBB to NextMBB is BranchProbToNext.

2854

addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);

2855

// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is

2856

// one as they are relative probabilities (and thus work more like weights),

2857

// and hence we need to normalize them to let the sum of them become one.

2858

SwitchBB->normalizeSuccProbs();

2859

2860

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,

2861

MVT::Other, getControlRoot(),

2862

Cmp, DAG.getBasicBlock(B.TargetBB));

2863

2864

// Avoid emitting unnecessary branches to the next block.

2865

if (NextMBB != NextBlock(SwitchBB))

2866

BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,

2867

DAG.getBasicBlock(NextMBB));

2868

2869

DAG.setRoot(BrAnd);

2870

}

2871

2872

void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {

2873

MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

2874

2875

// Retrieve successors. Look through artificial IR level blocks like

2876

// catchswitch for successors.

2877

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];

2878

const BasicBlock *EHPadBB = I.getSuccessor(1);

2879

2880

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2881

// have to do anything here to lower funclet bundles.

2882

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__))

2883

{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__))

2884

LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__))

2885

LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__))

2886

LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__))

2887

"Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2887, __extension__ __PRETTY_FUNCTION__));

2888

2889

const Value *Callee(I.getCalledOperand());

2890

const Function *Fn = dyn_cast<Function>(Callee);

2891

if (isa<InlineAsm>(Callee))

2892

visitInlineAsm(I, EHPadBB);

2893

else if (Fn && Fn->isIntrinsic()) {

2894

switch (Fn->getIntrinsicID()) {

2895

default:

2896

llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2896);

2897

case Intrinsic::donothing:

2898

// Ignore invokes to @llvm.donothing: jump directly to the next BB.

2899

case Intrinsic::seh_try_begin:

2900

case Intrinsic::seh_scope_begin:

2901

case Intrinsic::seh_try_end:

2902

case Intrinsic::seh_scope_end:

2903

break;

2904

case Intrinsic::experimental_patchpoint_void:

2905

case Intrinsic::experimental_patchpoint_i64:

2906

visitPatchpoint(I, EHPadBB);

2907

break;

2908

case Intrinsic::experimental_gc_statepoint:

2909

LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);

2910

break;

2911

case Intrinsic::wasm_rethrow: {

2912

// This is usually done in visitTargetIntrinsic, but this intrinsic is

2913

// special because it can be invoked, so we manually lower it to a DAG

2914

// node here.

2915

SmallVector<SDValue, 8> Ops;

2916

Ops.push_back(getRoot()); // inchain

2917

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

2918

Ops.push_back(

2919

DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),

2920

TLI.getPointerTy(DAG.getDataLayout())));

2921

SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain

2922

DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));

2923

break;

2924

}

2925

}

2926

} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {

2927

// Currently we do not lower any intrinsic calls with deopt operand bundles.

2928

// Eventually we will support lowering the @llvm.experimental.deoptimize

2929

// intrinsic, and right now there are no plans to support other intrinsics

2930

// with deopt state.

2931

LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);

2932

} else {

2933

LowerCallTo(I, getValue(Callee), false, false, EHPadBB);

2934

}

2935

2936

// If the value of the invoke is used outside of its defining block, make it

2937

// available as a virtual register.

2938

// We already took care of the exported value for the statepoint instruction

2939

// during call to the LowerStatepoint.

2940

if (!isa<GCStatepointInst>(I)) {

2941

CopyToExportRegsIfNeeded(&I);

2942

}

2943

2944

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;

2945

BranchProbabilityInfo *BPI = FuncInfo.BPI;

2946

BranchProbability EHPadBBProb =

2947

BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)

2948

: BranchProbability::getZero();

2949

findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

2950

2951

// Update successor info.

2952

addSuccessorWithProb(InvokeMBB, Return);

2953

for (auto &UnwindDest : UnwindDests) {

2954

UnwindDest.first->setIsEHPad();

2955

addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);

2956

}

2957

InvokeMBB->normalizeSuccProbs();

2958

2959

// Drop into normal successor.

2960

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),

2961

DAG.getBasicBlock(Return)));

2962

}

2963

2964

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {

2965

MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

2966

2967

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't

2968

// have to do anything here to lower funclet bundles.

2969

assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2971, __extension__ __PRETTY_FUNCTION__))

2970

{LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2971, __extension__ __PRETTY_FUNCTION__))

2971

"Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2971, __extension__ __PRETTY_FUNCTION__));

2972

2973

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2973, __extension__ __PRETTY_FUNCTION__));

2974

visitInlineAsm(I);

2975

CopyToExportRegsIfNeeded(&I);

2976

2977

// Retrieve successors.

2978

MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

2979

2980

// Update successor info.

2981

addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());

2982

for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {

2983

MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];

2984

addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());

2985

Target->setIsInlineAsmBrIndirectTarget();

2986

}

2987

CallBrMBB->normalizeSuccProbs();

2988

2989

// Drop into default successor.

2990

DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),

2991

MVT::Other, getControlRoot(),

2992

DAG.getBasicBlock(Return)));

2993

}

2994

2995

void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {

2996

llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2996);

2997

}

2998

2999

void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {

3000

assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3001, __extension__ __PRETTY_FUNCTION__))

3001

"Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3001, __extension__ __PRETTY_FUNCTION__));

3002

3003

// If there aren't registers to copy the values into (e.g., during SjLj

3004

// exceptions), then don't bother to create these DAG nodes.

3005

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3006

const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();

3007

if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&

3008

TLI.getExceptionSelectorRegister(PersonalityFn) == 0)

3009

return;

3010

3011

// If landingpad's return type is token type, we don't create DAG nodes

3012

// for its exception pointer and selector value. The extraction of exception

3013

// pointer or selector value from token type landingpads is not currently

3014

// supported.

3015

if (LP.getType()->isTokenTy())

3016

return;

3017

3018

SmallVector<EVT, 2> ValueVTs;

3019

SDLoc dl = getCurSDLoc();

3020

ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);

3021

assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3021, __extension__ __PRETTY_FUNCTION__));

3022

3023

// Get the two live-in registers as SDValues. The physregs have already been

3024

// copied into virtual registers.

3025

SDValue Ops[2];

3026

if (FuncInfo.ExceptionPointerVirtReg) {

3027

Ops[0] = DAG.getZExtOrTrunc(

3028

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3029

FuncInfo.ExceptionPointerVirtReg,

3030

TLI.getPointerTy(DAG.getDataLayout())),

3031

dl, ValueVTs[0]);

3032

} else {

3033

Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));

3034

}

3035

Ops[1] = DAG.getZExtOrTrunc(

3036

DAG.getCopyFromReg(DAG.getEntryNode(), dl,

3037

FuncInfo.ExceptionSelectorVirtReg,

3038

TLI.getPointerTy(DAG.getDataLayout())),

3039

dl, ValueVTs[1]);

3040

3041

// Merge into one.

3042

SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,

3043

DAG.getVTList(ValueVTs), Ops);

3044

setValue(&LP, Res);

3045

}

3046

3047

void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,

3048

MachineBasicBlock *Last) {

3049

// Update JTCases.

3050

for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)

3051

if (SL->JTCases[i].first.HeaderBB == First)

3052

SL->JTCases[i].first.HeaderBB = Last;

3053

3054

// Update BitTestCases.

3055

for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)

3056

if (SL->BitTestCases[i].Parent == First)

3057

SL->BitTestCases[i].Parent = Last;

3058

}

3059

3060

void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {

3061

MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

3062

3063

// Update machine-CFG edges with unique successors.

3064

SmallSet<BasicBlock*, 32> Done;

3065

for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {

3066

BasicBlock *BB = I.getSuccessor(i);

3067

bool Inserted = Done.insert(BB).second;

3068

if (!Inserted)

3069

continue;

3070

3071

MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];

3072

addSuccessorWithProb(IndirectBrMBB, Succ);

3073

}

3074

IndirectBrMBB->normalizeSuccProbs();

3075

3076

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),

3077

MVT::Other, getControlRoot(),

3078

getValue(I.getAddress())));

3079

}

3080

3081

void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {

3082

if (!DAG.getTarget().Options.TrapUnreachable)

3083

return;

3084

3085

// We may be able to ignore unreachable behind a noreturn call.

3086

if (DAG.getTarget().Options.NoTrapAfterNoreturn) {

3087

const BasicBlock &BB = *I.getParent();

3088

if (&I != &BB.front()) {

3089

BasicBlock::const_iterator PredI =

3090

std::prev(BasicBlock::const_iterator(&I));

3091

if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {

3092

if (Call->doesNotReturn())

3093

return;

3094

}

3095

}

3096

}

3097

3098

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

3099

}

3100

3101

void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {

3102

SDNodeFlags Flags;

3103

3104

SDValue Op = getValue(I.getOperand(0));

3105

SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),

3106

Op, Flags);

3107

setValue(&I, UnNodeValue);

3108

}

3109

3110

void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {

3111

SDNodeFlags Flags;

3112

if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {

3113

Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());

3114

Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());

3115

}

3116

if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))

3117

Flags.setExact(ExactOp->isExact());

3118

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3119

Flags.copyFMF(*FPOp);

3120

3121

SDValue Op1 = getValue(I.getOperand(0));

3122

SDValue Op2 = getValue(I.getOperand(1));

3123

SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),

3124

Op1, Op2, Flags);

3125

setValue(&I, BinNodeValue);

3126

}

3127

3128

void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {

3129

SDValue Op1 = getValue(I.getOperand(0));

3130

SDValue Op2 = getValue(I.getOperand(1));

3131

3132

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(

3133

Op1.getValueType(), DAG.getDataLayout());

3134

3135

// Coerce the shift amount to the right type if we can.

3136

if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {

3137

unsigned ShiftSize = ShiftTy.getSizeInBits();

3138

unsigned Op2Size = Op2.getValueSizeInBits();

3139

SDLoc DL = getCurSDLoc();

3140

3141

// If the operand is smaller than the shift count type, promote it.

3142

if (ShiftSize > Op2Size)

3143

Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);

3144

3145

// If the operand is larger than the shift count type but the shift

3146

// count type has enough bits to represent any shift value, truncate

3147

// it now. This is a common case and it exposes the truncate to

3148

// optimization early.

3149

else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))

3150

Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);

3151

// Otherwise we'll need to temporarily settle for some other convenient

3152

// type. Type legalization will make adjustments once the shiftee is split.

3153

else

3154

Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);

3155

}

3156

3157

bool nuw = false;

3158

bool nsw = false;

3159

bool exact = false;

3160

3161

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

3162

3163

if (const OverflowingBinaryOperator *OFBinOp =

3164

dyn_cast<const OverflowingBinaryOperator>(&I)) {

3165

nuw = OFBinOp->hasNoUnsignedWrap();

3166

nsw = OFBinOp->hasNoSignedWrap();

3167

}

3168

if (const PossiblyExactOperator *ExactOp =

3169

dyn_cast<const PossiblyExactOperator>(&I))

3170

exact = ExactOp->isExact();

3171

}

3172

SDNodeFlags Flags;

3173

Flags.setExact(exact);

3174

Flags.setNoSignedWrap(nsw);

3175

Flags.setNoUnsignedWrap(nuw);

3176

SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,

3177

Flags);

3178

setValue(&I, Res);

3179

}

3180

3181

void SelectionDAGBuilder::visitSDiv(const User &I) {

3182

SDValue Op1 = getValue(I.getOperand(0));

3183

SDValue Op2 = getValue(I.getOperand(1));

3184

3185

SDNodeFlags Flags;

3186

Flags.setExact(isa<PossiblyExactOperator>(&I) &&

3187

cast<PossiblyExactOperator>(&I)->isExact());

3188

setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,

3189

Op2, Flags));

3190

}

3191

3192

void SelectionDAGBuilder::visitICmp(const User &I) {

3193

ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;

3194

if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))

3195

predicate = IC->getPredicate();

3196

else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))

3197

predicate = ICmpInst::Predicate(IC->getPredicate());

3198

SDValue Op1 = getValue(I.getOperand(0));

3199

SDValue Op2 = getValue(I.getOperand(1));

3200

ISD::CondCode Opcode = getICmpCondCode(predicate);

3201

3202

auto &TLI = DAG.getTargetLoweringInfo();

3203

EVT MemVT =

3204

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3205

3206

// If a pointer's DAG type is larger than its memory type then the DAG values

3207

// are zero-extended. This breaks signed comparisons so truncate back to the

3208

// underlying type before doing the compare.

3209

if (Op1.getValueType() != MemVT) {

3210

Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);

3211

Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);

3212

}

3213

3214

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3215

I.getType());

3216

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));

3217

}

3218

3219

void SelectionDAGBuilder::visitFCmp(const User &I) {

3220

FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;

3221

if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))

3222

predicate = FC->getPredicate();

3223

else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))

3224

predicate = FCmpInst::Predicate(FC->getPredicate());

3225

SDValue Op1 = getValue(I.getOperand(0));

3226

SDValue Op2 = getValue(I.getOperand(1));

3227

3228

ISD::CondCode Condition = getFCmpCondCode(predicate);

3229

auto *FPMO = cast<FPMathOperator>(&I);

3230

if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)

3231

Condition = getFCmpCodeWithoutNaN(Condition);

3232

3233

SDNodeFlags Flags;

3234

Flags.copyFMF(*FPMO);

3235

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

3236

3237

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3238

I.getType());

3239

setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));

3240

}

3241

3242

// Check if the condition of the select has one use or two users that are both

3243

// selects with the same condition.

3244

static bool hasOnlySelectUsers(const Value *Cond) {

3245

return llvm::all_of(Cond->users(), [](const Value *V) {

3246

return isa<SelectInst>(V);

3247

});

3248

}

3249

3250

void SelectionDAGBuilder::visitSelect(const User &I) {

3251

SmallVector<EVT, 4> ValueVTs;

3252

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),

3253

ValueVTs);

3254

unsigned NumValues = ValueVTs.size();

3255

if (NumValues == 0) return;

3256

3257

SmallVector<SDValue, 4> Values(NumValues);

3258

SDValue Cond = getValue(I.getOperand(0));

3259

SDValue LHSVal = getValue(I.getOperand(1));

3260

SDValue RHSVal = getValue(I.getOperand(2));

3261

SmallVector<SDValue, 1> BaseOps(1, Cond);

3262

ISD::NodeType OpCode =

3263

Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

3264

3265

bool IsUnaryAbs = false;

3266

bool Negate = false;

3267

3268

SDNodeFlags Flags;

3269

if (auto *FPOp = dyn_cast<FPMathOperator>(&I))

3270

Flags.copyFMF(*FPOp);

3271

3272

// Min/max matching is only viable if all output VTs are the same.

3273

if (is_splat(ValueVTs)) {

3274

EVT VT = ValueVTs[0];

3275

LLVMContext &Ctx = *DAG.getContext();

3276

auto &TLI = DAG.getTargetLoweringInfo();

3277

3278

// We care about the legality of the operation after it has been type

3279

// legalized.

3280

while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)

3281

VT = TLI.getTypeToTransformTo(Ctx, VT);

3282

3283

// If the vselect is legal, assume we want to leave this as a vector setcc +

3284

// vselect. Otherwise, if this is going to be scalarized, we want to see if

3285

// min/max is legal on the scalar type.

3286

bool UseScalarMinMax = VT.isVector() &&

3287

!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

3288

3289

Value *LHS, *RHS;

3290

auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);

3291

ISD::NodeType Opc = ISD::DELETED_NODE;

3292

switch (SPR.Flavor) {

3293

case SPF_UMAX: Opc = ISD::UMAX; break;

3294

case SPF_UMIN: Opc = ISD::UMIN; break;

3295

case SPF_SMAX: Opc = ISD::SMAX; break;

3296

case SPF_SMIN: Opc = ISD::SMIN; break;

3297

case SPF_FMINNUM:

3298

switch (SPR.NaNBehavior) {

3299

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3299);

3300

case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break;

3301

case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;

3302

case SPNB_RETURNS_ANY: {

3303

if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))

3304

Opc = ISD::FMINNUM;

3305

else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))

3306

Opc = ISD::FMINIMUM;

3307

else if (UseScalarMinMax)

3308

Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?

3309

ISD::FMINNUM : ISD::FMINIMUM;

3310

break;

3311

}

3312

}

3313

break;

3314

case SPF_FMAXNUM:

3315

switch (SPR.NaNBehavior) {

3316

case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3316);

3317

case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break;

3318

case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;

3319

case SPNB_RETURNS_ANY:

3320

3321

if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))

3322

Opc = ISD::FMAXNUM;

3323

else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))

3324

Opc = ISD::FMAXIMUM;

3325

else if (UseScalarMinMax)

3326

Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?

3327

ISD::FMAXNUM : ISD::FMAXIMUM;

3328

break;

3329

}

3330

break;

3331

case SPF_NABS:

3332

Negate = true;

3333

LLVM_FALLTHROUGH[[gnu::fallthrough]];

3334

case SPF_ABS:

3335

IsUnaryAbs = true;

3336

Opc = ISD::ABS;

3337

break;

3338

default: break;

3339

}

3340

3341

if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&

3342

(TLI.isOperationLegalOrCustom(Opc, VT) ||

3343

(UseScalarMinMax &&

3344

TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&

3345

// If the underlying comparison instruction is used by any other

3346

// instruction, the consumed instructions won't be destroyed, so it is

3347

// not profitable to convert to a min/max.

3348

hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {

3349

OpCode = Opc;

3350

LHSVal = getValue(LHS);

3351

RHSVal = getValue(RHS);

3352

BaseOps.clear();

3353

}

3354

3355

if (IsUnaryAbs) {

3356

OpCode = Opc;

3357

LHSVal = getValue(LHS);

3358

BaseOps.clear();

3359

}

3360

}

3361

3362

if (IsUnaryAbs) {

3363

for (unsigned i = 0; i != NumValues; ++i) {

3364

SDLoc dl = getCurSDLoc();

3365

EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);

3366

Values[i] =

3367

DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));

3368

if (Negate)

3369

Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),

3370

Values[i]);

3371

}

3372

} else {

3373

for (unsigned i = 0; i != NumValues; ++i) {

3374

SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());

3375

Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));

3376

Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));

3377

Values[i] = DAG.getNode(

3378

OpCode, getCurSDLoc(),

3379

LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);

3380

}

3381

}

3382

3383

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3384

DAG.getVTList(ValueVTs), Values));

3385

}

3386

3387

void SelectionDAGBuilder::visitTrunc(const User &I) {

3388

// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).

3389

SDValue N = getValue(I.getOperand(0));

3390

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3391

I.getType());

3392

setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));

3393

}

3394

3395

void SelectionDAGBuilder::visitZExt(const User &I) {

3396

// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3397

// ZExt also can't be a cast to bool for same reason. So, nothing much to do

3398

SDValue N = getValue(I.getOperand(0));

3399

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3400

I.getType());

3401

setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));

3402

}

3403

3404

void SelectionDAGBuilder::visitSExt(const User &I) {

3405

// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).

3406

// SExt also can't be a cast to bool for same reason. So, nothing much to do

3407

SDValue N = getValue(I.getOperand(0));

3408

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3409

I.getType());

3410

setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));

3411

}

3412

3413

void SelectionDAGBuilder::visitFPTrunc(const User &I) {

3414

// FPTrunc is never a no-op cast, no need to check

3415

SDValue N = getValue(I.getOperand(0));

3416

SDLoc dl = getCurSDLoc();

3417

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3418

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3419

setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,

3420

DAG.getTargetConstant(

3421

0, dl, TLI.getPointerTy(DAG.getDataLayout()))));

3422

}

3423

3424

void SelectionDAGBuilder::visitFPExt(const User &I) {

3425

// FPExt is never a no-op cast, no need to check

3426

SDValue N = getValue(I.getOperand(0));

3427

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3428

I.getType());

3429

setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));

3430

}

3431

3432

void SelectionDAGBuilder::visitFPToUI(const User &I) {

3433

// FPToUI is never a no-op cast, no need to check

3434

SDValue N = getValue(I.getOperand(0));

3435

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3436

I.getType());

3437

setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));

3438

}

3439

3440

void SelectionDAGBuilder::visitFPToSI(const User &I) {

3441

// FPToSI is never a no-op cast, no need to check

3442

SDValue N = getValue(I.getOperand(0));

3443

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3444

I.getType());

3445

setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));

3446

}

3447

3448

void SelectionDAGBuilder::visitUIToFP(const User &I) {

3449

// UIToFP is never a no-op cast, no need to check

3450

SDValue N = getValue(I.getOperand(0));

3451

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3452

I.getType());

3453

setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));

3454

}

3455

3456

void SelectionDAGBuilder::visitSIToFP(const User &I) {

3457

// SIToFP is never a no-op cast, no need to check

3458

SDValue N = getValue(I.getOperand(0));

3459

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3460

I.getType());

3461

setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));

3462

}

3463

3464

void SelectionDAGBuilder::visitPtrToInt(const User &I) {

3465

// What to do depends on the size of the integer and the size of the pointer.

3466

// We can either truncate, zero extend, or no-op, accordingly.

3467

SDValue N = getValue(I.getOperand(0));

3468

auto &TLI = DAG.getTargetLoweringInfo();

3469

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3470

I.getType());

3471

EVT PtrMemVT =

3472

TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

3473

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3474

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);

3475

setValue(&I, N);

3476

}

3477

3478

void SelectionDAGBuilder::visitIntToPtr(const User &I) {

3479

// What to do depends on the size of the integer and the size of the pointer.

3480

// We can either truncate, zero extend, or no-op, accordingly.

3481

SDValue N = getValue(I.getOperand(0));

3482

auto &TLI = DAG.getTargetLoweringInfo();

3483

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3484

EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

3485

N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);

3486

N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);

3487

setValue(&I, N);

3488

}

3489

3490

void SelectionDAGBuilder::visitBitCast(const User &I) {

3491

SDValue N = getValue(I.getOperand(0));

3492

SDLoc dl = getCurSDLoc();

3493

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

3494

I.getType());

3495

3496

// BitCast assures us that source and destination are the same size so this is

3497

// either a BITCAST or a no-op.

3498

if (DestVT != N.getValueType())

3499

setValue(&I, DAG.getNode(ISD::BITCAST, dl,

3500

DestVT, N)); // convert types.

3501

// Check if the original LLVM IR Operand was a ConstantInt, because getValue()

3502

// might fold any kind of constant expression to an integer constant and that

3503

// is not what we are looking for. Only recognize a bitcast of a genuine

3504

// constant integer as an opaque constant.

3505

else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))

3506

setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,

3507

/*isOpaque*/true));

3508

else

3509

setValue(&I, N); // noop cast.

3510

}

3511

3512

void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {

3513

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3514

const Value *SV = I.getOperand(0);

3515

SDValue N = getValue(SV);

3516

EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3517

3518

unsigned SrcAS = SV->getType()->getPointerAddressSpace();

3519

unsigned DestAS = I.getType()->getPointerAddressSpace();

3520

3521

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))

3522

N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

3523

3524

setValue(&I, N);

3525

}

3526

3527

void SelectionDAGBuilder::visitInsertElement(const User &I) {

3528

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3529

SDValue InVec = getValue(I.getOperand(0));

3530

SDValue InVal = getValue(I.getOperand(1));

3531

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),

3532

TLI.getVectorIdxTy(DAG.getDataLayout()));

3533

setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),

3534

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3535

InVec, InVal, InIdx));

3536

}

3537

3538

void SelectionDAGBuilder::visitExtractElement(const User &I) {

3539

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3540

SDValue InVec = getValue(I.getOperand(0));

3541

SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),

3542

TLI.getVectorIdxTy(DAG.getDataLayout()));

3543

setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),

3544

TLI.getValueType(DAG.getDataLayout(), I.getType()),

3545

InVec, InIdx));

3546

}

3547

3548

void SelectionDAGBuilder::visitShuffleVector(const User &I) {

3549

SDValue Src1 = getValue(I.getOperand(0));

3550

SDValue Src2 = getValue(I.getOperand(1));

3551

ArrayRef<int> Mask;

3552

if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))

3553

Mask = SVI->getShuffleMask();

3554

else

3555

Mask = cast<ConstantExpr>(I).getShuffleMask();

3556

SDLoc DL = getCurSDLoc();

3557

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3558

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

3559

EVT SrcVT = Src1.getValueType();

3560

3561

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&

3562

VT.isScalableVector()) {

3563

// Canonical splat form of first element of first input vector.

3564

SDValue FirstElt =

3565

DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,

3566

DAG.getVectorIdxConstant(0, DL));

3567

setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));

3568

return;

3569

}

3570

3571

// For now, we only handle splats for scalable vectors.

3572

// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation

3573

// for targets that support a SPLAT_VECTOR for non-scalable vector types.

3574

assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3574, __extension__ __PRETTY_FUNCTION__));

3575

3576

unsigned SrcNumElts = SrcVT.getVectorNumElements();

3577

unsigned MaskNumElts = Mask.size();

3578

3579

if (SrcNumElts == MaskNumElts) {

3580

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));

3581

return;

3582

}

3583

3584

// Normalize the shuffle vector since mask and vector length don't match.

3585

if (SrcNumElts < MaskNumElts) {

3586

// Mask is longer than the source vectors. We can use concatenate vector to

3587

// make the mask and vectors lengths match.

3588

3589

if (MaskNumElts % SrcNumElts == 0) {

3590

// Mask length is a multiple of the source vector length.

3591

// Check if the shuffle is some kind of concatenation of the input

3592

// vectors.

3593

unsigned NumConcat = MaskNumElts / SrcNumElts;

3594

bool IsConcat = true;

3595

SmallVector<int, 8> ConcatSrcs(NumConcat, -1);

3596

for (unsigned i = 0; i != MaskNumElts; ++i) {

3597

int Idx = Mask[i];

3598

if (Idx < 0)

3599

continue;

3600

// Ensure the indices in each SrcVT sized piece are sequential and that

3601

// the same source is used for the whole piece.

3602

if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

3603

(ConcatSrcs[i / SrcNumElts] >= 0 &&

3604

ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {

3605

IsConcat = false;

3606

break;

3607

}

3608

// Remember which source this index came from.

3609

ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

3610

}

3611

3612

// The shuffle is concatenating multiple vectors together. Just emit

3613

// a CONCAT_VECTORS operation.

3614

if (IsConcat) {

3615

SmallVector<SDValue, 8> ConcatOps;

3616

for (auto Src : ConcatSrcs) {

3617

if (Src < 0)

3618

ConcatOps.push_back(DAG.getUNDEF(SrcVT));

3619

else if (Src == 0)

3620

ConcatOps.push_back(Src1);

3621

else

3622

ConcatOps.push_back(Src2);

3623

}

3624

setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));

3625

return;

3626

}

3627

}

3628

3629

unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);

3630

unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;

3631

EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),

3632

PaddedMaskNumElts);

3633

3634

// Pad both vectors with undefs to make them the same length as the mask.

3635

SDValue UndefVal = DAG.getUNDEF(SrcVT);

3636

3637

SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);

3638

SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);

3639

MOps1[0] = Src1;

3640

MOps2[0] = Src2;

3641

3642

Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);

3643

Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

3644

3645

// Readjust mask for new input vector length.

3646

SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);

3647

for (unsigned i = 0; i != MaskNumElts; ++i) {

3648

int Idx = Mask[i];

3649

if (Idx >= (int)SrcNumElts)

3650

Idx -= SrcNumElts - PaddedMaskNumElts;

3651

MappedOps[i] = Idx;

3652

}

3653

3654

SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

3655

3656

// If the concatenated vector was padded, extract a subvector with the

3657

// correct number of elements.

3658

if (MaskNumElts != PaddedMaskNumElts)

3659

Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,

3660

DAG.getVectorIdxConstant(0, DL));

3661

3662

setValue(&I, Result);

3663

return;

3664

}

3665

3666

if (SrcNumElts > MaskNumElts) {

3667

// Analyze the access pattern of the vector to see if we can extract

3668

// two subvectors and do the shuffle.

3669

int StartIdx[2] = { -1, -1 }; // StartIdx to extract from

3670

bool CanExtract = true;

3671

for (int Idx : Mask) {

3672

unsigned Input = 0;

3673

if (Idx < 0)

3674

continue;

3675

3676

if (Idx >= (int)SrcNumElts) {

3677

Input = 1;

3678

Idx -= SrcNumElts;

3679

}

3680

3681

// If all the indices come from the same MaskNumElts sized portion of

3682

// the sources we can use extract. Also make sure the extract wouldn't

3683

// extract past the end of the source.

3684

int NewStartIdx = alignDown(Idx, MaskNumElts);

3685

if (NewStartIdx + MaskNumElts > SrcNumElts ||

3686

(StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))

3687

CanExtract = false;

3688

// Make sure we always update StartIdx as we use it to track if all

3689

// elements are undef.

3690

StartIdx[Input] = NewStartIdx;

3691

}

3692

3693

if (StartIdx[0] < 0 && StartIdx[1] < 0) {

3694

setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.

3695

return;

3696

}

3697

if (CanExtract) {

3698

// Extract appropriate subvector and generate a vector shuffle

3699

for (unsigned Input = 0; Input < 2; ++Input) {

3700

SDValue &Src = Input == 0 ? Src1 : Src2;

3701

if (StartIdx[Input] < 0)

3702

Src = DAG.getUNDEF(VT);

3703

else {

3704

Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,

3705

DAG.getVectorIdxConstant(StartIdx[Input], DL));

3706

}

3707

}

3708

3709

// Calculate new mask.

3710

SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());

3711

for (int &Idx : MappedOps) {

3712

if (Idx >= (int)SrcNumElts)

3713

Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;

3714

else if (Idx >= 0)

3715

Idx -= StartIdx[0];

3716

}

3717

3718

setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));

3719

return;

3720

}

3721

}

3722

3723

// We can't use either concat vectors or extract subvectors so fall back to

3724

// replacing the shuffle with extract and build vector.

3725

// to insert and build vector.

3726

EVT EltVT = VT.getVectorElementType();

3727

SmallVector<SDValue,8> Ops;

3728

for (int Idx : Mask) {

3729

SDValue Res;

3730

3731

if (Idx < 0) {

3732

Res = DAG.getUNDEF(EltVT);

3733

} else {

3734

SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;

3735

if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

3736

3737

Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,

3738

DAG.getVectorIdxConstant(Idx, DL));

3739

}

3740

3741

Ops.push_back(Res);

3742

}

3743

3744

setValue(&I, DAG.getBuildVector(VT, DL, Ops));

3745

}

3746

3747

void SelectionDAGBuilder::visitInsertValue(const User &I) {

3748

ArrayRef<unsigned> Indices;

3749

if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))

3750

Indices = IV->getIndices();

3751

else

3752

Indices = cast<ConstantExpr>(&I)->getIndices();

3753

3754

const Value *Op0 = I.getOperand(0);

3755

const Value *Op1 = I.getOperand(1);

3756

Type *AggTy = I.getType();

3757

Type *ValTy = Op1->getType();

3758

bool IntoUndef = isa<UndefValue>(Op0);

3759

bool FromUndef = isa<UndefValue>(Op1);

3760

3761

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3762

3763

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3764

SmallVector<EVT, 4> AggValueVTs;

3765

ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);

3766

SmallVector<EVT, 4> ValValueVTs;

3767

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3768

3769

unsigned NumAggValues = AggValueVTs.size();

3770

unsigned NumValValues = ValValueVTs.size();

3771

SmallVector<SDValue, 4> Values(NumAggValues);

3772

3773

// Ignore an insertvalue that produces an empty object

3774

if (!NumAggValues) {

3775

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3776

return;

3777

}

3778

3779

SDValue Agg = getValue(Op0);

3780

unsigned i = 0;

3781

// Copy the beginning value(s) from the original aggregate.

3782

for (; i != LinearIndex; ++i)

3783

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3784

SDValue(Agg.getNode(), Agg.getResNo() + i);

3785

// Copy values from the inserted value(s).

3786

if (NumValValues) {

3787

SDValue Val = getValue(Op1);

3788

for (; i != LinearIndex + NumValValues; ++i)

3789

Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3790

SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);

3791

}

3792

// Copy remaining value(s) from the original aggregate.

3793

for (; i != NumAggValues; ++i)

3794

Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :

3795

SDValue(Agg.getNode(), Agg.getResNo() + i);

3796

3797

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3798

DAG.getVTList(AggValueVTs), Values));

3799

}

3800

3801

void SelectionDAGBuilder::visitExtractValue(const User &I) {

3802

ArrayRef<unsigned> Indices;

3803

if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))

3804

Indices = EV->getIndices();

3805

else

3806

Indices = cast<ConstantExpr>(&I)->getIndices();

3807

3808

const Value *Op0 = I.getOperand(0);

3809

Type *AggTy = Op0->getType();

3810

Type *ValTy = I.getType();

3811

bool OutOfUndef = isa<UndefValue>(Op0);

3812

3813

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

3814

3815

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3816

SmallVector<EVT, 4> ValValueVTs;

3817

ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

3818

3819

unsigned NumValValues = ValValueVTs.size();

3820

3821

// Ignore a extractvalue that produces an empty object

3822

if (!NumValValues) {

3823

setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));

3824

return;

3825

}

3826

3827

SmallVector<SDValue, 4> Values(NumValValues);

3828

3829

SDValue Agg = getValue(Op0);

3830

// Copy out the selected value(s).

3831

for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)

3832

Values[i - LinearIndex] =

3833

OutOfUndef ?

3834

DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :

3835

SDValue(Agg.getNode(), Agg.getResNo() + i);

3836

3837

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),

3838

DAG.getVTList(ValValueVTs), Values));

3839

}

3840

3841

void SelectionDAGBuilder::visitGetElementPtr(const User &I) {

3842

Value *Op0 = I.getOperand(0);

3843

// Note that the pointer operand may be a vector of pointers. Take the scalar

3844

// element which holds a pointer.

3845

unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();

3846

SDValue N = getValue(Op0);

3847

SDLoc dl = getCurSDLoc();

3848

auto &TLI = DAG.getTargetLoweringInfo();

3849

3850

// Normalize Vector GEP - all scalar operands should be converted to the

3851

// splat vector.

3852

bool IsVectorGEP = I.getType()->isVectorTy();

3853

ElementCount VectorElementCount =

3854

IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()

3855

: ElementCount::getFixed(0);

3856

3857

if (IsVectorGEP && !N.getValueType().isVector()) {

3858

LLVMContext &Context = *DAG.getContext();

3859

EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);

3860

if (VectorElementCount.isScalable())

3861

N = DAG.getSplatVector(VT, dl, N);

3862

else

3863

N = DAG.getSplatBuildVector(VT, dl, N);

3864

}

3865

3866

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);

3867

GTI != E; ++GTI) {

3868

const Value *Idx = GTI.getOperand();

3869

if (StructType *StTy = GTI.getStructTypeOrNull()) {

3870

unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();

3871

if (Field) {

3872

// N = N + Offset

3873

uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);

3874

3875

// In an inbounds GEP with an offset that is nonnegative even when

3876

// interpreted as signed, assume there is no unsigned overflow.

3877

SDNodeFlags Flags;

3878

if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())

3879

Flags.setNoUnsignedWrap(true);

3880

3881

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,

3882

DAG.getConstant(Offset, dl, N.getValueType()), Flags);

3883

}

3884

} else {

3885

// IdxSize is the width of the arithmetic according to IR semantics.

3886

// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth

3887

// (and fix up the result later).

3888

unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);

3889

MVT IdxTy = MVT::getIntegerVT(IdxSize);

3890

TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());

3891

// We intentionally mask away the high bits here; ElementSize may not

3892

// fit in IdxTy.

3893

APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());

3894

bool ElementScalable = ElementSize.isScalable();

3895

3896

// If this is a scalar constant or a splat vector of constants,

3897

// handle it quickly.

3898

const auto *C = dyn_cast<Constant>(Idx);

3899

if (C && isa<VectorType>(C->getType()))

3900

C = C->getSplatValue();

3901

3902

const auto *CI = dyn_cast_or_null<ConstantInt>(C);

3903

if (CI && CI->isZero())

3904

continue;

3905

if (CI && !ElementScalable) {

3906

APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);

3907

LLVMContext &Context = *DAG.getContext();

3908

SDValue OffsVal;

3909

if (IsVectorGEP)

3910

OffsVal = DAG.getConstant(

3911

Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));

3912

else

3913

OffsVal = DAG.getConstant(Offs, dl, IdxTy);

3914

3915

// In an inbounds GEP with an offset that is nonnegative even when

3916

// interpreted as signed, assume there is no unsigned overflow.

3917

SDNodeFlags Flags;

3918

if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())

3919

Flags.setNoUnsignedWrap(true);

3920

3921

OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

3922

3923

N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);

3924

continue;

3925

}

3926

3927

// N = N + Idx * ElementMul;

3928

SDValue IdxN = getValue(Idx);

3929

3930

if (!IdxN.getValueType().isVector() && IsVectorGEP) {

3931

EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),

3932

VectorElementCount);

3933

if (VectorElementCount.isScalable())

3934

IdxN = DAG.getSplatVector(VT, dl, IdxN);

3935

else

3936

IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);

3937

}

3938

3939

// If the index is smaller or larger than intptr_t, truncate or extend

3940

// it.

3941

IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

3942

3943

if (ElementScalable) {

3944

EVT VScaleTy = N.getValueType().getScalarType();

3945

SDValue VScale = DAG.getNode(

3946

ISD::VSCALE, dl, VScaleTy,

3947

DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));

3948

if (IsVectorGEP)

3949

VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);

3950

IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);

3951

} else {

3952

// If this is a multiply by a power of two, turn it into a shl

3953

// immediately. This is a very common case.

3954

if (ElementMul != 1) {

3955

if (ElementMul.isPowerOf2()) {

3956

unsigned Amt = ElementMul.logBase2();

3957

IdxN = DAG.getNode(ISD::SHL, dl,

3958

N.getValueType(), IdxN,

3959

DAG.getConstant(Amt, dl, IdxN.getValueType()));

3960

} else {

3961

SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,

3962

IdxN.getValueType());

3963

IdxN = DAG.getNode(ISD::MUL, dl,

3964

N.getValueType(), IdxN, Scale);

3965

}

3966

}

3967

}

3968

3969

N = DAG.getNode(ISD::ADD, dl,

3970

N.getValueType(), N, IdxN);

3971

}

3972

}

3973

3974

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);

3975

MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);

3976

if (IsVectorGEP) {

3977

PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);

3978

PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);

3979

}

3980

3981

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())

3982

N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

3983

3984

setValue(&I, N);

3985

}

3986

3987

void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {

3988

// If this is a fixed sized alloca in the entry block of the function,

3989

// allocate it statically on the stack.

3990

if (FuncInfo.StaticAllocaMap.count(&I))

3991

return; // getValue will auto-populate this.

3992

3993

SDLoc dl = getCurSDLoc();

3994

Type *Ty = I.getAllocatedType();

3995

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3996

auto &DL = DAG.getDataLayout();

3997

uint64_t TySize = DL.getTypeAllocSize(Ty);

3998

MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

3999

4000

SDValue AllocSize = getValue(I.getArraySize());

4001

4002

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());

4003

if (AllocSize.getValueType() != IntPtr)

4004

AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

4005

4006

AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,

4007

AllocSize,

4008

DAG.getConstant(TySize, dl, IntPtr));

4009

4010

// Handle alignment. If the requested alignment is less than or equal to

4011

// the stack alignment, ignore it. If the size is greater than or equal to

4012

// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.

4013

Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();

4014

if (*Alignment <= StackAlign)

4015

Alignment = None;

4016

4017

const uint64_t StackAlignMask = StackAlign.value() - 1U;

4018

// Round the size of the allocation up to the stack alignment size

4019

// by add SA-1 to the size. This doesn't overflow because we're computing

4020

// an address inside an alloca.

4021

SDNodeFlags Flags;

4022

Flags.setNoUnsignedWrap(true);

4023

AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,

4024

DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

4025

4026

// Mask out the low bits for alignment purposes.

4027

AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,

4028

DAG.getConstant(~StackAlignMask, dl, IntPtr));

4029

4030

SDValue Ops[] = {

4031

getRoot(), AllocSize,

4032

DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};

4033

SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);

4034

SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);

4035

setValue(&I, DSA);

4036

DAG.setRoot(DSA.getValue(1));

4037

4038

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4038, __extension__ __PRETTY_FUNCTION__));

4039

}

4040

4041

void SelectionDAGBuilder::visitLoad(const LoadInst &I) {

4042

if (I.isAtomic())

4043

return visitAtomicLoad(I);

4044

4045

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4046

const Value *SV = I.getOperand(0);

4047

if (TLI.supportSwiftError()) {

4048

// Swifterror values can come from either a function parameter with

4049

// swifterror attribute or an alloca with swifterror attribute.

4050

if (const Argument *Arg = dyn_cast<Argument>(SV)) {

4051

if (Arg->hasSwiftErrorAttr())

4052

return visitLoadFromSwiftError(I);

4053

}

4054

4055

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {

4056

if (Alloca->isSwiftError())

4057

return visitLoadFromSwiftError(I);

4058

}

4059

}

4060

4061

SDValue Ptr = getValue(SV);

4062

4063

Type *Ty = I.getType();

4064

Align Alignment = I.getAlign();

4065

4066

AAMDNodes AAInfo;

4067

I.getAAMetadata(AAInfo);

4068

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4069

4070

SmallVector<EVT, 4> ValueVTs, MemVTs;

4071

SmallVector<uint64_t, 4> Offsets;

4072

ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);

4073

unsigned NumValues = ValueVTs.size();

4074

if (NumValues == 0)

4075

return;

4076

4077

bool isVolatile = I.isVolatile();

4078

4079

SDValue Root;

4080

bool ConstantMemory = false;

4081

if (isVolatile)

4082

// Serialize volatile loads with other side effects.

4083

Root = getRoot();

4084

else if (NumValues > MaxParallelChains)

4085

Root = getMemoryRoot();

4086

else if (AA &&

4087

AA->pointsToConstantMemory(MemoryLocation(

4088

SV,

4089

LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),

4090

AAInfo))) {

4091

// Do not serialize (non-volatile) loads of constant memory with anything.

4092

Root = DAG.getEntryNode();

4093

ConstantMemory = true;

4094

} else {

4095

// Do not serialize non-volatile loads against each other.

4096

Root = DAG.getRoot();

4097

}

4098

4099

SDLoc dl = getCurSDLoc();

4100

4101

if (isVolatile)

4102

Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

4103

4104

// An aggregate load cannot wrap around the address space, so offsets to its

4105

// parts don't wrap either.

4106

SDNodeFlags Flags;

4107

Flags.setNoUnsignedWrap(true);

4108

4109

SmallVector<SDValue, 4> Values(NumValues);

4110

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4111

EVT PtrVT = Ptr.getValueType();

4112

4113

MachineMemOperand::Flags MMOFlags

4114

= TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4115

4116

unsigned ChainI = 0;

4117

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4118

// Serializing loads here may result in excessive register pressure, and

4119

// TokenFactor places arbitrary choke points on the scheduler. SD scheduling

4120

// could recover a bit by hoisting nodes upward in the chain by recognizing

4121

// they are side-effect free or do not alias. The optimizer should really

4122

// avoid this case by converting large object/array copies to llvm.memcpy

4123

// (MaxParallelChains should always remain as failsafe).

4124

if (ChainI == MaxParallelChains) {

4125

assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4125, __extension__ __PRETTY_FUNCTION__));

4126

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4127

makeArrayRef(Chains.data(), ChainI));

4128

Root = Chain;

4129

ChainI = 0;

4130

}

4131

SDValue A = DAG.getNode(ISD::ADD, dl,

4132

PtrVT, Ptr,

4133

DAG.getConstant(Offsets[i], dl, PtrVT),

4134

Flags);

4135

4136

SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,

4137

MachinePointerInfo(SV, Offsets[i]), Alignment,

4138

MMOFlags, AAInfo, Ranges);

4139

Chains[ChainI] = L.getValue(1);

4140

4141

if (MemVTs[i] != ValueVTs[i])

4142

L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

4143

4144

Values[i] = L;

4145

}

4146

4147

if (!ConstantMemory) {

4148

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4149

makeArrayRef(Chains.data(), ChainI));

4150

if (isVolatile)

4151

DAG.setRoot(Chain);

4152

else

4153

PendingLoads.push_back(Chain);

4154

}

4155

4156

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,

4157

DAG.getVTList(ValueVTs), Values));

4158

}

4159

4160

void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {

4161

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4162, __extension__ __PRETTY_FUNCTION__))

4162

"call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4162, __extension__ __PRETTY_FUNCTION__));

4163

4164

SmallVector<EVT, 4> ValueVTs;

4165

SmallVector<uint64_t, 4> Offsets;

4166

const Value *SrcV = I.getOperand(0);

4167

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4168

SrcV->getType(), ValueVTs, &Offsets);

4169

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4170, __extension__ __PRETTY_FUNCTION__))

4170

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4170, __extension__ __PRETTY_FUNCTION__));

4171

4172

SDValue Src = getValue(SrcV);

4173

// Create a virtual register, then update the virtual register.

4174

Register VReg =

4175

SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());

4176

// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue

4177

// Chain can be getRoot or getControlRoot.

4178

SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,

4179

SDValue(Src.getNode(), Src.getResNo()));

4180

DAG.setRoot(CopyNode);

4181

}

4182

4183

void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {

4184

assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4185, __extension__ __PRETTY_FUNCTION__))

4185

"call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4185, __extension__ __PRETTY_FUNCTION__));

4186

4187

assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4190, __extension__ __PRETTY_FUNCTION__))

4188

!I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4190, __extension__ __PRETTY_FUNCTION__))

4189

!I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4190, __extension__ __PRETTY_FUNCTION__))

4190

"Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4190, __extension__ __PRETTY_FUNCTION__));

4191

4192

const Value *SV = I.getOperand(0);

4193

Type *Ty = I.getType();

4194

AAMDNodes AAInfo;

4195

I.getAAMetadata(AAInfo);

4196

assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__))

4197

(!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__))

4198

!AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__))

4199

SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__))

4200

AAInfo))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__))

4201

"load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4201, __extension__ __PRETTY_FUNCTION__));

4202

4203

SmallVector<EVT, 4> ValueVTs;

4204

SmallVector<uint64_t, 4> Offsets;

4205

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,

4206

ValueVTs, &Offsets);

4207

assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4208, __extension__ __PRETTY_FUNCTION__))

4208

"expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4208, __extension__ __PRETTY_FUNCTION__));

4209

4210

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT

4211

SDValue L = DAG.getCopyFromReg(

4212

getRoot(), getCurSDLoc(),

4213

SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

4214

4215

setValue(&I, L);

4216

}

4217

4218

void SelectionDAGBuilder::visitStore(const StoreInst &I) {

4219

if (I.isAtomic())

4220

return visitAtomicStore(I);

4221

4222

const Value *SrcV = I.getOperand(0);

4223

const Value *PtrV = I.getOperand(1);

4224

4225

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4226

if (TLI.supportSwiftError()) {

4227

// Swifterror values can come from either a function parameter with

4228

// swifterror attribute or an alloca with swifterror attribute.

4229

if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {

4230

if (Arg->hasSwiftErrorAttr())

4231

return visitStoreToSwiftError(I);

4232

}

4233

4234

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {

4235

if (Alloca->isSwiftError())

4236

return visitStoreToSwiftError(I);

4237

}

4238

}

4239

4240

SmallVector<EVT, 4> ValueVTs, MemVTs;

4241

SmallVector<uint64_t, 4> Offsets;

4242

ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),

4243

SrcV->getType(), ValueVTs, &MemVTs, &Offsets);

4244

unsigned NumValues = ValueVTs.size();

4245

if (NumValues == 0)

4246

return;

4247

4248

// Get the lowered operands. Note that we do this after

4249

// checking if NumResults is zero, because with zero results

4250

// the operands won't have values in the map.

4251

SDValue Src = getValue(SrcV);

4252

SDValue Ptr = getValue(PtrV);

4253

4254

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();

4255

SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));

4256

SDLoc dl = getCurSDLoc();

4257

Align Alignment = I.getAlign();

4258

AAMDNodes AAInfo;

4259

I.getAAMetadata(AAInfo);

4260

4261

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4262

4263

// An aggregate load cannot wrap around the address space, so offsets to its

4264

// parts don't wrap either.

4265

SDNodeFlags Flags;

4266

Flags.setNoUnsignedWrap(true);

4267

4268

unsigned ChainI = 0;

4269

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {

4270

// See visitLoad comments.

4271

if (ChainI == MaxParallelChains) {

4272

SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4273

makeArrayRef(Chains.data(), ChainI));

4274

Root = Chain;

4275

ChainI = 0;

4276

}

4277

SDValue Add =

4278

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);

4279

SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);

4280

if (MemVTs[i] != ValueVTs[i])

4281

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);

4282

SDValue St =

4283

DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),

4284

Alignment, MMOFlags, AAInfo);

4285

Chains[ChainI] = St;

4286

}

4287

4288

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,

4289

makeArrayRef(Chains.data(), ChainI));

4290

DAG.setRoot(StoreNode);

4291

}

4292

4293

void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,

4294

bool IsCompressing) {

4295

SDLoc sdl = getCurSDLoc();

4296

4297

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4298

MaybeAlign &Alignment) {

4299

// llvm.masked.store.*(Src0, Ptr, alignment, Mask)

4300

Src0 = I.getArgOperand(0);

4301

Ptr = I.getArgOperand(1);

4302

Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();

4303

Mask = I.getArgOperand(3);

4304

};

4305

auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4306

MaybeAlign &Alignment) {

4307

// llvm.masked.compressstore.*(Src0, Ptr, Mask)

4308

Src0 = I.getArgOperand(0);

4309

Ptr = I.getArgOperand(1);

4310

Mask = I.getArgOperand(2);

4311

Alignment = None;

4312

};

4313

4314

Value *PtrOperand, *MaskOperand, *Src0Operand;

4315

MaybeAlign Alignment;

4316

if (IsCompressing)

4317

getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4318

else

4319

getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4320

4321

SDValue Ptr = getValue(PtrOperand);

4322

SDValue Src0 = getValue(Src0Operand);

4323

SDValue Mask = getValue(MaskOperand);

4324

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4325

4326

EVT VT = Src0.getValueType();

4327

if (!Alignment)

4328

Alignment = DAG.getEVTAlign(VT);

4329

4330

AAMDNodes AAInfo;

4331

I.getAAMetadata(AAInfo);

4332

4333

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4334

MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,

4335

// TODO: Make MachineMemOperands aware of scalable

4336

// vectors.

4337

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);

4338

SDValue StoreNode =

4339

DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,

4340

ISD::UNINDEXED, false /* Truncating */, IsCompressing);

4341

DAG.setRoot(StoreNode);

4342

setValue(&I, StoreNode);

4343

}

4344

4345

// Get a uniform base for the Gather/Scatter intrinsic.

4346

// The first argument of the Gather/Scatter intrinsic is a vector of pointers.

4347

// We try to represent it as a base pointer + vector of indices.

4348

// Usually, the vector of pointers comes from a 'getelementptr' instruction.

4349

// The first operand of the GEP may be a single pointer or a vector of pointers

4350

// Example:

4351

// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind

4352

// or

4353

// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind

4354

// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..

4355

//

4356

// When the first GEP operand is a single pointer - it is the uniform base we

4357

// are looking for. If first operand of the GEP is a splat vector - we

4358

// extract the splat value and use it as a uniform base.

4359

// In all other cases the function returns 'false'.

4360

static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,

4361

ISD::MemIndexType &IndexType, SDValue &Scale,

4362

SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {

4363

SelectionDAG& DAG = SDB->DAG;

4364

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4365

const DataLayout &DL = DAG.getDataLayout();

4366

4367

assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Uexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4367, __extension__ __PRETTY_FUNCTION__));

4368

4369

// Handle splat constant pointer.

4370

if (auto *C = dyn_cast<Constant>(Ptr)) {

4371

C = C->getSplatValue();

4372

if (!C)

4373

return false;

4374

4375

Base = SDB->getValue(C);

4376

4377

ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();

4378

EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);

4379

Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);

4380

IndexType = ISD::SIGNED_SCALED;

4381

Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4382

return true;

4383

}

4384

4385

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);

4386

if (!GEP || GEP->getParent() != CurBB)

4387

return false;

4388

4389

if (GEP->getNumOperands() != 2)

4390

return false;

4391

4392

const Value *BasePtr = GEP->getPointerOperand();

4393

const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

4394

4395

// Make sure the base is scalar and the index is a vector.

4396

if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())

4397

return false;

4398

4399

Base = SDB->getValue(BasePtr);

4400

Index = SDB->getValue(IndexVal);

4401

IndexType = ISD::SIGNED_SCALED;

4402

Scale = DAG.getTargetConstant(

4403

DL.getTypeAllocSize(GEP->getResultElementType()),

4404

SDB->getCurSDLoc(), TLI.getPointerTy(DL));

4405

return true;

4406

}

4407

4408

void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {

4409

SDLoc sdl = getCurSDLoc();

4410

4411

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)

4412

const Value *Ptr = I.getArgOperand(1);

4413

SDValue Src0 = getValue(I.getArgOperand(0));

4414

SDValue Mask = getValue(I.getArgOperand(3));

4415

EVT VT = Src0.getValueType();

4416

Align Alignment = cast<ConstantInt>(I.getArgOperand(2))

4417

->getMaybeAlignValue()

4418

.getValueOr(DAG.getEVTAlign(VT.getScalarType()));

4419

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4420

4421

AAMDNodes AAInfo;

4422

I.getAAMetadata(AAInfo);

4423

4424

SDValue Base;

4425

SDValue Index;

4426

ISD::MemIndexType IndexType;

4427

SDValue Scale;

4428

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4429

I.getParent());

4430

4431

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4432

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4433

MachinePointerInfo(AS), MachineMemOperand::MOStore,

4434

// TODO: Make MachineMemOperands aware of scalable

4435

// vectors.

4436

MemoryLocation::UnknownSize, Alignment, AAInfo);

4437

if (!UniformBase) {

4438

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4439

Index = getValue(Ptr);

4440

IndexType = ISD::SIGNED_UNSCALED;

4441

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4442

}

4443

4444

EVT IdxVT = Index.getValueType();

4445

EVT EltTy = IdxVT.getVectorElementType();

4446

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4447

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4448

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4449

}

4450

4451

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };

4452

SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,

4453

Ops, MMO, IndexType, false);

4454

DAG.setRoot(Scatter);

4455

setValue(&I, Scatter);

4456

}

4457

4458

void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {

4459

SDLoc sdl = getCurSDLoc();

4460

4461

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4462

MaybeAlign &Alignment) {

4463

// @llvm.masked.load.*(Ptr, alignment, Mask, Src0)

4464

Ptr = I.getArgOperand(0);

4465

Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();

4466

Mask = I.getArgOperand(2);

4467

Src0 = I.getArgOperand(3);

4468

};

4469

auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,

4470

MaybeAlign &Alignment) {

4471

// @llvm.masked.expandload.*(Ptr, Mask, Src0)

4472

Ptr = I.getArgOperand(0);

4473

Alignment = None;

4474

Mask = I.getArgOperand(1);

4475

Src0 = I.getArgOperand(2);

4476

};

4477

4478

Value *PtrOperand, *MaskOperand, *Src0Operand;

4479

MaybeAlign Alignment;

4480

if (IsExpanding)

4481

getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4482

else

4483

getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

4484

4485

SDValue Ptr = getValue(PtrOperand);

4486

SDValue Src0 = getValue(Src0Operand);

4487

SDValue Mask = getValue(MaskOperand);

4488

SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

4489

4490

EVT VT = Src0.getValueType();

4491

if (!Alignment)

4492

Alignment = DAG.getEVTAlign(VT);

4493

4494

AAMDNodes AAInfo;

4495

I.getAAMetadata(AAInfo);

4496

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4497

4498

// Do not serialize masked loads of constant memory with anything.

4499

MemoryLocation ML;

4500

if (VT.isScalableVector())

4501

ML = MemoryLocation::getAfter(PtrOperand);

4502

else

4503

ML = MemoryLocation(PtrOperand, LocationSize::precise(

4504

DAG.getDataLayout().getTypeStoreSize(I.getType())),

4505

AAInfo);

4506

bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

4507

4508

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

4509

4510

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4511

MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,

4512

// TODO: Make MachineMemOperands aware of scalable

4513

// vectors.

4514

VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);

4515

4516

SDValue Load =

4517

DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,

4518

ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);

4519

if (AddToChain)

4520

PendingLoads.push_back(Load.getValue(1));

4521

setValue(&I, Load);

4522

}

4523

4524

void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {

4525

SDLoc sdl = getCurSDLoc();

4526

4527

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)

4528

const Value *Ptr = I.getArgOperand(0);

4529

SDValue Src0 = getValue(I.getArgOperand(3));

4530

SDValue Mask = getValue(I.getArgOperand(2));

4531

4532

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4533

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4534

Align Alignment = cast<ConstantInt>(I.getArgOperand(1))

4535

->getMaybeAlignValue()

4536

.getValueOr(DAG.getEVTAlign(VT.getScalarType()));

4537

4538

AAMDNodes AAInfo;

4539

I.getAAMetadata(AAInfo);

4540

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

4541

4542

SDValue Root = DAG.getRoot();

4543

SDValue Base;

4544

SDValue Index;

4545

ISD::MemIndexType IndexType;

4546

SDValue Scale;

4547

bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,

4548

I.getParent());

4549

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();

4550

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4551

MachinePointerInfo(AS), MachineMemOperand::MOLoad,

4552

// TODO: Make MachineMemOperands aware of scalable

4553

// vectors.

4554

MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);

4555

4556

if (!UniformBase) {

4557

Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4558

Index = getValue(Ptr);

4559

IndexType = ISD::SIGNED_UNSCALED;

4560

Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));

4561

}

4562

4563

EVT IdxVT = Index.getValueType();

4564

EVT EltTy = IdxVT.getVectorElementType();

4565

if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {

4566

EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);

4567

Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);

4568

}

4569

4570

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };

4571

SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,

4572

Ops, MMO, IndexType, ISD::NON_EXTLOAD);

4573

4574

PendingLoads.push_back(Gather.getValue(1));

4575

setValue(&I, Gather);

4576

}

4577

4578

void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {

4579

SDLoc dl = getCurSDLoc();

4580

AtomicOrdering SuccessOrdering = I.getSuccessOrdering();

4581

AtomicOrdering FailureOrdering = I.getFailureOrdering();

4582

SyncScope::ID SSID = I.getSyncScopeID();

4583

4584

SDValue InChain = getRoot();

4585

4586

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();

4587

SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

4588

4589

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4590

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4591

4592

MachineFunction &MF = DAG.getMachineFunction();

4593

MachineMemOperand *MMO = MF.getMachineMemOperand(

4594

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4595

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,

4596

FailureOrdering);

4597

4598

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,

4599

dl, MemVT, VTs, InChain,

4600

getValue(I.getPointerOperand()),

4601

getValue(I.getCompareOperand()),

4602

getValue(I.getNewValOperand()), MMO);

4603

4604

SDValue OutChain = L.getValue(2);

4605

4606

setValue(&I, L);

4607

DAG.setRoot(OutChain);

4608

}

4609

4610

void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {

4611

SDLoc dl = getCurSDLoc();

4612

ISD::NodeType NT;

4613

switch (I.getOperation()) {

4614

default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4614);

4615

case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;

4616

case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;

4617

case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;

4618

case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;

4619

case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;

4620

case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;

4621

case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;

4622

case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;

4623

case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;

4624

case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;

4625

case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;

4626

case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;

4627

case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;

4628

}

4629

AtomicOrdering Ordering = I.getOrdering();

4630

SyncScope::ID SSID = I.getSyncScopeID();

4631

4632

SDValue InChain = getRoot();

4633

4634

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();

4635

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4636

auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

4637

4638

MachineFunction &MF = DAG.getMachineFunction();

4639

MachineMemOperand *MMO = MF.getMachineMemOperand(

4640

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4641

DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

4642

4643

SDValue L =

4644

DAG.getAtomic(NT, dl, MemVT, InChain,

4645

getValue(I.getPointerOperand()), getValue(I.getValOperand()),

4646

MMO);

4647

4648

SDValue OutChain = L.getValue(1);

4649

4650

setValue(&I, L);

4651

DAG.setRoot(OutChain);

4652

}

4653

4654

void SelectionDAGBuilder::visitFence(const FenceInst &I) {

4655

SDLoc dl = getCurSDLoc();

4656

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4657

SDValue Ops[3];

4658

Ops[0] = getRoot();

4659

Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,

4660

TLI.getFenceOperandTy(DAG.getDataLayout()));

4661

Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,

4662

TLI.getFenceOperandTy(DAG.getDataLayout()));

4663

DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));

4664

}

4665

4666

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {

4667

SDLoc dl = getCurSDLoc();

4668

AtomicOrdering Order = I.getOrdering();

4669

SyncScope::ID SSID = I.getSyncScopeID();

4670

4671

SDValue InChain = getRoot();

4672

4673

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4674

EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

4675

EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

4676

4677

if (!TLI.supportsUnalignedAtomics() &&

4678

I.getAlignment() < MemVT.getSizeInBits() / 8)

4679

report_fatal_error("Cannot generate unaligned atomic load");

4680

4681

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());

4682

4683

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(

4684

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4685

I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

4686

4687

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

4688

4689

SDValue Ptr = getValue(I.getPointerOperand());

4690

4691

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {

4692

// TODO: Once this is better exercised by tests, it should be merged with

4693

// the normal path for loads to prevent future divergence.

4694

SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);

4695

if (MemVT != VT)

4696

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4697

4698

setValue(&I, L);

4699

SDValue OutChain = L.getValue(1);

4700

if (!I.isUnordered())

4701

DAG.setRoot(OutChain);

4702

else

4703

PendingLoads.push_back(OutChain);

4704

return;

4705

}

4706

4707

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,

4708

Ptr, MMO);

4709

4710

SDValue OutChain = L.getValue(1);

4711

if (MemVT != VT)

4712

L = DAG.getPtrExtOrTrunc(L, dl, VT);

4713

4714

setValue(&I, L);

4715

DAG.setRoot(OutChain);

4716

}

4717

4718

void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {

4719

SDLoc dl = getCurSDLoc();

4720

4721

AtomicOrdering Ordering = I.getOrdering();

4722

SyncScope::ID SSID = I.getSyncScopeID();

4723

4724

SDValue InChain = getRoot();

4725

4726

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4727

EVT MemVT =

4728

TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

4729

4730

if (I.getAlignment() < MemVT.getSizeInBits() / 8)

4731

report_fatal_error("Cannot generate unaligned atomic store");

4732

4733

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

4734

4735

MachineFunction &MF = DAG.getMachineFunction();

4736

MachineMemOperand *MMO = MF.getMachineMemOperand(

4737

MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),

4738

I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

4739

4740

SDValue Val = getValue(I.getValueOperand());

4741

if (Val.getValueType() != MemVT)

4742

Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);

4743

SDValue Ptr = getValue(I.getPointerOperand());

4744

4745

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {

4746

// TODO: Once this is better exercised by tests, it should be merged with

4747

// the normal path for stores to prevent future divergence.

4748

SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);

4749

DAG.setRoot(S);

4750

return;

4751

}

4752

SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,

4753

Ptr, Val, MMO);

4754

4755

4756

DAG.setRoot(OutChain);

4757

}

4758

4759

/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC

4760

/// node.

4761

void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,

4762

unsigned Intrinsic) {

4763

// Ignore the callsite's attributes. A specific call site may be marked with

4764

// readnone, but the lowering code will expect the chain based on the

4765

// definition.

4766

const Function *F = I.getCalledFunction();

4767

bool HasChain = !F->doesNotAccessMemory();

4768

bool OnlyLoad = HasChain && F->onlyReadsMemory();

4769

4770

// Build the operand list.

4771

SmallVector<SDValue, 8> Ops;

4772

if (HasChain) { // If this intrinsic has side-effects, chainify it.

4773

if (OnlyLoad) {

4774

// We don't need to serialize loads against other loads.

4775

Ops.push_back(DAG.getRoot());

4776

} else {

4777

Ops.push_back(getRoot());

4778

}

4779

}

4780

4781

// Info is set by getTgtMemInstrinsic

4782

TargetLowering::IntrinsicInfo Info;

4783

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4784

bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,

4785

DAG.getMachineFunction(),

4786

Intrinsic);

4787

4788

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.

4789

if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||

4790

Info.opc == ISD::INTRINSIC_W_CHAIN)

4791

Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),

4792

TLI.getPointerTy(DAG.getDataLayout())));

4793

4794

// Add all operands of the call to the operand list.

4795

for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {

4796

const Value *Arg = I.getArgOperand(i);

4797

if (!I.paramHasAttr(i, Attribute::ImmArg)) {

4798

Ops.push_back(getValue(Arg));

4799

continue;

4800

}

4801

4802

// Use TargetConstant instead of a regular constant for immarg.

4803

EVT VT = TLI.getValueType(*DL, Arg->getType(), true);

4804

if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {

4805

assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4806, __extension__ __PRETTY_FUNCTION__))

4806

"large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 &&
"large intrinsic immediates not handled") ? void (0) : __assert_fail
("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4806, __extension__ __PRETTY_FUNCTION__));

4807

Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));

4808

} else {

4809

Ops.push_back(

4810

DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));

4811

}

4812

}

4813

4814

SmallVector<EVT, 4> ValueVTs;

4815

ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

4816

4817

if (HasChain)

4818

ValueVTs.push_back(MVT::Other);

4819

4820

SDVTList VTs = DAG.getVTList(ValueVTs);

4821

4822

// Propagate fast-math-flags from IR to node(s).

4823

SDNodeFlags Flags;

4824

if (auto *FPMO = dyn_cast<FPMathOperator>(&I))

4825

Flags.copyFMF(*FPMO);

4826

SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

4827

4828

// Create the node.

4829

SDValue Result;

4830

if (IsTgtIntrinsic) {

4831

// This is target intrinsic that touches memory

4832

AAMDNodes AAInfo;

4833

I.getAAMetadata(AAInfo);

4834

Result =

4835

DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,

4836

MachinePointerInfo(Info.ptrVal, Info.offset),

4837

Info.align, Info.flags, Info.size, AAInfo);

4838

} else if (!HasChain) {

4839

Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);

4840

} else if (!I.getType()->isVoidTy()) {

4841

Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);

4842

} else {

4843

Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);

4844

}

4845

4846

if (HasChain) {

4847

SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);

4848

if (OnlyLoad)

4849

PendingLoads.push_back(Chain);

4850

else

4851

DAG.setRoot(Chain);

4852

}

4853

4854

if (!I.getType()->isVoidTy()) {

4855

if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {

4856

EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);

4857

Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);

4858

} else

4859

Result = lowerRangeToAssertZExt(DAG, I, Result);

4860

4861

MaybeAlign Alignment = I.getRetAlign();

4862

if (!Alignment)

4863

Alignment = F->getAttributes().getRetAlignment();

4864

// Insert `assertalign` node if there's an alignment.

4865

if (InsertAssertAlign && Alignment) {

4866

Result =

4867

DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());

4868

}

4869

4870

setValue(&I, Result);

4871

}

4872

}

4873

4874

/// GetSignificand - Get the significand and build it into a floating-point

4875

/// number with exponent of 1:

4876

///

4877

/// Op = (Op & 0x007fffff) | 0x3f800000;

4878

///

4879

/// where Op is the hexadecimal representation of floating point value.

4880

static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {

4881

SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4882

DAG.getConstant(0x007fffff, dl, MVT::i32));

4883

SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,

4884

DAG.getConstant(0x3f800000, dl, MVT::i32));

4885

return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);

4886

}

4887

4888

/// GetExponent - Get the exponent:

4889

///

4890

/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);

4891

///

4892

/// where Op is the hexadecimal representation of floating point value.

4893

static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,

4894

const TargetLowering &TLI, const SDLoc &dl) {

4895

SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,

4896

DAG.getConstant(0x7f800000, dl, MVT::i32));

4897

SDValue t1 = DAG.getNode(

4898

ISD::SRL, dl, MVT::i32, t0,

4899

DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));

4900

SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,

4901

DAG.getConstant(127, dl, MVT::i32));

4902

return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);

4903

}

4904

4905

/// getF32Constant - Get 32-bit floating point constant.

4906

static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,

4907

const SDLoc &dl) {

4908

return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,

4909

MVT::f32);

4910

}

4911

4912

static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,

4913

SelectionDAG &DAG) {

4914

// TODO: What fast-math-flags should be set on the floating-point nodes?

4915

4916

// IntegerPartOfX = ((int32_t)(t0);

4917

SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

4918

4919

// FractionalPartOfX = t0 - (float)IntegerPartOfX;

4920

SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);

4921

SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

4922

4923

// IntegerPartOfX <<= 23;

4924

IntegerPartOfX = DAG.getNode(

4925

ISD::SHL, dl, MVT::i32, IntegerPartOfX,

4926

DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(

4927

DAG.getDataLayout())));

4928

4929

SDValue TwoToFractionalPartOfX;

4930

if (LimitFloatPrecision <= 6) {

4931

// For floating-point precision of 6:

4932

//

4933

// TwoToFractionalPartOfX =

4934

// 0.997535578f +

4935

// (0.735607626f + 0.252464424f * x) * x;

4936

//

4937

// error 0.0144103317, which is 6 bits

4938

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4939

getF32Constant(DAG, 0x3e814304, dl));

4940

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4941

getF32Constant(DAG, 0x3f3c50c8, dl));

4942

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4943

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4944

getF32Constant(DAG, 0x3f7f5e7e, dl));

4945

} else if (LimitFloatPrecision <= 12) {

4946

// For floating-point precision of 12:

4947

//

4948

// TwoToFractionalPartOfX =

4949

// 0.999892986f +

4950

// (0.696457318f +

4951

// (0.224338339f + 0.792043434e-1f * x) * x) * x;

4952

//

4953

// error 0.000107046256, which is 13 to 14 bits

4954

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4955

getF32Constant(DAG, 0x3da235e3, dl));

4956

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4957

getF32Constant(DAG, 0x3e65b8f3, dl));

4958

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4959

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4960

getF32Constant(DAG, 0x3f324b07, dl));

4961

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4962

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4963

getF32Constant(DAG, 0x3f7ff8fd, dl));

4964

} else { // LimitFloatPrecision <= 18

4965

// For floating-point precision of 18:

4966

//

4967

// TwoToFractionalPartOfX =

4968

// 0.999999982f +

4969

// (0.693148872f +

4970

// (0.240227044f +

4971

// (0.554906021e-1f +

4972

// (0.961591928e-2f +

4973

// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;

4974

// error 2.47208000*10^(-7), which is better than 18 bits

4975

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

4976

getF32Constant(DAG, 0x3924b03e, dl));

4977

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

4978

getF32Constant(DAG, 0x3ab24b87, dl));

4979

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

4980

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

4981

getF32Constant(DAG, 0x3c1d8c17, dl));

4982

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

4983

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

4984

getF32Constant(DAG, 0x3d634a1d, dl));

4985

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

4986

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

4987

getF32Constant(DAG, 0x3e75fe14, dl));

4988

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

4989

SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,

4990

getF32Constant(DAG, 0x3f317234, dl));

4991

SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);

4992

TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,

4993

getF32Constant(DAG, 0x3f800000, dl));

4994

}

4995

4996

// Add the exponent into the result in integer domain.

4997

SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);

4998

return DAG.getNode(ISD::BITCAST, dl, MVT::f32,

4999

DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));

5000

}

5001

5002

/// expandExp - Lower an exp intrinsic. Handles the special sequences for

5003

/// limited-precision mode.

5004

static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5005

const TargetLowering &TLI, SDNodeFlags Flags) {

5006

if (Op.getValueType() == MVT::f32 &&

5007

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5008

5009

// Put the exponent in the right bit position for later addition to the

5010

// final result:

5011

//

5012

// t0 = Op * log2(e)

5013

5014

// TODO: What fast-math-flags should be set here?

5015

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,

5016

DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));

5017

return getLimitedPrecisionExp2(t0, dl, DAG);

5018

}

5019

5020

// No special expansion.

5021

return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);

5022

}

5023

5024

/// expandLog - Lower a log intrinsic. Handles the special sequences for

5025

/// limited-precision mode.

5026

static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5027

const TargetLowering &TLI, SDNodeFlags Flags) {

5028

// TODO: What fast-math-flags should be set on the floating-point nodes?

5029

5030

if (Op.getValueType() == MVT::f32 &&

5031

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5032

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5033

5034

// Scale the exponent by log(2).

5035

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5036

SDValue LogOfExponent =

5037

DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5038

DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

5039

5040

// Get the significand and build it into a floating-point number with

5041

// exponent of 1.

5042

SDValue X = GetSignificand(DAG, Op1, dl);

5043

5044

SDValue LogOfMantissa;

5045

if (LimitFloatPrecision <= 6) {

5046

// For floating-point precision of 6:

5047

//

5048

// LogofMantissa =

5049

// -1.1609546f +

5050

// (1.4034025f - 0.23903021f * x) * x;

5051

//

5052

// error 0.0034276066, which is better than 8 bits

5053

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5054

getF32Constant(DAG, 0xbe74c456, dl));

5055

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5056

getF32Constant(DAG, 0x3fb3a2b1, dl));

5057

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5058

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5059

getF32Constant(DAG, 0x3f949a29, dl));

5060

} else if (LimitFloatPrecision <= 12) {

5061

// For floating-point precision of 12:

5062

//

5063

// LogOfMantissa =

5064

// -1.7417939f +

5065

// (2.8212026f +

5066

// (-1.4699568f +

5067

// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;

5068

//

5069

// error 0.000061011436, which is 14 bits

5070

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5071

getF32Constant(DAG, 0xbd67b6d6, dl));

5072

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5073

getF32Constant(DAG, 0x3ee4f4b8, dl));

5074

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5075

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5076

getF32Constant(DAG, 0x3fbc278b, dl));

5077

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5078

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5079

getF32Constant(DAG, 0x40348e95, dl));

5080

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5081

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5082

getF32Constant(DAG, 0x3fdef31a, dl));

5083

} else { // LimitFloatPrecision <= 18

5084

// For floating-point precision of 18:

5085

//

5086

// LogOfMantissa =

5087

// -2.1072184f +

5088

// (4.2372794f +

5089

// (-3.7029485f +

5090

// (2.2781945f +

5091

// (-0.87823314f +

5092

// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;

5093

//

5094

// error 0.0000023660568, which is better than 18 bits

5095

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5096

getF32Constant(DAG, 0xbc91e5ac, dl));

5097

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5098

getF32Constant(DAG, 0x3e4350aa, dl));

5099

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5100

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5101

getF32Constant(DAG, 0x3f60d3e3, dl));

5102

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5103

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5104

getF32Constant(DAG, 0x4011cdf0, dl));

5105

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5106

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5107

getF32Constant(DAG, 0x406cfd1c, dl));

5108

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5109

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5110

getF32Constant(DAG, 0x408797cb, dl));

5111

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5112

LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5113

getF32Constant(DAG, 0x4006dcab, dl));

5114

}

5115

5116

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);

5117

}

5118

5119

// No special expansion.

5120

return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);

5121

}

5122

5123

/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for

5124

/// limited-precision mode.

5125

static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5126

const TargetLowering &TLI, SDNodeFlags Flags) {

5127

// TODO: What fast-math-flags should be set on the floating-point nodes?

5128

5129

if (Op.getValueType() == MVT::f32 &&

5130

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5131

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5132

5133

// Get the exponent.

5134

SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

5135

5136

// Get the significand and build it into a floating-point number with

5137

// exponent of 1.

5138

SDValue X = GetSignificand(DAG, Op1, dl);

5139

5140

// Different possible minimax approximations of significand in

5141

// floating-point for various degrees of accuracy over [1,2].

5142

SDValue Log2ofMantissa;

5143

if (LimitFloatPrecision <= 6) {

5144

// For floating-point precision of 6:

5145

//

5146

// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;

5147

//

5148

// error 0.0049451742, which is more than 7 bits

5149

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5150

getF32Constant(DAG, 0xbeb08fe0, dl));

5151

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5152

getF32Constant(DAG, 0x40019463, dl));

5153

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5154

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5155

getF32Constant(DAG, 0x3fd6633d, dl));

5156

} else if (LimitFloatPrecision <= 12) {

5157

// For floating-point precision of 12:

5158

//

5159

// Log2ofMantissa =

5160

// -2.51285454f +

5161

// (4.07009056f +

5162

// (-2.12067489f +

5163

// (.645142248f - 0.816157886e-1f * x) * x) * x) * x;

5164

//

5165

// error 0.0000876136000, which is better than 13 bits

5166

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5167

getF32Constant(DAG, 0xbda7262e, dl));

5168

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5169

getF32Constant(DAG, 0x3f25280b, dl));

5170

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5171

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5172

getF32Constant(DAG, 0x4007b923, dl));

5173

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5174

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5175

getF32Constant(DAG, 0x40823e2f, dl));

5176

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5177

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5178

getF32Constant(DAG, 0x4020d29c, dl));

5179

} else { // LimitFloatPrecision <= 18

5180

// For floating-point precision of 18:

5181

//

5182

// Log2ofMantissa =

5183

// -3.0400495f +

5184

// (6.1129976f +

5185

// (-5.3420409f +

5186

// (3.2865683f +

5187

// (-1.2669343f +

5188

// (0.27515199f -

5189

// 0.25691327e-1f * x) * x) * x) * x) * x) * x;

5190

//

5191

// error 0.0000018516, which is better than 18 bits

5192

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5193

getF32Constant(DAG, 0xbcd2769e, dl));

5194

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5195

getF32Constant(DAG, 0x3e8ce0b9, dl));

5196

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5197

SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5198

getF32Constant(DAG, 0x3fa22ae7, dl));

5199

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5200

SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,

5201

getF32Constant(DAG, 0x40525723, dl));

5202

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5203

SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,

5204

getF32Constant(DAG, 0x40aaf200, dl));

5205

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5206

SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,

5207

getF32Constant(DAG, 0x40c39dad, dl));

5208

SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);

5209

Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,

5210

getF32Constant(DAG, 0x4042902c, dl));

5211

}

5212

5213

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);

5214

}

5215

5216

// No special expansion.

5217

return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);

5218

}

5219

5220

/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for

5221

/// limited-precision mode.

5222

static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5223

const TargetLowering &TLI, SDNodeFlags Flags) {

5224

// TODO: What fast-math-flags should be set on the floating-point nodes?

5225

5226

if (Op.getValueType() == MVT::f32 &&

5227

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5228

SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

5229

5230

// Scale the exponent by log10(2) [0.30102999f].

5231

SDValue Exp = GetExponent(DAG, Op1, TLI, dl);

5232

SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,

5233

getF32Constant(DAG, 0x3e9a209a, dl));

5234

5235

// Get the significand and build it into a floating-point number with

5236

// exponent of 1.

5237

SDValue X = GetSignificand(DAG, Op1, dl);

5238

5239

SDValue Log10ofMantissa;

5240

if (LimitFloatPrecision <= 6) {

5241

// For floating-point precision of 6:

5242

//

5243

// Log10ofMantissa =

5244

// -0.50419619f +

5245

// (0.60948995f - 0.10380950f * x) * x;

5246

//

5247

// error 0.0014886165, which is 6 bits

5248

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5249

getF32Constant(DAG, 0xbdd49a13, dl));

5250

SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,

5251

getF32Constant(DAG, 0x3f1c0789, dl));

5252

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5253

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,

5254

getF32Constant(DAG, 0x3f011300, dl));

5255

} else if (LimitFloatPrecision <= 12) {

5256

// For floating-point precision of 12:

5257

//

5258

// Log10ofMantissa =

5259

// -0.64831180f +

5260

// (0.91751397f +

5261

// (-0.31664806f + 0.47637168e-1f * x) * x) * x;

5262

//

5263

// error 0.00019228036, which is better than 12 bits

5264

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5265

getF32Constant(DAG, 0x3d431f31, dl));

5266

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5267

getF32Constant(DAG, 0x3ea21fb2, dl));

5268

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5269

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5270

getF32Constant(DAG, 0x3f6ae232, dl));

5271

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5272

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5273

getF32Constant(DAG, 0x3f25f7c3, dl));

5274

} else { // LimitFloatPrecision <= 18

5275

// For floating-point precision of 18:

5276

//

5277

// Log10ofMantissa =

5278

// -0.84299375f +

5279

// (1.5327582f +

5280

// (-1.0688956f +

5281

// (0.49102474f +

5282

// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;

5283

//

5284

// error 0.0000037995730, which is better than 18 bits

5285

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,

5286

getF32Constant(DAG, 0x3c5d51ce, dl));

5287

SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,

5288

getF32Constant(DAG, 0x3e00685a, dl));

5289

SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);

5290

SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,

5291

getF32Constant(DAG, 0x3efb6798, dl));

5292

SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);

5293

SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,

5294

getF32Constant(DAG, 0x3f88d192, dl));

5295

SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);

5296

SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,

5297

getF32Constant(DAG, 0x3fc4316c, dl));

5298

SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);

5299

Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,

5300

getF32Constant(DAG, 0x3f57ce70, dl));

5301

}

5302

5303

return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);

5304

}

5305

5306

// No special expansion.

5307

return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);

5308

}

5309

5310

/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for

5311

/// limited-precision mode.

5312

static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,

5313

const TargetLowering &TLI, SDNodeFlags Flags) {

5314

if (Op.getValueType() == MVT::f32 &&

5315

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)

5316

return getLimitedPrecisionExp2(Op, dl, DAG);

5317

5318

// No special expansion.

5319

return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);

5320

}

5321

5322

/// visitPow - Lower a pow intrinsic. Handles the special sequences for

5323

/// limited-precision mode with x == 10.0f.

5324

static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,

5325

SelectionDAG &DAG, const TargetLowering &TLI,

5326

SDNodeFlags Flags) {

5327

bool IsExp10 = false;

5328

if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&

5329

LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

5330

if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {

5331

APFloat Ten(10.0f);

5332

IsExp10 = LHSC->isExactlyValue(Ten);

5333

}

5334

}

5335

5336

// TODO: What fast-math-flags should be set on the FMUL node?

5337

if (IsExp10) {

5338

// Put the exponent in the right bit position for later addition to the

5339

// final result:

5340

//

5341

// #define LOG2OF10 3.3219281f

5342

// t0 = Op * LOG2OF10;

5343

SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,

5344

getF32Constant(DAG, 0x40549a78, dl));

5345

return getLimitedPrecisionExp2(t0, dl, DAG);

5346

}

5347

5348

// No special expansion.

5349

return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);

5350

}

5351

5352

/// ExpandPowI - Expand a llvm.powi intrinsic.

5353

static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,

5354

SelectionDAG &DAG) {

5355

// If RHS is a constant, we can expand this out to a multiplication tree,

5356

// otherwise we end up lowering to a call to __powidf2 (for example). When

5357

// optimizing for size, we only want to do this if the expansion would produce

5358

// a small number of multiplies, otherwise we do the full expansion.

5359

if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {

5360

// Get the exponent as a positive value.

5361

unsigned Val = RHSC->getSExtValue();

5362

if ((int)Val < 0) Val = -Val;

5363

5364

// powi(x, 0) -> 1.0

5365

if (Val == 0)

5366

return DAG.getConstantFP(1.0, DL, LHS.getValueType());

5367

5368

bool OptForSize = DAG.shouldOptForSize();

5369

if (!OptForSize ||

5370

// If optimizing for size, don't insert too many multiplies.

5371

// This inserts up to 5 multiplies.

5372

countPopulation(Val) + Log2_32(Val) < 7) {

5373

// We use the simple binary decomposition method to generate the multiply

5374

// sequence. There are more optimal ways to do this (for example,

5375

// powi(x,15) generates one more multiply than it should), but this has

5376

// the benefit of being both really simple and much better than a libcall.

5377

SDValue Res; // Logically starts equal to 1.0

5378

SDValue CurSquare = LHS;

5379

// TODO: Intrinsics should have fast-math-flags that propagate to these

5380

// nodes.

5381

while (Val) {

5382

if (Val & 1) {

5383

if (Res.getNode())

5384

Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);

5385

else

5386

Res = CurSquare; // 1.0*CurSquare.

5387

}

5388

5389

CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),

5390

CurSquare, CurSquare);

5391

Val >>= 1;

5392

}

5393

5394

// If the original was negative, invert the result, producing 1/(x*x*x).

5395

if (RHSC->getSExtValue() < 0)

5396

Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),

5397

DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);

5398

return Res;

5399

}

5400

}

5401

5402

// Otherwise, expand to a libcall.

5403

return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);

5404

}

5405

5406

static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,

5407

SDValue LHS, SDValue RHS, SDValue Scale,

5408

SelectionDAG &DAG, const TargetLowering &TLI) {

5409

EVT VT = LHS.getValueType();

5410

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

5411

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

5412

LLVMContext &Ctx = *DAG.getContext();

5413

5414

// If the type is legal but the operation isn't, this node might survive all

5415

// the way to operation legalization. If we end up there and we do not have

5416

// the ability to widen the type (if VT*2 is not legal), we cannot expand the

5417

// node.

5418

5419

// Coax the legalizer into expanding the node during type legalization instead

5420

// by bumping the size by one bit. This will force it to Promote, enabling the

5421

// early expansion and avoiding the need to expand later.

5422

5423

// We don't have to do this if Scale is 0; that can always be expanded, unless

5424

// it's a saturating signed operation. Those can experience true integer

5425

// division overflow, a case which we must avoid.

5426

5427

// FIXME: We wouldn't have to do this (or any of the early

5428

// expansion/promotion) if it was possible to expand a libcall of an

5429

// illegal type during operation legalization. But it's not, so things

5430

// get a bit hacky.

5431

unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();

5432

if ((ScaleInt > 0 || (Saturating && Signed)) &&

5433

(TLI.isTypeLegal(VT) ||

5434

(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {

5435

TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(

5436

Opcode, VT, ScaleInt);

5437

if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {

5438

EVT PromVT;

5439

if (VT.isScalarInteger())

5440

PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);

5441

else if (VT.isVector()) {

5442

PromVT = VT.getVectorElementType();

5443

PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);

5444

PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());

5445

} else

5446

llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5446);

5447

if (Signed) {

5448

LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);

5449

RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);

5450

} else {

5451

LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);

5452

RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);

5453

}

5454

EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());

5455

// For saturating operations, we need to shift up the LHS to get the

5456

// proper saturation width, and then shift down again afterwards.

5457

if (Saturating)

5458

LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,

5459

DAG.getConstant(1, DL, ShiftTy));

5460

SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);

5461

if (Saturating)

5462

Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,

5463

DAG.getConstant(1, DL, ShiftTy));

5464

return DAG.getZExtOrTrunc(Res, DL, VT);

5465

}

5466

}

5467

5468

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);

5469

}

5470

5471

// getUnderlyingArgRegs - Find underlying registers used for a truncated,

5472

// bitcasted, or split argument. Returns a list of <Register, size in bits>

5473

static void

5474

getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,

5475

const SDValue &N) {

5476

switch (N.getOpcode()) {

5477

case ISD::CopyFromReg: {

5478

SDValue Op = N.getOperand(1);

5479

Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),

5480

Op.getValueType().getSizeInBits());

5481

return;

5482

}

5483

case ISD::BITCAST:

5484

case ISD::AssertZext:

5485

case ISD::AssertSext:

5486

case ISD::TRUNCATE:

5487

getUnderlyingArgRegs(Regs, N.getOperand(0));

5488

return;

5489

case ISD::BUILD_PAIR:

5490

case ISD::BUILD_VECTOR:

5491

case ISD::CONCAT_VECTORS:

5492

for (SDValue Op : N->op_values())

5493

getUnderlyingArgRegs(Regs, Op);

5494

return;

5495

default:

5496

return;

5497

}

5498

}

5499

5500

/// If the DbgValueInst is a dbg_value of a function argument, create the

5501

/// corresponding DBG_VALUE machine instruction for it now. At the end of

5502

/// instruction selection, they will be inserted to the entry BB.

5503

/// We don't currently support this for variadic dbg_values, as they shouldn't

5504

/// appear for function arguments or in the prologue.

5505

bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(

5506

const Value *V, DILocalVariable *Variable, DIExpression *Expr,

5507

DILocation *DL, bool IsDbgDeclare, const SDValue &N) {

5508

const Argument *Arg = dyn_cast<Argument>(V);

5509

if (!Arg)

5510

return false;

5511

5512

MachineFunction &MF = DAG.getMachineFunction();

5513

const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

5514

5515

// Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind

5516

// we've been asked to pursue.

5517

auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,

5518

bool Indirect) {

5519

if (Reg.isVirtual() && MF.useDebugInstrRef()) {

5520

// For VRegs, in instruction referencing mode, create a DBG_INSTR_REF

5521

// pointing at the VReg, which will be patched up later.

5522

auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);

5523

auto MIB = BuildMI(MF, DL, Inst);

5524

MIB.addReg(Reg, RegState::Debug);

5525

MIB.addImm(0);

5526

MIB.addMetadata(Variable);

5527

auto *NewDIExpr = FragExpr;

5528

// We don't have an "Indirect" field in DBG_INSTR_REF, fold that into

5529

// the DIExpression.

5530

if (Indirect)

5531

NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);

5532

MIB.addMetadata(NewDIExpr);

5533

return MIB;

5534

} else {

5535

// Create a completely standard DBG_VALUE.

5536

auto &Inst = TII->get(TargetOpcode::DBG_VALUE);

5537

return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);

5538

}

5539

};

5540

5541

if (!IsDbgDeclare) {

5542

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5543

// should only emit as ArgDbgValue if the dbg.value intrinsic is found in

5544

// the entry block.

5545

bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();

5546

if (!IsInEntryBlock)

5547

return false;

5548

5549

// ArgDbgValues are hoisted to the beginning of the entry block. So we

5550

// should only emit as ArgDbgValue if the dbg.value intrinsic describes a

5551

// variable that also is a param.

5552

//

5553

// Although, if we are at the top of the entry block already, we can still

5554

// emit using ArgDbgValue. This might catch some situations when the

5555

// dbg.value refers to an argument that isn't used in the entry block, so

5556

// any CopyToReg node would be optimized out and the only way to express

5557

// this DBG_VALUE is by using the physical reg (or FI) as done in this

5558

// method. ArgDbgValues are hoisted to the beginning of the entry block. So

5559

// we should only emit as ArgDbgValue if the Variable is an argument to the

5560

// current function, and the dbg.value intrinsic is found in the entry

5561

// block.

5562

bool VariableIsFunctionInputArg = Variable->isParameter() &&

5563

!DL->getInlinedAt();

5564

bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;

5565

if (!IsInPrologue && !VariableIsFunctionInputArg)

5566

return false;

5567

5568

// Here we assume that a function argument on IR level only can be used to

5569

// describe one input parameter on source level. If we for example have

5570

// source code like this

5571

//

5572

// struct A { long x, y; };

5573

// void foo(struct A a, long b) {

5574

// ...

5575

// b = a.x;

5576

// ...

5577

// }

5578

//

5579

// and IR like this

5580

//

5581

// define void @foo(i32 %a1, i32 %a2, i32 %b) {

5582

// entry:

5583

// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment

5584

// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment

5585

// call void @llvm.dbg.value(metadata i32 %b, "b",

5586

// ...

5587

// call void @llvm.dbg.value(metadata i32 %a1, "b"

5588

// ...

5589

//

5590

// then the last dbg.value is describing a parameter "b" using a value that

5591

// is an argument. But since we already has used %a1 to describe a parameter

5592

// we should not handle that last dbg.value here (that would result in an

5593

// incorrect hoisting of the DBG_VALUE to the function entry).

5594

// Notice that we allow one dbg.value per IR level argument, to accommodate

5595

// for the situation with fragments above.

5596

if (VariableIsFunctionInputArg) {

5597

unsigned ArgNo = Arg->getArgNo();

5598

if (ArgNo >= FuncInfo.DescribedArgs.size())

5599

FuncInfo.DescribedArgs.resize(ArgNo + 1, false);

5600

else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))

5601

return false;

5602

FuncInfo.DescribedArgs.set(ArgNo);

5603

}

5604

}

5605

5606

bool IsIndirect = false;

5607

Optional<MachineOperand> Op;

5608

// Some arguments' frame index is recorded during argument lowering.

5609

int FI = FuncInfo.getArgumentFrameIndex(Arg);

5610

if (FI != std::numeric_limits<int>::max())

5611

Op = MachineOperand::CreateFI(FI);

5612

5613

SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;

5614

if (!Op && N.getNode()) {

5615

getUnderlyingArgRegs(ArgRegsAndSizes, N);

5616

Register Reg;

5617

if (ArgRegsAndSizes.size() == 1)

5618

Reg = ArgRegsAndSizes.front().first;

5619

5620

if (Reg && Reg.isVirtual()) {

5621

MachineRegisterInfo &RegInfo = MF.getRegInfo();

5622

Register PR = RegInfo.getLiveInPhysReg(Reg);

5623

if (PR)

5624

Reg = PR;

5625

}

5626

if (Reg) {

5627

Op = MachineOperand::CreateReg(Reg, false);

5628

IsIndirect = IsDbgDeclare;

5629

}

5630

}

5631

5632

if (!Op && N.getNode()) {

5633

// Check if frame index is available.

5634

SDValue LCandidate = peekThroughBitcasts(N);

5635

if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))

5636

if (FrameIndexSDNode *FINode =

5637

dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))

5638

Op = MachineOperand::CreateFI(FINode->getIndex());

5639

}

5640

5641

if (!Op) {

5642

// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg

5643

auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>

5644

SplitRegs) {

5645

unsigned Offset = 0;

5646

for (auto RegAndSize : SplitRegs) {

5647

// If the expression is already a fragment, the current register

5648

// offset+size might extend beyond the fragment. In this case, only

5649

// the register bits that are inside the fragment are relevant.

5650

int RegFragmentSizeInBits = RegAndSize.second;

5651

if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {

5652

uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;

5653

// The register is entirely outside the expression fragment,

5654

// so is irrelevant for debug info.

5655

if (Offset >= ExprFragmentSizeInBits)

5656

break;

5657

// The register is partially outside the expression fragment, only

5658

// the low bits within the fragment are relevant for debug info.

5659

if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {

5660

RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;

5661

}

5662

}

5663

5664

auto FragmentExpr = DIExpression::createFragmentExpression(

5665

Expr, Offset, RegFragmentSizeInBits);

5666

Offset += RegAndSize.second;

5667

// If a valid fragment expression cannot be created, the variable's

5668

// correct value cannot be determined and so it is set as Undef.

5669

if (!FragmentExpr) {

5670

SDDbgValue *SDV = DAG.getConstantDbgValue(

5671

Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);

5672

DAG.AddDbgValue(SDV, false);

5673

continue;

5674

}

5675

MachineInstr *NewMI =

5676

MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, IsDbgDeclare);

5677

FuncInfo.ArgDbgValues.push_back(NewMI);

5678

}

5679

};

5680

5681

// Check if ValueMap has reg number.

5682

DenseMap<const Value *, Register>::const_iterator

5683

VMI = FuncInfo.ValueMap.find(V);

5684

if (VMI != FuncInfo.ValueMap.end()) {

5685

const auto &TLI = DAG.getTargetLoweringInfo();

5686

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,

5687

V->getType(), None);

5688

if (RFV.occupiesMultipleRegs()) {

5689

splitMultiRegDbgValue(RFV.getRegsAndSizes());

5690

return true;

5691

}

5692

5693

Op = MachineOperand::CreateReg(VMI->second, false);

5694

IsIndirect = IsDbgDeclare;

5695

} else if (ArgRegsAndSizes.size() > 1) {

5696

// This was split due to the calling convention, and no virtual register

5697

// mapping exists for the value.

5698

splitMultiRegDbgValue(ArgRegsAndSizes);

5699

return true;

5700

}

5701

}

5702

5703

if (!Op)

5704

return false;

5705

5706

assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5707, __extension__ __PRETTY_FUNCTION__))

5707

"Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5707, __extension__ __PRETTY_FUNCTION__));

5708

MachineInstr *NewMI = nullptr;

5709

5710

if (Op->isReg())

5711

NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);

5712

else

5713

NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,

5714

Variable, Expr);

5715

5716

FuncInfo.ArgDbgValues.push_back(NewMI);

5717

return true;

5718

}

5719

5720

/// Return the appropriate SDDbgValue based on N.

5721

SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,

5722

DILocalVariable *Variable,

5723

DIExpression *Expr,

5724

const DebugLoc &dl,

5725

unsigned DbgSDNodeOrder) {

5726

if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {

5727

// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe

5728

// stack slot locations.

5729

//

5730

// Consider "int x = 0; int *px = &x;". There are two kinds of interesting

5731

// debug values here after optimization:

5732

//

5733

// dbg.value(i32* %px, !"int *px", !DIExpression()), and

5734

// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))

5735

//

5736

// Both describe the direct values of their associated variables.

5737

return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),

5738

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5739

}

5740

return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),

5741

/*IsIndirect*/ false, dl, DbgSDNodeOrder);

5742

}

5743

5744

static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {

5745

switch (Intrinsic) {

5746

case Intrinsic::smul_fix:

5747

return ISD::SMULFIX;

5748

case Intrinsic::umul_fix:

5749

return ISD::UMULFIX;

5750

case Intrinsic::smul_fix_sat:

5751

return ISD::SMULFIXSAT;

5752

case Intrinsic::umul_fix_sat:

5753

return ISD::UMULFIXSAT;

5754

case Intrinsic::sdiv_fix:

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1114, column 10 Called C++ object pointer is null

Bug Summary

Annotated Source Code

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp