/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1

//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements the TargetLowering class.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "llvm/CodeGen/TargetLowering.h"

14

#include "llvm/ADT/STLExtras.h"

15

#include "llvm/CodeGen/CallingConvLower.h"

16

#include "llvm/CodeGen/MachineFrameInfo.h"

17

#include "llvm/CodeGen/MachineFunction.h"

18

#include "llvm/CodeGen/MachineJumpTableInfo.h"

19

#include "llvm/CodeGen/MachineRegisterInfo.h"

20

#include "llvm/CodeGen/SelectionDAG.h"

21

#include "llvm/CodeGen/TargetRegisterInfo.h"

22

#include "llvm/CodeGen/TargetSubtargetInfo.h"

23

#include "llvm/IR/DataLayout.h"

24

#include "llvm/IR/DerivedTypes.h"

25

#include "llvm/IR/GlobalVariable.h"

26

#include "llvm/IR/LLVMContext.h"

27

#include "llvm/MC/MCAsmInfo.h"

28

#include "llvm/MC/MCExpr.h"

29

#include "llvm/Support/DivisionByConstantInfo.h"

30

#include "llvm/Support/ErrorHandling.h"

31

#include "llvm/Support/KnownBits.h"

32

#include "llvm/Support/MathExtras.h"

33

#include "llvm/Target/TargetLoweringObjectFile.h"

34

#include "llvm/Target/TargetMachine.h"

35

#include <cctype>

36

using namespace llvm;

37

38

/// NOTE: The TargetMachine owns TLOF.

39

TargetLowering::TargetLowering(const TargetMachine &tm)

40

: TargetLoweringBase(tm) {}

41

42

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {

43

return nullptr;

44

}

45

46

bool TargetLowering::isPositionIndependent() const {

47

return getTargetMachine().isPositionIndependent();

48

}

49

50

/// Check whether a given call node is in tail position within its function. If

51

/// so, it sets Chain to the input chain of the tail call.

52

bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,

53

SDValue &Chain) const {

54

const Function &F = DAG.getMachineFunction().getFunction();

55

56

// First, check if tail calls have been disabled in this function.

57

if (F.getFnAttribute("disable-tail-calls").getValueAsBool())

58

return false;

59

60

// Conservatively require the attributes of the call to match those of

61

// the return. Ignore following attributes because they don't affect the

62

// call sequence.

63

AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);

64

for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,

65

Attribute::DereferenceableOrNull, Attribute::NoAlias,

66

Attribute::NonNull})

67

CallerAttrs.removeAttribute(Attr);

68

69

if (CallerAttrs.hasAttributes())

70

return false;

71

72

// It's not safe to eliminate the sign / zero extension of the return value.

73

if (CallerAttrs.contains(Attribute::ZExt) ||

74

CallerAttrs.contains(Attribute::SExt))

75

return false;

76

77

// Check if the only use is a function return node.

78

return isUsedByReturnOnly(Node, Chain);

79

}

80

81

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,

82

const uint32_t *CallerPreservedMask,

83

const SmallVectorImpl<CCValAssign> &ArgLocs,

84

const SmallVectorImpl<SDValue> &OutVals) const {

85

for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {

86

const CCValAssign &ArgLoc = ArgLocs[I];

87

if (!ArgLoc.isRegLoc())

88

continue;

89

MCRegister Reg = ArgLoc.getLocReg();

90

// Only look at callee saved registers.

91

if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))

92

continue;

93

// Check that we pass the value used for the caller.

94

// (We look for a CopyFromReg reading a virtual register that is used

95

// for the function live-in value of register Reg)

96

SDValue Value = OutVals[I];

97

if (Value->getOpcode() != ISD::CopyFromReg)

98

return false;

99

Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();

100

if (MRI.getLiveInPhysReg(ArgReg) != Reg)

101

return false;

102

}

103

return true;

104

}

105

106

/// Set CallLoweringInfo attribute flags based on a call instruction

107

/// and called function attributes.

108

void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,

109

unsigned ArgIdx) {

110

IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);

111

IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);

112

IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);

113

IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);

114

IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);

115

IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);

116

IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);

117

IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);

118

IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);

119

IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);

120

IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);

121

IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);

122

Alignment = Call->getParamStackAlign(ArgIdx);

123

IndirectType = nullptr;

124

assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__))

125

"multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__));

126

if (IsByVal) {

127

IndirectType = Call->getParamByValType(ArgIdx);

128

if (!Alignment)

129

Alignment = Call->getParamAlign(ArgIdx);

130

}

131

if (IsPreallocated)

132

IndirectType = Call->getParamPreallocatedType(ArgIdx);

133

if (IsInAlloca)

134

IndirectType = Call->getParamInAllocaType(ArgIdx);

135

}

136

137

/// Generate a libcall taking the given operands as arguments and returning a

138

/// result of type RetVT.

139

std::pair<SDValue, SDValue>

140

TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,

141

ArrayRef<SDValue> Ops,

142

MakeLibCallOptions CallOptions,

143

const SDLoc &dl,

144

SDValue InChain) const {

145

if (!InChain)

146

InChain = DAG.getEntryNode();

147

148

TargetLowering::ArgListTy Args;

149

Args.reserve(Ops.size());

150

151

TargetLowering::ArgListEntry Entry;

152

for (unsigned i = 0; i < Ops.size(); ++i) {

153

SDValue NewOp = Ops[i];

154

Entry.Node = NewOp;

155

Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());

156

Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),

157

CallOptions.IsSExt);

158

Entry.IsZExt = !Entry.IsSExt;

159

160

if (CallOptions.IsSoften &&

161

!shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {

162

Entry.IsSExt = Entry.IsZExt = false;

163

}

164

Args.push_back(Entry);

165

}

166

167

if (LC == RTLIB::UNKNOWN_LIBCALL)

168

report_fatal_error("Unsupported library call operation!");

169

SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),

170

getPointerTy(DAG.getDataLayout()));

171

172

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());

173

TargetLowering::CallLoweringInfo CLI(DAG);

174

bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);

175

bool zeroExtend = !signExtend;

176

177

if (CallOptions.IsSoften &&

178

!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {

179

signExtend = zeroExtend = false;

180

}

181

182

CLI.setDebugLoc(dl)

183

.setChain(InChain)

184

.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))

185

.setNoReturn(CallOptions.DoesNotReturn)

186

.setDiscardResult(!CallOptions.IsReturnValueUsed)

187

.setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)

188

.setSExtResult(signExtend)

189

.setZExtResult(zeroExtend);

190

return LowerCallTo(CLI);

191

}

192

193

bool TargetLowering::findOptimalMemOpLowering(

194

std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,

195

unsigned SrcAS, const AttributeList &FuncAttributes) const {

196

if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())

197

return false;

198

199

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

200

201

if (VT == MVT::Other) {

202

// Use the largest integer type whose alignment constraints are satisfied.

203

// We only need to check DstAlign here as SrcAlign is always greater or

204

// equal to DstAlign (or zero).

205

VT = MVT::i64;

206

if (Op.isFixedDstAlign())

207

while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

208

!allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))

209

VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);

210

assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 210, __extension__ __PRETTY_FUNCTION__));

211

212

// Find the largest legal integer type.

213

MVT LVT = MVT::i64;

214

while (!isTypeLegal(LVT))

215

LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);

216

assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 216, __extension__ __PRETTY_FUNCTION__));

217

218

// If the type we've chosen is larger than the largest legal integer type

219

// then use that instead.

220

if (VT.bitsGT(LVT))

221

VT = LVT;

222

}

223

224

unsigned NumMemOps = 0;

225

uint64_t Size = Op.size();

226

while (Size) {

227

unsigned VTSize = VT.getSizeInBits() / 8;

228

while (VTSize > Size) {

229

// For now, only use non-vector load / store's for the left-over pieces.

230

EVT NewVT = VT;

231

unsigned NewVTSize;

232

233

bool Found = false;

234

if (VT.isVector() || VT.isFloatingPoint()) {

235

NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;

236

if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&

237

isSafeMemOpType(NewVT.getSimpleVT()))

238

Found = true;

239

else if (NewVT == MVT::i64 &&

240

isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&

241

isSafeMemOpType(MVT::f64)) {

242

// i64 is usually not legal on 32-bit targets, but f64 may be.

243

NewVT = MVT::f64;

244

Found = true;

245

}

246

}

247

248

if (!Found) {

249

do {

250

NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);

251

if (NewVT == MVT::i8)

252

break;

253

} while (!isSafeMemOpType(NewVT.getSimpleVT()));

254

}

255

NewVTSize = NewVT.getSizeInBits() / 8;

256

257

// If the new VT cannot cover all of the remaining bits, then consider

258

// issuing a (or a pair of) unaligned and overlapping load / store.

259

bool Fast;

260

if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&

261

allowsMisalignedMemoryAccesses(

262

VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),

263

MachineMemOperand::MONone, &Fast) &&

264

Fast)

265

VTSize = Size;

266

else {

267

VT = NewVT;

268

VTSize = NewVTSize;

269

}

270

}

271

272

if (++NumMemOps > Limit)

273

return false;

274

275

MemOps.push_back(VT);

276

Size -= VTSize;

277

}

278

279

return true;

280

}

281

282

/// Soften the operands of a comparison. This code is shared among BR_CC,

283

/// SELECT_CC, and SETCC handlers.

284

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

285

SDValue &NewLHS, SDValue &NewRHS,

286

ISD::CondCode &CCCode,

287

const SDLoc &dl, const SDValue OldLHS,

288

const SDValue OldRHS) const {

289

SDValue Chain;

290

return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,

291

OldRHS, Chain);

292

}

293

294

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

295

SDValue &NewLHS, SDValue &NewRHS,

296

ISD::CondCode &CCCode,

297

const SDLoc &dl, const SDValue OldLHS,

298

const SDValue OldRHS,

299

SDValue &Chain,

300

bool IsSignaling) const {

301

// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc

302

// not supporting it. We can update this code when libgcc provides such

303

// functions.

304

305

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__))

306

&& "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__));

307

308

// Expand into one or more soft-fp libcall(s).

309

RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;

310

bool ShouldInvertCC = false;

311

switch (CCCode) {

312

case ISD::SETEQ:

313

case ISD::SETOEQ:

314

LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

315

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

316

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

317

break;

318

case ISD::SETNE:

319

case ISD::SETUNE:

320

LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :

321

(VT == MVT::f64) ? RTLIB::UNE_F64 :

322

(VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;

323

break;

324

case ISD::SETGE:

325

case ISD::SETOGE:

326

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

327

(VT == MVT::f64) ? RTLIB::OGE_F64 :

328

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

329

break;

330

case ISD::SETLT:

331

case ISD::SETOLT:

332

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

333

(VT == MVT::f64) ? RTLIB::OLT_F64 :

334

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

335

break;

336

case ISD::SETLE:

337

case ISD::SETOLE:

338

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

339

(VT == MVT::f64) ? RTLIB::OLE_F64 :

340

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

341

break;

342

case ISD::SETGT:

343

case ISD::SETOGT:

344

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

345

(VT == MVT::f64) ? RTLIB::OGT_F64 :

346

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

347

break;

348

case ISD::SETO:

349

ShouldInvertCC = true;

350

LLVM_FALLTHROUGH[[gnu::fallthrough]];

351

case ISD::SETUO:

352

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

353

(VT == MVT::f64) ? RTLIB::UO_F64 :

354

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

355

break;

356

case ISD::SETONE:

357

// SETONE = O && UNE

358

ShouldInvertCC = true;

359

LLVM_FALLTHROUGH[[gnu::fallthrough]];

360

case ISD::SETUEQ:

361

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

362

(VT == MVT::f64) ? RTLIB::UO_F64 :

363

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

364

LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

365

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

366

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

367

break;

368

default:

369

// Invert CC for unordered comparisons

370

ShouldInvertCC = true;

371

switch (CCCode) {

372

case ISD::SETULT:

373

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

374

(VT == MVT::f64) ? RTLIB::OGE_F64 :

375

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

376

break;

377

case ISD::SETULE:

378

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

379

(VT == MVT::f64) ? RTLIB::OGT_F64 :

380

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

381

break;

382

case ISD::SETUGT:

383

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

384

(VT == MVT::f64) ? RTLIB::OLE_F64 :

385

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

386

break;

387

case ISD::SETUGE:

388

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

389

(VT == MVT::f64) ? RTLIB::OLT_F64 :

390

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

391

break;

392

default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 392);

393

}

394

}

395

396

// Use the target specific return value for comparions lib calls.

397

EVT RetVT = getCmpLibcallReturnType();

398

SDValue Ops[2] = {NewLHS, NewRHS};

399

TargetLowering::MakeLibCallOptions CallOptions;

400

EVT OpsVT[2] = { OldLHS.getValueType(),

401

OldRHS.getValueType() };

402

CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);

403

auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);

404

NewLHS = Call.first;

405

NewRHS = DAG.getConstant(0, dl, RetVT);

406

407

CCCode = getCmpLibcallCC(LC1);

408

if (ShouldInvertCC) {

409

assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 409, __extension__ __PRETTY_FUNCTION__));

410

CCCode = getSetCCInverse(CCCode, RetVT);

411

}

412

413

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {

414

// Update Chain.

415

Chain = Call.second;

416

} else {

417

EVT SetCCVT =

418

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);

419

SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);

420

auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);

421

CCCode = getCmpLibcallCC(LC2);

422

if (ShouldInvertCC)

423

CCCode = getSetCCInverse(CCCode, RetVT);

424

NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);

425

if (Chain)

426

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,

427

Call2.second);

428

NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,

429

Tmp.getValueType(), Tmp, NewLHS);

430

NewRHS = SDValue();

431

}

432

}

433

434

/// Return the entry encoding for a jump table in the current function. The

435

/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.

436

unsigned TargetLowering::getJumpTableEncoding() const {

437

// In non-pic modes, just use the address of a block.

438

if (!isPositionIndependent())

439

return MachineJumpTableInfo::EK_BlockAddress;

440

441

// In PIC mode, if the target supports a GPRel32 directive, use it.

442

if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)

443

return MachineJumpTableInfo::EK_GPRel32BlockAddress;

444

445

// Otherwise, use a label difference.

446

return MachineJumpTableInfo::EK_LabelDifference32;

447

}

448

449

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,

450

SelectionDAG &DAG) const {

451

// If our PIC model is GP relative, use the global offset table as the base.

452

unsigned JTEncoding = getJumpTableEncoding();

453

454

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||

455

(JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))

456

return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

457

458

return Table;

459

}

460

461

/// This returns the relocation base for the given PIC jumptable, the same as

462

/// getPICJumpTableRelocBase, but as an MCExpr.

463

const MCExpr *

464

TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,

465

unsigned JTI,MCContext &Ctx) const{

466

// The normal PIC reloc base is the label at the start of the jump table.

467

return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);

468

}

469

470

bool

471

TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

472

const TargetMachine &TM = getTargetMachine();

473

const GlobalValue *GV = GA->getGlobal();

474

475

// If the address is not even local to this DSO we will have to load it from

476

// a got and then add the offset.

477

if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))

478

return false;

479

480

// If the code is position independent we will have to add a base register.

481

if (isPositionIndependent())

482

return false;

483

484

// Otherwise we can do it.

485

return true;

486

}

487

488

//===----------------------------------------------------------------------===//

489

// Optimization Methods

490

//===----------------------------------------------------------------------===//

491

492

/// If the specified instruction has a constant integer operand and there are

493

/// bits set in that constant that are not demanded, then clear those bits and

494

/// return true.

495

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

496

const APInt &DemandedBits,

497

const APInt &DemandedElts,

498

TargetLoweringOpt &TLO) const {

499

SDLoc DL(Op);

500

unsigned Opcode = Op.getOpcode();

501

502

// Do target-specific constant optimization.

503

if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

504

return TLO.New.getNode();

505

506

// FIXME: ISD::SELECT, ISD::SELECT_CC

507

switch (Opcode) {

508

default:

509

break;

510

case ISD::XOR:

511

case ISD::AND:

512

case ISD::OR: {

513

auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));

514

if (!Op1C || Op1C->isOpaque())

515

return false;

516

517

// If this is a 'not' op, don't touch it because that's a canonical form.

518

const APInt &C = Op1C->getAPIntValue();

519

if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))

520

return false;

521

522

if (!C.isSubsetOf(DemandedBits)) {

523

EVT VT = Op.getValueType();

524

SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);

525

SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);

526

return TLO.CombineTo(Op, NewOp);

527

}

528

529

break;

530

}

531

}

532

533

return false;

534

}

535

536

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

537

const APInt &DemandedBits,

538

TargetLoweringOpt &TLO) const {

539

EVT VT = Op.getValueType();

540

APInt DemandedElts = VT.isVector()

541

? APInt::getAllOnes(VT.getVectorNumElements())

542

: APInt(1, 1);

543

return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);

544

}

545

546

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.

547

/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be

548

/// generalized for targets with other types of implicit widening casts.

549

bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,

550

const APInt &Demanded,

551

TargetLoweringOpt &TLO) const {

552

assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__))

553

"ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__));

554

assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__))

555

"ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__));

556

557

SelectionDAG &DAG = TLO.DAG;

558

SDLoc dl(Op);

559

560

// Early return, as this function cannot handle vector types.

561

if (Op.getValueType().isVector())

562

return false;

563

564

// Don't do this if the node has another user, which may require the

565

// full value.

566

if (!Op.getNode()->hasOneUse())

567

return false;

568

569

// Search for the smallest integer type with free casts to and from

570

// Op's type. For expedience, just check power-of-2 integer types.

571

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

572

unsigned DemandedSize = Demanded.getActiveBits();

573

unsigned SmallVTBits = DemandedSize;

574

if (!isPowerOf2_32(SmallVTBits))

575

SmallVTBits = NextPowerOf2(SmallVTBits);

576

for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {

577

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);

578

if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&

579

TLI.isZExtFree(SmallVT, Op.getValueType())) {

580

// We found a type with free casts.

581

SDValue X = DAG.getNode(

582

Op.getOpcode(), dl, SmallVT,

583

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),

584

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));

585

assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 585, __extension__ __PRETTY_FUNCTION__));

586

SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);

587

return TLO.CombineTo(Op, Z);

588

}

589

}

590

return false;

591

}

592

593

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

594

DAGCombinerInfo &DCI) const {

595

SelectionDAG &DAG = DCI.DAG;

596

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

597

!DCI.isBeforeLegalizeOps());

598

KnownBits Known;

599

600

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);

601

if (Simplified) {

602

DCI.AddToWorklist(Op.getNode());

603

DCI.CommitTargetLoweringOpt(TLO);

604

}

605

return Simplified;

606

}

607

608

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

609

KnownBits &Known,

610

TargetLoweringOpt &TLO,

611

unsigned Depth,

612

bool AssumeSingleUse) const {

613

EVT VT = Op.getValueType();

614

615

// TODO: We can probably do more work on calculating the known bits and

616

// simplifying the operations for scalable vectors, but for now we just

617

// bail out.

618

if (VT.isScalableVector()) {

619

// Pretend we don't know anything for now.

620

Known = KnownBits(DemandedBits.getBitWidth());

621

return false;

622

}

623

624

APInt DemandedElts = VT.isVector()

625

? APInt::getAllOnes(VT.getVectorNumElements())

626

: APInt(1, 1);

627

return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,

628

AssumeSingleUse);

629

}

630

631

// TODO: Can we merge SelectionDAG::GetDemandedBits into this?

632

// TODO: Under what circumstances can we create nodes? Constant folding?

633

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

634

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

635

SelectionDAG &DAG, unsigned Depth) const {

636

// Limit search depth.

637

if (Depth >= SelectionDAG::MaxRecursionDepth)

638

return SDValue();

639

640

// Ignore UNDEFs.

641

if (Op.isUndef())

642

return SDValue();

643

644

// Not demanding any bits/elts from Op.

645

if (DemandedBits == 0 || DemandedElts == 0)

646

return DAG.getUNDEF(Op.getValueType());

647

648

unsigned NumElts = DemandedElts.getBitWidth();

649

unsigned BitWidth = DemandedBits.getBitWidth();

650

KnownBits LHSKnown, RHSKnown;

651

switch (Op.getOpcode()) {

652

case ISD::BITCAST: {

653

SDValue Src = peekThroughBitcasts(Op.getOperand(0));

654

EVT SrcVT = Src.getValueType();

655

EVT DstVT = Op.getValueType();

656

if (SrcVT == DstVT)

657

return Src;

658

659

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

660

unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

661

if (NumSrcEltBits == NumDstEltBits)

662

if (SDValue V = SimplifyMultipleUseDemandedBits(

663

Src, DemandedBits, DemandedElts, DAG, Depth + 1))

664

return DAG.getBitcast(DstVT, V);

665

666

// TODO - bigendian once we have test coverage.

667

if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&

668

DAG.getDataLayout().isLittleEndian()) {

669

unsigned Scale = NumDstEltBits / NumSrcEltBits;

670

unsigned NumSrcElts = SrcVT.getVectorNumElements();

671

APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);

672

APInt DemandedSrcElts = APInt::getZero(NumSrcElts);

673

for (unsigned i = 0; i != Scale; ++i) {

674

unsigned Offset = i * NumSrcEltBits;

675

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

676

if (!Sub.isZero()) {

677

DemandedSrcBits |= Sub;

678

for (unsigned j = 0; j != NumElts; ++j)

679

if (DemandedElts[j])

680

DemandedSrcElts.setBit((j * Scale) + i);

681

}

682

}

683

684

if (SDValue V = SimplifyMultipleUseDemandedBits(

685

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

686

return DAG.getBitcast(DstVT, V);

687

}

688

689

// TODO - bigendian once we have test coverage.

690

if ((NumSrcEltBits % NumDstEltBits) == 0 &&

691

DAG.getDataLayout().isLittleEndian()) {

692

unsigned Scale = NumSrcEltBits / NumDstEltBits;

693

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

694

APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);

695

APInt DemandedSrcElts = APInt::getZero(NumSrcElts);

696

for (unsigned i = 0; i != NumElts; ++i)

697

if (DemandedElts[i]) {

698

unsigned Offset = (i % Scale) * NumDstEltBits;

699

DemandedSrcBits.insertBits(DemandedBits, Offset);

700

DemandedSrcElts.setBit(i / Scale);

701

}

702

703

if (SDValue V = SimplifyMultipleUseDemandedBits(

704

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

705

return DAG.getBitcast(DstVT, V);

706

}

707

708

break;

709

}

710

case ISD::AND: {

711

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

712

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

713

714

// If all of the demanded bits are known 1 on one side, return the other.

715

// These bits cannot contribute to the result of the 'and' in this

716

// context.

717

if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))

718

return Op.getOperand(0);

719

if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))

720

return Op.getOperand(1);

721

break;

722

}

723

case ISD::OR: {

724

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

725

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

726

727

// If all of the demanded bits are known zero on one side, return the

728

// other. These bits cannot contribute to the result of the 'or' in this

729

// context.

730

if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))

731

return Op.getOperand(0);

732

if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))

733

return Op.getOperand(1);

734

break;

735

}

736

case ISD::XOR: {

737

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

738

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

739

740

// If all of the demanded bits are known zero on one side, return the

741

// other.

742

if (DemandedBits.isSubsetOf(RHSKnown.Zero))

743

return Op.getOperand(0);

744

if (DemandedBits.isSubsetOf(LHSKnown.Zero))

745

return Op.getOperand(1);

746

break;

747

}

748

case ISD::SHL: {

749

// If we are only demanding sign bits then we can use the shift source

750

// directly.

751

if (const APInt *MaxSA =

752

DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

753

SDValue Op0 = Op.getOperand(0);

754

unsigned ShAmt = MaxSA->getZExtValue();

755

unsigned NumSignBits =

756

DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

757

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

758

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

759

return Op0;

760

}

761

break;

762

}

763

case ISD::SETCC: {

764

SDValue Op0 = Op.getOperand(0);

765

SDValue Op1 = Op.getOperand(1);

766

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

767

// If (1) we only need the sign-bit, (2) the setcc operands are the same

768

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

769

// -1, we may be able to bypass the setcc.

770

if (DemandedBits.isSignMask() &&

771

Op0.getScalarValueSizeInBits() == BitWidth &&

772

getBooleanContents(Op0.getValueType()) ==

773

BooleanContent::ZeroOrNegativeOneBooleanContent) {

774

// If we're testing X < 0, then this compare isn't needed - just use X!

775

// FIXME: We're limiting to integer types here, but this should also work

776

// if we don't care about FP signed-zero. The use of SETLT with FP means

777

// that we don't care about NaNs.

778

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

779

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

780

return Op0;

781

}

782

break;

783

}

784

case ISD::SIGN_EXTEND_INREG: {

785

// If none of the extended bits are demanded, eliminate the sextinreg.

786

SDValue Op0 = Op.getOperand(0);

787

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

788

unsigned ExBits = ExVT.getScalarSizeInBits();

789

if (DemandedBits.getActiveBits() <= ExBits)

790

return Op0;

791

// If the input is already sign extended, just drop the extension.

792

unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

793

if (NumSignBits >= (BitWidth - ExBits + 1))

794

return Op0;

795

break;

796

}

797

case ISD::ANY_EXTEND_VECTOR_INREG:

798

case ISD::SIGN_EXTEND_VECTOR_INREG:

799

case ISD::ZERO_EXTEND_VECTOR_INREG: {

800

// If we only want the lowest element and none of extended bits, then we can

801

// return the bitcasted source vector.

802

SDValue Src = Op.getOperand(0);

803

EVT SrcVT = Src.getValueType();

804

EVT DstVT = Op.getValueType();

805

if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&

806

DAG.getDataLayout().isLittleEndian() &&

807

DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {

808

return DAG.getBitcast(DstVT, Src);

809

}

810

break;

811

}

812

case ISD::INSERT_VECTOR_ELT: {

813

// If we don't demand the inserted element, return the base vector.

814

SDValue Vec = Op.getOperand(0);

815

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

816

EVT VecVT = Vec.getValueType();

817

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&

818

!DemandedElts[CIdx->getZExtValue()])

819

return Vec;

820

break;

821

}

822

case ISD::INSERT_SUBVECTOR: {

823

SDValue Vec = Op.getOperand(0);

824

SDValue Sub = Op.getOperand(1);

825

uint64_t Idx = Op.getConstantOperandVal(2);

826

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

827

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

828

// If we don't demand the inserted subvector, return the base vector.

829

if (DemandedSubElts == 0)

830

return Vec;

831

// If this simply widens the lowest subvector, see if we can do it earlier.

832

if (Idx == 0 && Vec.isUndef()) {

833

if (SDValue NewSub = SimplifyMultipleUseDemandedBits(

834

Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))

835

return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),

836

Op.getOperand(0), NewSub, Op.getOperand(2));

837

}

838

break;

839

}

840

case ISD::VECTOR_SHUFFLE: {

841

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

842

843

// If all the demanded elts are from one operand and are inline,

844

// then we can use the operand directly.

845

bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;

846

for (unsigned i = 0; i != NumElts; ++i) {

847

int M = ShuffleMask[i];

848

if (M < 0 || !DemandedElts[i])

849

continue;

850

AllUndef = false;

851

IdentityLHS &= (M == (int)i);

852

IdentityRHS &= ((M - NumElts) == i);

853

}

854

855

if (AllUndef)

856

return DAG.getUNDEF(Op.getValueType());

857

if (IdentityLHS)

858

return Op.getOperand(0);

859

if (IdentityRHS)

860

return Op.getOperand(1);

861

break;

862

}

863

default:

864

if (Op.getOpcode() >= ISD::BUILTIN_OP_END)

865

if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(

866

Op, DemandedBits, DemandedElts, DAG, Depth))

867

return V;

868

break;

869

}

870

return SDValue();

871

}

872

873

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

874

SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,

875

unsigned Depth) const {

876

EVT VT = Op.getValueType();

877

APInt DemandedElts = VT.isVector()

878

? APInt::getAllOnes(VT.getVectorNumElements())

879

: APInt(1, 1);

880

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

881

Depth);

882

}

883

884

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(

885

SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,

886

unsigned Depth) const {

887

APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());

888

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

889

Depth);

890

}

891

892

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the

893

/// result of Op are ever used downstream. If we can use this information to

894

/// simplify Op, create a new simplified DAG node and return true, returning the

895

/// original and new nodes in Old and New. Otherwise, analyze the expression and

896

/// return a mask of Known bits for the expression (used to simplify the

897

/// caller). The Known bits may only be accurate for those bits in the

898

/// OriginalDemandedBits and OriginalDemandedElts.

899

bool TargetLowering::SimplifyDemandedBits(

900

SDValue Op, const APInt &OriginalDemandedBits,

901

const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,

902

unsigned Depth, bool AssumeSingleUse) const {

903

unsigned BitWidth = OriginalDemandedBits.getBitWidth();

904

assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))

905

"Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__));

906

907

// Don't know anything.

908

Known = KnownBits(BitWidth);

909

910

// TODO: We can probably do more work on calculating the known bits and

911

// simplifying the operations for scalable vectors, but for now we just

912

// bail out.

913

if (Op.getValueType().isScalableVector())

914

return false;

915

916

unsigned NumElts = OriginalDemandedElts.getBitWidth();

917

assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))

918

NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))

919

"Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__));

920

921

APInt DemandedBits = OriginalDemandedBits;

922

APInt DemandedElts = OriginalDemandedElts;

923

SDLoc dl(Op);

924

auto &DL = TLO.DAG.getDataLayout();

925

926

// Undef operand.

927

if (Op.isUndef())

928

return false;

929

930

if (Op.getOpcode() == ISD::Constant) {

931

// We know all of the bits for a constant!

932

Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());

933

return false;

934

}

935

936

if (Op.getOpcode() == ISD::ConstantFP) {

937

// We know all of the bits for a floating point constant!

938

Known = KnownBits::makeConstant(

939

cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());

940

return false;

941

}

942

943

// Other users may use these bits.

944

EVT VT = Op.getValueType();

945

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {

946

if (Depth != 0) {

947

// If not at the root, Just compute the Known bits to

948

// simplify things downstream.

949

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

950

return false;

951

}

952

// If this is the root being simplified, allow it to have multiple uses,

953

// just set the DemandedBits/Elts to all bits.

954

DemandedBits = APInt::getAllOnes(BitWidth);

955

DemandedElts = APInt::getAllOnes(NumElts);

956

} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {

957

// Not demanding any bits/elts from Op.

958

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

959

} else if (Depth >= SelectionDAG::MaxRecursionDepth) {

960

// Limit search depth.

961

return false;

962

}

963

964

KnownBits Known2;

965

switch (Op.getOpcode()) {

966

case ISD::TargetConstant:

967

llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 967);

968

case ISD::SCALAR_TO_VECTOR: {

969

if (!DemandedElts[0])

970

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

971

972

KnownBits SrcKnown;

973

SDValue Src = Op.getOperand(0);

974

unsigned SrcBitWidth = Src.getScalarValueSizeInBits();

975

APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);

976

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))

977

return true;

978

979

// Upper elements are undef, so only get the knownbits if we just demand

980

// the bottom element.

981

if (DemandedElts == 1)

982

Known = SrcKnown.anyextOrTrunc(BitWidth);

983

break;

984

}

985

case ISD::BUILD_VECTOR:

986

// Collect the known bits that are shared by every demanded element.

987

// TODO: Call SimplifyDemandedBits for non-constant demanded elements.

988

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

989

return false; // Don't fall through, will infinitely loop.

990

case ISD::LOAD: {

991

auto *LD = cast<LoadSDNode>(Op);

992

if (getTargetConstantFromLoad(LD)) {

993

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

994

return false; // Don't fall through, will infinitely loop.

995

}

996

if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {

997

// If this is a ZEXTLoad and we are looking at the loaded value.

998

EVT MemVT = LD->getMemoryVT();

999

unsigned MemBits = MemVT.getScalarSizeInBits();

1000

Known.Zero.setBitsFrom(MemBits);

1001

return false; // Don't fall through, will infinitely loop.

1002

}

1003

break;

1004

}

1005

case ISD::INSERT_VECTOR_ELT: {

1006

SDValue Vec = Op.getOperand(0);

1007

SDValue Scl = Op.getOperand(1);

1008

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

1009

EVT VecVT = Vec.getValueType();

1010

1011

// If index isn't constant, assume we need all vector elements AND the

1012

// inserted element.

1013

APInt DemandedVecElts(DemandedElts);

1014

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {

1015

unsigned Idx = CIdx->getZExtValue();

1016

DemandedVecElts.clearBit(Idx);

1017

1018

// Inserted element is not required.

1019

if (!DemandedElts[Idx])

1020

return TLO.CombineTo(Op, Vec);

1021

}

1022

1023

KnownBits KnownScl;

1024

unsigned NumSclBits = Scl.getScalarValueSizeInBits();

1025

APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);

1026

if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))

1027

return true;

1028

1029

Known = KnownScl.anyextOrTrunc(BitWidth);

1030

1031

KnownBits KnownVec;

1032

if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,

1033

Depth + 1))

1034

return true;

1035

1036

if (!!DemandedVecElts)

1037

Known = KnownBits::commonBits(Known, KnownVec);

1038

1039

return false;

1040

}

1041

case ISD::INSERT_SUBVECTOR: {

1042

// Demand any elements from the subvector and the remainder from the src its

1043

// inserted into.

1044

SDValue Src = Op.getOperand(0);

1045

SDValue Sub = Op.getOperand(1);

1046

uint64_t Idx = Op.getConstantOperandVal(2);

1047

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

1048

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

1049

APInt DemandedSrcElts = DemandedElts;

1050

DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);

1051

1052

KnownBits KnownSub, KnownSrc;

1053

if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,

1054

Depth + 1))

1055

return true;

1056

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,

1057

Depth + 1))

1058

return true;

1059

1060

Known.Zero.setAllBits();

1061

Known.One.setAllBits();

1062

if (!!DemandedSubElts)

1063

Known = KnownBits::commonBits(Known, KnownSub);

1064

if (!!DemandedSrcElts)

1065

Known = KnownBits::commonBits(Known, KnownSrc);

1066

1067

// Attempt to avoid multi-use src if we don't need anything from it.

1068

if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||

1069

!DemandedSrcElts.isAllOnes()) {

1070

SDValue NewSub = SimplifyMultipleUseDemandedBits(

1071

Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);

1072

SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1073

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1074

if (NewSub || NewSrc) {

1075

NewSub = NewSub ? NewSub : Sub;

1076

NewSrc = NewSrc ? NewSrc : Src;

1077

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,

1078

Op.getOperand(2));

1079

return TLO.CombineTo(Op, NewOp);

1080

}

1081

}

1082

break;

1083

}

1084

case ISD::EXTRACT_SUBVECTOR: {

1085

// Offset the demanded elts by the subvector index.

1086

SDValue Src = Op.getOperand(0);

1087

if (Src.getValueType().isScalableVector())

1088

break;

1089

uint64_t Idx = Op.getConstantOperandVal(1);

1090

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

1091

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

1092

1093

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,

1094

Depth + 1))

1095

return true;

1096

1097

// Attempt to avoid multi-use src if we don't need anything from it.

1098

if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {

1099

SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

1100

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1101

if (DemandedSrc) {

1102

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,

1103

Op.getOperand(1));

1104

return TLO.CombineTo(Op, NewOp);

1105

}

1106

}

1107

break;

1108

}

1109

case ISD::CONCAT_VECTORS: {

1110

Known.Zero.setAllBits();

1111

Known.One.setAllBits();

1112

EVT SubVT = Op.getOperand(0).getValueType();

1113

unsigned NumSubVecs = Op.getNumOperands();

1114

unsigned NumSubElts = SubVT.getVectorNumElements();

1115

for (unsigned i = 0; i != NumSubVecs; ++i) {

1116

APInt DemandedSubElts =

1117

DemandedElts.extractBits(NumSubElts, i * NumSubElts);

1118

if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,

1119

Known2, TLO, Depth + 1))

1120

return true;

1121

// Known bits are shared by every demanded subvector element.

1122

if (!!DemandedSubElts)

1123

Known = KnownBits::commonBits(Known, Known2);

1124

}

1125

break;

1126

}

1127

case ISD::VECTOR_SHUFFLE: {

1128

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

1129

1130

// Collect demanded elements from shuffle operands..

1131

APInt DemandedLHS(NumElts, 0);

1132

APInt DemandedRHS(NumElts, 0);

1133

for (unsigned i = 0; i != NumElts; ++i) {

1134

if (!DemandedElts[i])

1135

continue;

1136

int M = ShuffleMask[i];

1137

if (M < 0) {

1138

// For UNDEF elements, we don't know anything about the common state of

1139

// the shuffle result.

1140

DemandedLHS.clearAllBits();

1141

DemandedRHS.clearAllBits();

1142

break;

1143

}

1144

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__));

1145

if (M < (int)NumElts)

1146

DemandedLHS.setBit(M);

1147

else

1148

DemandedRHS.setBit(M - NumElts);

1149

}

1150

1151

if (!!DemandedLHS || !!DemandedRHS) {

1152

SDValue Op0 = Op.getOperand(0);

1153

SDValue Op1 = Op.getOperand(1);

1154

1155

Known.Zero.setAllBits();

1156

Known.One.setAllBits();

1157

if (!!DemandedLHS) {

1158

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,

1159

Depth + 1))

1160

return true;

1161

Known = KnownBits::commonBits(Known, Known2);

1162

}

1163

if (!!DemandedRHS) {

1164

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,

1165

Depth + 1))

1166

return true;

1167

Known = KnownBits::commonBits(Known, Known2);

1168

}

1169

1170

// Attempt to avoid multi-use ops if we don't need anything from them.

1171

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1172

Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);

1173

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1174

Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);

1175

if (DemandedOp0 || DemandedOp1) {

1176

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1177

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1178

SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);

1179

return TLO.CombineTo(Op, NewOp);

1180

}

1181

}

1182

break;

1183

}

1184

case ISD::AND: {

1185

SDValue Op0 = Op.getOperand(0);

1186

SDValue Op1 = Op.getOperand(1);

1187

1188

// If the RHS is a constant, check to see if the LHS would be zero without

1189

// using the bits from the RHS. Below, we use knowledge about the RHS to

1190

// simplify the LHS, here we're using information from the LHS to simplify

1191

// the RHS.

1192

if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {

1193

// Do not increment Depth here; that can cause an infinite loop.

1194

KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);

1195

// If the LHS already has zeros where RHSC does, this 'and' is dead.

1196

if ((LHSKnown.Zero & DemandedBits) ==

1197

(~RHSC->getAPIntValue() & DemandedBits))

1198

return TLO.CombineTo(Op, Op0);

1199

1200

// If any of the set bits in the RHS are known zero on the LHS, shrink

1201

// the constant.

1202

if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,

1203

DemandedElts, TLO))

1204

return true;

1205

1206

// Bitwise-not (xor X, -1) is a special case: we don't usually shrink its

1207

// constant, but if this 'and' is only clearing bits that were just set by

1208

// the xor, then this 'and' can be eliminated by shrinking the mask of

1209

// the xor. For example, for a 32-bit X:

1210

// and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1

1211

if (isBitwiseNot(Op0) && Op0.hasOneUse() &&

1212

LHSKnown.One == ~RHSC->getAPIntValue()) {

1213

SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);

1214

return TLO.CombineTo(Op, Xor);

1215

}

1216

}

1217

1218

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1219

Depth + 1))

1220

return true;

1221

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1221, __extension__ __PRETTY_FUNCTION__));

1222

if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,

1223

Known2, TLO, Depth + 1))

1224

return true;

1225

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__));

1226

1227

// Attempt to avoid multi-use ops if we don't need anything from them.

1228

if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {

1229

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1230

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1231

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1232

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1233

if (DemandedOp0 || DemandedOp1) {

1234

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1235

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1236

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1237

return TLO.CombineTo(Op, NewOp);

1238

}

1239

}

1240

1241

// If all of the demanded bits are known one on one side, return the other.

1242

// These bits cannot contribute to the result of the 'and'.

1243

if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))

1244

return TLO.CombineTo(Op, Op0);

1245

if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))

1246

return TLO.CombineTo(Op, Op1);

1247

// If all of the demanded bits in the inputs are known zeros, return zero.

1248

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1249

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));

1250

// If the RHS is a constant, see if we can simplify it.

1251

if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,

1252

TLO))

1253

return true;

1254

// If the operation can be done in a smaller type, do so.

1255

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1256

return true;

1257

1258

Known &= Known2;

1259

break;

1260

}

1261

case ISD::OR: {

1262

SDValue Op0 = Op.getOperand(0);

1263

SDValue Op1 = Op.getOperand(1);

1264

1265

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1266

Depth + 1))

1267

return true;

1268

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1268, __extension__ __PRETTY_FUNCTION__));

1269

if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,

1270

Known2, TLO, Depth + 1))

1271

return true;

1272

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1272, __extension__ __PRETTY_FUNCTION__));

1273

1274

// Attempt to avoid multi-use ops if we don't need anything from them.

1275

if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {

1276

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1277

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1278

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1279

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1280

if (DemandedOp0 || DemandedOp1) {

1281

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1282

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1283

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1284

return TLO.CombineTo(Op, NewOp);

1285

}

1286

}

1287

1288

// If all of the demanded bits are known zero on one side, return the other.

1289

// These bits cannot contribute to the result of the 'or'.

1290

if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))

1291

return TLO.CombineTo(Op, Op0);

1292

if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))

1293

return TLO.CombineTo(Op, Op1);

1294

// If the RHS is a constant, see if we can simplify it.

1295

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1296

return true;

1297

// If the operation can be done in a smaller type, do so.

1298

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1299

return true;

1300

1301

Known |= Known2;

1302

break;

1303

}

1304

case ISD::XOR: {

1305

SDValue Op0 = Op.getOperand(0);

1306

SDValue Op1 = Op.getOperand(1);

1307

1308

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1309

Depth + 1))

1310

return true;

1311

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__));

1312

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,

1313

Depth + 1))

1314

return true;

1315

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1315, __extension__ __PRETTY_FUNCTION__));

1316

1317

// Attempt to avoid multi-use ops if we don't need anything from them.

1318

if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {

1319

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1320

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1321

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1322

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1323

if (DemandedOp0 || DemandedOp1) {

1324

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1325

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1326

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1327

return TLO.CombineTo(Op, NewOp);

1328

}

1329

}

1330

1331

// If all of the demanded bits are known zero on one side, return the other.

1332

// These bits cannot contribute to the result of the 'xor'.

1333

if (DemandedBits.isSubsetOf(Known.Zero))

1334

return TLO.CombineTo(Op, Op0);

1335

if (DemandedBits.isSubsetOf(Known2.Zero))

1336

return TLO.CombineTo(Op, Op1);

1337

// If the operation can be done in a smaller type, do so.

1338

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1339

return true;

1340

1341

// If all of the unknown bits are known to be zero on one side or the other

1342

// turn this into an *inclusive* or.

1343

// e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0

1344

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1345

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

1346

1347

ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);

1348

if (C) {

1349

// If one side is a constant, and all of the set bits in the constant are

1350

// also known set on the other side, turn this into an AND, as we know

1351

// the bits will be cleared.

1352

// e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2

1353

// NB: it is okay if more bits are known than are requested

1354

if (C->getAPIntValue() == Known2.One) {

1355

SDValue ANDC =

1356

TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);

1357

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));

1358

}

1359

1360

// If the RHS is a constant, see if we can change it. Don't alter a -1

1361

// constant because that's a 'not' op, and that is better for combining

1362

// and codegen.

1363

if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {

1364

// We're flipping all demanded bits. Flip the undemanded bits too.

1365

SDValue New = TLO.DAG.getNOT(dl, Op0, VT);

1366

return TLO.CombineTo(Op, New);

1367

}

1368

}

1369

1370

// If we can't turn this into a 'not', try to shrink the constant.

1371

if (!C || !C->isAllOnes())

1372

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1373

return true;

1374

1375

Known ^= Known2;

1376

break;

1377

}

1378

case ISD::SELECT:

1379

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,

1380

Depth + 1))

1381

return true;

1382

if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,

1383

Depth + 1))

1384

return true;

1385

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1385, __extension__ __PRETTY_FUNCTION__));

1386

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1386, __extension__ __PRETTY_FUNCTION__));

1387

1388

// If the operands are constants, see if we can simplify them.

1389

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1390

return true;

1391

1392

// Only known if known in both the LHS and RHS.

1393

Known = KnownBits::commonBits(Known, Known2);

1394

break;

1395

case ISD::SELECT_CC:

1396

if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,

1397

Depth + 1))

1398

return true;

1399

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,

1400

Depth + 1))

1401

return true;

1402

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1402, __extension__ __PRETTY_FUNCTION__));

1403

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1403, __extension__ __PRETTY_FUNCTION__));

1404

1405

// If the operands are constants, see if we can simplify them.

1406

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1407

return true;

1408

1409

// Only known if known in both the LHS and RHS.

1410

Known = KnownBits::commonBits(Known, Known2);

1411

break;

1412

case ISD::SETCC: {

1413

SDValue Op0 = Op.getOperand(0);

1414

SDValue Op1 = Op.getOperand(1);

1415

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

1416

// If (1) we only need the sign-bit, (2) the setcc operands are the same

1417

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

1418

// -1, we may be able to bypass the setcc.

1419

if (DemandedBits.isSignMask() &&

1420

Op0.getScalarValueSizeInBits() == BitWidth &&

1421

getBooleanContents(Op0.getValueType()) ==

1422

BooleanContent::ZeroOrNegativeOneBooleanContent) {

1423

// If we're testing X < 0, then this compare isn't needed - just use X!

1424

// FIXME: We're limiting to integer types here, but this should also work

1425

// if we don't care about FP signed-zero. The use of SETLT with FP means

1426

// that we don't care about NaNs.

1427

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

1428

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

1429

return TLO.CombineTo(Op, Op0);

1430

1431

// TODO: Should we check for other forms of sign-bit comparisons?

1432

// Examples: X <= -1, X >= 0

1433

}

1434

if (getBooleanContents(Op0.getValueType()) ==

1435

TargetLowering::ZeroOrOneBooleanContent &&

1436

BitWidth > 1)

1437

Known.Zero.setBitsFrom(1);

1438

break;

1439

}

1440

case ISD::SHL: {

1441

SDValue Op0 = Op.getOperand(0);

1442

SDValue Op1 = Op.getOperand(1);

1443

EVT ShiftVT = Op1.getValueType();

1444

1445

if (const APInt *SA =

1446

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1447

unsigned ShAmt = SA->getZExtValue();

1448

if (ShAmt == 0)

1449

return TLO.CombineTo(Op, Op0);

1450

1451

// If this is ((X >>u C1) << ShAmt), see if we can simplify this into a

1452

// single shift. We can do this if the bottom bits (which are shifted

1453

// out) are never demanded.

1454

// TODO - support non-uniform vector amounts.

1455

if (Op0.getOpcode() == ISD::SRL) {

1456

if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {

1457

if (const APInt *SA2 =

1458

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1459

unsigned C1 = SA2->getZExtValue();

1460

unsigned Opc = ISD::SHL;

1461

int Diff = ShAmt - C1;

1462

if (Diff < 0) {

1463

Diff = -Diff;

1464

Opc = ISD::SRL;

1465

}

1466

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1467

return TLO.CombineTo(

1468

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1469

}

1470

}

1471

}

1472

1473

// Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits

1474

// are not demanded. This will likely allow the anyext to be folded away.

1475

// TODO - support non-uniform vector amounts.

1476

if (Op0.getOpcode() == ISD::ANY_EXTEND) {

1477

SDValue InnerOp = Op0.getOperand(0);

1478

EVT InnerVT = InnerOp.getValueType();

1479

unsigned InnerBits = InnerVT.getScalarSizeInBits();

1480

if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&

1481

isTypeDesirableForOp(ISD::SHL, InnerVT)) {

1482

EVT ShTy = getShiftAmountTy(InnerVT, DL);

1483

if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))

1484

ShTy = InnerVT;

1485

SDValue NarrowShl =

1486

TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,

1487

TLO.DAG.getConstant(ShAmt, dl, ShTy));

1488

return TLO.CombineTo(

1489

Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));

1490

}

1491

1492

// Repeat the SHL optimization above in cases where an extension

1493

// intervenes: (shl (anyext (shr x, c1)), c2) to

1494

// (shl (anyext x), c2-c1). This requires that the bottom c1 bits

1495

// aren't demanded (as above) and that the shifted upper c1 bits of

1496

// x aren't demanded.

1497

// TODO - support non-uniform vector amounts.

1498

if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&

1499

InnerOp.hasOneUse()) {

1500

if (const APInt *SA2 =

1501

TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {

1502

unsigned InnerShAmt = SA2->getZExtValue();

1503

if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&

1504

DemandedBits.getActiveBits() <=

1505

(InnerBits - InnerShAmt + ShAmt) &&

1506

DemandedBits.countTrailingZeros() >= ShAmt) {

1507

SDValue NewSA =

1508

TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);

1509

SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,

1510

InnerOp.getOperand(0));

1511

return TLO.CombineTo(

1512

Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));

1513

}

1514

}

1515

}

1516

}

1517

1518

APInt InDemandedMask = DemandedBits.lshr(ShAmt);

1519

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1520

Depth + 1))

1521

return true;

1522

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__));

1523

Known.Zero <<= ShAmt;

1524

Known.One <<= ShAmt;

1525

// low bits known zero.

1526

Known.Zero.setLowBits(ShAmt);

1527

1528

// Try shrinking the operation as long as the shift amount will still be

1529

// in range.

1530

if ((ShAmt < DemandedBits.getActiveBits()) &&

1531

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1532

return true;

1533

}

1534

1535

// If we are only demanding sign bits then we can use the shift source

1536

// directly.

1537

if (const APInt *MaxSA =

1538

TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

1539

unsigned ShAmt = MaxSA->getZExtValue();

1540

unsigned NumSignBits =

1541

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1542

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1543

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

1544

return TLO.CombineTo(Op, Op0);

1545

}

1546

break;

1547

}

1548

case ISD::SRL: {

1549

SDValue Op0 = Op.getOperand(0);

1550

SDValue Op1 = Op.getOperand(1);

1551

EVT ShiftVT = Op1.getValueType();

1552

1553

if (const APInt *SA =

1554

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1555

unsigned ShAmt = SA->getZExtValue();

1556

if (ShAmt == 0)

1557

return TLO.CombineTo(Op, Op0);

1558

1559

// If this is ((X << C1) >>u ShAmt), see if we can simplify this into a

1560

// single shift. We can do this if the top bits (which are shifted out)

1561

// are never demanded.

1562

// TODO - support non-uniform vector amounts.

1563

if (Op0.getOpcode() == ISD::SHL) {

1564

if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {

1565

if (const APInt *SA2 =

1566

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1567

unsigned C1 = SA2->getZExtValue();

1568

unsigned Opc = ISD::SRL;

1569

int Diff = ShAmt - C1;

1570

if (Diff < 0) {

1571

Diff = -Diff;

1572

Opc = ISD::SHL;

1573

}

1574

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1575

return TLO.CombineTo(

1576

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1577

}

1578

}

1579

}

1580

1581

APInt InDemandedMask = (DemandedBits << ShAmt);

1582

1583

// If the shift is exact, then it does demand the low bits (and knows that

1584

// they are zero).

1585

if (Op->getFlags().hasExact())

1586

InDemandedMask.setLowBits(ShAmt);

1587

1588

// Compute the new bits that are at the top now.

1589

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1590

Depth + 1))

1591

return true;

1592

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1592, __extension__ __PRETTY_FUNCTION__));

1593

Known.Zero.lshrInPlace(ShAmt);

1594

Known.One.lshrInPlace(ShAmt);

1595

// High bits known zero.

1596

Known.Zero.setHighBits(ShAmt);

1597

}

1598

break;

1599

}

1600

case ISD::SRA: {

1601

SDValue Op0 = Op.getOperand(0);

1602

SDValue Op1 = Op.getOperand(1);

1603

EVT ShiftVT = Op1.getValueType();

1604

1605

// If we only want bits that already match the signbit then we don't need

1606

// to shift.

1607

unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1608

if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=

1609

NumHiDemandedBits)

1610

return TLO.CombineTo(Op, Op0);

1611

1612

// If this is an arithmetic shift right and only the low-bit is set, we can

1613

// always convert this into a logical shr, even if the shift amount is

1614

// variable. The low bit of the shift cannot be an input sign bit unless

1615

// the shift amount is >= the size of the datatype, which is undefined.

1616

if (DemandedBits.isOne())

1617

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

1618

1619

if (const APInt *SA =

1620

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1621

unsigned ShAmt = SA->getZExtValue();

1622

if (ShAmt == 0)

1623

return TLO.CombineTo(Op, Op0);

1624

1625

APInt InDemandedMask = (DemandedBits << ShAmt);

1626

1627

// If the shift is exact, then it does demand the low bits (and knows that

1628

// they are zero).

1629

if (Op->getFlags().hasExact())

1630

InDemandedMask.setLowBits(ShAmt);

1631

1632

// If any of the demanded bits are produced by the sign extension, we also

1633

// demand the input sign bit.

1634

if (DemandedBits.countLeadingZeros() < ShAmt)

1635

InDemandedMask.setSignBit();

1636

1637

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1638

Depth + 1))

1639

return true;

1640

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__));

1641

Known.Zero.lshrInPlace(ShAmt);

1642

Known.One.lshrInPlace(ShAmt);

1643

1644

// If the input sign bit is known to be zero, or if none of the top bits

1645

// are demanded, turn this into an unsigned shift right.

1646

if (Known.Zero[BitWidth - ShAmt - 1] ||

1647

DemandedBits.countLeadingZeros() >= ShAmt) {

1648

SDNodeFlags Flags;

1649

Flags.setExact(Op->getFlags().hasExact());

1650

return TLO.CombineTo(

1651

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));

1652

}

1653

1654

int Log2 = DemandedBits.exactLogBase2();

1655

if (Log2 >= 0) {

1656

// The bit must come from the sign.

1657

SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);

1658

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));

1659

}

1660

1661

if (Known.One[BitWidth - ShAmt - 1])

1662

// New bits are known one.

1663

Known.One.setHighBits(ShAmt);

1664

1665

// Attempt to avoid multi-use ops if we don't need anything from them.

1666

if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {

1667

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1668

Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);

1669

if (DemandedOp0) {

1670

SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);

1671

return TLO.CombineTo(Op, NewOp);

1672

}

1673

}

1674

}

1675

break;

1676

}

1677

case ISD::FSHL:

1678

case ISD::FSHR: {

1679

SDValue Op0 = Op.getOperand(0);

1680

SDValue Op1 = Op.getOperand(1);

1681

SDValue Op2 = Op.getOperand(2);

1682

bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

1683

1684

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {

1685

unsigned Amt = SA->getAPIntValue().urem(BitWidth);

1686

1687

// For fshl, 0-shift returns the 1st arg.

1688

// For fshr, 0-shift returns the 2nd arg.

1689

if (Amt == 0) {

1690

if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,

1691

Known, TLO, Depth + 1))

1692

return true;

1693

break;

1694

}

1695

1696

// fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))

1697

// fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)

1698

APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));

1699

APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);

1700

if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,

1701

Depth + 1))

1702

return true;

1703

if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,

1704

Depth + 1))

1705

return true;

1706

1707

Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));

1708

Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));

1709

Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1710

Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1711

Known.One |= Known2.One;

1712

Known.Zero |= Known2.Zero;

1713

}

1714

1715

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1716

if (isPowerOf2_32(BitWidth)) {

1717

APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);

1718

if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,

1719

Known2, TLO, Depth + 1))

1720

return true;

1721

}

1722

break;

1723

}

1724

case ISD::ROTL:

1725

case ISD::ROTR: {

1726

SDValue Op0 = Op.getOperand(0);

1727

SDValue Op1 = Op.getOperand(1);

1728

1729

// If we're rotating an 0/-1 value, then it stays an 0/-1 value.

1730

if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))

1731

return TLO.CombineTo(Op, Op0);

1732

1733

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1734

if (isPowerOf2_32(BitWidth)) {

1735

APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);

1736

if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,

1737

Depth + 1))

1738

return true;

1739

}

1740

break;

1741

}

1742

case ISD::UMIN: {

1743

// Check if one arg is always less than (or equal) to the other arg.

1744

SDValue Op0 = Op.getOperand(0);

1745

SDValue Op1 = Op.getOperand(1);

1746

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1747

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1748

Known = KnownBits::umin(Known0, Known1);

1749

if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))

1750

return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);

1751

if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))

1752

return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);

1753

break;

1754

}

1755

case ISD::UMAX: {

1756

// Check if one arg is always greater than (or equal) to the other arg.

1757

SDValue Op0 = Op.getOperand(0);

1758

SDValue Op1 = Op.getOperand(1);

1759

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1760

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1761

Known = KnownBits::umax(Known0, Known1);

1762

if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))

1763

return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);

1764

if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))

1765

return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);

1766

break;

1767

}

1768

case ISD::BITREVERSE: {

1769

SDValue Src = Op.getOperand(0);

1770

APInt DemandedSrcBits = DemandedBits.reverseBits();

1771

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1772

Depth + 1))

1773

return true;

1774

Known.One = Known2.One.reverseBits();

1775

Known.Zero = Known2.Zero.reverseBits();

1776

break;

1777

}

1778

case ISD::BSWAP: {

1779

SDValue Src = Op.getOperand(0);

1780

APInt DemandedSrcBits = DemandedBits.byteSwap();

1781

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1782

Depth + 1))

1783

return true;

1784

Known.One = Known2.One.byteSwap();

1785

Known.Zero = Known2.Zero.byteSwap();

1786

break;

1787

}

1788

case ISD::CTPOP: {

1789

// If only 1 bit is demanded, replace with PARITY as long as we're before

1790

// op legalization.

1791

// FIXME: Limit to scalars for now.

1792

if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())

1793

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,

1794

Op.getOperand(0)));

1795

1796

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

1797

break;

1798

}

1799

case ISD::SIGN_EXTEND_INREG: {

1800

SDValue Op0 = Op.getOperand(0);

1801

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

1802

unsigned ExVTBits = ExVT.getScalarSizeInBits();

1803

1804

// If we only care about the highest bit, don't bother shifting right.

1805

if (DemandedBits.isSignMask()) {

1806

unsigned NumSignBits =

1807

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1808

bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;

1809

// However if the input is already sign extended we expect the sign

1810

// extension to be dropped altogether later and do not simplify.

1811

if (!AlreadySignExtended) {

1812

// Compute the correct shift amount type, which must be getShiftAmountTy

1813

// for scalar types after legalization.

1814

EVT ShiftAmtTy = VT;

1815

if (TLO.LegalTypes() && !ShiftAmtTy.isVector())

1816

ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

1817

1818

SDValue ShiftAmt =

1819

TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);

1820

return TLO.CombineTo(Op,

1821

TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));

1822

}

1823

}

1824

1825

// If none of the extended bits are demanded, eliminate the sextinreg.

1826

if (DemandedBits.getActiveBits() <= ExVTBits)

1827

return TLO.CombineTo(Op, Op0);

1828

1829

APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

1830

1831

// Since the sign extended bits are demanded, we know that the sign

1832

// bit is demanded.

1833

InputDemandedBits.setBit(ExVTBits - 1);

1834

1835

if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))

1836

return true;

1837

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1837, __extension__ __PRETTY_FUNCTION__));

1838

1839

// If the sign bit of the input is known set or clear, then we know the

1840

// top bits of the result.

1841

1842

// If the input sign bit is known zero, convert this into a zero extension.

1843

if (Known.Zero[ExVTBits - 1])

1844

return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

1845

1846

APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);

1847

if (Known.One[ExVTBits - 1]) { // Input sign bit known set

1848

Known.One.setBitsFrom(ExVTBits);

1849

Known.Zero &= Mask;

1850

} else { // Input sign bit unknown

1851

Known.Zero &= Mask;

1852

Known.One &= Mask;

1853

}

1854

break;

1855

}

1856

case ISD::BUILD_PAIR: {

1857

EVT HalfVT = Op.getOperand(0).getValueType();

1858

unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

1859

1860

APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);

1861

APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

1862

1863

KnownBits KnownLo, KnownHi;

1864

1865

if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))

1866

return true;

1867

1868

if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))

1869

return true;

1870

1871

Known.Zero = KnownLo.Zero.zext(BitWidth) |

1872

KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

1873

1874

Known.One = KnownLo.One.zext(BitWidth) |

1875

KnownHi.One.zext(BitWidth).shl(HalfBitWidth);

1876

break;

1877

}

1878

case ISD::ZERO_EXTEND:

1879

case ISD::ZERO_EXTEND_VECTOR_INREG: {

1880

SDValue Src = Op.getOperand(0);

1881

EVT SrcVT = Src.getValueType();

1882

unsigned InBits = SrcVT.getScalarSizeInBits();

1883

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1884

bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

1885

1886

// If none of the top bits are demanded, convert this into an any_extend.

1887

if (DemandedBits.getActiveBits() <= InBits) {

1888

// If we only need the non-extended bits of the bottom element

1889

// then we can just bitcast to the result.

1890

if (IsVecInReg && DemandedElts == 1 &&

1891

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1892

TLO.DAG.getDataLayout().isLittleEndian())

1893

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1894

1895

unsigned Opc =

1896

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1897

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1898

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1899

}

1900

1901

APInt InDemandedBits = DemandedBits.trunc(InBits);

1902

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1903

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1904

Depth + 1))

1905

return true;

1906

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1906, __extension__ __PRETTY_FUNCTION__));

1907

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1907, __extension__ __PRETTY_FUNCTION__));

1908

Known = Known.zext(BitWidth);

1909

1910

// Attempt to avoid multi-use ops if we don't need anything from them.

1911

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1912

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1913

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1914

break;

1915

}

1916

case ISD::SIGN_EXTEND:

1917

case ISD::SIGN_EXTEND_VECTOR_INREG: {

1918

SDValue Src = Op.getOperand(0);

1919

EVT SrcVT = Src.getValueType();

1920

unsigned InBits = SrcVT.getScalarSizeInBits();

1921

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1922

bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

1923

1924

// If none of the top bits are demanded, convert this into an any_extend.

1925

if (DemandedBits.getActiveBits() <= InBits) {

1926

// If we only need the non-extended bits of the bottom element

1927

// then we can just bitcast to the result.

1928

if (IsVecInReg && DemandedElts == 1 &&

1929

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1930

TLO.DAG.getDataLayout().isLittleEndian())

1931

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1932

1933

unsigned Opc =

1934

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1935

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1936

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1937

}

1938

1939

APInt InDemandedBits = DemandedBits.trunc(InBits);

1940

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1941

1942

// Since some of the sign extended bits are demanded, we know that the sign

1943

// bit is demanded.

1944

InDemandedBits.setBit(InBits - 1);

1945

1946

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1947

Depth + 1))

1948

return true;

1949

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__));

1950

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1950, __extension__ __PRETTY_FUNCTION__));

1951

1952

// If the sign bit is known one, the top bits match.

1953

Known = Known.sext(BitWidth);

1954

1955

// If the sign bit is known zero, convert this to a zero extend.

1956

if (Known.isNonNegative()) {

1957

unsigned Opc =

1958

IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;

1959

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1960

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1961

}

1962

1963

// Attempt to avoid multi-use ops if we don't need anything from them.

1964

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1965

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1966

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1967

break;

1968

}

1969

case ISD::ANY_EXTEND:

1970

case ISD::ANY_EXTEND_VECTOR_INREG: {

1971

SDValue Src = Op.getOperand(0);

1972

EVT SrcVT = Src.getValueType();

1973

unsigned InBits = SrcVT.getScalarSizeInBits();

1974

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1975

bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

1976

1977

// If we only need the bottom element then we can just bitcast.

1978

// TODO: Handle ANY_EXTEND?

1979

if (IsVecInReg && DemandedElts == 1 &&

1980

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1981

TLO.DAG.getDataLayout().isLittleEndian())

1982

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1983

1984

APInt InDemandedBits = DemandedBits.trunc(InBits);

1985

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1986

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1987

Depth + 1))

1988

return true;

1989

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1989, __extension__ __PRETTY_FUNCTION__));

1990

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1990, __extension__ __PRETTY_FUNCTION__));

1991

Known = Known.anyext(BitWidth);

1992

1993

// Attempt to avoid multi-use ops if we don't need anything from them.

1994

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1995

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1996

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1997

break;

1998

}

1999

case ISD::TRUNCATE: {

2000

SDValue Src = Op.getOperand(0);

2001

2002

// Simplify the input, using demanded bit information, and compute the known

2003

// zero/one bits live out.

2004

unsigned OperandBitWidth = Src.getScalarValueSizeInBits();

2005

APInt TruncMask = DemandedBits.zext(OperandBitWidth);

2006

if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,

2007

Depth + 1))

2008

return true;

2009

Known = Known.trunc(BitWidth);

2010

2011

// Attempt to avoid multi-use ops if we don't need anything from them.

2012

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

2013

Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))

2014

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

2015

2016

// If the input is only used by this truncate, see if we can shrink it based

2017

// on the known demanded bits.

2018

if (Src.getNode()->hasOneUse()) {

2019

switch (Src.getOpcode()) {

2020

default:

2021

break;

2022

case ISD::SRL:

2023

// Shrink SRL by a constant if none of the high bits shifted in are

2024

// demanded.

2025

if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))

2026

// Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is

2027

// undesirable.

2028

break;

2029

2030

const APInt *ShAmtC =

2031

TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);

2032

if (!ShAmtC || ShAmtC->uge(BitWidth))

2033

break;

2034

uint64_t ShVal = ShAmtC->getZExtValue();

2035

2036

APInt HighBits =

2037

APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);

2038

HighBits.lshrInPlace(ShVal);

2039

HighBits = HighBits.trunc(BitWidth);

2040

2041

if (!(HighBits & DemandedBits)) {

2042

// None of the shifted in bits are needed. Add a truncate of the

2043

// shift input, then shift it.

2044

SDValue NewShAmt = TLO.DAG.getConstant(

2045

ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));

2046

SDValue NewTrunc =

2047

TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));

2048

return TLO.CombineTo(

2049

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));

2050

}

2051

break;

2052

}

2053

}

2054

2055

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__));

2056

break;

2057

}

2058

case ISD::AssertZext: {

2059

// AssertZext demands all of the high bits, plus any of the low bits

2060

// demanded by its users.

2061

EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

2062

APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());

2063

if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,

2064

TLO, Depth + 1))

2065

return true;

2066

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2066, __extension__ __PRETTY_FUNCTION__));

2067

2068

Known.Zero |= ~InMask;

2069

break;

2070

}

2071

case ISD::EXTRACT_VECTOR_ELT: {

2072

SDValue Src = Op.getOperand(0);

2073

SDValue Idx = Op.getOperand(1);

2074

ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();

2075

unsigned EltBitWidth = Src.getScalarValueSizeInBits();

2076

2077

if (SrcEltCnt.isScalable())

2078

return false;

2079

2080

// Demand the bits from every vector element without a constant index.

2081

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2082

APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);

2083

if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))

2084

if (CIdx->getAPIntValue().ult(NumSrcElts))

2085

DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

2086

2087

// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know

2088

// anything about the extended bits.

2089

APInt DemandedSrcBits = DemandedBits;

2090

if (BitWidth > EltBitWidth)

2091

DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

2092

2093

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,

2094

Depth + 1))

2095

return true;

2096

2097

// Attempt to avoid multi-use ops if we don't need anything from them.

2098

if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {

2099

if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

2100

Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {

2101

SDValue NewOp =

2102

TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);

2103

return TLO.CombineTo(Op, NewOp);

2104

}

2105

}

2106

2107

Known = Known2;

2108

if (BitWidth > EltBitWidth)

2109

Known = Known.anyext(BitWidth);

2110

break;

2111

}

2112

case ISD::BITCAST: {

2113

SDValue Src = Op.getOperand(0);

2114

EVT SrcVT = Src.getValueType();

2115

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

2116

2117

// If this is an FP->Int bitcast and if the sign bit is the only

2118

// thing demanded, turn this into a FGETSIGN.

2119

if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&

2120

DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&

2121

SrcVT.isFloatingPoint()) {

2122

bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);

2123

bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);

2124

if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&

2125

SrcVT != MVT::f128) {

2126

// Cannot eliminate/lower SHL for f128 yet.

2127

EVT Ty = OpVTLegal ? VT : MVT::i32;

2128

// Make a FGETSIGN + SHL to move the sign bit into the appropriate

2129

// place. We expect the SHL to be eliminated by other optimizations.

2130

SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);

2131

unsigned OpVTSizeInBits = Op.getValueSizeInBits();

2132

if (!OpVTLegal && OpVTSizeInBits > 32)

2133

Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);

2134

unsigned ShVal = Op.getValueSizeInBits() - 1;

2135

SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);

2136

return TLO.CombineTo(Op,

2137

TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));

2138

}

2139

}

2140

2141

// Bitcast from a vector using SimplifyDemanded Bits/VectorElts.

2142

// Demand the elt/bit if any of the original elts/bits are demanded.

2143

// TODO - bigendian once we have test coverage.

2144

if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&

2145

TLO.DAG.getDataLayout().isLittleEndian()) {

2146

unsigned Scale = BitWidth / NumSrcEltBits;

2147

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2148

APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);

2149

APInt DemandedSrcElts = APInt::getZero(NumSrcElts);

2150

for (unsigned i = 0; i != Scale; ++i) {

2151

unsigned Offset = i * NumSrcEltBits;

2152

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

2153

if (!Sub.isZero()) {

2154

DemandedSrcBits |= Sub;

2155

for (unsigned j = 0; j != NumElts; ++j)

2156

if (DemandedElts[j])

2157

DemandedSrcElts.setBit((j * Scale) + i);

2158

}

2159

}

2160

2161

APInt KnownSrcUndef, KnownSrcZero;

2162

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2163

KnownSrcZero, TLO, Depth + 1))

2164

return true;

2165

2166

KnownBits KnownSrcBits;

2167

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2168

KnownSrcBits, TLO, Depth + 1))

2169

return true;

2170

} else if ((NumSrcEltBits % BitWidth) == 0 &&

2171

TLO.DAG.getDataLayout().isLittleEndian()) {

2172

unsigned Scale = NumSrcEltBits / BitWidth;

2173

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

2174

APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);

2175

APInt DemandedSrcElts = APInt::getZero(NumSrcElts);

2176

for (unsigned i = 0; i != NumElts; ++i)

2177

if (DemandedElts[i]) {

2178

unsigned Offset = (i % Scale) * BitWidth;

2179

DemandedSrcBits.insertBits(DemandedBits, Offset);

2180

DemandedSrcElts.setBit(i / Scale);

2181

}

2182

2183

if (SrcVT.isVector()) {

2184

APInt KnownSrcUndef, KnownSrcZero;

2185

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2186

KnownSrcZero, TLO, Depth + 1))

2187

return true;

2188

}

2189

2190

KnownBits KnownSrcBits;

2191

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2192

KnownSrcBits, TLO, Depth + 1))

2193

return true;

2194

}

2195

2196

// If this is a bitcast, let computeKnownBits handle it. Only do this on a

2197

// recursive call where Known may be useful to the caller.

2198

if (Depth > 0) {

2199

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2200

return false;

2201

}

2202

break;

2203

}

2204

case ISD::ADD:

2205

case ISD::MUL:

2206

case ISD::SUB: {

2207

// Add, Sub, and Mul don't demand any bits in positions beyond that

2208

// of the highest bit demanded of them.

2209

SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);

2210

SDNodeFlags Flags = Op.getNode()->getFlags();

2211

unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();

2212

APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

2213

if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,

2214

Depth + 1) ||

2215

SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,

2216

Depth + 1) ||

2217

// See if the operation should be performed at a smaller bit width.

2218

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {

2219

if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {

2220

// Disable the nsw and nuw flags. We can no longer guarantee that we

2221

// won't wrap after simplification.

2222

Flags.setNoSignedWrap(false);

2223

Flags.setNoUnsignedWrap(false);

2224

SDValue NewOp =

2225

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2226

return TLO.CombineTo(Op, NewOp);

2227

}

2228

return true;

2229

}

2230

2231

// Attempt to avoid multi-use ops if we don't need anything from them.

2232

if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {

2233

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

2234

Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2235

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

2236

Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2237

if (DemandedOp0 || DemandedOp1) {

2238

Flags.setNoSignedWrap(false);

2239

Flags.setNoUnsignedWrap(false);

2240

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

2241

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

2242

SDValue NewOp =

2243

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2244

return TLO.CombineTo(Op, NewOp);

2245

}

2246

}

2247

2248

// If we have a constant operand, we may be able to turn it into -1 if we

2249

// do not demand the high bits. This can make the constant smaller to

2250

// encode, allow more general folding, or match specialized instruction

2251

// patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that

2252

// is probably not useful (and could be detrimental).

2253

ConstantSDNode *C = isConstOrConstSplat(Op1);

2254

APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);

2255

if (C && !C->isAllOnes() && !C->isOne() &&

2256

(C->getAPIntValue() | HighMask).isAllOnes()) {

2257

SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);

2258

// Disable the nsw and nuw flags. We can no longer guarantee that we

2259

// won't wrap after simplification.

2260

Flags.setNoSignedWrap(false);

2261

Flags.setNoUnsignedWrap(false);

2262

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);

2263

return TLO.CombineTo(Op, NewOp);

2264

}

2265

2266

LLVM_FALLTHROUGH[[gnu::fallthrough]];

2267

}

2268

default:

2269

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2270

if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,

2271

Known, TLO, Depth))

2272

return true;

2273

break;

2274

}

2275

2276

// Just use computeKnownBits to compute output bits.

2277

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2278

break;

2279

}

2280

2281

// If we know the value of all of the demanded bits, return this as a

2282

// constant.

2283

if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {

2284

// Avoid folding to a constant if any OpaqueConstant is involved.

2285

const SDNode *N = Op.getNode();

2286

for (SDNode *Op :

2287

llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {

2288

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))

2289

if (C->isOpaque())

2290

return false;

2291

}

2292

if (VT.isInteger())

2293

return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));

2294

if (VT.isFloatingPoint())

2295

return TLO.CombineTo(

2296

Op,

2297

TLO.DAG.getConstantFP(

2298

APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));

2299

}

2300

2301

return false;

2302

}

2303

2304

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,

2305

const APInt &DemandedElts,

2306

APInt &KnownUndef,

2307

APInt &KnownZero,

2308

DAGCombinerInfo &DCI) const {

2309

SelectionDAG &DAG = DCI.DAG;

2310

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

2311

!DCI.isBeforeLegalizeOps());

2312

2313

bool Simplified =

2314

SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);

2315

if (Simplified) {

2316

DCI.AddToWorklist(Op.getNode());

2317

DCI.CommitTargetLoweringOpt(TLO);

2318

}

2319

2320

return Simplified;

2321

}

2322

2323

/// Given a vector binary operation and known undefined elements for each input

2324

/// operand, compute whether each element of the output is undefined.

2325

static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,

2326

const APInt &UndefOp0,

2327

const APInt &UndefOp1) {

2328

EVT VT = BO.getValueType();

2329

assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__))

2330

"Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__));

2331

2332

EVT EltVT = VT.getVectorElementType();

2333

unsigned NumElts = VT.getVectorNumElements();

2334

assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__))

2335

UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__));

2336

2337

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,

2338

const APInt &UndefVals) {

2339

if (UndefVals[Index])

2340

return DAG.getUNDEF(EltVT);

2341

2342

if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {

2343

// Try hard to make sure that the getNode() call is not creating temporary

2344

// nodes. Ignore opaque integers because they do not constant fold.

2345

SDValue Elt = BV->getOperand(Index);

2346

auto *C = dyn_cast<ConstantSDNode>(Elt);

2347

if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))

2348

return Elt;

2349

}

2350

2351

return SDValue();

2352

};

2353

2354

APInt KnownUndef = APInt::getZero(NumElts);

2355

for (unsigned i = 0; i != NumElts; ++i) {

2356

// If both inputs for this element are either constant or undef and match

2357

// the element type, compute the constant/undef result for this element of

2358

// the vector.

2359

// TODO: Ideally we would use FoldConstantArithmetic() here, but that does

2360

// not handle FP constants. The code within getNode() should be refactored

2361

// to avoid the danger of creating a bogus temporary node here.

2362

SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);

2363

SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);

2364

if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)

2365

if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())

2366

KnownUndef.setBit(i);

2367

}

2368

return KnownUndef;

2369

}

2370

2371

bool TargetLowering::SimplifyDemandedVectorElts(

2372

SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,

2373

APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,

2374

bool AssumeSingleUse) const {

2375

EVT VT = Op.getValueType();

2376

unsigned Opcode = Op.getOpcode();

2377

APInt DemandedElts = OriginalDemandedElts;

2378

unsigned NumElts = DemandedElts.getBitWidth();

2379

assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__));

2380

2381

KnownUndef = KnownZero = APInt::getZero(NumElts);

2382

2383

// TODO: For now we assume we know nothing about scalable vectors.

2384

if (VT.isScalableVector())

2385

return false;

2386

2387

assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__))

2388

"Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__));

2389

2390

// Undef operand.

2391

if (Op.isUndef()) {

2392

KnownUndef.setAllBits();

2393

return false;

2394

}

2395

2396

// If Op has other users, assume that all elements are needed.

2397

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)

2398

DemandedElts.setAllBits();

2399

2400

// Not demanding any elements from Op.

2401

if (DemandedElts == 0) {

2402

KnownUndef.setAllBits();

2403

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2404

}

2405

2406

// Limit search depth.

2407

if (Depth >= SelectionDAG::MaxRecursionDepth)

2408

return false;

2409

2410

SDLoc DL(Op);

2411

unsigned EltSizeInBits = VT.getScalarSizeInBits();

2412

2413

// Helper for demanding the specified elements and all the bits of both binary

2414

// operands.

2415

auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {

2416

SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,

2417

TLO.DAG, Depth + 1);

2418

SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,

2419

TLO.DAG, Depth + 1);

2420

if (NewOp0 || NewOp1) {

2421

SDValue NewOp = TLO.DAG.getNode(

2422

Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);

2423

return TLO.CombineTo(Op, NewOp);

2424

}

2425

return false;

2426

};

2427

2428

switch (Opcode) {

2429

case ISD::SCALAR_TO_VECTOR: {

2430

if (!DemandedElts[0]) {

2431

KnownUndef.setAllBits();

2432

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2433

}

2434

SDValue ScalarSrc = Op.getOperand(0);

2435

if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {

2436

SDValue Src = ScalarSrc.getOperand(0);

2437

SDValue Idx = ScalarSrc.getOperand(1);

2438

EVT SrcVT = Src.getValueType();

2439

2440

ElementCount SrcEltCnt = SrcVT.getVectorElementCount();

2441

2442

if (SrcEltCnt.isScalable())

2443

return false;

2444

2445

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2446

if (isNullConstant(Idx)) {

2447

APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);

2448

APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);

2449

APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);

2450

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2451

TLO, Depth + 1))

2452

return true;

2453

}

2454

}

2455

KnownUndef.setHighBits(NumElts - 1);

2456

break;

2457

}

2458

case ISD::BITCAST: {

2459

SDValue Src = Op.getOperand(0);

2460

EVT SrcVT = Src.getValueType();

2461

2462

// We only handle vectors here.

2463

// TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?

2464

if (!SrcVT.isVector())

2465

break;

2466

2467

// Fast handling of 'identity' bitcasts.

2468

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2469

if (NumSrcElts == NumElts)

2470

return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,

2471

KnownZero, TLO, Depth + 1);

2472

2473

APInt SrcDemandedElts, SrcZero, SrcUndef;

2474

2475

// Bitcast from 'large element' src vector to 'small element' vector, we

2476

// must demand a source element if any DemandedElt maps to it.

2477

if ((NumElts % NumSrcElts) == 0) {

2478

unsigned Scale = NumElts / NumSrcElts;

2479

SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);

2480

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2481

TLO, Depth + 1))

2482

return true;

2483

2484

// Try calling SimplifyDemandedBits, converting demanded elts to the bits

2485

// of the large element.

2486

// TODO - bigendian once we have test coverage.

2487

if (TLO.DAG.getDataLayout().isLittleEndian()) {

2488

unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();

2489

APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);

2490

for (unsigned i = 0; i != NumElts; ++i)

2491

if (DemandedElts[i]) {

2492

unsigned Ofs = (i % Scale) * EltSizeInBits;

2493

SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);

2494

}

2495

2496

KnownBits Known;

2497

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,

2498

TLO, Depth + 1))

2499

return true;

2500

}

2501

2502

// If the src element is zero/undef then all the output elements will be -

2503

// only demanded elements are guaranteed to be correct.

2504

for (unsigned i = 0; i != NumSrcElts; ++i) {

2505

if (SrcDemandedElts[i]) {

2506

if (SrcZero[i])

2507

KnownZero.setBits(i * Scale, (i + 1) * Scale);

2508

if (SrcUndef[i])

2509

KnownUndef.setBits(i * Scale, (i + 1) * Scale);

2510

}

2511

}

2512

}

2513

2514

// Bitcast from 'small element' src vector to 'large element' vector, we

2515

// demand all smaller source elements covered by the larger demanded element

2516

// of this vector.

2517

if ((NumSrcElts % NumElts) == 0) {

2518

unsigned Scale = NumSrcElts / NumElts;

2519

SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);

2520

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2521

TLO, Depth + 1))

2522

return true;

2523

2524

// If all the src elements covering an output element are zero/undef, then

2525

// the output element will be as well, assuming it was demanded.

2526

for (unsigned i = 0; i != NumElts; ++i) {

2527

if (DemandedElts[i]) {

2528

if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())

2529

KnownZero.setBit(i);

2530

if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())

2531

KnownUndef.setBit(i);

2532

}

2533

}

2534

}

2535

break;

2536

}

2537

case ISD::BUILD_VECTOR: {

2538

// Check all elements and simplify any unused elements with UNDEF.

2539

if (!DemandedElts.isAllOnes()) {

2540

// Don't simplify BROADCASTS.

2541

if (llvm::any_of(Op->op_values(),

2542

[&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {

2543

SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());

2544

bool Updated = false;

2545

for (unsigned i = 0; i != NumElts; ++i) {

2546

if (!DemandedElts[i] && !Ops[i].isUndef()) {

2547

Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());

2548

KnownUndef.setBit(i);

2549

Updated = true;

2550

}

2551

}

2552

if (Updated)

2553

return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));

2554

}

2555

}

2556

for (unsigned i = 0; i != NumElts; ++i) {

2557

SDValue SrcOp = Op.getOperand(i);

2558

if (SrcOp.isUndef()) {

2559

KnownUndef.setBit(i);

2560

} else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&

2561

(isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {

2562

KnownZero.setBit(i);

2563

}

2564

}

2565

break;

2566

}

2567

case ISD::CONCAT_VECTORS: {

2568

EVT SubVT = Op.getOperand(0).getValueType();

2569

unsigned NumSubVecs = Op.getNumOperands();

2570

unsigned NumSubElts = SubVT.getVectorNumElements();

2571

for (unsigned i = 0; i != NumSubVecs; ++i) {

2572

SDValue SubOp = Op.getOperand(i);

2573

APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);

2574

APInt SubUndef, SubZero;

2575

if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,

2576

Depth + 1))

2577

return true;

2578

KnownUndef.insertBits(SubUndef, i * NumSubElts);

2579

KnownZero.insertBits(SubZero, i * NumSubElts);

2580

}

2581

break;

2582

}

2583

case ISD::INSERT_SUBVECTOR: {

2584

// Demand any elements from the subvector and the remainder from the src its

2585

// inserted into.

2586

SDValue Src = Op.getOperand(0);

2587

SDValue Sub = Op.getOperand(1);

2588

uint64_t Idx = Op.getConstantOperandVal(2);

2589

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

2590

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

2591

APInt DemandedSrcElts = DemandedElts;

2592

DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);

2593

2594

APInt SubUndef, SubZero;

2595

if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,

2596

Depth + 1))

2597

return true;

2598

2599

// If none of the src operand elements are demanded, replace it with undef.

2600

if (!DemandedSrcElts && !Src.isUndef())

2601

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,

2602

TLO.DAG.getUNDEF(VT), Sub,

2603

Op.getOperand(2)));

2604

2605

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,

2606

TLO, Depth + 1))

2607

return true;

2608

KnownUndef.insertBits(SubUndef, Idx);

2609

KnownZero.insertBits(SubZero, Idx);

2610

2611

// Attempt to avoid multi-use ops if we don't need anything from them.

2612

if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {

2613

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2614

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2615

SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(

2616

Sub, DemandedSubElts, TLO.DAG, Depth + 1);

2617

if (NewSrc || NewSub) {

2618

NewSrc = NewSrc ? NewSrc : Src;

2619

NewSub = NewSub ? NewSub : Sub;

2620

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2621

NewSub, Op.getOperand(2));

2622

return TLO.CombineTo(Op, NewOp);

2623

}

2624

}

2625

break;

2626

}

2627

case ISD::EXTRACT_SUBVECTOR: {

2628

// Offset the demanded elts by the subvector index.

2629

SDValue Src = Op.getOperand(0);

2630

if (Src.getValueType().isScalableVector())

2631

break;

2632

uint64_t Idx = Op.getConstantOperandVal(1);

2633

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2634

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

2635

2636

APInt SrcUndef, SrcZero;

2637

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2638

Depth + 1))

2639

return true;

2640

KnownUndef = SrcUndef.extractBits(NumElts, Idx);

2641

KnownZero = SrcZero.extractBits(NumElts, Idx);

2642

2643

// Attempt to avoid multi-use ops if we don't need anything from them.

2644

if (!DemandedElts.isAllOnes()) {

2645

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2646

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2647

if (NewSrc) {

2648

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2649

Op.getOperand(1));

2650

return TLO.CombineTo(Op, NewOp);

2651

}

2652

}

2653

break;

2654

}

2655

case ISD::INSERT_VECTOR_ELT: {

2656

SDValue Vec = Op.getOperand(0);

2657

SDValue Scl = Op.getOperand(1);

2658

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

2659

2660

// For a legal, constant insertion index, if we don't need this insertion

2661

// then strip it, else remove it from the demanded elts.

2662

if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {

2663

unsigned Idx = CIdx->getZExtValue();

2664

if (!DemandedElts[Idx])

2665

return TLO.CombineTo(Op, Vec);

2666

2667

APInt DemandedVecElts(DemandedElts);

2668

DemandedVecElts.clearBit(Idx);

2669

if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,

2670

KnownZero, TLO, Depth + 1))

2671

return true;

2672

2673

KnownUndef.setBitVal(Idx, Scl.isUndef());

2674

2675

KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));

2676

break;

2677

}

2678

2679

APInt VecUndef, VecZero;

2680

if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,

2681

Depth + 1))

2682

return true;

2683

// Without knowing the insertion index we can't set KnownUndef/KnownZero.

2684

break;

2685

}

2686

case ISD::VSELECT: {

2687

// Try to transform the select condition based on the current demanded

2688

// elements.

2689

// TODO: If a condition element is undef, we can choose from one arm of the

2690

// select (and if one arm is undef, then we can propagate that to the

2691

// result).

2692

// TODO - add support for constant vselect masks (see IR version of this).

2693

APInt UnusedUndef, UnusedZero;

2694

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,

2695

UnusedZero, TLO, Depth + 1))

2696

return true;

2697

2698

// See if we can simplify either vselect operand.

2699

APInt DemandedLHS(DemandedElts);

2700

APInt DemandedRHS(DemandedElts);

2701

APInt UndefLHS, ZeroLHS;

2702

APInt UndefRHS, ZeroRHS;

2703

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,

2704

ZeroLHS, TLO, Depth + 1))

2705

return true;

2706

if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,

2707

ZeroRHS, TLO, Depth + 1))

2708

return true;

2709

2710

KnownUndef = UndefLHS & UndefRHS;

2711

KnownZero = ZeroLHS & ZeroRHS;

2712

break;

2713

}

2714

case ISD::VECTOR_SHUFFLE: {

2715

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

2716

2717

// Collect demanded elements from shuffle operands..

2718

APInt DemandedLHS(NumElts, 0);

2719

APInt DemandedRHS(NumElts, 0);

2720

for (unsigned i = 0; i != NumElts; ++i) {

2721

int M = ShuffleMask[i];

2722

if (M < 0 || !DemandedElts[i])

2723

continue;

2724

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2724, __extension__ __PRETTY_FUNCTION__));

2725

if (M < (int)NumElts)

2726

DemandedLHS.setBit(M);

2727

else

2728

DemandedRHS.setBit(M - NumElts);

2729

}

2730

2731

// See if we can simplify either shuffle operand.

2732

APInt UndefLHS, ZeroLHS;

2733

APInt UndefRHS, ZeroRHS;

2734

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,

2735

ZeroLHS, TLO, Depth + 1))

2736

return true;

2737

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,

2738

ZeroRHS, TLO, Depth + 1))

2739

return true;

2740

2741

// Simplify mask using undef elements from LHS/RHS.

2742

bool Updated = false;

2743

bool IdentityLHS = true, IdentityRHS = true;

2744

SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());

2745

for (unsigned i = 0; i != NumElts; ++i) {

2746

int &M = NewMask[i];

2747

if (M < 0)

2748

continue;

2749

if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||

2750

(M >= (int)NumElts && UndefRHS[M - NumElts])) {

2751

Updated = true;

2752

M = -1;

2753

}

2754

IdentityLHS &= (M < 0) || (M == (int)i);

2755

IdentityRHS &= (M < 0) || ((M - NumElts) == i);

2756

}

2757

2758

// Update legal shuffle masks based on demanded elements if it won't reduce

2759

// to Identity which can cause premature removal of the shuffle mask.

2760

if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {

2761

SDValue LegalShuffle =

2762

buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),

2763

NewMask, TLO.DAG);

2764

if (LegalShuffle)

2765

return TLO.CombineTo(Op, LegalShuffle);

2766

}

2767

2768

// Propagate undef/zero elements from LHS/RHS.

2769

for (unsigned i = 0; i != NumElts; ++i) {

2770

int M = ShuffleMask[i];

2771

if (M < 0) {

2772

KnownUndef.setBit(i);

2773

} else if (M < (int)NumElts) {

2774

if (UndefLHS[M])

2775

KnownUndef.setBit(i);

2776

if (ZeroLHS[M])

2777

KnownZero.setBit(i);

2778

} else {

2779

if (UndefRHS[M - NumElts])

2780

KnownUndef.setBit(i);

2781

if (ZeroRHS[M - NumElts])

2782

KnownZero.setBit(i);

2783

}

2784

}

2785

break;

2786

}

2787

case ISD::ANY_EXTEND_VECTOR_INREG:

2788

case ISD::SIGN_EXTEND_VECTOR_INREG:

2789

case ISD::ZERO_EXTEND_VECTOR_INREG: {

2790

APInt SrcUndef, SrcZero;

2791

SDValue Src = Op.getOperand(0);

2792

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2793

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);

2794

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2795

Depth + 1))

2796

return true;

2797

KnownZero = SrcZero.zextOrTrunc(NumElts);

2798

KnownUndef = SrcUndef.zextOrTrunc(NumElts);

2799

2800

if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&

2801

Op.getValueSizeInBits() == Src.getValueSizeInBits() &&

2802

DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {

2803

// aext - if we just need the bottom element then we can bitcast.

2804

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

2805

}

2806

2807

if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {

2808

// zext(undef) upper bits are guaranteed to be zero.

2809

if (DemandedElts.isSubsetOf(KnownUndef))

2810

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2811

KnownUndef.clearAllBits();

2812

}

2813

break;

2814

}

2815

2816

// TODO: There are more binop opcodes that could be handled here - MIN,

2817

// MAX, saturated math, etc.

2818

case ISD::OR:

2819

case ISD::XOR:

2820

case ISD::ADD:

2821

case ISD::SUB:

2822

case ISD::FADD:

2823

case ISD::FSUB:

2824

case ISD::FMUL:

2825

case ISD::FDIV:

2826

case ISD::FREM: {

2827

SDValue Op0 = Op.getOperand(0);

2828

SDValue Op1 = Op.getOperand(1);

2829

2830

APInt UndefRHS, ZeroRHS;

2831

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2832

Depth + 1))

2833

return true;

2834

APInt UndefLHS, ZeroLHS;

2835

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2836

Depth + 1))

2837

return true;

2838

2839

KnownZero = ZeroLHS & ZeroRHS;

2840

KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

2841

2842

// Attempt to avoid multi-use ops if we don't need anything from them.

2843

// TODO - use KnownUndef to relax the demandedelts?

2844

if (!DemandedElts.isAllOnes())

2845

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2846

return true;

2847

break;

2848

}

2849

case ISD::SHL:

2850

case ISD::SRL:

2851

case ISD::SRA:

2852

case ISD::ROTL:

2853

case ISD::ROTR: {

2854

SDValue Op0 = Op.getOperand(0);

2855

SDValue Op1 = Op.getOperand(1);

2856

2857

APInt UndefRHS, ZeroRHS;

2858

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2859

Depth + 1))

2860

return true;

2861

APInt UndefLHS, ZeroLHS;

2862

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2863

Depth + 1))

2864

return true;

2865

2866

KnownZero = ZeroLHS;

2867

KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

2868

2869

// Attempt to avoid multi-use ops if we don't need anything from them.

2870

// TODO - use KnownUndef to relax the demandedelts?

2871

if (!DemandedElts.isAllOnes())

2872

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2873

return true;

2874

break;

2875

}

2876

case ISD::MUL:

2877

case ISD::AND: {

2878

SDValue Op0 = Op.getOperand(0);

2879

SDValue Op1 = Op.getOperand(1);

2880

2881

APInt SrcUndef, SrcZero;

2882

if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,

2883

Depth + 1))

2884

return true;

2885

if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,

2886

TLO, Depth + 1))

2887

return true;

2888

2889

// If either side has a zero element, then the result element is zero, even

2890

// if the other is an UNDEF.

2891

// TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros

2892

// and then handle 'and' nodes with the rest of the binop opcodes.

2893

KnownZero |= SrcZero;

2894

KnownUndef &= SrcUndef;

2895

KnownUndef &= ~KnownZero;

2896

2897

// Attempt to avoid multi-use ops if we don't need anything from them.

2898

// TODO - use KnownUndef to relax the demandedelts?

2899

if (!DemandedElts.isAllOnes())

2900

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2901

return true;

2902

break;

2903

}

2904

case ISD::TRUNCATE:

2905

case ISD::SIGN_EXTEND:

2906

case ISD::ZERO_EXTEND:

2907

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,

2908

KnownZero, TLO, Depth + 1))

2909

return true;

2910

2911

if (Op.getOpcode() == ISD::ZERO_EXTEND) {

2912

// zext(undef) upper bits are guaranteed to be zero.

2913

if (DemandedElts.isSubsetOf(KnownUndef))

2914

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2915

KnownUndef.clearAllBits();

2916

}

2917

break;

2918

default: {

2919

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2920

if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,

2921

KnownZero, TLO, Depth))

2922

return true;

2923

} else {

2924

KnownBits Known;

2925

APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);

2926

if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,

2927

TLO, Depth, AssumeSingleUse))

2928

return true;

2929

}

2930

break;

2931

}

2932

}

2933

assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2933, __extension__ __PRETTY_FUNCTION__));

2934

2935

// Constant fold all undef cases.

2936

// TODO: Handle zero cases as well.

2937

if (DemandedElts.isSubsetOf(KnownUndef))

2938

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2939

2940

return false;

2941

}

2942

2943

/// Determine which of the bits specified in Mask are known to be either zero or

2944

/// one and return them in the Known.

2945

void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,

2946

KnownBits &Known,

2947

const APInt &DemandedElts,

2948

const SelectionDAG &DAG,

2949

unsigned Depth) const {

2950

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))

2951

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))

2952

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))

2953

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))

2954

"Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))

2955

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__));

2956

Known.resetAll();

2957

}

2958

2959

void TargetLowering::computeKnownBitsForTargetInstr(

2960

GISelKnownBits &Analysis, Register R, KnownBits &Known,

2961

const APInt &DemandedElts, const MachineRegisterInfo &MRI,

2962

unsigned Depth) const {

2963

Known.resetAll();

2964

}

2965

2966

void TargetLowering::computeKnownBitsForFrameIndex(

2967

const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {

2968

// The low bits are known zero if the pointer is aligned.

2969

Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));

2970

}

2971

2972

Align TargetLowering::computeKnownAlignForTargetInstr(

2973

GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,

2974

unsigned Depth) const {

2975

return Align(1);

2976

}

2977

2978

/// This method can be implemented by targets that want to expose additional

2979

/// information about sign bits to the DAG Combiner.

2980

unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,

2981

const APInt &,

2982

const SelectionDAG &,

2983

unsigned Depth) const {

2984

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))

2985

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))

2986

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))

2987

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))

2988

"Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))

2989

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__));

2990

return 1;

2991

}

2992

2993

unsigned TargetLowering::computeNumSignBitsForTargetInstr(

2994

GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,

2995

const MachineRegisterInfo &MRI, unsigned Depth) const {

2996

return 1;

2997

}

2998

2999

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(

3000

SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,

3001

TargetLoweringOpt &TLO, unsigned Depth) const {

3002

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))

3003

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))

3004

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))

3005

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))

3006

"Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))

3007

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__));

3008

return false;

3009

}

3010

3011

bool TargetLowering::SimplifyDemandedBitsForTargetNode(

3012

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3013

KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {

3014

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))

3015

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))

3016

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))

3017

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))

3018

"Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))

3019

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__));

3020

computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);

3021

return false;

3022

}

3023

3024

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(

3025

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3026

SelectionDAG &DAG, unsigned Depth) const {

3027

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3028

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3029

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3030

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3031

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3032

"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))

3033

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__));

3034

return SDValue();

3035

}

3036

3037

SDValue

3038

TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,

3039

SDValue N1, MutableArrayRef<int> Mask,

3040

SelectionDAG &DAG) const {

3041

bool LegalMask = isShuffleMaskLegal(Mask, VT);

3042

if (!LegalMask) {

3043

std::swap(N0, N1);

3044

ShuffleVectorSDNode::commuteMask(Mask);

3045

LegalMask = isShuffleMaskLegal(Mask, VT);

3046

}

3047

3048

if (!LegalMask)

3049

return SDValue();

3050

3051

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);

3052

}

3053

3054

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {

3055

return nullptr;

3056

}

3057

3058

bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(

3059

SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,

3060

bool PoisonOnly, unsigned Depth) const {

3061

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3062

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3063

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3064

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3065

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3066

"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))

3067

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__));

3068

return false;

3069

}

3070

3071

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,

3072

const SelectionDAG &DAG,

3073

bool SNaN,

3074

unsigned Depth) const {

3075

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))

3076

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))

3077

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))

3078

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))

3079

"Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))

3080

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__));

3081

return false;

3082

}

3083

3084

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must

3085

// work with truncating build vectors and vectors with elements of less than

3086

// 8 bits.

3087

bool TargetLowering::isConstTrueVal(const SDNode *N) const {

3088

if (!N)

3089

return false;

3090

3091

APInt CVal;

3092

if (auto *CN = dyn_cast<ConstantSDNode>(N)) {

3093

CVal = CN->getAPIntValue();

3094

} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {

3095

auto *CN = BV->getConstantSplatNode();

3096

if (!CN)

3097

return false;

3098

3099

// If this is a truncating build vector, truncate the splat value.

3100

// Otherwise, we may fail to match the expected values below.

3101

unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();

3102

CVal = CN->getAPIntValue();

3103

if (BVEltWidth < CVal.getBitWidth())

3104

CVal = CVal.trunc(BVEltWidth);

3105

} else {

3106

return false;

3107

}

3108

3109

switch (getBooleanContents(N->getValueType(0))) {

3110

case UndefinedBooleanContent:

3111

return CVal[0];

3112

case ZeroOrOneBooleanContent:

3113

return CVal.isOne();

3114

case ZeroOrNegativeOneBooleanContent:

3115

return CVal.isAllOnes();

3116

}

3117

3118

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3118);

3119

}

3120

3121

bool TargetLowering::isConstFalseVal(const SDNode *N) const {

3122

if (!N)

3123

return false;

3124

3125

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);

3126

if (!CN) {

3127

const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);

3128

if (!BV)

3129

return false;

3130

3131

// Only interested in constant splats, we don't care about undef

3132

// elements in identifying boolean constants and getConstantSplatNode

3133

// returns NULL if all ops are undef;

3134

CN = BV->getConstantSplatNode();

3135

if (!CN)

3136

return false;

3137

}

3138

3139

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)

3140

return !CN->getAPIntValue()[0];

3141

3142

return CN->isZero();

3143

}

3144

3145

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,

3146

bool SExt) const {

3147

if (VT == MVT::i1)

3148

return N->isOne();

3149

3150

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);

3151

switch (Cnt) {

3152

case TargetLowering::ZeroOrOneBooleanContent:

3153

// An extended value of 1 is always true, unless its original type is i1,

3154

// in which case it will be sign extended to -1.

3155

return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));

3156

case TargetLowering::UndefinedBooleanContent:

3157

case TargetLowering::ZeroOrNegativeOneBooleanContent:

3158

return N->isAllOnes() && SExt;

3159

}

3160

llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3160);

3161

}

3162

3163

/// This helper function of SimplifySetCC tries to optimize the comparison when

3164

/// either operand of the SetCC node is a bitwise-and instruction.

3165

SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,

3166

ISD::CondCode Cond, const SDLoc &DL,

3167

DAGCombinerInfo &DCI) const {

3168

// Match these patterns in any of their permutations:

3169

// (X & Y) == Y

3170

// (X & Y) != Y

3171

if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)

3172

std::swap(N0, N1);

3173

3174

EVT OpVT = N0.getValueType();

3175

if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||

3176

(Cond != ISD::SETEQ && Cond != ISD::SETNE))

3177

return SDValue();

3178

3179

SDValue X, Y;

3180

if (N0.getOperand(0) == N1) {

3181

X = N0.getOperand(1);

3182

Y = N0.getOperand(0);

3183

} else if (N0.getOperand(1) == N1) {

3184

X = N0.getOperand(0);

3185

Y = N0.getOperand(1);

3186

} else {

3187

return SDValue();

3188

}

3189

3190

SelectionDAG &DAG = DCI.DAG;

3191

SDValue Zero = DAG.getConstant(0, DL, OpVT);

3192

if (DAG.isKnownToBeAPowerOfTwo(Y)) {

3193

// Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.

3194

// Note that where Y is variable and is known to have at most one bit set

3195

// (for example, if it is Z & 1) we cannot do this; the expressions are not

3196

// equivalent when Y == 0.

3197

assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3197, __extension__ __PRETTY_FUNCTION__));

3198

Cond = ISD::getSetCCInverse(Cond, OpVT);

3199

if (DCI.isBeforeLegalizeOps() ||

3200

isCondCodeLegal(Cond, N0.getSimpleValueType()))

3201

return DAG.getSetCC(DL, VT, N0, Zero, Cond);

3202

} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {

3203

// If the target supports an 'and-not' or 'and-complement' logic operation,

3204

// try to use that to make a comparison operation more efficient.

3205

// But don't do this transform if the mask is a single bit because there are

3206

// more efficient ways to deal with that case (for example, 'bt' on x86 or

3207

// 'rlwinm' on PPC).

3208

3209

// Bail out if the compare operand that we want to turn into a zero is

3210

// already a zero (otherwise, infinite loop).

3211

auto *YConst = dyn_cast<ConstantSDNode>(Y);

3212

if (YConst && YConst->isZero())

3213

return SDValue();

3214

3215

// Transform this into: ~X & Y == 0.

3216

SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);

3217

SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);

3218

return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);

3219

}

3220

3221

return SDValue();

3222

}

3223

3224

/// There are multiple IR patterns that could be checking whether certain

3225

/// truncation of a signed number would be lossy or not. The pattern which is

3226

/// best at IR level, may not lower optimally. Thus, we want to unfold it.

3227

/// We are looking for the following pattern: (KeptBits is a constant)

3228

/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)

3229

/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.

3230

/// KeptBits also can't be 1, that would have been folded to %x dstcond 0

3231

/// We will unfold it into the natural trunc+sext pattern:

3232

/// ((%x << C) a>> C) dstcond %x

3233

/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)

3234

SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(

3235

EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,

3236

const SDLoc &DL) const {

3237

// We must be comparing with a constant.

3238

ConstantSDNode *C1;

3239

if (!(C1 = dyn_cast<ConstantSDNode>(N1)))

3240

return SDValue();

3241

3242

// N0 should be: add %x, (1 << (KeptBits-1))

3243

if (N0->getOpcode() != ISD::ADD)

3244

return SDValue();

3245

3246

// And we must be 'add'ing a constant.

3247

ConstantSDNode *C01;

3248

if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))

3249

return SDValue();

3250

3251

SDValue X = N0->getOperand(0);

3252

EVT XVT = X.getValueType();

3253

3254

// Validate constants ...

3255

3256

APInt I1 = C1->getAPIntValue();

3257

3258

ISD::CondCode NewCond;

3259

if (Cond == ISD::CondCode::SETULT) {

3260

NewCond = ISD::CondCode::SETEQ;

3261

} else if (Cond == ISD::CondCode::SETULE) {

3262

NewCond = ISD::CondCode::SETEQ;

3263

// But need to 'canonicalize' the constant.

3264

I1 += 1;

3265

} else if (Cond == ISD::CondCode::SETUGT) {

3266

NewCond = ISD::CondCode::SETNE;

3267

// But need to 'canonicalize' the constant.

3268

I1 += 1;

3269

} else if (Cond == ISD::CondCode::SETUGE) {

3270

NewCond = ISD::CondCode::SETNE;

3271

} else

3272

return SDValue();

3273

3274

APInt I01 = C01->getAPIntValue();

3275

3276

auto checkConstants = [&I1, &I01]() -> bool {

3277

// Both of them must be power-of-two, and the constant from setcc is bigger.

3278

return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();

3279

};

3280

3281

if (checkConstants()) {

3282

// Great, e.g. got icmp ult i16 (add i16 %x, 128), 256

3283

} else {

3284

// What if we invert constants? (and the target predicate)

3285

I1.negate();

3286

I01.negate();

3287

assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3287, __extension__ __PRETTY_FUNCTION__));

3288

NewCond = getSetCCInverse(NewCond, XVT);

3289

if (!checkConstants())

3290

return SDValue();

3291

// Great, e.g. got icmp uge i16 (add i16 %x, -128), -256

3292

}

3293

3294

// They are power-of-two, so which bit is set?

3295

const unsigned KeptBits = I1.logBase2();

3296

const unsigned KeptBitsMinusOne = I01.logBase2();

3297

3298

// Magic!

3299

if (KeptBits != (KeptBitsMinusOne + 1))

3300

return SDValue();

3301

assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3301, __extension__ __PRETTY_FUNCTION__));

3302

3303

// We don't want to do this in every single case.

3304

SelectionDAG &DAG = DCI.DAG;

3305

if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(

3306

XVT, KeptBits))

3307

return SDValue();

3308

3309

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;

3310

assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3310, __extension__ __PRETTY_FUNCTION__));

3311

3312

// Unfold into: ((%x << C) a>> C) cond %x

3313

// Where 'cond' will be either 'eq' or 'ne'.

3314

SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);

3315

SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);

3316

SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);

3317

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

3318

3319

return T2;

3320

}

3321

3322

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3323

SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(

3324

EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,

3325

DAGCombinerInfo &DCI, const SDLoc &DL) const {

3326

assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))

3327

isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))

3328

"Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__));

3329

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__))

3330

"Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__));

3331

3332

unsigned NewShiftOpcode;

3333

SDValue X, C, Y;

3334

3335

SelectionDAG &DAG = DCI.DAG;

3336

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3337

3338

// Look for '(C l>>/<< Y)'.

3339

auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {

3340

// The shift should be one-use.

3341

if (!V.hasOneUse())

3342

return false;

3343

unsigned OldShiftOpcode = V.getOpcode();

3344

switch (OldShiftOpcode) {

3345

case ISD::SHL:

3346

NewShiftOpcode = ISD::SRL;

3347

break;

3348

case ISD::SRL:

3349

NewShiftOpcode = ISD::SHL;

3350

break;

3351

default:

3352

return false; // must be a logical shift.

3353

}

3354

// We should be shifting a constant.

3355

// FIXME: best to use isConstantOrConstantVector().

3356

C = V.getOperand(0);

3357

ConstantSDNode *CC =

3358

isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3359

if (!CC)

3360

return false;

3361

Y = V.getOperand(1);

3362

3363

ConstantSDNode *XC =

3364

isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3365

return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(

3366

X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);

3367

};

3368

3369

// LHS of comparison should be an one-use 'and'.

3370

if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())

3371

return SDValue();

3372

3373

X = N0.getOperand(0);

3374

SDValue Mask = N0.getOperand(1);

3375

3376

// 'and' is commutative!

3377

if (!Match(Mask)) {

3378

std::swap(X, Mask);

3379

if (!Match(Mask))

3380

return SDValue();

3381

}

3382

3383

EVT VT = X.getValueType();

3384

3385

// Produce:

3386

// ((X 'OppositeShiftOpcode' Y) & C) Cond 0

3387

SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);

3388

SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);

3389

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);

3390

return T2;

3391

}

3392

3393

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as

3394

/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to

3395

/// handle the commuted versions of these patterns.

3396

SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,

3397

ISD::CondCode Cond, const SDLoc &DL,

3398

DAGCombinerInfo &DCI) const {

3399

unsigned BOpcode = N0.getOpcode();

3400

assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__))

3401

"Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__));

3402

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__));

3403

3404

// (X + Y) == X --> Y == 0

3405

// (X - Y) == X --> Y == 0

3406

// (X ^ Y) == X --> Y == 0

3407

SelectionDAG &DAG = DCI.DAG;

3408

EVT OpVT = N0.getValueType();

3409

SDValue X = N0.getOperand(0);

3410

SDValue Y = N0.getOperand(1);

3411

if (X == N1)

3412

return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

3413

3414

if (Y != N1)

3415

return SDValue();

3416

3417

// (X + Y) == Y --> X == 0

3418

// (X ^ Y) == Y --> X == 0

3419

if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)

3420

return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

3421

3422

// The shift would not be valid if the operands are boolean (i1).

3423

if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)

3424

return SDValue();

3425

3426

// (X - Y) == Y --> X == Y << 1

3427

EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),

3428

!DCI.isBeforeLegalize());

3429

SDValue One = DAG.getConstant(1, DL, ShiftVT);

3430

SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);

3431

if (!DCI.isCalledByLegalizer())

3432

DCI.AddToWorklist(YShl1.getNode());

3433

return DAG.getSetCC(DL, VT, X, YShl1, Cond);

3434

}

3435

3436

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,

3437

SDValue N0, const APInt &C1,

3438

ISD::CondCode Cond, const SDLoc &dl,

3439

SelectionDAG &DAG) {

3440

// Look through truncs that don't change the value of a ctpop.

3441

// FIXME: Add vector support? Need to be careful with setcc result type below.

3442

SDValue CTPOP = N0;

3443

if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&

3444

N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))

3445

CTPOP = N0.getOperand(0);

3446

3447

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())

3448

return SDValue();

3449

3450

EVT CTVT = CTPOP.getValueType();

3451

SDValue CTOp = CTPOP.getOperand(0);

3452

3453

// If this is a vector CTPOP, keep the CTPOP if it is legal.

3454

// TODO: Should we check if CTPOP is legal(or custom) for scalars?

3455

if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))

3456

return SDValue();

3457

3458

// (ctpop x) u< 2 -> (x & x-1) == 0

3459

// (ctpop x) u> 1 -> (x & x-1) != 0

3460

if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {

3461

unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);

3462

if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))

3463

return SDValue();

3464

if (C1 == 0 && (Cond == ISD::SETULT))

3465

return SDValue(); // This is handled elsewhere.

3466

3467

unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

3468

3469

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3470

SDValue Result = CTOp;

3471

for (unsigned i = 0; i < Passes; i++) {

3472

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);

3473

Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);

3474

}

3475

ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;

3476

return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);

3477

}

3478

3479

// If ctpop is not supported, expand a power-of-2 comparison based on it.

3480

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {

3481

// For scalars, keep CTPOP if it is legal or custom.

3482

if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))

3483

return SDValue();

3484

// This is based on X86's custom lowering for CTPOP which produces more

3485

// instructions than the expansion here.

3486

3487

// (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)

3488

// (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)

3489

SDValue Zero = DAG.getConstant(0, dl, CTVT);

3490

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3491

assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3491, __extension__ __PRETTY_FUNCTION__));

3492

ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);

3493

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

3494

SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

3495

SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);

3496

SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);

3497

unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;

3498

return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);

3499

}

3500

3501

return SDValue();

3502

}

3503

3504

/// Try to simplify a setcc built with the specified operands and cc. If it is

3505

/// unable to simplify it, return a null SDValue.

3506

SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,

3507

ISD::CondCode Cond, bool foldBooleans,

3508

DAGCombinerInfo &DCI,

3509

const SDLoc &dl) const {

3510

SelectionDAG &DAG = DCI.DAG;

3511

const DataLayout &Layout = DAG.getDataLayout();

3512

EVT OpVT = N0.getValueType();

3513

3514

// Constant fold or commute setcc.

3515

if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))

3516

return Fold;

3517

3518

// Ensure that the constant occurs on the RHS and fold constant comparisons.

3519

// TODO: Handle non-splat vector constants. All undef causes trouble.

3520

// FIXME: We can't yet fold constant scalable vector splats, so avoid an

3521

// infinite loop here when we encounter one.

3522

ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);

3523

if (isConstOrConstSplat(N0) &&

3524

(!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&

3525

(DCI.isBeforeLegalizeOps() ||

3526

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))

3527

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3528

3529

// If we have a subtract with the same 2 non-constant operands as this setcc

3530

// -- but in reverse order -- then try to commute the operands of this setcc

3531

// to match. A matching pair of setcc (cmp) and sub may be combined into 1

3532

// instruction on some targets.

3533

if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&

3534

(DCI.isBeforeLegalizeOps() ||

3535

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&

3536

DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&

3537

!DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))

3538

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3539

3540

if (auto *N1C = isConstOrConstSplat(N1)) {

3541

const APInt &C1 = N1C->getAPIntValue();

3542

3543

// Optimize some CTPOP cases.

3544

if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))

3545

return V;

3546

3547

// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an

3548

// equality comparison, then we're just comparing whether X itself is

3549

// zero.

3550

if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&

3551

N0.getOperand(0).getOpcode() == ISD::CTLZ &&

3552

isPowerOf2_32(N0.getScalarValueSizeInBits())) {

3553

if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {

3554

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3555

ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {

3556

if ((C1 == 0) == (Cond == ISD::SETEQ)) {

3557

// (srl (ctlz x), 5) == 0 -> X != 0

3558

// (srl (ctlz x), 5) != 1 -> X != 0

3559

Cond = ISD::SETNE;

3560

} else {

3561

// (srl (ctlz x), 5) != 0 -> X == 0

3562

// (srl (ctlz x), 5) == 1 -> X == 0

3563

Cond = ISD::SETEQ;

3564

}

3565

SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());

3566

return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,

3567

Cond);

3568

}

3569

}

3570

}

3571

}

3572

3573

// FIXME: Support vectors.

3574

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

3575

const APInt &C1 = N1C->getAPIntValue();

3576

3577

// (zext x) == C --> x == (trunc C)

3578

// (sext x) == C --> x == (trunc C)

3579

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3580

DCI.isBeforeLegalize() && N0->hasOneUse()) {

3581

unsigned MinBits = N0.getValueSizeInBits();

3582

SDValue PreExt;

3583

bool Signed = false;

3584

if (N0->getOpcode() == ISD::ZERO_EXTEND) {

3585

// ZExt

3586

MinBits = N0->getOperand(0).getValueSizeInBits();

3587

PreExt = N0->getOperand(0);

3588

} else if (N0->getOpcode() == ISD::AND) {

3589

// DAGCombine turns costly ZExts into ANDs

3590

if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))

3591

if ((C->getAPIntValue()+1).isPowerOf2()) {

3592

MinBits = C->getAPIntValue().countTrailingOnes();

3593

PreExt = N0->getOperand(0);

3594

}

3595

} else if (N0->getOpcode() == ISD::SIGN_EXTEND) {

3596

// SExt

3597

MinBits = N0->getOperand(0).getValueSizeInBits();

3598

PreExt = N0->getOperand(0);

3599

Signed = true;

3600

} else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {

3601

// ZEXTLOAD / SEXTLOAD

3602

if (LN0->getExtensionType() == ISD::ZEXTLOAD) {

3603

MinBits = LN0->getMemoryVT().getSizeInBits();

3604

PreExt = N0;

3605

} else if (LN0->getExtensionType() == ISD::SEXTLOAD) {

3606

Signed = true;

3607

MinBits = LN0->getMemoryVT().getSizeInBits();

3608

PreExt = N0;

3609

}

3610

}

3611

3612

// Figure out how many bits we need to preserve this constant.

3613

unsigned ReqdBits = Signed ?

3614

C1.getBitWidth() - C1.getNumSignBits() + 1 :

3615

C1.getActiveBits();

3616

3617

// Make sure we're not losing bits from the constant.

3618

if (MinBits > 0 &&

3619

MinBits < C1.getBitWidth() &&

3620

MinBits >= ReqdBits) {

3621

EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);

3622

if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {

3623

// Will get folded away.

3624

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);

3625

if (MinBits == 1 && C1 == 1)

3626

// Invert the condition.

3627

return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),

3628

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3629

SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);

3630

return DAG.getSetCC(dl, VT, Trunc, C, Cond);

3631

}

3632

3633

// If truncating the setcc operands is not desirable, we can still

3634

// simplify the expression in some cases:

3635

// setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)

3636

// setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))

3637

// setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))

3638

// setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)

3639

// setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))

3640

// setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)

3641

SDValue TopSetCC = N0->getOperand(0);

3642

unsigned N0Opc = N0->getOpcode();

3643

bool SExt = (N0Opc == ISD::SIGN_EXTEND);

3644

if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&

3645

TopSetCC.getOpcode() == ISD::SETCC &&

3646

(N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&

3647

(isConstFalseVal(N1C) ||

3648

isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

3649

3650

bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||

3651

(!N1C->isZero() && Cond == ISD::SETNE);

3652

3653

if (!Inverse)

3654

return TopSetCC;

3655

3656

ISD::CondCode InvCond = ISD::getSetCCInverse(

3657

cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),

3658

TopSetCC.getOperand(0).getValueType());

3659

return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),

3660

TopSetCC.getOperand(1),

3661

InvCond);

3662

}

3663

}

3664

}

3665

3666

// If the LHS is '(and load, const)', the RHS is 0, the test is for

3667

// equality or unsigned, and all 1 bits of the const are in the same

3668

// partial word, see if we can shorten the load.

3669

if (DCI.isBeforeLegalize() &&

3670

!ISD::isSignedIntSetCC(Cond) &&

3671

N0.getOpcode() == ISD::AND && C1 == 0 &&

3672

N0.getNode()->hasOneUse() &&

3673

isa<LoadSDNode>(N0.getOperand(0)) &&

3674

N0.getOperand(0).getNode()->hasOneUse() &&

3675

isa<ConstantSDNode>(N0.getOperand(1))) {

3676

LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));

3677

APInt bestMask;

3678

unsigned bestWidth = 0, bestOffset = 0;

3679

if (Lod->isSimple() && Lod->isUnindexed()) {

3680

unsigned origWidth = N0.getValueSizeInBits();

3681

unsigned maskWidth = origWidth;

3682

// We can narrow (e.g.) 16-bit extending loads on 32-bit target to

3683

// 8 bits, but have to be careful...

3684

if (Lod->getExtensionType() != ISD::NON_EXTLOAD)

3685

origWidth = Lod->getMemoryVT().getSizeInBits();

3686

const APInt &Mask = N0.getConstantOperandAPInt(1);

3687

for (unsigned width = origWidth / 2; width>=8; width /= 2) {

3688

APInt newMask = APInt::getLowBitsSet(maskWidth, width);

3689

for (unsigned offset=0; offset<origWidth/width; offset++) {

3690

if (Mask.isSubsetOf(newMask)) {

3691

if (Layout.isLittleEndian())

3692

bestOffset = (uint64_t)offset * (width/8);

3693

else

3694

bestOffset = (origWidth/width - offset - 1) * (width/8);

3695

bestMask = Mask.lshr(offset * (width/8) * 8);

3696

bestWidth = width;

3697

break;

3698

}

3699

newMask <<= width;

3700

}

3701

}

3702

}

3703

if (bestWidth) {

3704

EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);

3705

if (newVT.isRound() &&

3706

shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {

3707

SDValue Ptr = Lod->getBasePtr();

3708

if (bestOffset != 0)

3709

Ptr =

3710

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);

3711

SDValue NewLoad =

3712

DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,

3713

Lod->getPointerInfo().getWithOffset(bestOffset),

3714

Lod->getOriginalAlign());

3715

return DAG.getSetCC(dl, VT,

3716

DAG.getNode(ISD::AND, dl, newVT, NewLoad,

3717

DAG.getConstant(bestMask.trunc(bestWidth),

3718

dl, newVT)),

3719

DAG.getConstant(0LL, dl, newVT), Cond);

3720

}

3721

}

3722

}

3723

3724

// If the LHS is a ZERO_EXTEND, perform the comparison on the input.

3725

if (N0.getOpcode() == ISD::ZERO_EXTEND) {

3726

unsigned InSize = N0.getOperand(0).getValueSizeInBits();

3727

3728

// If the comparison constant has bits in the upper part, the

3729

// zero-extended value could never match.

3730

if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),

3731

C1.getBitWidth() - InSize))) {

3732

switch (Cond) {

3733

case ISD::SETUGT:

3734

case ISD::SETUGE:

3735

case ISD::SETEQ:

3736

return DAG.getConstant(0, dl, VT);

3737

case ISD::SETULT:

3738

case ISD::SETULE:

3739

case ISD::SETNE:

3740

return DAG.getConstant(1, dl, VT);

3741

case ISD::SETGT:

3742

case ISD::SETGE:

3743

// True if the sign bit of C1 is set.

3744

return DAG.getConstant(C1.isNegative(), dl, VT);

3745

case ISD::SETLT:

3746

case ISD::SETLE:

3747

// True if the sign bit of C1 isn't set.

3748

return DAG.getConstant(C1.isNonNegative(), dl, VT);

3749

default:

3750

break;

3751

}

3752

}

3753

3754

// Otherwise, we can perform the comparison with the low bits.

3755

switch (Cond) {

3756

case ISD::SETEQ:

3757

case ISD::SETNE:

3758

case ISD::SETUGT:

3759

case ISD::SETUGE:

3760

case ISD::SETULT:

3761

case ISD::SETULE: {

3762

EVT newVT = N0.getOperand(0).getValueType();

3763

if (DCI.isBeforeLegalizeOps() ||

3764

(isOperationLegal(ISD::SETCC, newVT) &&

3765

isCondCodeLegal(Cond, newVT.getSimpleVT()))) {

3766

EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);

3767

SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

3768

3769

SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),

3770

NewConst, Cond);

3771

return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());

3772

}

3773

break;

3774

}

3775

default:

3776

break; // todo, be more careful with signed comparisons

3777

}

3778

} else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&

3779

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3780

!isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),

3781

OpVT)) {

3782

EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();

3783

unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();

3784

EVT ExtDstTy = N0.getValueType();

3785

unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

3786

3787

// If the constant doesn't fit into the number of bits for the source of

3788

// the sign extension, it is impossible for both sides to be equal.

3789

if (C1.getMinSignedBits() > ExtSrcTyBits)

3790

return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

3791

3792

assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__))

3793

ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__));

3794

APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);

3795

SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),

3796

DAG.getConstant(Imm, dl, ExtDstTy));

3797

if (!DCI.isCalledByLegalizer())

3798

DCI.AddToWorklist(ZextOp.getNode());

3799

// Otherwise, make this a use of a zext.

3800

return DAG.getSetCC(dl, VT, ZextOp,

3801

DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);

3802

} else if ((N1C->isZero() || N1C->isOne()) &&

3803

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3804

// SETCC (SETCC), [0|1], [EQ|NE] -> SETCC

3805

if (N0.getOpcode() == ISD::SETCC &&

3806

isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&

3807

(N0.getValueType() == MVT::i1 ||

3808

getBooleanContents(N0.getOperand(0).getValueType()) ==

3809

ZeroOrOneBooleanContent)) {

3810

bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());

3811

if (TrueWhenTrue)

3812

return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);

3813

// Invert the condition.

3814

ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();

3815

CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());

3816

if (DCI.isBeforeLegalizeOps() ||

3817

isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))

3818

return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);

3819

}

3820

3821

if ((N0.getOpcode() == ISD::XOR ||

3822

(N0.getOpcode() == ISD::AND &&

3823

N0.getOperand(0).getOpcode() == ISD::XOR &&

3824

N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&

3825

isOneConstant(N0.getOperand(1))) {

3826

// If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We

3827

// can only do this if the top bits are known zero.

3828

unsigned BitWidth = N0.getValueSizeInBits();

3829

if (DAG.MaskedValueIsZero(N0,

3830

APInt::getHighBitsSet(BitWidth,

3831

BitWidth-1))) {

3832

// Okay, get the un-inverted input value.

3833

SDValue Val;

3834

if (N0.getOpcode() == ISD::XOR) {

3835

Val = N0.getOperand(0);

3836

} else {

3837

assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__))

3838

N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__));

3839

// ((X^1)&1)^1 -> X & 1

3840

Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),

3841

N0.getOperand(0).getOperand(0),

3842

N0.getOperand(1));

3843

}

3844

3845

return DAG.getSetCC(dl, VT, Val, N1,

3846

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3847

}

3848

} else if (N1C->isOne()) {

3849

SDValue Op0 = N0;

3850

if (Op0.getOpcode() == ISD::TRUNCATE)

3851

Op0 = Op0.getOperand(0);

3852

3853

if ((Op0.getOpcode() == ISD::XOR) &&

3854

Op0.getOperand(0).getOpcode() == ISD::SETCC &&

3855

Op0.getOperand(1).getOpcode() == ISD::SETCC) {

3856

SDValue XorLHS = Op0.getOperand(0);

3857

SDValue XorRHS = Op0.getOperand(1);

3858

// Ensure that the input setccs return an i1 type or 0/1 value.

3859

if (Op0.getValueType() == MVT::i1 ||

3860

(getBooleanContents(XorLHS.getOperand(0).getValueType()) ==

3861

ZeroOrOneBooleanContent &&

3862

getBooleanContents(XorRHS.getOperand(0).getValueType()) ==

3863

ZeroOrOneBooleanContent)) {

3864

// (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)

3865

Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;

3866

return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);

3867

}

3868

}

3869

if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {

3870

// If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.

3871

if (Op0.getValueType().bitsGT(VT))

3872

Op0 = DAG.getNode(ISD::AND, dl, VT,

3873

DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),

3874

DAG.getConstant(1, dl, VT));

3875

else if (Op0.getValueType().bitsLT(VT))

3876

Op0 = DAG.getNode(ISD::AND, dl, VT,

3877

DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),

3878

DAG.getConstant(1, dl, VT));

3879

3880

return DAG.getSetCC(dl, VT, Op0,

3881

DAG.getConstant(0, dl, Op0.getValueType()),

3882

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3883

}

3884

if (Op0.getOpcode() == ISD::AssertZext &&

3885

cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)

3886

return DAG.getSetCC(dl, VT, Op0,

3887

DAG.getConstant(0, dl, Op0.getValueType()),

3888

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3889

}

3890

}

3891

3892

// Given:

3893

// icmp eq/ne (urem %x, %y), 0

3894

// Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':

3895

// icmp eq/ne %x, 0

3896

if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&

3897

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3898

KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));

3899

KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));

3900

if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)

3901

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);

3902

}

3903

3904

// Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0

3905

// and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0

3906

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3907

N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&

3908

N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&

3909

N1C && N1C->isAllOnes()) {

3910

return DAG.getSetCC(dl, VT, N0.getOperand(0),

3911

DAG.getConstant(0, dl, OpVT),

3912

Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);

3913

}

3914

3915

if (SDValue V =

3916

optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))

3917

return V;

3918

}

3919

3920

// These simplifications apply to splat vectors as well.

3921

// TODO: Handle more splat vector cases.

3922

if (auto *N1C = isConstOrConstSplat(N1)) {

3923

const APInt &C1 = N1C->getAPIntValue();

3924

3925

APInt MinVal, MaxVal;

3926

unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();

3927

if (ISD::isSignedIntSetCC(Cond)) {

3928

MinVal = APInt::getSignedMinValue(OperandBitSize);

3929

MaxVal = APInt::getSignedMaxValue(OperandBitSize);

3930

} else {

3931

MinVal = APInt::getMinValue(OperandBitSize);

3932

MaxVal = APInt::getMaxValue(OperandBitSize);

3933

}

3934

3935

// Canonicalize GE/LE comparisons to use GT/LT comparisons.

3936

if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {

3937

// X >= MIN --> true

3938

if (C1 == MinVal)

3939

return DAG.getBoolConstant(true, dl, VT, OpVT);

3940

3941

if (!VT.isVector()) { // TODO: Support this for vectors.

3942

// X >= C0 --> X > (C0 - 1)

3943

APInt C = C1 - 1;

3944

ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;

3945

if ((DCI.isBeforeLegalizeOps() ||

3946

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3947

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3948

isLegalICmpImmediate(C.getSExtValue())))) {

3949

return DAG.getSetCC(dl, VT, N0,

3950

DAG.getConstant(C, dl, N1.getValueType()),

3951

NewCC);

3952

}

3953

}

3954

}

3955

3956

if (Cond == ISD::SETLE || Cond == ISD::SETULE) {

3957

// X <= MAX --> true

3958

if (C1 == MaxVal)

3959

return DAG.getBoolConstant(true, dl, VT, OpVT);

3960

3961

// X <= C0 --> X < (C0 + 1)

3962

if (!VT.isVector()) { // TODO: Support this for vectors.

3963

APInt C = C1 + 1;

3964

ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;

3965

if ((DCI.isBeforeLegalizeOps() ||

3966

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3967

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3968

isLegalICmpImmediate(C.getSExtValue())))) {

3969

return DAG.getSetCC(dl, VT, N0,

3970

DAG.getConstant(C, dl, N1.getValueType()),

3971

NewCC);

3972

}

3973

}

3974

}

3975

3976

if (Cond == ISD::SETLT || Cond == ISD::SETULT) {

3977

if (C1 == MinVal)

3978

return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

3979

3980

// TODO: Support this for vectors after legalize ops.

3981

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3982

// Canonicalize setlt X, Max --> setne X, Max

3983

if (C1 == MaxVal)

3984

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3985

3986

// If we have setult X, 1, turn it into seteq X, 0

3987

if (C1 == MinVal+1)

3988

return DAG.getSetCC(dl, VT, N0,

3989

DAG.getConstant(MinVal, dl, N0.getValueType()),

3990

ISD::SETEQ);

3991

}

3992

}

3993

3994

if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {

3995

if (C1 == MaxVal)

3996

return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

3997

3998

// TODO: Support this for vectors after legalize ops.

3999

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4000

// Canonicalize setgt X, Min --> setne X, Min

4001

if (C1 == MinVal)

4002

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

4003

4004

// If we have setugt X, Max-1, turn it into seteq X, Max

4005

if (C1 == MaxVal-1)

4006

return DAG.getSetCC(dl, VT, N0,

4007

DAG.getConstant(MaxVal, dl, N0.getValueType()),

4008

ISD::SETEQ);

4009

}

4010

}

4011

4012

if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {

4013

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

4014

if (C1.isZero())

4015

if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(

4016

VT, N0, N1, Cond, DCI, dl))

4017

return CC;

4018

4019

// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).

4020

// For example, when high 32-bits of i64 X are known clear:

4021

// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0

4022

// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1

4023

bool CmpZero = N1C->getAPIntValue().isZero();

4024

bool CmpNegOne = N1C->getAPIntValue().isAllOnes();

4025

if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {

4026

// Match or(lo,shl(hi,bw/2)) pattern.

4027

auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {

4028

unsigned EltBits = V.getScalarValueSizeInBits();

4029

if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)

4030

return false;

4031

SDValue LHS = V.getOperand(0);

4032

SDValue RHS = V.getOperand(1);

4033

APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);

4034

// Unshifted element must have zero upperbits.

4035

if (RHS.getOpcode() == ISD::SHL &&

4036

isa<ConstantSDNode>(RHS.getOperand(1)) &&

4037

RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4038

DAG.MaskedValueIsZero(LHS, HiBits)) {

4039

Lo = LHS;

4040

Hi = RHS.getOperand(0);

4041

return true;

4042

}

4043

if (LHS.getOpcode() == ISD::SHL &&

4044

isa<ConstantSDNode>(LHS.getOperand(1)) &&

4045

LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4046

DAG.MaskedValueIsZero(RHS, HiBits)) {

4047

Lo = RHS;

4048

Hi = LHS.getOperand(0);

4049

return true;

4050

}

4051

return false;

4052

};

4053

4054

auto MergeConcat = [&](SDValue Lo, SDValue Hi) {

4055

unsigned EltBits = N0.getScalarValueSizeInBits();

4056

unsigned HalfBits = EltBits / 2;

4057

APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);

4058

SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);

4059

SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);

4060

SDValue NewN0 =

4061

DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);

4062

SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;

4063

return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);

4064

};

4065

4066

SDValue Lo, Hi;

4067

if (IsConcat(N0, Lo, Hi))

4068

return MergeConcat(Lo, Hi);

4069

4070

if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {

4071

SDValue Lo0, Lo1, Hi0, Hi1;

4072

if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&

4073

IsConcat(N0.getOperand(1), Lo1, Hi1)) {

4074

return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),

4075

DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));

4076

}

4077

}

4078

}

4079

}

4080

4081

// If we have "setcc X, C0", check to see if we can shrink the immediate

4082

// by changing cc.

4083

// TODO: Support this for vectors after legalize ops.

4084

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4085

// SETUGT X, SINTMAX -> SETLT X, 0

4086

// SETUGE X, SINTMIN -> SETLT X, 0

4087

if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||

4088

(Cond == ISD::SETUGE && C1.isMinSignedValue()))

4089

return DAG.getSetCC(dl, VT, N0,

4090

DAG.getConstant(0, dl, N1.getValueType()),

4091

ISD::SETLT);

4092

4093

// SETULT X, SINTMIN -> SETGT X, -1

4094

// SETULE X, SINTMAX -> SETGT X, -1

4095

if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||

4096

(Cond == ISD::SETULE && C1.isMaxSignedValue()))

4097

return DAG.getSetCC(dl, VT, N0,

4098

DAG.getAllOnesConstant(dl, N1.getValueType()),

4099

ISD::SETGT);

4100

}

4101

}

4102

4103

// Back to non-vector simplifications.

4104

// TODO: Can we do these for vector splats?

4105

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

4106

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4107

const APInt &C1 = N1C->getAPIntValue();

4108

EVT ShValTy = N0.getValueType();

4109

4110

// Fold bit comparisons when we can. This will result in an

4111

// incorrect value when boolean false is negative one, unless

4112

// the bitsize is 1 in which case the false value is the same

4113

// in practice regardless of the representation.

4114

if ((VT.getSizeInBits() == 1 ||

4115

getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&

4116

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4117

(VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&

4118

N0.getOpcode() == ISD::AND) {

4119

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4120

EVT ShiftTy =

4121

getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4122

if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3

4123

// Perform the xform if the AND RHS is a single bit.

4124

unsigned ShCt = AndRHS->getAPIntValue().logBase2();

4125

if (AndRHS->getAPIntValue().isPowerOf2() &&

4126

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4127

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4128

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4129

DAG.getConstant(ShCt, dl, ShiftTy)));

4130

}

4131

} else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {

4132

// (X & 8) == 8 --> (X & 8) >> 3

4133

// Perform the xform if C1 is a single bit.

4134

unsigned ShCt = C1.logBase2();

4135

if (C1.isPowerOf2() &&

4136

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4137

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4138

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4139

DAG.getConstant(ShCt, dl, ShiftTy)));

4140

}

4141

}

4142

}

4143

}

4144

4145

if (C1.getMinSignedBits() <= 64 &&

4146

!isLegalICmpImmediate(C1.getSExtValue())) {

4147

EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4148

// (X & -256) == 256 -> (X >> 8) == 1

4149

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4150

N0.getOpcode() == ISD::AND && N0.hasOneUse()) {

4151

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4152

const APInt &AndRHSC = AndRHS->getAPIntValue();

4153

if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {

4154

unsigned ShiftBits = AndRHSC.countTrailingZeros();

4155

if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4156

SDValue Shift =

4157

DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),

4158

DAG.getConstant(ShiftBits, dl, ShiftTy));

4159

SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);

4160

return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);

4161

}

4162

}

4163

}

4164

} else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||

4165

Cond == ISD::SETULE || Cond == ISD::SETUGT) {

4166

bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);

4167

// X < 0x100000000 -> (X >> 32) < 1

4168

// X >= 0x100000000 -> (X >> 32) >= 1

4169

// X <= 0x0ffffffff -> (X >> 32) < 1

4170

// X > 0x0ffffffff -> (X >> 32) >= 1

4171

unsigned ShiftBits;

4172

APInt NewC = C1;

4173

ISD::CondCode NewCond = Cond;

4174

if (AdjOne) {

4175

ShiftBits = C1.countTrailingOnes();

4176

NewC = NewC + 1;

4177

NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;

4178

} else {

4179

ShiftBits = C1.countTrailingZeros();

4180

}

4181

NewC.lshrInPlace(ShiftBits);

4182

if (ShiftBits && NewC.getMinSignedBits() <= 64 &&

4183

isLegalICmpImmediate(NewC.getSExtValue()) &&

4184

!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4185

SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4186

DAG.getConstant(ShiftBits, dl, ShiftTy));

4187

SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);

4188

return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);

4189

}

4190

}

4191

}

4192

}

4193

4194

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {

4195

auto *CFP = cast<ConstantFPSDNode>(N1);

4196

assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4196, __extension__ __PRETTY_FUNCTION__));

4197

4198

// Otherwise, we know the RHS is not a NaN. Simplify the node to drop the

4199

// constant if knowing that the operand is non-nan is enough. We prefer to

4200

// have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to

4201

// materialize 0.0.

4202

if (Cond == ISD::SETO || Cond == ISD::SETUO)

4203

return DAG.getSetCC(dl, VT, N0, N0, Cond);

4204

4205

// setcc (fneg x), C -> setcc swap(pred) x, -C

4206

if (N0.getOpcode() == ISD::FNEG) {

4207

ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);

4208

if (DCI.isBeforeLegalizeOps() ||

4209

isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {

4210

SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);

4211

return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);

4212

}

4213

}

4214

4215

// If the condition is not legal, see if we can find an equivalent one

4216

// which is legal.

4217

if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {

4218

// If the comparison was an awkward floating-point == or != and one of

4219

// the comparison operands is infinity or negative infinity, convert the

4220

// condition to a less-awkward <= or >=.

4221

if (CFP->getValueAPF().isInfinity()) {

4222

bool IsNegInf = CFP->getValueAPF().isNegative();

4223

ISD::CondCode NewCond = ISD::SETCC_INVALID;

4224

switch (Cond) {

4225

case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;

4226

case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;

4227

case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;

4228

case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;

4229

default: break;

4230

}

4231

if (NewCond != ISD::SETCC_INVALID &&

4232

isCondCodeLegal(NewCond, N0.getSimpleValueType()))

4233

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4234

}

4235

}

4236

}

4237

4238

if (N0 == N1) {

4239

// The sext(setcc()) => setcc() optimization relies on the appropriate

4240

// constant being emitted.

4241

assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__))

4242

"Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__));

4243

4244

bool EqTrue = ISD::isTrueWhenEqual(Cond);

4245

unsigned UOF = ISD::getUnorderedFlavor(Cond);

4246

if (UOF == 2) // FP operators that are undefined on NaNs.

4247

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4248

if (UOF == unsigned(EqTrue))

4249

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4250

// Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO

4251

// if it is not already.

4252

ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;

4253

if (NewCond != Cond &&

4254

(DCI.isBeforeLegalizeOps() ||

4255

isCondCodeLegal(NewCond, N0.getSimpleValueType())))

4256

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4257

}

4258

4259

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4260

N0.getValueType().isInteger()) {

4261

if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||

4262

N0.getOpcode() == ISD::XOR) {

4263

// Simplify (X+Y) == (X+Z) --> Y == Z

4264

if (N0.getOpcode() == N1.getOpcode()) {

4265

if (N0.getOperand(0) == N1.getOperand(0))

4266

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);

4267

if (N0.getOperand(1) == N1.getOperand(1))

4268

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);

4269

if (isCommutativeBinOp(N0.getOpcode())) {

4270

// If X op Y == Y op X, try other combinations.

4271

if (N0.getOperand(0) == N1.getOperand(1))

4272

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),

4273

Cond);

4274

if (N0.getOperand(1) == N1.getOperand(0))

4275

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),

4276

Cond);

4277

}

4278

}

4279

4280

// If RHS is a legal immediate value for a compare instruction, we need

4281

// to be careful about increasing register pressure needlessly.

4282

bool LegalRHSImm = false;

4283

4284

if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {

4285

if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4286

// Turn (X+C1) == C2 --> X == C2-C1

4287

if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {

4288

return DAG.getSetCC(dl, VT, N0.getOperand(0),

4289

DAG.getConstant(RHSC->getAPIntValue()-

4290

LHSR->getAPIntValue(),

4291

dl, N0.getValueType()), Cond);

4292

}

4293

4294

// Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.

4295

if (N0.getOpcode() == ISD::XOR)

4296

// If we know that all of the inverted bits are zero, don't bother

4297

// performing the inversion.

4298

if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))

4299

return

4300

DAG.getSetCC(dl, VT, N0.getOperand(0),

4301

DAG.getConstant(LHSR->getAPIntValue() ^

4302

RHSC->getAPIntValue(),

4303

dl, N0.getValueType()),

4304

Cond);

4305

}

4306

4307

// Turn (C1-X) == C2 --> X == C1-C2

4308

if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {

4309

if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {

4310

return

4311

DAG.getSetCC(dl, VT, N0.getOperand(1),

4312

DAG.getConstant(SUBC->getAPIntValue() -

4313

RHSC->getAPIntValue(),

4314

dl, N0.getValueType()),

4315

Cond);

4316

}

4317

}

4318

4319

// Could RHSC fold directly into a compare?

4320

if (RHSC->getValueType(0).getSizeInBits() <= 64)

4321

LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());

4322

}

4323

4324

// (X+Y) == X --> Y == 0 and similar folds.

4325

// Don't do this if X is an immediate that can fold into a cmp

4326

// instruction and X+Y has other uses. It could be an induction variable

4327

// chain, and the transform would increase register pressure.

4328

if (!LegalRHSImm || N0.hasOneUse())

4329

if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))

4330

return V;

4331

}

4332

4333

if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||

4334

N1.getOpcode() == ISD::XOR)

4335

if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))

4336

return V;

4337

4338

if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))

4339

return V;

4340

}

4341

4342

// Fold remainder of division by a constant.

4343

if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&

4344

N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

4345

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

4346

4347

// When division is cheap or optimizing for minimum size,

4348

// fall through to DIVREM creation by skipping this fold.

4349

if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {

4350

if (N0.getOpcode() == ISD::UREM) {

4351

if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))

4352

return Folded;

4353

} else if (N0.getOpcode() == ISD::SREM) {

4354

if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))

4355

return Folded;

4356

}

4357

}

4358

}

4359

4360

// Fold away ALL boolean setcc's.

4361

if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {

4362

SDValue Temp;

4363

switch (Cond) {

4364

default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4364);

4365

case ISD::SETEQ: // X == Y -> ~(X^Y)

4366

Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4367

N0 = DAG.getNOT(dl, Temp, OpVT);

4368

if (!DCI.isCalledByLegalizer())

4369

DCI.AddToWorklist(Temp.getNode());

4370

break;

4371

case ISD::SETNE: // X != Y --> (X^Y)

4372

N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4373

break;

4374

case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y

4375

case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y

4376

Temp = DAG.getNOT(dl, N0, OpVT);

4377

N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);

4378

if (!DCI.isCalledByLegalizer())

4379

DCI.AddToWorklist(Temp.getNode());

4380

break;

4381

case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X

4382

case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X

4383

Temp = DAG.getNOT(dl, N1, OpVT);

4384

N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);

4385

if (!DCI.isCalledByLegalizer())

4386

DCI.AddToWorklist(Temp.getNode());

4387

break;

4388

case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y

4389

case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y

4390

Temp = DAG.getNOT(dl, N0, OpVT);

4391

N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);

4392

if (!DCI.isCalledByLegalizer())

4393

DCI.AddToWorklist(Temp.getNode());

4394

break;

4395

case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X

4396

case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X

4397

Temp = DAG.getNOT(dl, N1, OpVT);

4398

N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);

4399

break;

4400

}

4401

if (VT.getScalarType() != MVT::i1) {

4402

if (!DCI.isCalledByLegalizer())

4403

DCI.AddToWorklist(N0.getNode());

4404

// FIXME: If running after legalize, we probably can't do this.

4405

ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));

4406

N0 = DAG.getNode(ExtendCode, dl, VT, N0);

4407

}

4408

return N0;

4409

}

4410

4411

// Could not fold it.

4412

return SDValue();

4413

}

4414

4415

/// Returns true (and the GlobalValue and the offset) if the node is a

4416

/// GlobalAddress + offset.

4417

bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,

4418

int64_t &Offset) const {

4419

4420

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

4421

4422

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {

4423

GA = GASD->getGlobal();

4424

Offset += GASD->getOffset();

4425

return true;

4426

}

4427

4428

if (N->getOpcode() == ISD::ADD) {

4429

SDValue N1 = N->getOperand(0);

4430

SDValue N2 = N->getOperand(1);

4431

if (isGAPlusOffset(N1.getNode(), GA, Offset)) {

4432

if (auto *V = dyn_cast<ConstantSDNode>(N2)) {

4433

Offset += V->getSExtValue();

4434

return true;

4435

}

4436

} else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {

4437

if (auto *V = dyn_cast<ConstantSDNode>(N1)) {

4438

Offset += V->getSExtValue();

4439

return true;

4440

}

4441

}

4442

}

4443

4444

return false;

4445

}

4446

4447

SDValue TargetLowering::PerformDAGCombine(SDNode *N,

4448

DAGCombinerInfo &DCI) const {

4449

// Default implementation: no optimization.

4450

return SDValue();

4451

}

4452

4453

//===----------------------------------------------------------------------===//

4454

// Inline Assembler Implementation Methods

4455

//===----------------------------------------------------------------------===//

4456

4457

TargetLowering::ConstraintType

4458

TargetLowering::getConstraintType(StringRef Constraint) const {

4459

unsigned S = Constraint.size();

4460

4461

if (S == 1) {

4462

switch (Constraint[0]) {

4463

default: break;

4464

case 'r':

4465

return C_RegisterClass;

4466

case 'm': // memory

4467

case 'o': // offsetable

4468

case 'V': // not offsetable

4469

return C_Memory;

4470

case 'n': // Simple Integer

4471

case 'E': // Floating Point Constant

4472

case 'F': // Floating Point Constant

4473

return C_Immediate;

4474

case 'i': // Simple Integer or Relocatable Constant

4475

case 's': // Relocatable Constant

4476

case 'p': // Address.

4477

case 'X': // Allow ANY value.

4478

case 'I': // Target registers.

4479

case 'J':

4480

case 'K':

4481

case 'L':

4482

case 'M':

4483

case 'N':

4484

case 'O':

4485

case 'P':

4486

case '<':

4487

case '>':

4488

return C_Other;

4489

}

4490

}

4491

4492

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {

4493

if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"

4494

return C_Memory;

4495

return C_Register;

4496

}

4497

return C_Unknown;

4498

}

4499

4500

/// Try to replace an X constraint, which matches anything, with another that

4501

/// has more specific requirements based on the type of the corresponding

4502

/// operand.

4503

const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {

4504

if (ConstraintVT.isInteger())

4505

return "r";

4506

if (ConstraintVT.isFloatingPoint())

4507

return "f"; // works for many targets

4508

return nullptr;

4509

}

4510

4511

SDValue TargetLowering::LowerAsmOutputForConstraint(

4512

SDValue &Chain, SDValue &Flag, const SDLoc &DL,

4513

const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {

4514

return SDValue();

4515

}

4516

4517

/// Lower the specified operand into the Ops vector.

4518

/// If it is invalid, don't add anything to Ops.

4519

void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,

4520

std::string &Constraint,

4521

std::vector<SDValue> &Ops,

4522

SelectionDAG &DAG) const {

4523

4524

if (Constraint.length() > 1) return;

4525

4526

char ConstraintLetter = Constraint[0];

4527

switch (ConstraintLetter) {

4528

default: break;

4529

case 'X': // Allows any operand; labels (basic block) use this.

4530

if (Op.getOpcode() == ISD::BasicBlock ||

4531

Op.getOpcode() == ISD::TargetBlockAddress) {

4532

Ops.push_back(Op);

4533

return;

4534

}

4535

LLVM_FALLTHROUGH[[gnu::fallthrough]];

4536

case 'i': // Simple Integer or Relocatable Constant

4537

case 'n': // Simple Integer

4538

case 's': { // Relocatable Constant

4539

4540

GlobalAddressSDNode *GA;

4541

ConstantSDNode *C;

4542

BlockAddressSDNode *BA;

4543

uint64_t Offset = 0;

4544

4545

// Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),

4546

// etc., since getelementpointer is variadic. We can't use

4547

// SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible

4548

// while in this case the GA may be furthest from the root node which is

4549

// likely an ISD::ADD.

4550

while (1) {

4551

if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4552

Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),

4553

GA->getValueType(0),

4554

Offset + GA->getOffset()));

4555

return;

4556

}

4557

if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {

4558

// gcc prints these as sign extended. Sign extend value to 64 bits

4559

// now; without this it would get ZExt'd later in

4560

// ScheduleDAGSDNodes::EmitNode, which is very generic.

4561

bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;

4562

BooleanContent BCont = getBooleanContents(MVT::i64);

4563

ISD::NodeType ExtOpc =

4564

IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;

4565

int64_t ExtVal =

4566

ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();

4567

Ops.push_back(

4568

DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));

4569

return;

4570

}

4571

if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4572

Ops.push_back(DAG.getTargetBlockAddress(

4573

BA->getBlockAddress(), BA->getValueType(0),

4574

Offset + BA->getOffset(), BA->getTargetFlags()));

4575

return;

4576

}

4577

const unsigned OpCode = Op.getOpcode();

4578

if (OpCode == ISD::ADD || OpCode == ISD::SUB) {

4579

if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))

4580

Op = Op.getOperand(1);

4581

// Subtraction is not commutative.

4582

else if (OpCode == ISD::ADD &&

4583

(C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))

4584

Op = Op.getOperand(0);

4585

else

4586

return;

4587

Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();

4588

continue;

4589

}

4590

return;

4591

}

4592

break;

4593

}

4594

}

4595

}

4596

4597

std::pair<unsigned, const TargetRegisterClass *>

4598

TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,

4599

StringRef Constraint,

4600

MVT VT) const {

4601

if (Constraint.empty() || Constraint[0] != '{')

4602

return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));

4603

assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4603, __extension__ __PRETTY_FUNCTION__));

4604

4605

// Remove the braces from around the name.

4606

StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

4607

4608

std::pair<unsigned, const TargetRegisterClass *> R =

4609

std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

4610

4611

// Figure out which register class contains this reg.

4612

for (const TargetRegisterClass *RC : RI->regclasses()) {

4613

// If none of the value types for this register class are valid, we

4614

// can't use it. For example, 64-bit reg classes on 32-bit targets.

4615

if (!isLegalRC(*RI, *RC))

4616

continue;

4617

4618

for (const MCPhysReg &PR : *RC) {

4619

if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {

4620

std::pair<unsigned, const TargetRegisterClass *> S =

4621

std::make_pair(PR, RC);

4622

4623

// If this register class has the requested value type, return it,

4624

// otherwise keep searching and return the first class found

4625

// if no other is found which explicitly has the requested type.

4626

if (RI->isTypeLegalForClass(*RC, VT))

4627

return S;

4628

if (!R.second)

4629

R = S;

4630

}

4631

}

4632

}

4633

4634

return R;

4635

}

4636

4637

//===----------------------------------------------------------------------===//

4638

// Constraint Selection.

4639

4640

/// Return true of this is an input operand that is a matching constraint like

4641

/// "4".

4642

bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {

4643

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4643, __extension__ __PRETTY_FUNCTION__));

4644

return isdigit(static_cast<unsigned char>(ConstraintCode[0]));

4645

}

4646

4647

/// If this is an input matching constraint, this method returns the output

4648

/// operand it matches.

4649

unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {

4650

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4650, __extension__ __PRETTY_FUNCTION__));

4651

return atoi(ConstraintCode.c_str());

4652

}

4653

4654

/// Split up the constraint string from the inline assembly value into the

4655

/// specific constraints and their prefixes, and also tie in the associated

4656

/// operand values.

4657

/// If this returns an empty vector, and if the constraint string itself

4658

/// isn't empty, there was an error parsing.

4659

TargetLowering::AsmOperandInfoVector

4660

TargetLowering::ParseConstraints(const DataLayout &DL,

4661

const TargetRegisterInfo *TRI,

4662

const CallBase &Call) const {

4663

/// Information about all of the constraints.

4664

AsmOperandInfoVector ConstraintOperands;

4665

const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());

4666

unsigned maCount = 0; // Largest number of multiple alternative constraints.

4667

4668

// Do a prepass over the constraints, canonicalizing them, and building up the

4669

// ConstraintOperands list.

4670

unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.

4671

unsigned ResNo = 0; // ResNo - The result number of the next output.

4672

4673

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {

4674

ConstraintOperands.emplace_back(std::move(CI));

4675

AsmOperandInfo &OpInfo = ConstraintOperands.back();

4676

4677

// Update multiple alternative constraint count.

4678

if (OpInfo.multipleAlternatives.size() > maCount)

4679

maCount = OpInfo.multipleAlternatives.size();

4680

4681

OpInfo.ConstraintVT = MVT::Other;

4682

4683

// Compute the value type for each operand.

4684

switch (OpInfo.Type) {

4685

case InlineAsm::isOutput:

4686

// Indirect outputs just consume an argument.

4687

if (OpInfo.isIndirect) {

4688

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4689

break;

4690

}

4691

4692

// The return value of the call is this value. As such, there is no

4693

// corresponding argument.

4694

assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4694, __extension__ __PRETTY_FUNCTION__));

4695

if (StructType *STy = dyn_cast<StructType>(Call.getType())) {

4696

OpInfo.ConstraintVT =

4697

getSimpleValueType(DL, STy->getElementType(ResNo));

4698

} else {

4699

assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4699, __extension__ __PRETTY_FUNCTION__));

4700

OpInfo.ConstraintVT =

4701

getAsmOperandValueType(DL, Call.getType()).getSimpleVT();

4702

}

4703

++ResNo;

4704

break;

4705

case InlineAsm::isInput:

4706

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4707

break;

4708

case InlineAsm::isClobber:

4709

// Nothing to do.

4710

break;

4711

}

4712

4713

if (OpInfo.CallOperandVal) {

4714

llvm::Type *OpTy = OpInfo.CallOperandVal->getType();

4715

if (OpInfo.isIndirect) {

4716

llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);

4717

if (!PtrTy)

4718

report_fatal_error("Indirect operand for inline asm not a pointer!");

4719

OpTy = PtrTy->getElementType();

4720

}

4721

4722

// Look for vector wrapped in a struct. e.g. { <16 x i8> }.

4723

if (StructType *STy = dyn_cast<StructType>(OpTy))

4724

if (STy->getNumElements() == 1)

4725

OpTy = STy->getElementType(0);

4726

4727

// If OpTy is not a single value, it may be a struct/union that we

4728

// can tile with integers.

4729

if (!OpTy->isSingleValueType() && OpTy->isSized()) {

4730

unsigned BitSize = DL.getTypeSizeInBits(OpTy);

4731

switch (BitSize) {

4732

default: break;

4733

case 1:

4734

case 8:

4735

case 16:

4736

case 32:

4737

case 64:

4738

case 128:

4739

OpInfo.ConstraintVT =

4740

MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);

4741

break;

4742

}

4743

} else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {

4744

unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());

4745

OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);

4746

} else {

4747

OpInfo.ConstraintVT = MVT::getVT(OpTy, true);

4748

}

4749

}

4750

}

4751

4752

// If we have multiple alternative constraints, select the best alternative.

4753

if (!ConstraintOperands.empty()) {

4754

if (maCount) {

4755

unsigned bestMAIndex = 0;

4756

int bestWeight = -1;

4757

// weight: -1 = invalid match, and 0 = so-so match to 5 = good match.

4758

int weight = -1;

4759

unsigned maIndex;

4760

// Compute the sums of the weights for each alternative, keeping track

4761

// of the best (highest weight) one so far.

4762

for (maIndex = 0; maIndex < maCount; ++maIndex) {

4763

int weightSum = 0;

4764

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4765

cIndex != eIndex; ++cIndex) {

4766

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4767

if (OpInfo.Type == InlineAsm::isClobber)

4768

continue;

4769

4770

// If this is an output operand with a matching input operand,

4771

// look up the matching input. If their types mismatch, e.g. one

4772

// is an integer, the other is floating point, or their sizes are

4773

// different, flag it as an maCantMatch.

4774

if (OpInfo.hasMatchingInput()) {

4775

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4776

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4777

if ((OpInfo.ConstraintVT.isInteger() !=

4778

Input.ConstraintVT.isInteger()) ||

4779

(OpInfo.ConstraintVT.getSizeInBits() !=

4780

Input.ConstraintVT.getSizeInBits())) {

4781

weightSum = -1; // Can't match.

4782

break;

4783

}

4784

}

4785

}

4786

weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);

4787

if (weight == -1) {

4788

weightSum = -1;

4789

break;

4790

}

4791

weightSum += weight;

4792

}

4793

// Update best.

4794

if (weightSum > bestWeight) {

4795

bestWeight = weightSum;

4796

bestMAIndex = maIndex;

4797

}

4798

}

4799

4800

// Now select chosen alternative in each constraint.

4801

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4802

cIndex != eIndex; ++cIndex) {

4803

AsmOperandInfo &cInfo = ConstraintOperands[cIndex];

4804

if (cInfo.Type == InlineAsm::isClobber)

4805

continue;

4806

cInfo.selectAlternative(bestMAIndex);

4807

}

4808

}

4809

}

4810

4811

// Check and hook up tied operands, choose constraint code to use.

4812

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4813

cIndex != eIndex; ++cIndex) {

4814

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4815

4816

// If this is an output operand with a matching input operand, look up the

4817

// matching input. If their types mismatch, e.g. one is an integer, the

4818

// other is floating point, or their sizes are different, flag it as an

4819

// error.

4820

if (OpInfo.hasMatchingInput()) {

4821

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4822

4823

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4824

std::pair<unsigned, const TargetRegisterClass *> MatchRC =

4825

getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,

4826

OpInfo.ConstraintVT);

4827

std::pair<unsigned, const TargetRegisterClass *> InputRC =

4828

getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,

4829

Input.ConstraintVT);

4830

if ((OpInfo.ConstraintVT.isInteger() !=

4831

Input.ConstraintVT.isInteger()) ||

4832

(MatchRC.second != InputRC.second)) {

4833

report_fatal_error("Unsupported asm: input constraint"

4834

" with a matching output constraint of"

4835

" incompatible type!");

4836

}

4837

}

4838

}

4839

}

4840

4841

return ConstraintOperands;

4842

}

4843

4844

/// Return an integer indicating how general CT is.

4845

static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {

4846

switch (CT) {

4847

case TargetLowering::C_Immediate:

4848

case TargetLowering::C_Other:

4849

case TargetLowering::C_Unknown:

4850

return 0;

4851

case TargetLowering::C_Register:

4852

return 1;

4853

case TargetLowering::C_RegisterClass:

4854

return 2;

4855

case TargetLowering::C_Memory:

4856

return 3;

4857

}

4858

llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4858);

4859

}

4860

4861

/// Examine constraint type and operand type and determine a weight value.

4862

/// This object must already have been set up with the operand type

4863

/// and the current alternative constraint selected.

4864

TargetLowering::ConstraintWeight

4865

TargetLowering::getMultipleConstraintMatchWeight(

4866

AsmOperandInfo &info, int maIndex) const {

4867

InlineAsm::ConstraintCodeVector *rCodes;

4868

if (maIndex >= (int)info.multipleAlternatives.size())

4869

rCodes = &info.Codes;

4870

else

4871

rCodes = &info.multipleAlternatives[maIndex].Codes;

4872

ConstraintWeight BestWeight = CW_Invalid;

4873

4874

// Loop over the options, keeping track of the most general one.

4875

for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {

4876

ConstraintWeight weight =

4877

getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());

4878

if (weight > BestWeight)

4879

BestWeight = weight;

4880

}

4881

4882

return BestWeight;

4883

}

4884

4885

/// Examine constraint type and operand type and determine a weight value.

4886

/// This object must already have been set up with the operand type

4887

/// and the current alternative constraint selected.

4888

TargetLowering::ConstraintWeight

4889

TargetLowering::getSingleConstraintMatchWeight(

4890

AsmOperandInfo &info, const char *constraint) const {

4891

ConstraintWeight weight = CW_Invalid;

4892

Value *CallOperandVal = info.CallOperandVal;

4893

// If we don't have a value, we can't do a match,

4894

// but allow it at the lowest weight.

4895

if (!CallOperandVal)

4896

return CW_Default;

4897

// Look at the constraint type.

4898

switch (*constraint) {

4899

case 'i': // immediate integer.

4900

case 'n': // immediate integer with a known value.

4901

if (isa<ConstantInt>(CallOperandVal))

4902

weight = CW_Constant;

4903

break;

4904

case 's': // non-explicit intregal immediate.

4905

if (isa<GlobalValue>(CallOperandVal))

4906

weight = CW_Constant;

4907

break;

4908

case 'E': // immediate float if host format.

4909

case 'F': // immediate float.

4910

if (isa<ConstantFP>(CallOperandVal))

4911

weight = CW_Constant;

4912

break;

4913

case '<': // memory operand with autodecrement.

4914

case '>': // memory operand with autoincrement.

4915

case 'm': // memory operand.

4916

case 'o': // offsettable memory operand

4917

case 'V': // non-offsettable memory operand

4918

weight = CW_Memory;

4919

break;

4920

case 'r': // general register.

4921

case 'g': // general register, memory operand or immediate integer.

4922

// note: Clang converts "g" to "imr".

4923

if (CallOperandVal->getType()->isIntegerTy())

4924

weight = CW_Register;

4925

break;

4926

case 'X': // any operand.

4927

default:

4928

weight = CW_Default;

4929

break;

4930

}

4931

return weight;

4932

}

4933

4934

/// If there are multiple different constraints that we could pick for this

4935

/// operand (e.g. "imr") try to pick the 'best' one.

4936

/// This is somewhat tricky: constraints fall into four classes:

4937

/// Other -> immediates and magic values

4938

/// Register -> one specific register

4939

/// RegisterClass -> a group of regs

4940

/// Memory -> memory

4941

/// Ideally, we would pick the most specific constraint possible: if we have

4942

/// something that fits into a register, we would pick it. The problem here

4943

/// is that if we have something that could either be in a register or in

4944

/// memory that use of the register could cause selection of *other*

4945

/// operands to fail: they might only succeed if we pick memory. Because of

4946

/// this the heuristic we use is:

4947

///

4948

/// 1) If there is an 'other' constraint, and if the operand is valid for

4949

/// that constraint, use it. This makes us take advantage of 'i'

4950

/// constraints when available.

4951

/// 2) Otherwise, pick the most general constraint present. This prefers

4952

/// 'm' over 'r', for example.

4953

///

4954

static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,

4955

const TargetLowering &TLI,

4956

SDValue Op, SelectionDAG *DAG) {

4957

assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4957, __extension__ __PRETTY_FUNCTION__));

4958

unsigned BestIdx = 0;

4959

TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;

4960

int BestGenerality = -1;

4961

4962

// Loop over the options, keeping track of the most general one.

4963

for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {

4964

TargetLowering::ConstraintType CType =

4965

TLI.getConstraintType(OpInfo.Codes[i]);

4966

4967

// Indirect 'other' or 'immediate' constraints are not allowed.

4968

if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||

4969

CType == TargetLowering::C_Register ||

4970

CType == TargetLowering::C_RegisterClass))

4971

continue;

4972

4973

// If this is an 'other' or 'immediate' constraint, see if the operand is

4974

// valid for it. For example, on X86 we might have an 'rI' constraint. If

4975

// the operand is an integer in the range [0..31] we want to use I (saving a

4976

// load of a register), otherwise we must use 'r'.

4977

if ((CType == TargetLowering::C_Other ||

4978

CType == TargetLowering::C_Immediate) && Op.getNode()) {

4979

assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))

4980

"Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__));

4981

std::vector<SDValue> ResultOps;

4982

TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],

4983

ResultOps, *DAG);

4984

if (!ResultOps.empty()) {

4985

BestType = CType;

4986

BestIdx = i;

4987

break;

4988

}

4989

}

4990

4991

// Things with matching constraints can only be registers, per gcc

4992

// documentation. This mainly affects "g" constraints.

4993

if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())

4994

continue;

4995

4996

// This constraint letter is more general than the previous one, use it.

4997

int Generality = getConstraintGenerality(CType);

4998

if (Generality > BestGenerality) {

4999

BestType = CType;

5000

BestIdx = i;

5001

BestGenerality = Generality;

5002

}

5003

}

5004

5005

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];

5006

OpInfo.ConstraintType = BestType;

5007

}

5008

5009

/// Determines the constraint code and constraint type to use for the specific

5010

/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.

5011

void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

5012

SDValue Op,

5013

SelectionDAG *DAG) const {

5014

assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5014, __extension__ __PRETTY_FUNCTION__));

5015

5016

// Single-letter constraints ('r') are very common.

5017

if (OpInfo.Codes.size() == 1) {

5018

OpInfo.ConstraintCode = OpInfo.Codes[0];

5019

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5020

} else {

5021

ChooseConstraint(OpInfo, *this, Op, DAG);

5022

}

5023

5024

// 'X' matches anything.

5025

if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {

5026

// Labels and constants are handled elsewhere ('X' is the only thing

5027

// that matches labels). For Functions, the type here is the type of

5028

// the result, which is not what we want to look at; leave them alone.

5029

Value *v = OpInfo.CallOperandVal;

5030

if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {

5031

OpInfo.CallOperandVal = v;

5032

return;

5033

}

5034

5035

if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)

5036

return;

5037

5038

// Otherwise, try to resolve it to something we know about by looking at

5039

// the actual operand type.

5040

if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {

5041

OpInfo.ConstraintCode = Repl;

5042

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5043

}

5044

}

5045

}

5046

5047

/// Given an exact SDIV by a constant, create a multiplication

5048

/// with the multiplicative inverse of the constant.

5049

static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,

5050

const SDLoc &dl, SelectionDAG &DAG,

5051

SmallVectorImpl<SDNode *> &Created) {

5052

SDValue Op0 = N->getOperand(0);

5053

SDValue Op1 = N->getOperand(1);

5054

EVT VT = N->getValueType(0);

5055

EVT SVT = VT.getScalarType();

5056

EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());

5057

EVT ShSVT = ShVT.getScalarType();

5058

5059

bool UseSRA = false;

5060

SmallVector<SDValue, 16> Shifts, Factors;

5061

5062

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5063

if (C->isZero())

5064

return false;

5065

APInt Divisor = C->getAPIntValue();

5066

unsigned Shift = Divisor.countTrailingZeros();

5067

if (Shift) {

5068

Divisor.ashrInPlace(Shift);

5069

UseSRA = true;

5070

}

5071

// Calculate the multiplicative inverse, using Newton's method.

5072

APInt t;

5073

APInt Factor = Divisor;

5074

while ((t = Divisor * Factor) != 1)

5075

Factor *= APInt(Divisor.getBitWidth(), 2) - t;

5076

Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));

5077

Factors.push_back(DAG.getConstant(Factor, dl, SVT));

5078

return true;

5079

};

5080

5081

// Collect all magic values from the build vector.

5082

if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))

5083

return SDValue();

5084

5085

SDValue Shift, Factor;

5086

if (Op1.getOpcode() == ISD::BUILD_VECTOR) {

5087

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5088

Factor = DAG.getBuildVector(VT, dl, Factors);

5089

} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {

5090

assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))

5091

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))

5092

"vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__));

5093

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5094

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5095

} else {

5096

assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5096, __extension__ __PRETTY_FUNCTION__));

5097

Shift = Shifts[0];

5098

Factor = Factors[0];

5099

}

5100

5101

SDValue Res = Op0;

5102

5103

// Shift the value upfront if it is even, so the LSB is one.

5104

if (UseSRA) {

5105

// TODO: For UDIV use SRL instead of SRA.

5106

SDNodeFlags Flags;

5107

Flags.setExact(true);

5108

Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);

5109

Created.push_back(Res.getNode());

5110

}

5111

5112

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);

5113

}

5114

5115

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,

5116

SelectionDAG &DAG,

5117

SmallVectorImpl<SDNode *> &Created) const {

5118

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

5119

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5120

if (TLI.isIntDivCheap(N->getValueType(0), Attr))

5121

return SDValue(N, 0); // Lower SDIV as SDIV

5122

return SDValue();

5123

}

5124

5125

/// Given an ISD::SDIV node expressing a divide by constant,

5126

/// return a DAG expression to select that will generate the same value by

5127

/// multiplying by a magic number.

5128

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5129

SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,

5130

bool IsAfterLegalization,

5131

SmallVectorImpl<SDNode *> &Created) const {

5132

SDLoc dl(N);

5133

EVT VT = N->getValueType(0);

5134

EVT SVT = VT.getScalarType();

5135

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5136

EVT ShSVT = ShVT.getScalarType();

5137

unsigned EltBits = VT.getScalarSizeInBits();

5138

EVT MulVT;

5139

5140

// Check to see if we can do this.

5141

// FIXME: We should be more aggressive here.

5142

if (!isTypeLegal(VT)) {

5143

// Limit this to simple scalars for now.

5144

if (VT.isVector() || !VT.isSimple())

5145

return SDValue();

5146

5147

// If this type will be promoted to a large enough type with a legal

5148

// multiply operation, we can go ahead and do this transform.

5149

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5150

return SDValue();

5151

5152

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5153

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5154

!isOperationLegal(ISD::MUL, MulVT))

5155

return SDValue();

5156

}

5157

5158

// If the sdiv has an 'exact' bit we can use a simpler lowering.

5159

if (N->getFlags().hasExact())

5160

return BuildExactSDIV(*this, N, dl, DAG, Created);

5161

5162

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

5163

5164

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5165

if (C->isZero())

5166

return false;

5167

5168

const APInt &Divisor = C->getAPIntValue();

5169

SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);

5170

int NumeratorFactor = 0;

5171

int ShiftMask = -1;

5172

5173

if (Divisor.isOne() || Divisor.isAllOnes()) {

5174

// If d is +1/-1, we just multiply the numerator by +1/-1.

5175

NumeratorFactor = Divisor.getSExtValue();

5176

magics.Magic = 0;

5177

magics.ShiftAmount = 0;

5178

ShiftMask = 0;

5179

} else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {

5180

// If d > 0 and m < 0, add the numerator.

5181

NumeratorFactor = 1;

5182

} else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {

5183

// If d < 0 and m > 0, subtract the numerator.

5184

NumeratorFactor = -1;

5185

}

5186

5187

MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));

5188

Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));

5189

Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));

5190

ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));

5191

return true;

5192

};

5193

5194

SDValue N0 = N->getOperand(0);

5195

SDValue N1 = N->getOperand(1);

5196

5197

// Collect the shifts / magic values from each element.

5198

if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))

5199

return SDValue();

5200

5201

SDValue MagicFactor, Factor, Shift, ShiftMask;

5202

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5203

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5204

Factor = DAG.getBuildVector(VT, dl, Factors);

5205

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5206

ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);

5207

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5208

assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))

5209

Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))

5210

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))

5211

"vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__));

5212

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5213

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5214

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5215

ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);

5216

} else {

5217

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5217, __extension__ __PRETTY_FUNCTION__));

5218

MagicFactor = MagicFactors[0];

5219

Factor = Factors[0];

5220

Shift = Shifts[0];

5221

ShiftMask = ShiftMasks[0];

5222

}

5223

5224

// Multiply the numerator (operand 0) by the magic value.

5225

// FIXME: We should support doing a MUL in a wider type.

5226

auto GetMULHS = [&](SDValue X, SDValue Y) {

5227

// If the type isn't legal, use a wider mul of the the type calculated

5228

// earlier.

5229

if (!isTypeLegal(VT)) {

5230

X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);

5231

Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);

5232

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5233

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5234

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5235

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5236

}

5237

5238

if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))

5239

return DAG.getNode(ISD::MULHS, dl, VT, X, Y);

5240

if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {

5241

SDValue LoHi =

5242

DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5243

return SDValue(LoHi.getNode(), 1);

5244

}

5245

return SDValue();

5246

};

5247

5248

SDValue Q = GetMULHS(N0, MagicFactor);

5249

if (!Q)

5250

return SDValue();

5251

5252

Created.push_back(Q.getNode());

5253

5254

// (Optionally) Add/subtract the numerator using Factor.

5255

Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);

5256

Created.push_back(Factor.getNode());

5257

Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);

5258

Created.push_back(Q.getNode());

5259

5260

// Shift right algebraic by shift value.

5261

Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);

5262

Created.push_back(Q.getNode());

5263

5264

// Extract the sign bit, mask it and add it to the quotient.

5265

SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);

5266

SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);

5267

Created.push_back(T.getNode());

5268

T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);

5269

Created.push_back(T.getNode());

5270

return DAG.getNode(ISD::ADD, dl, VT, Q, T);

5271

}

5272

5273

/// Given an ISD::UDIV node expressing a divide by constant,

5274

/// return a DAG expression to select that will generate the same value by

5275

/// multiplying by a magic number.

5276

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5277

SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,

5278

bool IsAfterLegalization,

5279

SmallVectorImpl<SDNode *> &Created) const {

5280

SDLoc dl(N);

5281

EVT VT = N->getValueType(0);

5282

EVT SVT = VT.getScalarType();

5283

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5284

EVT ShSVT = ShVT.getScalarType();

5285

unsigned EltBits = VT.getScalarSizeInBits();

5286

EVT MulVT;

5287

5288

// Check to see if we can do this.

5289

// FIXME: We should be more aggressive here.

5290

if (!isTypeLegal(VT)) {

5291

// Limit this to simple scalars for now.

5292

if (VT.isVector() || !VT.isSimple())

5293

return SDValue();

5294

5295

// If this type will be promoted to a large enough type with a legal

5296

// multiply operation, we can go ahead and do this transform.

5297

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5298

return SDValue();

5299

5300

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5301

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5302

!isOperationLegal(ISD::MUL, MulVT))

5303

return SDValue();

5304

}

5305

5306

bool UseNPQ = false;

5307

SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

5308

5309

auto BuildUDIVPattern = [&](ConstantSDNode *C) {

5310

if (C->isZero())

5311

return false;

5312

// FIXME: We should use a narrower constant when the upper

5313

// bits are known to be zero.

5314

const APInt& Divisor = C->getAPIntValue();

5315

UnsignedDivisonByConstantInfo magics = UnsignedDivisonByConstantInfo::get(Divisor);

5316

unsigned PreShift = 0, PostShift = 0;

5317

5318

// If the divisor is even, we can avoid using the expensive fixup by

5319

// shifting the divided value upfront.

5320

if (magics.IsAdd != 0 && !Divisor[0]) {

5321

PreShift = Divisor.countTrailingZeros();

5322

// Get magic number for the shifted divisor.

5323

magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);

5324

assert(magics.IsAdd == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.IsAdd == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.IsAdd == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5324, __extension__ __PRETTY_FUNCTION__));

5325

}

5326

5327

APInt Magic = magics.Magic;

5328

5329

unsigned SelNPQ;

5330

if (magics.IsAdd == 0 || Divisor.isOne()) {

5331

assert(magics.ShiftAmount < Divisor.getBitWidth() &&(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__))

5332

"We shouldn't generate an undefined shift!")(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__));

5333

PostShift = magics.ShiftAmount;

5334

SelNPQ = false;

5335

} else {

5336

PostShift = magics.ShiftAmount - 1;

5337

SelNPQ = true;

5338

}

5339

5340

PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));

5341

MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));

5342

NPQFactors.push_back(

5343

DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)

5344

: APInt::getZero(EltBits),

5345

dl, SVT));

5346

PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));

5347

UseNPQ |= SelNPQ;

5348

return true;

5349

};

5350

5351

SDValue N0 = N->getOperand(0);

5352

SDValue N1 = N->getOperand(1);

5353

5354

// Collect the shifts/magic values from each element.

5355

if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))

5356

return SDValue();

5357

5358

SDValue PreShift, PostShift, MagicFactor, NPQFactor;

5359

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5360

PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);

5361

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5362

NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);

5363

PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);

5364

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5365

assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))

5366

NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))

5367

"Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__));

5368

PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);

5369

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5370

NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);

5371

PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);

5372

} else {

5373

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5373, __extension__ __PRETTY_FUNCTION__));

5374

PreShift = PreShifts[0];

5375

MagicFactor = MagicFactors[0];

5376

PostShift = PostShifts[0];

5377

}

5378

5379

SDValue Q = N0;

5380

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);

5381

Created.push_back(Q.getNode());

5382

5383

// FIXME: We should support doing a MUL in a wider type.

5384

auto GetMULHU = [&](SDValue X, SDValue Y) {

5385

// If the type isn't legal, use a wider mul of the the type calculated

5386

// earlier.

5387

if (!isTypeLegal(VT)) {

5388

X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);

5389

Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);

5390

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5391

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5392

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5393

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5394

}

5395

5396

if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))

5397

return DAG.getNode(ISD::MULHU, dl, VT, X, Y);

5398

if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {

5399

SDValue LoHi =

5400

DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5401

return SDValue(LoHi.getNode(), 1);

5402

}

5403

return SDValue(); // No mulhu or equivalent

5404

};

5405

5406

// Multiply the numerator (operand 0) by the magic value.

5407

Q = GetMULHU(Q, MagicFactor);

5408

if (!Q)

5409

return SDValue();

5410

5411

Created.push_back(Q.getNode());

5412

5413

if (UseNPQ) {

5414

SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);

5415

Created.push_back(NPQ.getNode());

5416

5417

// For vectors we might have a mix of non-NPQ/NPQ paths, so use

5418

// MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.

5419

if (VT.isVector())

5420

NPQ = GetMULHU(NPQ, NPQFactor);

5421

else

5422

NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

5423

5424

Created.push_back(NPQ.getNode());

5425

5426

Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);

5427

Created.push_back(Q.getNode());

5428

}

5429

5430

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);

5431

Created.push_back(Q.getNode());

5432

5433

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5434

5435

SDValue One = DAG.getConstant(1, dl, VT);

5436

SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);

5437

return DAG.getSelect(dl, VT, IsOne, N0, Q);

5438

}

5439

5440

/// If all values in Values that *don't* match the predicate are same 'splat'

5441

/// value, then replace all values with that splat value.

5442

/// Else, if AlternativeReplacement was provided, then replace all values that

5443

/// do match predicate with AlternativeReplacement value.

5444

static void

5445

turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,

5446

std::function<bool(SDValue)> Predicate,

5447

SDValue AlternativeReplacement = SDValue()) {

5448

SDValue Replacement;

5449

// Is there a value for which the Predicate does *NOT* match? What is it?

5450

auto SplatValue = llvm::find_if_not(Values, Predicate);

5451

if (SplatValue != Values.end()) {

5452

// Does Values consist only of SplatValue's and values matching Predicate?

5453

if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {

5454

return Value == *SplatValue || Predicate(Value);

5455

})) // Then we shall replace values matching predicate with SplatValue.

5456

Replacement = *SplatValue;

5457

}

5458

if (!Replacement) {

5459

// Oops, we did not find the "baseline" splat value.

5460

if (!AlternativeReplacement)

5461

return; // Nothing to do.

5462

// Let's replace with provided value then.

5463

Replacement = AlternativeReplacement;

5464

}

5465

std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);

5466

}

5467

5468

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE

5469

/// where the divisor is constant and the comparison target is zero,

5470

/// return a DAG expression that will generate the same comparison result

5471

/// using only multiplications, additions and shifts/rotations.

5472

/// Ref: "Hacker's Delight" 10-17.

5473

SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,

5474

SDValue CompTargetNode,

5475

ISD::CondCode Cond,

5476

DAGCombinerInfo &DCI,

5477

const SDLoc &DL) const {

5478

SmallVector<SDNode *, 5> Built;

5479

if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5480

DCI, DL, Built)) {

5481

for (SDNode *N : Built)

5482

DCI.AddToWorklist(N);

5483

return Folded;

5484

}

5485

5486

return SDValue();

5487

}

5488

5489

SDValue

5490

TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,

5491

SDValue CompTargetNode, ISD::CondCode Cond,

5492

DAGCombinerInfo &DCI, const SDLoc &DL,

5493

SmallVectorImpl<SDNode *> &Created) const {

5494

// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)

5495

// - D must be constant, with D = D0 * 2^K where D0 is odd

5496

// - P is the multiplicative inverse of D0 modulo 2^W

5497

// - Q = floor(((2^W) - 1) / D)

5498

// where W is the width of the common type of N and D.

5499

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__))

5500

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__));

5501

5502

SelectionDAG &DAG = DCI.DAG;

5503

5504

EVT VT = REMNode.getValueType();

5505

EVT SVT = VT.getScalarType();

5506

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5507

EVT ShSVT = ShVT.getScalarType();

5508

5509

// If MUL is unavailable, we cannot proceed in any case.

5510

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5511

return SDValue();

5512

5513

bool ComparingWithAllZeros = true;

5514

bool AllComparisonsWithNonZerosAreTautological = true;

5515

bool HadTautologicalLanes = false;

5516

bool AllLanesAreTautological = true;

5517

bool HadEvenDivisor = false;

5518

bool AllDivisorsArePowerOfTwo = true;

5519

bool HadTautologicalInvertedLanes = false;

5520

SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

5521

5522

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {

5523

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5524

if (CDiv->isZero())

5525

return false;

5526

5527

const APInt &D = CDiv->getAPIntValue();

5528

const APInt &Cmp = CCmp->getAPIntValue();

5529

5530

ComparingWithAllZeros &= Cmp.isZero();

5531

5532

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5533

// if C2 is not less than C1, the comparison is always false.

5534

// But we will only be able to produce the comparison that will give the

5535

// opposive tautological answer. So this lane would need to be fixed up.

5536

bool TautologicalInvertedLane = D.ule(Cmp);

5537

HadTautologicalInvertedLanes |= TautologicalInvertedLane;

5538

5539

// If all lanes are tautological (either all divisors are ones, or divisor

5540

// is not greater than the constant we are comparing with),

5541

// we will prefer to avoid the fold.

5542

bool TautologicalLane = D.isOne() || TautologicalInvertedLane;

5543

HadTautologicalLanes |= TautologicalLane;

5544

AllLanesAreTautological &= TautologicalLane;

5545

5546

// If we are comparing with non-zero, we need'll need to subtract said

5547

// comparison value from the LHS. But there is no point in doing that if

5548

// every lane where we are comparing with non-zero is tautological..

5549

if (!Cmp.isZero())

5550

AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

5551

5552

// Decompose D into D0 * 2^K

5553

unsigned K = D.countTrailingZeros();

5554

assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOne() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5554, __extension__ __PRETTY_FUNCTION__));

5555

APInt D0 = D.lshr(K);

5556

5557

// D is even if it has trailing zeros.

5558

HadEvenDivisor |= (K != 0);

5559

// D is a power-of-two if D0 is one.

5560

// If all divisors are power-of-two, we will prefer to avoid the fold.

5561

AllDivisorsArePowerOfTwo &= D0.isOne();

5562

5563

// P = inv(D0, 2^W)

5564

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5565

unsigned W = D.getBitWidth();

5566

APInt P = D0.zext(W + 1)

5567

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5568

.trunc(W);

5569

assert(!P.isZero() && "No multiplicative inverse!")(static_cast <bool> (!P.isZero() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isZero() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5569, __extension__ __PRETTY_FUNCTION__)); // unreachable

5570

assert((D0 * P).isOne() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOne() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOne() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5570, __extension__ __PRETTY_FUNCTION__));

5571

5572

// Q = floor((2^W - 1) u/ D)

5573

// R = ((2^W - 1) u% D)

5574

APInt Q, R;

5575

APInt::udivrem(APInt::getAllOnes(W), D, Q, R);

5576

5577

// If we are comparing with zero, then that comparison constant is okay,

5578

// else it may need to be one less than that.

5579

if (Cmp.ugt(R))

5580

Q -= 1;

5581

5582

assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5583, __extension__ __PRETTY_FUNCTION__))

5583

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5583, __extension__ __PRETTY_FUNCTION__));

5584

5585

// If the lane is tautological the result can be constant-folded.

5586

if (TautologicalLane) {

5587

// Set P and K amount to a bogus values so we can try to splat them.

5588

P = 0;

5589

K = -1;

5590

// And ensure that comparison constant is tautological,

5591

// it will always compare true/false.

5592

Q = -1;

5593

}

5594

5595

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5596

KAmts.push_back(

5597

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5598

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5599

return true;

5600

};

5601

5602

SDValue N = REMNode.getOperand(0);

5603

SDValue D = REMNode.getOperand(1);

5604

5605

// Collect the values from each element.

5606

if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))

5607

return SDValue();

5608

5609

// If all lanes are tautological, the result can be constant-folded.

5610

if (AllLanesAreTautological)

5611

return SDValue();

5612

5613

// If this is a urem by a powers-of-two, avoid the fold since it can be

5614

// best implemented as a bit test.

5615

if (AllDivisorsArePowerOfTwo)

5616

return SDValue();

5617

5618

SDValue PVal, KVal, QVal;

5619

if (D.getOpcode() == ISD::BUILD_VECTOR) {

5620

if (HadTautologicalLanes) {

5621

// Try to turn PAmts into a splat, since we don't care about the values

5622

// that are currently '0'. If we can't, just keep '0'`s.

5623

turnVectorIntoSplatVector(PAmts, isNullConstant);

5624

// Try to turn KAmts into a splat, since we don't care about the values

5625

// that are currently '-1'. If we can't, change them to '0'`s.

5626

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5627

DAG.getConstant(0, DL, ShSVT));

5628

}

5629

5630

PVal = DAG.getBuildVector(VT, DL, PAmts);

5631

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5632

QVal = DAG.getBuildVector(VT, DL, QAmts);

5633

} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {

5634

assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__))

5635

"Expected matchBinaryPredicate to return one element for "(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__))

5636

"SPLAT_VECTORs")(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__));

5637

PVal = DAG.getSplatVector(VT, DL, PAmts[0]);

5638

KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);

5639

QVal = DAG.getSplatVector(VT, DL, QAmts[0]);

5640

} else {

5641

PVal = PAmts[0];

5642

KVal = KAmts[0];

5643

QVal = QAmts[0];

5644

}

5645

5646

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {

5647

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))

5648

return SDValue(); // FIXME: Could/should use `ISD::ADD`?

5649

assert(CompTargetNode.getValueType() == N.getValueType() &&(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5650, __extension__ __PRETTY_FUNCTION__))

5650

"Expecting that the types on LHS and RHS of comparisons match.")(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5650, __extension__ __PRETTY_FUNCTION__));

5651

N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);

5652

}

5653

5654

// (mul N, P)

5655

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5656

Created.push_back(Op0.getNode());

5657

5658

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5659

// divisors as a performance improvement, since rotating by 0 is a no-op.

5660

if (HadEvenDivisor) {

5661

// We need ROTR to do this.

5662

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5663

return SDValue();

5664

// UREM: (rotr (mul N, P), K)

5665

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);

5666

Created.push_back(Op0.getNode());

5667

}

5668

5669

// UREM: (setule/setugt (rotr (mul N, P), K), Q)

5670

SDValue NewCC =

5671

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5672

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5673

if (!HadTautologicalInvertedLanes)

5674

return NewCC;

5675

5676

// If any lanes previously compared always-false, the NewCC will give

5677

// always-true result for them, so we need to fixup those lanes.

5678

// Or the other way around for inequality predicate.

5679

assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5679, __extension__ __PRETTY_FUNCTION__));

5680

Created.push_back(NewCC.getNode());

5681

5682

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5683

// if C2 is not less than C1, the comparison is always false.

5684

// But we have produced the comparison that will give the

5685

// opposive tautological answer. So these lanes would need to be fixed up.

5686

SDValue TautologicalInvertedChannels =

5687

DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);

5688

Created.push_back(TautologicalInvertedChannels.getNode());

5689

5690

// NOTE: we avoid letting illegal types through even if we're before legalize

5691

// ops – legalization has a hard time producing good code for this.

5692

if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {

5693

// If we have a vector select, let's replace the comparison results in the

5694

// affected lanes with the correct tautological result.

5695

SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,

5696

DL, SETCCVT, SETCCVT);

5697

return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,

5698

Replacement, NewCC);

5699

}

5700

5701

// Else, we can just invert the comparison result in the appropriate lanes.

5702

//

5703

// NOTE: see the note above VSELECT above.

5704

if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))

5705

return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,

5706

TautologicalInvertedChannels);

5707

5708

return SDValue(); // Don't know how to lower.

5709

}

5710

5711

/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE

5712

/// where the divisor is constant and the comparison target is zero,

5713

/// return a DAG expression that will generate the same comparison result

5714

/// using only multiplications, additions and shifts/rotations.

5715

/// Ref: "Hacker's Delight" 10-17.

5716

SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,

5717

SDValue CompTargetNode,

5718

ISD::CondCode Cond,

5719

DAGCombinerInfo &DCI,

5720

const SDLoc &DL) const {

5721

SmallVector<SDNode *, 7> Built;

5722

if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5723

DCI, DL, Built)) {

5724

assert(Built.size() <= 7 && "Max size prediction failed.")(static_cast <bool> (Built.size() <= 7 && "Max size prediction failed."
) ? void (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5724, __extension__ __PRETTY_FUNCTION__));

5725

for (SDNode *N : Built)

5726

DCI.AddToWorklist(N);

5727

return Folded;

5728

}

5729

5730

return SDValue();

5731

}

5732

5733

SDValue

5734

TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,

5735

SDValue CompTargetNode, ISD::CondCode Cond,

5736

DAGCombinerInfo &DCI, const SDLoc &DL,

5737

SmallVectorImpl<SDNode *> &Created) const {

5738

// Fold:

5739

// (seteq/ne (srem N, D), 0)

5740

// To:

5741

// (setule/ugt (rotr (add (mul N, P), A), K), Q)

5742

//

5743

// - D must be constant, with D = D0 * 2^K where D0 is odd

5744

// - P is the multiplicative inverse of D0 modulo 2^W

5745

// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))

5746

// - Q = floor((2 * A) / (2^K))

5747

// where W is the width of the common type of N and D.

5748

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__))

5749

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__));

5750

5751

SelectionDAG &DAG = DCI.DAG;

5752

5753

EVT VT = REMNode.getValueType();

5754

EVT SVT = VT.getScalarType();

5755

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5756

EVT ShSVT = ShVT.getScalarType();

5757

5758

// If we are after ops legalization, and MUL is unavailable, we can not

5759

// proceed.

5760

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5761

return SDValue();

5762

5763

// TODO: Could support comparing with non-zero too.

5764

ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);

5765

if (!CompTarget || !CompTarget->isZero())

5766

return SDValue();

5767

5768

bool HadIntMinDivisor = false;

5769

bool HadOneDivisor = false;

5770

bool AllDivisorsAreOnes = true;

5771

bool HadEvenDivisor = false;

5772

bool NeedToApplyOffset = false;

5773

bool AllDivisorsArePowerOfTwo = true;

5774

SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

5775

5776

auto BuildSREMPattern = [&](ConstantSDNode *C) {

5777

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5778

if (C->isZero())

5779

return false;

5780

5781

// FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

5782

5783

// WARNING: this fold is only valid for positive divisors!

5784

APInt D = C->getAPIntValue();

5785

if (D.isNegative())

5786

D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`

5787

5788

HadIntMinDivisor |= D.isMinSignedValue();

5789

5790

// If all divisors are ones, we will prefer to avoid the fold.

5791

HadOneDivisor |= D.isOne();

5792

AllDivisorsAreOnes &= D.isOne();

5793

5794

// Decompose D into D0 * 2^K

5795

unsigned K = D.countTrailingZeros();

5796

assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOne() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5796, __extension__ __PRETTY_FUNCTION__));

5797

APInt D0 = D.lshr(K);

5798

5799

if (!D.isMinSignedValue()) {

5800

// D is even if it has trailing zeros; unless it's INT_MIN, in which case

5801

// we don't care about this lane in this fold, we'll special-handle it.

5802

HadEvenDivisor |= (K != 0);

5803

}

5804

5805

// D is a power-of-two if D0 is one. This includes INT_MIN.

5806

// If all divisors are power-of-two, we will prefer to avoid the fold.

5807

AllDivisorsArePowerOfTwo &= D0.isOne();

5808

5809

// P = inv(D0, 2^W)

5810

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5811

unsigned W = D.getBitWidth();

5812

APInt P = D0.zext(W + 1)

5813

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5814

.trunc(W);

5815

assert(!P.isZero() && "No multiplicative inverse!")(static_cast <bool> (!P.isZero() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isZero() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5815, __extension__ __PRETTY_FUNCTION__)); // unreachable

5816

assert((D0 * P).isOne() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOne() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOne() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5816, __extension__ __PRETTY_FUNCTION__));

5817

5818

// A = floor((2^(W - 1) - 1) / D0) & -2^K

5819

APInt A = APInt::getSignedMaxValue(W).udiv(D0);

5820

A.clearLowBits(K);

5821

5822

if (!D.isMinSignedValue()) {

5823

// If divisor INT_MIN, then we don't care about this lane in this fold,

5824

// we'll special-handle it.

5825

NeedToApplyOffset |= A != 0;

5826

}

5827

5828

// Q = floor((2 * A) / (2^K))

5829

APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));

5830

5831

assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&(static_cast <bool> (APInt::getAllOnes(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5832, __extension__ __PRETTY_FUNCTION__))

5832

"We are expecting that A is always less than all-ones for SVT")(static_cast <bool> (APInt::getAllOnes(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5832, __extension__ __PRETTY_FUNCTION__));

5833

assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5834, __extension__ __PRETTY_FUNCTION__))

5834

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5834, __extension__ __PRETTY_FUNCTION__));

5835

5836

// If the divisor is 1 the result can be constant-folded. Likewise, we

5837

// don't care about INT_MIN lanes, those can be set to undef if appropriate.

5838

if (D.isOne()) {

5839

// Set P, A and K to a bogus values so we can try to splat them.

5840

P = 0;

5841

A = -1;

5842

K = -1;

5843

5844

// x ?% 1 == 0 <--> true <--> x u<= -1

5845

Q = -1;

5846

}

5847

5848

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5849

AAmts.push_back(DAG.getConstant(A, DL, SVT));

5850

KAmts.push_back(

5851

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5852

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5853

return true;

5854

};

5855

5856

SDValue N = REMNode.getOperand(0);

5857

SDValue D = REMNode.getOperand(1);

5858

5859

// Collect the values from each element.

5860

if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))

5861

return SDValue();

5862

5863

// If this is a srem by a one, avoid the fold since it can be constant-folded.

5864

if (AllDivisorsAreOnes)

5865

return SDValue();

5866

5867

// If this is a srem by a powers-of-two (including INT_MIN), avoid the fold

5868

// since it can be best implemented as a bit test.

5869

if (AllDivisorsArePowerOfTwo)

5870

return SDValue();

5871

5872

SDValue PVal, AVal, KVal, QVal;

5873

if (D.getOpcode() == ISD::BUILD_VECTOR) {

5874

if (HadOneDivisor) {

5875

// Try to turn PAmts into a splat, since we don't care about the values

5876

// that are currently '0'. If we can't, just keep '0'`s.

5877

turnVectorIntoSplatVector(PAmts, isNullConstant);

5878

// Try to turn AAmts into a splat, since we don't care about the

5879

// values that are currently '-1'. If we can't, change them to '0'`s.

5880

turnVectorIntoSplatVector(AAmts, isAllOnesConstant,

5881

DAG.getConstant(0, DL, SVT));

5882

// Try to turn KAmts into a splat, since we don't care about the values

5883

// that are currently '-1'. If we can't, change them to '0'`s.

5884

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5885

DAG.getConstant(0, DL, ShSVT));

5886

}

5887

5888

PVal = DAG.getBuildVector(VT, DL, PAmts);

5889

AVal = DAG.getBuildVector(VT, DL, AAmts);

5890

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5891

QVal = DAG.getBuildVector(VT, DL, QAmts);

5892

} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {

5893

assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))

5894

QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))

5895

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))

5896

"vectors")(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__));

5897

PVal = DAG.getSplatVector(VT, DL, PAmts[0]);

5898

AVal = DAG.getSplatVector(VT, DL, AAmts[0]);

5899

KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);

5900

QVal = DAG.getSplatVector(VT, DL, QAmts[0]);

5901

} else {

5902

assert(isa<ConstantSDNode>(D) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(D) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(D) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5902, __extension__ __PRETTY_FUNCTION__));

5903

PVal = PAmts[0];

5904

AVal = AAmts[0];

5905

KVal = KAmts[0];

5906

QVal = QAmts[0];

5907

}

5908

5909

// (mul N, P)

5910

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5911

Created.push_back(Op0.getNode());

5912

5913

if (NeedToApplyOffset) {

5914

// We need ADD to do this.

5915

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))

5916

return SDValue();

5917

5918

// (add (mul N, P), A)

5919

Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);

5920

Created.push_back(Op0.getNode());

5921

}

5922

5923

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5924

// divisors as a performance improvement, since rotating by 0 is a no-op.

5925

if (HadEvenDivisor) {

5926

// We need ROTR to do this.

5927

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5928

return SDValue();

5929

// SREM: (rotr (add (mul N, P), A), K)

5930

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);

5931

Created.push_back(Op0.getNode());

5932

}

5933

5934

// SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)

5935

SDValue Fold =

5936

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5937

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5938

5939

// If we didn't have lanes with INT_MIN divisor, then we're done.

5940

if (!HadIntMinDivisor)

5941

return Fold;

5942

5943

// That fold is only valid for positive divisors. Which effectively means,

5944

// it is invalid for INT_MIN divisors. So if we have such a lane,

5945

// we must fix-up results for said lanes.

5946

assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5946, __extension__ __PRETTY_FUNCTION__));

5947

5948

// NOTE: we avoid letting illegal types through even if we're before legalize

5949

// ops – legalization has a hard time producing good code for the code that

5950

// follows.

5951

if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||

5952

!isOperationLegalOrCustom(ISD::AND, VT) ||

5953

!isOperationLegalOrCustom(Cond, VT) ||

5954

!isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))

5955

return SDValue();

5956

5957

Created.push_back(Fold.getNode());

5958

5959

SDValue IntMin = DAG.getConstant(

5960

APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);

5961

SDValue IntMax = DAG.getConstant(

5962

APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);

5963

SDValue Zero =

5964

DAG.getConstant(APInt::getZero(SVT.getScalarSizeInBits()), DL, VT);

5965

5966

// Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.

5967

SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);

5968

Created.push_back(DivisorIsIntMin.getNode());

5969

5970

// (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0

5971

SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);

5972

Created.push_back(Masked.getNode());

5973

SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);

5974

Created.push_back(MaskedIsZero.getNode());

5975

5976

// To produce final result we need to blend 2 vectors: 'SetCC' and

5977

// 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick

5978

// from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is

5979

// constant-folded, select can get lowered to a shuffle with constant mask.

5980

SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,

5981

MaskedIsZero, Fold);

5982

5983

return Blended;

5984

}

5985

5986

bool TargetLowering::

5987

verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {

5988

if (!isa<ConstantSDNode>(Op.getOperand(0))) {

5989

DAG.getContext()->emitError("argument to '__builtin_return_address' must "

5990

"be a constant integer");

5991

return true;

5992

}

5993

5994

return false;

5995

}

5996

5997

SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,

5998

const DenormalMode &Mode) const {

5999

SDLoc DL(Op);

6000

EVT VT = Op.getValueType();

6001

EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

6002

SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);

6003

// Testing it with denormal inputs to avoid wrong estimate.

6004

if (Mode.Input == DenormalMode::IEEE) {

6005

// This is specifically a check for the handling of denormal inputs,

6006

// not the result.

6007

6008

// Test = fabs(X) < SmallestNormal

6009

const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);

6010

APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);

6011

SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);

6012

SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);

6013

return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);

6014

}

6015

// Test = X == 0.0

6016

return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);

6017

}

6018

6019

SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,

6020

bool LegalOps, bool OptForSize,

6021

NegatibleCost &Cost,

6022

unsigned Depth) const {

6023

// fneg is removable even if it has multiple uses.

6024

if (Op.getOpcode() == ISD::FNEG) {

6025

Cost = NegatibleCost::Cheaper;

6026

return Op.getOperand(0);

6027

}

6028

6029

// Don't recurse exponentially.

6030

if (Depth > SelectionDAG::MaxRecursionDepth)

6031

return SDValue();

6032

6033

// Pre-increment recursion depth for use in recursive calls.

6034

++Depth;

6035

const SDNodeFlags Flags = Op->getFlags();

6036

const TargetOptions &Options = DAG.getTarget().Options;

6037

EVT VT = Op.getValueType();

6038

unsigned Opcode = Op.getOpcode();

6039

6040

// Don't allow anything with multiple uses unless we know it is free.

6041

if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {

6042

bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&

6043

isFPExtFree(VT, Op.getOperand(0).getValueType());

6044

if (!IsFreeExtend)

6045

return SDValue();

6046

}

6047

6048

auto RemoveDeadNode = [&](SDValue N) {

6049

if (N && N.getNode()->use_empty())

6050

DAG.RemoveDeadNode(N.getNode());

6051

};

6052

6053

SDLoc DL(Op);

6054

6055

// Because getNegatedExpression can delete nodes we need a handle to keep

6056

// temporary nodes alive in case the recursion manages to create an identical

6057

// node.

6058

std::list<HandleSDNode> Handles;

6059

6060

switch (Opcode) {

6061

case ISD::ConstantFP: {

6062

// Don't invert constant FP values after legalization unless the target says

6063

// the negated constant is legal.

6064

bool IsOpLegal =

6065

isOperationLegal(ISD::ConstantFP, VT) ||

6066

isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,

6067

OptForSize);

6068

6069

if (LegalOps && !IsOpLegal)

6070

break;

6071

6072

APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();

6073

V.changeSign();

6074

SDValue CFP = DAG.getConstantFP(V, DL, VT);

6075

6076

// If we already have the use of the negated floating constant, it is free

6077

// to negate it even it has multiple uses.

6078

if (!Op.hasOneUse() && CFP.use_empty())

6079

break;

6080

Cost = NegatibleCost::Neutral;

6081

return CFP;

6082

}

6083

case ISD::BUILD_VECTOR: {

6084

// Only permit BUILD_VECTOR of constants.

6085

if (llvm::any_of(Op->op_values(), [&](SDValue N) {

6086

return !N.isUndef() && !isa<ConstantFPSDNode>(N);

6087

}))

6088

break;

6089

6090

bool IsOpLegal =

6091

(isOperationLegal(ISD::ConstantFP, VT) &&

6092

isOperationLegal(ISD::BUILD_VECTOR, VT)) ||

6093

llvm::all_of(Op->op_values(), [&](SDValue N) {

6094

return N.isUndef() ||

6095

isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,

6096

OptForSize);

6097

});

6098

6099

if (LegalOps && !IsOpLegal)

6100

break;

6101

6102

SmallVector<SDValue, 4> Ops;

6103

for (SDValue C : Op->op_values()) {

6104

if (C.isUndef()) {

6105

Ops.push_back(C);

6106

continue;

6107

}

6108

APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();

6109

V.changeSign();

6110

Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));

6111

}

6112

Cost = NegatibleCost::Neutral;

6113

return DAG.getBuildVector(VT, DL, Ops);

6114

}

6115

case ISD::FADD: {

6116

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6117

break;

6118

6119

// After operation legalization, it might not be legal to create new FSUBs.

6120

if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))

6121

break;

6122

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6123

6124

// fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)

6125

NegatibleCost CostX = NegatibleCost::Expensive;

6126

SDValue NegX =

6127

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6128

// Prevent this node from being deleted by the next call.

6129

if (NegX)

6130

Handles.emplace_back(NegX);

6131

6132

// fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)

6133

NegatibleCost CostY = NegatibleCost::Expensive;

6134

SDValue NegY =

6135

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6136

6137

// We're done with the handles.

6138

Handles.clear();

6139

6140

// Negate the X if its cost is less or equal than Y.

6141

if (NegX && (CostX <= CostY)) {

6142

Cost = CostX;

6143

SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);

6144

if (NegY != N)

6145

RemoveDeadNode(NegY);

6146

return N;

6147

}

6148

6149

// Negate the Y if it is not expensive.

6150

if (NegY) {

6151

Cost = CostY;

6152

SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);

6153

if (NegX != N)

6154

RemoveDeadNode(NegX);

6155

return N;

6156

}

6157

break;

6158

}

6159

case ISD::FSUB: {

6160

// We can't turn -(A-B) into B-A when we honor signed zeros.

6161

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6162

break;

6163

6164

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6165

// fold (fneg (fsub 0, Y)) -> Y

6166

if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))

6167

if (C->isZero()) {

6168

Cost = NegatibleCost::Cheaper;

6169

return Y;

6170

}

6171

6172

// fold (fneg (fsub X, Y)) -> (fsub Y, X)

6173

Cost = NegatibleCost::Neutral;

6174

return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);

6175

}

6176

case ISD::FMUL:

6177

case ISD::FDIV: {

6178

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6179

6180

// fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)

6181

NegatibleCost CostX = NegatibleCost::Expensive;

6182

SDValue NegX =

6183

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6184

// Prevent this node from being deleted by the next call.

6185

if (NegX)

6186

Handles.emplace_back(NegX);

6187

6188

// fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))

6189

NegatibleCost CostY = NegatibleCost::Expensive;

6190

SDValue NegY =

6191

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6192

6193

// We're done with the handles.

6194

Handles.clear();

6195

6196

// Negate the X if its cost is less or equal than Y.

6197

if (NegX && (CostX <= CostY)) {

6198

Cost = CostX;

6199

SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);

6200

if (NegY != N)

6201

RemoveDeadNode(NegY);

6202

return N;

6203

}

6204

6205

// Ignore X * 2.0 because that is expected to be canonicalized to X + X.

6206

if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))

6207

if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)

6208

break;

6209

6210

// Negate the Y if it is not expensive.

6211

if (NegY) {

6212

Cost = CostY;

6213

SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);

6214

if (NegX != N)

6215

RemoveDeadNode(NegX);

6216

return N;

6217

}

6218

break;

6219

}

6220

case ISD::FMA:

6221

case ISD::FMAD: {

6222

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6223

break;

6224

6225

SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);

6226

NegatibleCost CostZ = NegatibleCost::Expensive;

6227

SDValue NegZ =

6228

getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);

6229

// Give up if fail to negate the Z.

6230

if (!NegZ)

6231

break;

6232

6233

// Prevent this node from being deleted by the next two calls.

6234

Handles.emplace_back(NegZ);

6235

6236

// fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))

6237

NegatibleCost CostX = NegatibleCost::Expensive;

6238

SDValue NegX =

6239

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6240

// Prevent this node from being deleted by the next call.

6241

if (NegX)

6242

Handles.emplace_back(NegX);

6243

6244

// fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))

6245

NegatibleCost CostY = NegatibleCost::Expensive;

6246

SDValue NegY =

6247

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6248

6249

// We're done with the handles.

6250

Handles.clear();

6251

6252

// Negate the X if its cost is less or equal than Y.

6253

if (NegX && (CostX <= CostY)) {

6254

Cost = std::min(CostX, CostZ);

6255

SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);

6256

if (NegY != N)

6257

RemoveDeadNode(NegY);

6258

return N;

6259

}

6260

6261

// Negate the Y if it is not expensive.

6262

if (NegY) {

6263

Cost = std::min(CostY, CostZ);

6264

SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);

6265

if (NegX != N)

6266

RemoveDeadNode(NegX);

6267

return N;

6268

}

6269

break;

6270

}

6271

6272

case ISD::FP_EXTEND:

6273

case ISD::FSIN:

6274

if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,

6275

OptForSize, Cost, Depth))

6276

return DAG.getNode(Opcode, DL, VT, NegV);

6277

break;

6278

case ISD::FP_ROUND:

6279

if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,

6280

OptForSize, Cost, Depth))

6281

return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));

6282

break;

6283

}

6284

6285

return SDValue();

6286

}

6287

6288

//===----------------------------------------------------------------------===//

6289

// Legalization Utilities

6290

//===----------------------------------------------------------------------===//

6291

6292

bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,

6293

SDValue LHS, SDValue RHS,

6294

SmallVectorImpl<SDValue> &Result,

6295

EVT HiLoVT, SelectionDAG &DAG,

6296

MulExpansionKind Kind, SDValue LL,

6297

SDValue LH, SDValue RL, SDValue RH) const {

6298

assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6299, __extension__ __PRETTY_FUNCTION__))

6299

Opcode == ISD::SMUL_LOHI)(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6299, __extension__ __PRETTY_FUNCTION__));

6300

6301

bool HasMULHS = (Kind == MulExpansionKind::Always) ||

6302

isOperationLegalOrCustom(ISD::MULHS, HiLoVT);

6303

bool HasMULHU = (Kind == MulExpansionKind::Always) ||

6304

isOperationLegalOrCustom(ISD::MULHU, HiLoVT);

6305

bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||

6306

isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);

6307

bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||

6308

isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);

6309

6310

if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)

6311

return false;

6312

6313

unsigned OuterBitSize = VT.getScalarSizeInBits();

6314

unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();

6315

6316

// LL, LH, RL, and RH must be either all NULL or all set to a value.

6317

assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
!RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6318, __extension__ __PRETTY_FUNCTION__))

6318

(!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()))(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
!RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6318, __extension__ __PRETTY_FUNCTION__));

6319

6320

SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);

6321

auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,

6322

bool Signed) -> bool {

6323

if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {

6324

Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);

6325

Hi = SDValue(Lo.getNode(), 1);

6326

return true;

6327

}

6328

if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {

6329

Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);

6330

Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);

6331

return true;

6332

}

6333

return false;

6334

};

6335

6336

SDValue Lo, Hi;

6337

6338

if (!LL.getNode() && !RL.getNode() &&

6339

isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {

6340

LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);

6341

RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);

6342

}

6343

6344

if (!LL.getNode())

6345

return false;

6346

6347

APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);

6348

if (DAG.MaskedValueIsZero(LHS, HighMask) &&

6349

DAG.MaskedValueIsZero(RHS, HighMask)) {

6350

// The inputs are both zero-extended.

6351

if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {

6352

Result.push_back(Lo);

6353

Result.push_back(Hi);

6354

if (Opcode != ISD::MUL) {

6355

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);

6356

Result.push_back(Zero);

6357

Result.push_back(Zero);

6358

}

6359

return true;

6360

}

6361

}

6362

6363

if (!VT.isVector() && Opcode == ISD::MUL &&

6364

DAG.ComputeNumSignBits(LHS) > InnerBitSize &&

6365

DAG.ComputeNumSignBits(RHS) > InnerBitSize) {

6366

// The input values are both sign-extended.

6367

// TODO non-MUL case?

6368

if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {

6369

Result.push_back(Lo);

6370

Result.push_back(Hi);

6371

return true;

6372

}

6373

}

6374

6375

unsigned ShiftAmount = OuterBitSize - InnerBitSize;

6376

EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());

6377

if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {

6378

// FIXME getShiftAmountTy does not always return a sensible result when VT

6379

// is an illegal type, and so the type may be too small to fit the shift

6380

// amount. Override it with i32. The shift will have to be legalized.

6381

ShiftAmountTy = MVT::i32;

6382

}

6383

SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);

6384

6385

if (!LH.getNode() && !RH.getNode() &&

6386

isOperationLegalOrCustom(ISD::SRL, VT) &&

6387

isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {

6388

LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);

6389

LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);

6390

RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);

6391

RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);

6392

}

6393

6394

if (!LH.getNode())

6395

return false;

6396

6397

if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))

6398

return false;

6399

6400

Result.push_back(Lo);

6401

6402

if (Opcode == ISD::MUL) {

6403

RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);

6404

LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);

6405

Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);

6406

Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);

6407

Result.push_back(Hi);

6408

return true;

6409

}

6410

6411

// Compute the full width result.

6412

auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {

6413

Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);

6414

Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);

6415

Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);

6416

return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);

6417

};

6418

6419

SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);

6420

if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))

6421

return false;

6422

6423

// This is effectively the add part of a multiply-add of half-sized operands,

6424

// so it cannot overflow.

6425

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

6426

6427

if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))

6428

return false;

6429

6430

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);

6431

EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

6432

6433

bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&

6434

isOperationLegalOrCustom(ISD::ADDE, VT));

6435

if (UseGlue)

6436

Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,

6437

Merge(Lo, Hi));

6438

else

6439

Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,

6440

Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));

6441

6442

SDValue Carry = Next.getValue(1);

6443

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6444

Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

6445

6446

if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))

6447

return false;

6448

6449

if (UseGlue)

6450

Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,

6451

Carry);

6452

else

6453

Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,

6454

Zero, Carry);

6455

6456

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

6457

6458

if (Opcode == ISD::SMUL_LOHI) {

6459

SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,

6460

DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));

6461

Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);

6462

6463

NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,

6464

DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));

6465

Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);

6466

}

6467

6468

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6469

Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

6470

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6471

return true;

6472

}

6473

6474

bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,

6475

SelectionDAG &DAG, MulExpansionKind Kind,

6476

SDValue LL, SDValue LH, SDValue RL,

6477

SDValue RH) const {

6478

SmallVector<SDValue, 2> Result;

6479

bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),

6480

N->getOperand(0), N->getOperand(1), Result, HiLoVT,

6481

DAG, Kind, LL, LH, RL, RH);

6482

if (Ok) {

6483

assert(Result.size() == 2)(static_cast <bool> (Result.size() == 2) ? void (0) : __assert_fail
("Result.size() == 2", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6483, __extension__ __PRETTY_FUNCTION__));

6484

Lo = Result[0];

6485

Hi = Result[1];

6486

}

6487

return Ok;

6488

}

6489

6490

// Check that (every element of) Z is undef or not an exact multiple of BW.

6491

static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {

6492

return ISD::matchUnaryPredicate(

6493

Z,

6494

[=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },

6495

true);

6496

}

6497

6498

bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,

6499

SelectionDAG &DAG) const {

6500

EVT VT = Node->getValueType(0);

6501

6502

if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||

6503

!isOperationLegalOrCustom(ISD::SRL, VT) ||

6504

!isOperationLegalOrCustom(ISD::SUB, VT) ||

6505

!isOperationLegalOrCustomOrPromote(ISD::OR, VT)))

6506

return false;

6507

6508

SDValue X = Node->getOperand(0);

6509

SDValue Y = Node->getOperand(1);

6510

SDValue Z = Node->getOperand(2);

6511

6512

unsigned BW = VT.getScalarSizeInBits();

6513

bool IsFSHL = Node->getOpcode() == ISD::FSHL;

6514

SDLoc DL(SDValue(Node, 0));

6515

6516

EVT ShVT = Z.getValueType();

6517

6518

// If a funnel shift in the other direction is more supported, use it.

6519

unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;

6520

if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&

6521

isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {

6522

if (isNonZeroModBitWidthOrUndef(Z, BW)) {

6523

// fshl X, Y, Z -> fshr X, Y, -Z

6524

// fshr X, Y, Z -> fshl X, Y, -Z

6525

SDValue Zero = DAG.getConstant(0, DL, ShVT);

6526

Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);

6527

} else {

6528

// fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z

6529

// fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z

6530

SDValue One = DAG.getConstant(1, DL, ShVT);

6531

if (IsFSHL) {

6532

Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);

6533

X = DAG.getNode(ISD::SRL, DL, VT, X, One);

6534

} else {

6535

X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);

6536

Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);

6537

}

6538

Z = DAG.getNOT(DL, Z, ShVT);

6539

}

6540

Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);

6541

return true;

6542

}

6543

6544

SDValue ShX, ShY;

6545

SDValue ShAmt, InvShAmt;

6546

if (isNonZeroModBitWidthOrUndef(Z, BW)) {

6547

// fshl: X << C | Y >> (BW - C)

6548

// fshr: X << (BW - C) | Y >> C

6549

// where C = Z % BW is not zero

6550

SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);

6551

ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);

6552

InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);

6553

ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);

6554

ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);

6555

} else {

6556

// fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))

6557

// fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)

6558

SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);

6559

if (isPowerOf2_32(BW)) {

6560

// Z % BW -> Z & (BW - 1)

6561

ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);

6562

// (BW - 1) - (Z % BW) -> ~Z & (BW - 1)

6563

InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);

6564

} else {

6565

SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);

6566

ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);

6567

InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);

6568

}

6569

6570

SDValue One = DAG.getConstant(1, DL, ShVT);

6571

if (IsFSHL) {

6572

ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);

6573

SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);

6574

ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);

6575

} else {

6576

SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);

6577

ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);

6578

ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);

6579

}

6580

}

6581

Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);

6582

return true;

6583

}

6584

6585

// TODO: Merge with expandFunnelShift.

6586

bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,

6587

SDValue &Result, SelectionDAG &DAG) const {

6588

EVT VT = Node->getValueType(0);

6589

unsigned EltSizeInBits = VT.getScalarSizeInBits();

6590

bool IsLeft = Node->getOpcode() == ISD::ROTL;

6591

SDValue Op0 = Node->getOperand(0);

6592

SDValue Op1 = Node->getOperand(1);

6593

SDLoc DL(SDValue(Node, 0));

6594

6595

EVT ShVT = Op1.getValueType();

6596

SDValue Zero = DAG.getConstant(0, DL, ShVT);

6597

6598

// If a rotate in the other direction is supported, use it.

6599

unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;

6600

if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {

6601

SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);

6602

Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);

6603

return true;

6604

}

6605

6606

if (!AllowVectorOps && VT.isVector() &&

6607

(!isOperationLegalOrCustom(ISD::SHL, VT) ||

6608

!isOperationLegalOrCustom(ISD::SRL, VT) ||

6609

!isOperationLegalOrCustom(ISD::SUB, VT) ||

6610

!isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||

6611

!isOperationLegalOrCustomOrPromote(ISD::AND, VT)))

6612

return false;

6613

6614

unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;

6615

unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;

6616

SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);

6617

SDValue ShVal;

6618

SDValue HsVal;

6619

if (isPowerOf2_32(EltSizeInBits)) {

6620

// (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))

6621

// (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))

6622

SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);

6623

SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);

6624

ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);

6625

SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);

6626

HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);

6627

} else {

6628

// (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))

6629

// (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))

6630

SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);

6631

SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);

6632

ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);

6633

SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);

6634

SDValue One = DAG.getConstant(1, DL, ShVT);

6635

HsVal =

6636

DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);

6637

}

6638

Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);

6639

return true;

6640

}

6641

6642

void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,

6643

SelectionDAG &DAG) const {

6644

assert(Node->getNumOperands() == 3 && "Not a double-shift!")(static_cast <bool> (Node->getNumOperands() == 3 &&
"Not a double-shift!") ? void (0) : __assert_fail ("Node->getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6644, __extension__ __PRETTY_FUNCTION__));

6645

EVT VT = Node->getValueType(0);

6646

unsigned VTBits = VT.getScalarSizeInBits();

6647

assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected")(static_cast <bool> (isPowerOf2_32(VTBits) && "Power-of-two integer type expected"
) ? void (0) : __assert_fail ("isPowerOf2_32(VTBits) && \"Power-of-two integer type expected\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6647, __extension__ __PRETTY_FUNCTION__));

6648

6649

bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;

6650

bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;

6651

SDValue ShOpLo = Node->getOperand(0);

6652

SDValue ShOpHi = Node->getOperand(1);

6653

SDValue ShAmt = Node->getOperand(2);

6654

EVT ShAmtVT = ShAmt.getValueType();

6655

EVT ShAmtCCVT =

6656

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);

6657

SDLoc dl(Node);

6658

6659

// ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and

6660

// ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized

6661

// away during isel.

6662

SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,

6663

DAG.getConstant(VTBits - 1, dl, ShAmtVT));

6664

SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,

6665

DAG.getConstant(VTBits - 1, dl, ShAmtVT))

6666

: DAG.getConstant(0, dl, VT);

6667

6668

SDValue Tmp2, Tmp3;

6669

if (IsSHL) {

6670

Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);

6671

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);

6672

} else {

6673

Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);

6674

Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);

6675

}

6676

6677

// If the shift amount is larger or equal than the width of a part we don't

6678

// use the result from the FSHL/FSHR. Insert a test and select the appropriate

6679

// values for large shift amounts.

6680

SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,

6681

DAG.getConstant(VTBits, dl, ShAmtVT));

6682

SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,

6683

DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);

6684

6685

if (IsSHL) {

6686

Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);

6687

Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);

6688

} else {

6689

Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);

6690

Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);

6691

}

6692

}

6693

6694

bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,

6695

SelectionDAG &DAG) const {

6696

unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;

6697

SDValue Src = Node->getOperand(OpNo);

6698

EVT SrcVT = Src.getValueType();

6699

EVT DstVT = Node->getValueType(0);

6700

SDLoc dl(SDValue(Node, 0));

6701

6702

// FIXME: Only f32 to i64 conversions are supported.

6703

if (SrcVT != MVT::f32 || DstVT != MVT::i64)

6704

return false;

6705

6706

if (Node->isStrictFPOpcode())

6707

// When a NaN is converted to an integer a trap is allowed. We can't

6708

// use this expansion here because it would eliminate that trap. Other

6709

// traps are also allowed and cannot be eliminated. See

6710

// IEEE 754-2008 sec 5.8.

6711

return false;

6712

6713

// Expand f32 -> i64 conversion

6714

// This algorithm comes from compiler-rt's implementation of fixsfdi:

6715

// https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c

6716

unsigned SrcEltBits = SrcVT.getScalarSizeInBits();

6717

EVT IntVT = SrcVT.changeTypeToInteger();

6718

EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());

6719

6720

SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);

6721

SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);

6722

SDValue Bias = DAG.getConstant(127, dl, IntVT);

6723

SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);

6724

SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);

6725

SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);

6726

6727

SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);

6728

6729

SDValue ExponentBits = DAG.getNode(

6730

ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),

6731

DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));

6732

SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);

6733

6734

SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,

6735

DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),

6736

DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));

6737

Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);

6738

6739

SDValue R = DAG.getNode(ISD::OR, dl, IntVT,

6740

DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),

6741

DAG.getConstant(0x00800000, dl, IntVT));

6742

6743

R = DAG.getZExtOrTrunc(R, dl, DstVT);

6744

6745

R = DAG.getSelectCC(

6746

dl, Exponent, ExponentLoBit,

6747

DAG.getNode(ISD::SHL, dl, DstVT, R,

6748

DAG.getZExtOrTrunc(

6749

DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),

6750

dl, IntShVT)),

6751

DAG.getNode(ISD::SRL, dl, DstVT, R,

6752

DAG.getZExtOrTrunc(

6753

DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),

6754

dl, IntShVT)),

6755

ISD::SETGT);

6756

6757

SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,

6758

DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);

6759

6760

Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),

6761

DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);

6762

return true;

6763

}

6764

6765

bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,

6766

SDValue &Chain,

6767

SelectionDAG &DAG) const {

6768

SDLoc dl(SDValue(Node, 0));

6769

unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;

6770

SDValue Src = Node->getOperand(OpNo);

6771

6772

EVT SrcVT = Src.getValueType();

6773

EVT DstVT = Node->getValueType(0);

6774

EVT SetCCVT =

6775

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);

6776

EVT DstSetCCVT =

6777

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);

6778

6779

// Only expand vector types if we have the appropriate vector bit operations.

6780

unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :

6781

ISD::FP_TO_SINT;

6782

if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||

6783

!isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))

6784

return false;

6785

6786

// If the maximum float value is smaller then the signed integer range,

6787

// the destination signmask can't be represented by the float, so we can

6788

// just use FP_TO_SINT directly.

6789

const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);

6790

APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));

6791

APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());

6792

if (APFloat::opOverflow &

6793

APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {

6794

if (Node->isStrictFPOpcode()) {

6795

Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },

6796

{ Node->getOperand(0), Src });

6797

Chain = Result.getValue(1);

6798

} else

6799

Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);

6800

return true;

6801

}

6802

6803

// Don't expand it if there isn't cheap fsub instruction.

6804

if (!isOperationLegalOrCustom(

6805

Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))

6806

return false;

6807

6808

SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);

6809

SDValue Sel;

6810

6811

if (Node->isStrictFPOpcode()) {

6812

Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,

6813

Node->getOperand(0), /*IsSignaling*/ true);

6814

Chain = Sel.getValue(1);

6815

} else {

6816

Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);

6817

}

6818

6819

bool Strict = Node->isStrictFPOpcode() ||

6820

shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);

6821

6822

if (Strict) {

6823

// Expand based on maximum range of FP_TO_SINT, if the value exceeds the

6824

// signmask then offset (the result of which should be fully representable).

6825

// Sel = Src < 0x8000000000000000

6826

// FltOfs = select Sel, 0, 0x8000000000000000

6827

// IntOfs = select Sel, 0, 0x8000000000000000

6828

// Result = fp_to_sint(Src - FltOfs) ^ IntOfs

6829

6830

// TODO: Should any fast-math-flags be set for the FSUB?

6831

SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,

6832

DAG.getConstantFP(0.0, dl, SrcVT), Cst);

6833

Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);

6834

SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,

6835

DAG.getConstant(0, dl, DstVT),

6836

DAG.getConstant(SignMask, dl, DstVT));

6837

SDValue SInt;

6838

if (Node->isStrictFPOpcode()) {

6839

SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },

6840

{ Chain, Src, FltOfs });

6841

SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },

6842

{ Val.getValue(1), Val });

6843

Chain = SInt.getValue(1);

6844

} else {

6845

SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);

6846

SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);

6847

}

6848

Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);

6849

} else {

6850

// Expand based on maximum range of FP_TO_SINT:

6851

// True = fp_to_sint(Src)

6852

// False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)

6853

// Result = select (Src < 0x8000000000000000), True, False

6854

6855

SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);

6856

// TODO: Should any fast-math-flags be set for the FSUB?

6857

SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,

6858

DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));

6859

False = DAG.getNode(ISD::XOR, dl, DstVT, False,

6860

DAG.getConstant(SignMask, dl, DstVT));

6861

Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);

6862

Result = DAG.getSelect(dl, DstVT, Sel, True, False);

6863

}

6864

return true;

6865

}

6866

6867

bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,

6868

SDValue &Chain,

6869

SelectionDAG &DAG) const {

6870

// This transform is not correct for converting 0 when rounding mode is set

6871

// to round toward negative infinity which will produce -0.0. So disable under

6872

// strictfp.

6873

if (Node->isStrictFPOpcode())

6874

return false;

6875

6876

SDValue Src = Node->getOperand(0);

6877

EVT SrcVT = Src.getValueType();

6878

EVT DstVT = Node->getValueType(0);

6879

6880

if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)

6881

return false;

6882

6883

// Only expand vector types if we have the appropriate vector bit operations.

6884

if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||

6885

!isOperationLegalOrCustom(ISD::FADD, DstVT) ||

6886

!isOperationLegalOrCustom(ISD::FSUB, DstVT) ||

6887

!isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||

6888

!isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))

6889

return false;

6890

6891

SDLoc dl(SDValue(Node, 0));

6892

EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());

6893

6894

// Implementation of unsigned i64 to f64 following the algorithm in

6895

// __floatundidf in compiler_rt. This implementation performs rounding

6896

// correctly in all rounding modes with the exception of converting 0

6897

// when rounding toward negative infinity. In that case the fsub will produce

6898

// -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.

6899

SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, SrcVT);

6900

SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(

6901

BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl, DstVT);

6902

SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, SrcVT);

6903

SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF)0x00000000FFFFFFFFUL, dl, SrcVT);

6904

SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);

6905

6906

SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);

6907

SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);

6908

SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);

6909

SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);

6910

SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);

6911

SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);

6912

SDValue HiSub =

6913

DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);

6914

Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);

6915

return true;

6916

}

6917

6918

SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,

6919

SelectionDAG &DAG) const {

6920

SDLoc dl(Node);

6921

unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?

6922

ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;

6923

EVT VT = Node->getValueType(0);

6924

6925

if (VT.isScalableVector())

6926

report_fatal_error(

6927

"Expanding fminnum/fmaxnum for scalable vectors is undefined.");

6928

6929

if (isOperationLegalOrCustom(NewOp, VT)) {

6930

SDValue Quiet0 = Node->getOperand(0);

6931

SDValue Quiet1 = Node->getOperand(1);

6932

6933

if (!Node->getFlags().hasNoNaNs()) {

6934

// Insert canonicalizes if it's possible we need to quiet to get correct

6935

// sNaN behavior.

6936

if (!DAG.isKnownNeverSNaN(Quiet0)) {

6937

Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,

6938

Node->getFlags());

6939

}

6940

if (!DAG.isKnownNeverSNaN(Quiet1)) {

6941

Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,

6942

Node->getFlags());

6943

}

6944

}

6945

6946

return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());

6947

}

6948

6949

// If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that

6950

// instead if there are no NaNs.

6951

if (Node->getFlags().hasNoNaNs()) {

6952

unsigned IEEE2018Op =

6953

Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;

6954

if (isOperationLegalOrCustom(IEEE2018Op, VT)) {

6955

return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),

6956

Node->getOperand(1), Node->getFlags());

6957

}

6958

}

6959

6960

// If none of the above worked, but there are no NaNs, then expand to

6961

// a compare/select sequence. This is required for correctness since

6962

// InstCombine might have canonicalized a fcmp+select sequence to a

6963

// FMINNUM/FMAXNUM node. If we were to fall through to the default

6964

// expansion to libcall, we might introduce a link-time dependency

6965

// on libm into a file that originally did not have one.

6966

if (Node->getFlags().hasNoNaNs()) {

6967

ISD::CondCode Pred =

6968

Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;

6969

SDValue Op1 = Node->getOperand(0);

6970

SDValue Op2 = Node->getOperand(1);

6971

SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);

6972

// Copy FMF flags, but always set the no-signed-zeros flag

6973

// as this is implied by the FMINNUM/FMAXNUM semantics.

6974

SDNodeFlags Flags = Node->getFlags();

6975

Flags.setNoSignedZeros(true);

6976

SelCC->setFlags(Flags);

6977

return SelCC;

6978

}

6979

6980

return SDValue();

6981

}

6982

6983

bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,

6984

SelectionDAG &DAG) const {

6985

SDLoc dl(Node);

6986

EVT VT = Node->getValueType(0);

6987

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

6988

SDValue Op = Node->getOperand(0);

6989

unsigned Len = VT.getScalarSizeInBits();

6990

assert(VT.isInteger() && "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6990, __extension__ __PRETTY_FUNCTION__));

6991

6992

// TODO: Add support for irregular type lengths.

6993

if (!(Len <= 128 && Len % 8 == 0))

6994

return false;

6995

6996

// Only expand vector types if we have the appropriate vector bit operations.

6997

if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||

6998

!isOperationLegalOrCustom(ISD::SUB, VT) ||

6999

!isOperationLegalOrCustom(ISD::SRL, VT) ||

7000

(Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||

7001

!isOperationLegalOrCustomOrPromote(ISD::AND, VT)))

7002

return false;

7003

7004

// This is the "best" algorithm from

7005

// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel

7006

SDValue Mask55 =

7007

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);

7008

SDValue Mask33 =

7009

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);

7010

SDValue Mask0F =

7011

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);

7012

SDValue Mask01 =

7013

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);

7014

7015

// v = v - ((v >> 1) & 0x55555555...)

7016

Op = DAG.getNode(ISD::SUB, dl, VT, Op,

7017

DAG.getNode(ISD::AND, dl, VT,

7018

DAG.getNode(ISD::SRL, dl, VT, Op,

7019

DAG.getConstant(1, dl, ShVT)),

7020

Mask55));

7021

// v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)

7022

Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),

7023

DAG.getNode(ISD::AND, dl, VT,

7024

DAG.getNode(ISD::SRL, dl, VT, Op,

7025

DAG.getConstant(2, dl, ShVT)),

7026

Mask33));

7027

// v = (v + (v >> 4)) & 0x0F0F0F0F...

7028

Op = DAG.getNode(ISD::AND, dl, VT,

7029

DAG.getNode(ISD::ADD, dl, VT, Op,

7030

DAG.getNode(ISD::SRL, dl, VT, Op,

7031

DAG.getConstant(4, dl, ShVT))),

7032

Mask0F);

7033

// v = (v * 0x01010101...) >> (Len - 8)

7034

if (Len > 8)

7035

Op =

7036

DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),

7037

DAG.getConstant(Len - 8, dl, ShVT));

7038

7039

Result = Op;

7040

return true;

7041

}

7042

7043

bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,

7044

SelectionDAG &DAG) const {

7045

SDLoc dl(Node);

7046

EVT VT = Node->getValueType(0);

7047

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

7048

SDValue Op = Node->getOperand(0);

7049

unsigned NumBitsPerElt = VT.getScalarSizeInBits();

7050

7051

// If the non-ZERO_UNDEF version is supported we can use that instead.

7052

if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&

7053

isOperationLegalOrCustom(ISD::CTLZ, VT)) {

7054

Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);

7055

return true;

7056

}

7057

7058

// If the ZERO_UNDEF version is supported use that and handle the zero case.

7059

if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {

7060

EVT SetCCVT =

7061

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

7062

SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);

7063

SDValue Zero = DAG.getConstant(0, dl, VT);

7064

SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);

7065

Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,

7066

DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);

7067

return true;

7068

}

7069

7070

// Only expand vector types if we have the appropriate vector bit operations.

7071

if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||

7072

!isOperationLegalOrCustom(ISD::CTPOP, VT) ||

7073

!isOperationLegalOrCustom(ISD::SRL, VT) ||

7074

!isOperationLegalOrCustomOrPromote(ISD::OR, VT)))

7075

return false;

7076

7077

// for now, we do this:

7078

// x = x | (x >> 1);

7079

// x = x | (x >> 2);

7080

// ...

7081

// x = x | (x >>16);

7082

// x = x | (x >>32); // for 64-bit input

7083

// return popcount(~x);

7084

//

7085

// Ref: "Hacker's Delight" by Henry Warren

7086

for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {

7087

SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);

7088

Op = DAG.getNode(ISD::OR, dl, VT, Op,

7089

DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));

7090

}

7091

Op = DAG.getNOT(dl, Op, VT);

7092

Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);

7093

return true;

7094

}

7095

7096

bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,

7097

SelectionDAG &DAG) const {

7098

SDLoc dl(Node);

7099

EVT VT = Node->getValueType(0);

7100

SDValue Op = Node->getOperand(0);

7101

unsigned NumBitsPerElt = VT.getScalarSizeInBits();

7102

7103

// If the non-ZERO_UNDEF version is supported we can use that instead.

7104

if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&

7105

isOperationLegalOrCustom(ISD::CTTZ, VT)) {

7106

Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);

7107

return true;

7108

}

7109

7110

// If the ZERO_UNDEF version is supported use that and handle the zero case.

7111

if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {

7112

EVT SetCCVT =

7113

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

7114

SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);

7115

SDValue Zero = DAG.getConstant(0, dl, VT);

7116

SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);

7117

Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,

7118

DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);

7119

return true;

7120

}

7121

7122

// Only expand vector types if we have the appropriate vector bit operations.

7123

if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||

7124

(!isOperationLegalOrCustom(ISD::CTPOP, VT) &&

7125

!isOperationLegalOrCustom(ISD::CTLZ, VT)) ||

7126

!isOperationLegalOrCustom(ISD::SUB, VT) ||

7127

!isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||

7128

!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))

7129

return false;

7130

7131

// for now, we use: { return popcount(~x & (x - 1)); }

7132

// unless the target has ctlz but not ctpop, in which case we use:

7133

// { return 32 - nlz(~x & (x-1)); }

7134

// Ref: "Hacker's Delight" by Henry Warren

7135

SDValue Tmp = DAG.getNode(

7136

ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),

7137

DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));

7138

7139

// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.

7140

if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {

7141

Result =

7142

DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),

7143

DAG.getNode(ISD::CTLZ, dl, VT, Tmp));

7144

return true;

7145

}

7146

7147

Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);

7148

return true;

7149

}

7150

7151

bool TargetLowering::expandABS(SDNode *N, SDValue &Result,

7152

SelectionDAG &DAG, bool IsNegative) const {

7153

SDLoc dl(N);

7154

EVT VT = N->getValueType(0);

7155

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

7156

SDValue Op = N->getOperand(0);

7157

7158

// abs(x) -> smax(x,sub(0,x))

7159

if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&

7160

isOperationLegal(ISD::SMAX, VT)) {

7161

SDValue Zero = DAG.getConstant(0, dl, VT);

7162

Result = DAG.getNode(ISD::SMAX, dl, VT, Op,

7163

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7164

return true;

7165

}

7166

7167

// abs(x) -> umin(x,sub(0,x))

7168

if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&

7169

isOperationLegal(ISD::UMIN, VT)) {

7170

SDValue Zero = DAG.getConstant(0, dl, VT);

7171

Result = DAG.getNode(ISD::UMIN, dl, VT, Op,

7172

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7173

return true;

7174

}

7175

7176

// 0 - abs(x) -> smin(x, sub(0,x))

7177

if (IsNegative && isOperationLegal(ISD::SUB, VT) &&

7178

isOperationLegal(ISD::SMIN, VT)) {

7179

SDValue Zero = DAG.getConstant(0, dl, VT);

7180

Result = DAG.getNode(ISD::SMIN, dl, VT, Op,

7181

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7182

return true;

7183

}

7184

7185

// Only expand vector types if we have the appropriate vector operations.

7186

if (VT.isVector() &&

7187

(!isOperationLegalOrCustom(ISD::SRA, VT) ||

7188

(!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||

7189

(IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||

7190

!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))

7191

return false;

7192

7193

SDValue Shift =

7194

DAG.getNode(ISD::SRA, dl, VT, Op,

7195

DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));

7196

if (!IsNegative) {

7197

SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);

7198

Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);

7199

} else {

7200

// 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))

7201

SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);

7202

Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);

7203

}

7204

return true;

7205

}

7206

7207

SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {

7208

SDLoc dl(N);

7209

EVT VT = N->getValueType(0);

7210

SDValue Op = N->getOperand(0);

7211

7212

if (!VT.isSimple())

7213

return SDValue();

7214

7215

EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());

7216

SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;

7217

switch (VT.getSimpleVT().getScalarType().SimpleTy) {

7218

default:

7219

return SDValue();

7220

case MVT::i16:

7221

// Use a rotate by 8. This can be further expanded if necessary.

7222

return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7223

case MVT::i32:

7224

Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7225

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7226

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7227

Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7228

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,

7229

DAG.getConstant(0xFF0000, dl, VT));

7230

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));

7231

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);

7232

Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);

7233

return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);

7234

case MVT::i64:

7235

Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));

7236

Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));

7237

Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7238

Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7239

Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7240

Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7241

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));

7242

Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));

7243

Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,

7244

DAG.getConstant(255ULL<<48, dl, VT));

7245

Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,

7246

DAG.getConstant(255ULL<<40, dl, VT));

7247

Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,

7248

DAG.getConstant(255ULL<<32, dl, VT));

7249

Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,

7250

DAG.getConstant(255ULL<<24, dl, VT));

7251

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,

7252

DAG.getConstant(255ULL<<16, dl, VT));

7253

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,

7254

DAG.getConstant(255ULL<<8 , dl, VT));

7255

Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);

7256

Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);

7257

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);

7258

Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);

7259

Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);

7260

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);

7261

return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);

7262

}

7263

}

7264

7265

SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {

7266

SDLoc dl(N);

7267

EVT VT = N->getValueType(0);

7268

SDValue Op = N->getOperand(0);

7269

EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());

7270

unsigned Sz = VT.getScalarSizeInBits();

7271

7272

SDValue Tmp, Tmp2, Tmp3;

7273

7274

// If we can, perform BSWAP first and then the mask+swap the i4, then i2

7275

// and finally the i1 pairs.

7276

// TODO: We can easily support i4/i2 legal types if any target ever does.

7277

if (Sz >= 8 && isPowerOf2_32(Sz)) {

7278

// Create the masks - repeating the pattern every byte.

7279

APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));

7280

APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));

7281

APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));

7282

7283

// BSWAP if the type is wider than a single byte.

7284

Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);

7285

7286

// swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)

7287

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));

7288

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));

7289

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));

7290

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));

7291

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7292

7293

// swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)

7294

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));

7295

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));

7296

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));

7297

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));

7298

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7299

7300

// swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)

7301

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));

7302

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));

7303

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));

7304

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));

7305

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7306

return Tmp;

7307

}

7308

7309

Tmp = DAG.getConstant(0, dl, VT);

7310

for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {

7311

if (I < J)

7312

Tmp2 =

7313

DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));

7314

else

7315

Tmp2 =

7316

DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));

7317

7318

APInt Shift(Sz, 1);

7319

Shift <<= J;

7320

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));

7321

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);

7322

}

7323

7324

return Tmp;

7325

}

7326

7327

std::pair<SDValue, SDValue>

7328

TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,

7329

SelectionDAG &DAG) const {

7330

SDLoc SL(LD);

7331

SDValue Chain = LD->getChain();

7332

SDValue BasePTR = LD->getBasePtr();

7333

EVT SrcVT = LD->getMemoryVT();

7334

EVT DstVT = LD->getValueType(0);

7335

ISD::LoadExtType ExtType = LD->getExtensionType();

7336

7337

if (SrcVT.isScalableVector())

7338

report_fatal_error("Cannot scalarize scalable vector loads");

7339

7340

unsigned NumElem = SrcVT.getVectorNumElements();

7341

7342

EVT SrcEltVT = SrcVT.getScalarType();

7343

EVT DstEltVT = DstVT.getScalarType();

7344

7345

// A vector must always be stored in memory as-is, i.e. without any padding

7346

// between the elements, since various code depend on it, e.g. in the

7347

// handling of a bitcast of a vector type to int, which may be done with a

7348

// vector store followed by an integer load. A vector that does not have

7349

// elements that are byte-sized must therefore be stored as an integer

7350

// built out of the extracted vector elements.

7351

if (!SrcEltVT.isByteSized()) {

7352

unsigned NumLoadBits = SrcVT.getStoreSizeInBits();

7353

EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);

7354

7355

unsigned NumSrcBits = SrcVT.getSizeInBits();

7356

EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);

7357

7358

unsigned SrcEltBits = SrcEltVT.getSizeInBits();

7359

SDValue SrcEltBitMask = DAG.getConstant(

7360

APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);

7361

7362

// Load the whole vector and avoid masking off the top bits as it makes

7363

// the codegen worse.

7364

SDValue Load =

7365

DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,

7366

LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),

7367

LD->getMemOperand()->getFlags(), LD->getAAInfo());

7368

7369

SmallVector<SDValue, 8> Vals;

7370

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7371

unsigned ShiftIntoIdx =

7372

(DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);

7373

SDValue ShiftAmount =

7374

DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),

7375

LoadVT, SL, /*LegalTypes=*/false);

7376

SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);

7377

SDValue Elt =

7378

DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);

7379

SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);

7380

7381

if (ExtType != ISD::NON_EXTLOAD) {

7382

unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);

7383

Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);

7384

}

7385

7386

Vals.push_back(Scalar);

7387

}

7388

7389

SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

7390

return std::make_pair(Value, Load.getValue(1));

7391

}

7392

7393

unsigned Stride = SrcEltVT.getSizeInBits() / 8;

7394

assert(SrcEltVT.isByteSized())(static_cast <bool> (SrcEltVT.isByteSized()) ? void (0)
: __assert_fail ("SrcEltVT.isByteSized()", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7394, __extension__ __PRETTY_FUNCTION__));

7395

7396

SmallVector<SDValue, 8> Vals;

7397

SmallVector<SDValue, 8> LoadChains;

7398

7399

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7400

SDValue ScalarLoad =

7401

DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,

7402

LD->getPointerInfo().getWithOffset(Idx * Stride),

7403

SrcEltVT, LD->getOriginalAlign(),

7404

LD->getMemOperand()->getFlags(), LD->getAAInfo());

7405

7406

BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));

7407

7408

Vals.push_back(ScalarLoad.getValue(0));

7409

LoadChains.push_back(ScalarLoad.getValue(1));

7410

}

7411

7412

SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);

7413

SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

7414

7415

return std::make_pair(Value, NewChain);

7416

}

7417

7418

SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,

7419

SelectionDAG &DAG) const {

7420

SDLoc SL(ST);

7421

7422

SDValue Chain = ST->getChain();

7423

SDValue BasePtr = ST->getBasePtr();

7424

SDValue Value = ST->getValue();

7425

EVT StVT = ST->getMemoryVT();

7426

7427

if (StVT.isScalableVector())

7428

report_fatal_error("Cannot scalarize scalable vector stores");

7429

7430

// The type of the data we want to save

7431

EVT RegVT = Value.getValueType();

7432

EVT RegSclVT = RegVT.getScalarType();

7433

7434

// The type of data as saved in memory.

7435

EVT MemSclVT = StVT.getScalarType();

7436

7437

unsigned NumElem = StVT.getVectorNumElements();

7438

7439

// A vector must always be stored in memory as-is, i.e. without any padding

7440

// between the elements, since various code depend on it, e.g. in the

7441

// handling of a bitcast of a vector type to int, which may be done with a

7442

// vector store followed by an integer load. A vector that does not have

7443

// elements that are byte-sized must therefore be stored as an integer

7444

// built out of the extracted vector elements.

7445

if (!MemSclVT.isByteSized()) {

7446

unsigned NumBits = StVT.getSizeInBits();

7447

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);

7448

7449

SDValue CurrVal = DAG.getConstant(0, SL, IntVT);

7450

7451

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7452

SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,

7453

DAG.getVectorIdxConstant(Idx, SL));

7454

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);

7455

SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);

7456

unsigned ShiftIntoIdx =

7457

(DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);

7458

SDValue ShiftAmount =

7459

DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);

7460

SDValue ShiftedElt =

7461

DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);

7462

CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);

7463

}

7464

7465

return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),

7466

ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),

7467

ST->getAAInfo());

7468

}

7469

7470

// Store Stride in bytes

7471

unsigned Stride = MemSclVT.getSizeInBits() / 8;

7472

assert(Stride && "Zero stride!")(static_cast <bool> (Stride && "Zero stride!") ?
void (0) : __assert_fail ("Stride && \"Zero stride!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7472, __extension__ __PRETTY_FUNCTION__));

7473

// Extract each of the elements from the original vector and save them into

7474

// memory individually.

7475

SmallVector<SDValue, 8> Stores;

7476

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7477

SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,

7478

DAG.getVectorIdxConstant(Idx, SL));

7479

7480

SDValue Ptr =

7481

DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));

7482

7483

// This scalar TruncStore may be illegal, but we legalize it later.

7484

SDValue Store = DAG.getTruncStore(

7485

Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),

7486

MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),

7487

ST->getAAInfo());

7488

7489

Stores.push_back(Store);

7490

}

7491

7492

return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);

7493

}

7494

7495

std::pair<SDValue, SDValue>

7496

TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {

7497

assert(LD->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7498, __extension__ __PRETTY_FUNCTION__))

7498

"unaligned indexed loads not implemented!")(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7498, __extension__ __PRETTY_FUNCTION__));

7499

SDValue Chain = LD->getChain();

7500

SDValue Ptr = LD->getBasePtr();

7501

EVT VT = LD->getValueType(0);

7502

EVT LoadedVT = LD->getMemoryVT();

7503

SDLoc dl(LD);

7504

auto &MF = DAG.getMachineFunction();

7505

7506

if (VT.isFloatingPoint() || VT.isVector()) {

7507

EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());

7508

if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {

7509

if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&

7510

LoadedVT.isVector()) {

7511

// Scalarize the load and let the individual components be handled.

7512

return scalarizeVectorLoad(LD, DAG);

7513

}

7514

7515

// Expand to a (misaligned) integer load of the same size,

7516

// then bitconvert to floating point or vector.

7517

SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,

7518

LD->getMemOperand());

7519

SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);

7520

if (LoadedVT != VT)

7521

Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :

7522

ISD::ANY_EXTEND, dl, VT, Result);

7523

7524

return std::make_pair(Result, newLoad.getValue(1));

7525

}

7526

7527

// Copy the value to a (aligned) stack slot using (unaligned) integer

7528

// loads and stores, then do a (aligned) load from the stack slot.

7529

MVT RegVT = getRegisterType(*DAG.getContext(), intVT);

7530

unsigned LoadedBytes = LoadedVT.getStoreSize();

7531

unsigned RegBytes = RegVT.getSizeInBits() / 8;

7532

unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;

7533

7534

// Make sure the stack slot is also aligned for the register type.

7535

SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);

7536

auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();

7537

SmallVector<SDValue, 8> Stores;

7538

SDValue StackPtr = StackBase;

7539

unsigned Offset = 0;

7540

7541

EVT PtrVT = Ptr.getValueType();

7542

EVT StackPtrVT = StackPtr.getValueType();

7543

7544

SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);

7545

SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

7546

7547

// Do all but one copies using the full register width.

7548

for (unsigned i = 1; i < NumRegs; i++) {

7549

// Load one integer register's worth from the original location.

7550

SDValue Load = DAG.getLoad(

7551

RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),

7552

LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),

7553

LD->getAAInfo());

7554

// Follow the load with a store to the stack slot. Remember the store.

7555

Stores.push_back(DAG.getStore(

7556

Load.getValue(1), dl, Load, StackPtr,

7557

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));

7558

// Increment the pointers.

7559

Offset += RegBytes;

7560

7561

Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);

7562

StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);

7563

}

7564

7565

// The last copy may be partial. Do an extending load.

7566

EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),

7567

8 * (LoadedBytes - Offset));

7568

SDValue Load =

7569

DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,

7570

LD->getPointerInfo().getWithOffset(Offset), MemVT,

7571

LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),

7572

LD->getAAInfo());

7573

// Follow the load with a store to the stack slot. Remember the store.

7574

// On big-endian machines this requires a truncating store to ensure

7575

// that the bits end up in the right place.

7576

Stores.push_back(DAG.getTruncStore(

7577

Load.getValue(1), dl, Load, StackPtr,

7578

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));

7579

7580

// The order of the stores doesn't matter - say it with a TokenFactor.

7581

SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

7582

7583

// Finally, perform the original load only redirected to the stack slot.

7584

Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,

7585

MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),

7586

LoadedVT);

7587

7588

// Callers expect a MERGE_VALUES node.

7589

return std::make_pair(Load, TF);

7590

}

7591

7592

assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7593, __extension__ __PRETTY_FUNCTION__))

7593

"Unaligned load of unsupported type.")(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7593, __extension__ __PRETTY_FUNCTION__));

7594

7595

// Compute the new VT that is half the size of the old one. This is an

7596

// integer MVT.

7597

unsigned NumBits = LoadedVT.getSizeInBits();

7598

EVT NewLoadedVT;

7599

NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);

7600

NumBits >>= 1;

7601

7602

Align Alignment = LD->getOriginalAlign();

7603

unsigned IncrementSize = NumBits / 8;

7604

ISD::LoadExtType HiExtType = LD->getExtensionType();

7605

7606

// If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.

7607

if (HiExtType == ISD::NON_EXTLOAD)

7608

HiExtType = ISD::ZEXTLOAD;

7609

7610

// Load the value in two parts

7611

SDValue Lo, Hi;

7612

if (DAG.getDataLayout().isLittleEndian()) {

7613

Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),

7614

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7615

LD->getAAInfo());

7616

7617

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7618

Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,

7619

LD->getPointerInfo().getWithOffset(IncrementSize),

7620

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7621

LD->getAAInfo());

7622

} else {

7623

Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),

7624

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7625

LD->getAAInfo());

7626

7627

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7628

Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,

7629

LD->getPointerInfo().getWithOffset(IncrementSize),

7630

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7631

LD->getAAInfo());

7632

}

7633

7634

// aggregate the two parts

7635

SDValue ShiftAmount =

7636

DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),

7637

DAG.getDataLayout()));

7638

SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);

7639

Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);

7640

7641

SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),

7642

Hi.getValue(1));

7643

7644

return std::make_pair(Result, TF);

7645

}

7646

7647

SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,

7648

SelectionDAG &DAG) const {

7649

assert(ST->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7650, __extension__ __PRETTY_FUNCTION__))

7650

"unaligned indexed stores not implemented!")(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7650, __extension__ __PRETTY_FUNCTION__));

7651

SDValue Chain = ST->getChain();

7652

SDValue Ptr = ST->getBasePtr();

7653

SDValue Val = ST->getValue();

7654

EVT VT = Val.getValueType();

7655

Align Alignment = ST->getOriginalAlign();

7656

auto &MF = DAG.getMachineFunction();

7657

EVT StoreMemVT = ST->getMemoryVT();

7658

7659

SDLoc dl(ST);

7660

if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {

7661

EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());

7662

if (isTypeLegal(intVT)) {

7663

if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&

7664

StoreMemVT.isVector()) {

7665

// Scalarize the store and let the individual components be handled.

7666

SDValue Result = scalarizeVectorStore(ST, DAG);

7667

return Result;

7668

}

7669

// Expand to a bitconvert of the value to the integer type of the

7670

// same size, then a (misaligned) int store.

7671

// FIXME: Does not handle truncating floating point stores!

7672

SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);

7673

Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),

7674

Alignment, ST->getMemOperand()->getFlags());

7675

return Result;

7676

}

7677

// Do a (aligned) store to a stack slot, then copy from the stack slot

7678

// to the final destination using (unaligned) integer loads and stores.

7679

MVT RegVT = getRegisterType(

7680

*DAG.getContext(),

7681

EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));

7682

EVT PtrVT = Ptr.getValueType();

7683

unsigned StoredBytes = StoreMemVT.getStoreSize();

7684

unsigned RegBytes = RegVT.getSizeInBits() / 8;

7685

unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;

7686

7687

// Make sure the stack slot is also aligned for the register type.

7688

SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);

7689

auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

7690

7691

// Perform the original store, only redirected to the stack slot.

7692

SDValue Store = DAG.getTruncStore(

7693

Chain, dl, Val, StackPtr,

7694

MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);

7695

7696

EVT StackPtrVT = StackPtr.getValueType();

7697

7698

SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);

7699

SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

7700

SmallVector<SDValue, 8> Stores;

7701

unsigned Offset = 0;

7702

7703

// Do all but one copies using the full register width.

7704

for (unsigned i = 1; i < NumRegs; i++) {

7705

// Load one integer register's worth from the stack slot.

7706

SDValue Load = DAG.getLoad(

7707

RegVT, dl, Store, StackPtr,

7708

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));

7709

// Store it to the final location. Remember the store.

7710

Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,

7711

ST->getPointerInfo().getWithOffset(Offset),

7712

ST->getOriginalAlign(),

7713

ST->getMemOperand()->getFlags()));

7714

// Increment the pointers.

7715

Offset += RegBytes;

7716

StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);

7717

Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);

7718

}

7719

7720

// The last store may be partial. Do a truncating store. On big-endian

7721

// machines this requires an extending load from the stack slot to ensure

7722

// that the bits are in the right place.

7723

EVT LoadMemVT =

7724

EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));

7725

7726

// Load from the stack slot.

7727

SDValue Load = DAG.getExtLoad(

7728

ISD::EXTLOAD, dl, RegVT, Store, StackPtr,

7729

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);

7730

7731

Stores.push_back(

7732

DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,

7733

ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,

7734

ST->getOriginalAlign(),

7735

ST->getMemOperand()->getFlags(), ST->getAAInfo()));

7736

// The order of the stores doesn't matter - say it with a TokenFactor.

7737

SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

7738

return Result;

7739

}

7740

7741

assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7742, __extension__ __PRETTY_FUNCTION__))

7742

"Unaligned store of unknown type.")(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7742, __extension__ __PRETTY_FUNCTION__));

7743

// Get the half-size VT

7744

EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());

7745

unsigned NumBits = NewStoredVT.getFixedSizeInBits();

7746

unsigned IncrementSize = NumBits / 8;

7747

7748

// Divide the stored value in two parts.

7749

SDValue ShiftAmount = DAG.getConstant(

7750

NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));

7751

SDValue Lo = Val;

7752

SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);

7753

7754

// Store the two parts

7755

SDValue Store1, Store2;

7756

Store1 = DAG.getTruncStore(Chain, dl,

7757

DAG.getDataLayout().isLittleEndian() ? Lo : Hi,

7758

Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,

7759

ST->getMemOperand()->getFlags());

7760

7761

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7762

Store2 = DAG.getTruncStore(

7763

Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,

7764

ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,

7765

ST->getMemOperand()->getFlags(), ST->getAAInfo());

7766

7767

SDValue Result =

7768

DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);

7769

return Result;

7770

}

7771

7772

SDValue

7773

TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,

7774

const SDLoc &DL, EVT DataVT,

7775

SelectionDAG &DAG,

7776

bool IsCompressedMemory) const {

7777

SDValue Increment;

7778

EVT AddrVT = Addr.getValueType();

7779

EVT MaskVT = Mask.getValueType();

7780

assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7781, __extension__ __PRETTY_FUNCTION__))

7781

"Incompatible types of Data and Mask")(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7781, __extension__ __PRETTY_FUNCTION__));

7782

if (IsCompressedMemory) {

7783

if (DataVT.isScalableVector())

7784

report_fatal_error(

7785

"Cannot currently handle compressed memory with scalable vectors");

7786

// Incrementing the pointer according to number of '1's in the mask.

7787

EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());

7788

SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);

7789

if (MaskIntVT.getSizeInBits() < 32) {

7790

MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);

7791

MaskIntVT = MVT::i32;

7792

}

7793

7794

// Count '1's with POPCNT.

7795

Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);

7796

Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);

7797

// Scale is an element size in bytes.

7798

SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,

7799

AddrVT);

7800

Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);

7801

} else if (DataVT.isScalableVector()) {

7802

Increment = DAG.getVScale(DL, AddrVT,

7803

APInt(AddrVT.getFixedSizeInBits(),

7804

DataVT.getStoreSize().getKnownMinSize()));

7805

} else

7806

Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);

7807

7808

return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);

7809

}

7810

7811

static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,

7812

EVT VecVT, const SDLoc &dl,

7813

unsigned NumSubElts) {

7814

if (!VecVT.isScalableVector() && isa<ConstantSDNode>(Idx))

7815

return Idx;

7816

7817

EVT IdxVT = Idx.getValueType();

7818

unsigned NElts = VecVT.getVectorMinNumElements();

7819

if (VecVT.isScalableVector()) {

7820

// If this is a constant index and we know the value plus the number of the

7821

// elements in the subvector minus one is less than the minimum number of

7822

// elements then it's safe to return Idx.

7823

if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))

7824

if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)

7825

return Idx;

7826

SDValue VS =

7827

DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));

7828

unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;

7829

SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,

7830

DAG.getConstant(NumSubElts, dl, IdxVT));

7831

return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);

7832

}

7833

if (isPowerOf2_32(NElts) && NumSubElts == 1) {

7834

APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));

7835

return DAG.getNode(ISD::AND, dl, IdxVT, Idx,

7836

DAG.getConstant(Imm, dl, IdxVT));

7837

}

7838

unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;

7839

return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,

7840

DAG.getConstant(MaxIndex, dl, IdxVT));

7841

}

7842

7843

SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,

7844

SDValue VecPtr, EVT VecVT,

7845

SDValue Index) const {

7846

return getVectorSubVecPointer(

7847

DAG, VecPtr, VecVT,

7848

EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),

7849

Index);

7850

}

7851

7852

SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG,

7853

SDValue VecPtr, EVT VecVT,

7854

EVT SubVecVT,

7855

SDValue Index) const {

7856

SDLoc dl(Index);

7857

// Make sure the index type is big enough to compute in.

7858

Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());

7859

7860

EVT EltVT = VecVT.getVectorElementType();

7861

7862

// Calculate the element offset and add it to the pointer.

7863

unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.

7864

assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
(0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7865, __extension__ __PRETTY_FUNCTION__))

7865

"Converting bits to bytes lost precision")(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
(0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7865, __extension__ __PRETTY_FUNCTION__));

7866

7867

// Scalable vectors don't need clamping as these are checked at compile time

7868

if (SubVecVT.isFixedLengthVector()) {

7869

assert(SubVecVT.getVectorElementType() == EltVT &&(static_cast <bool> (SubVecVT.getVectorElementType() ==
EltVT && "Sub-vector must be a fixed vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a fixed vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7870, __extension__ __PRETTY_FUNCTION__))

7870

"Sub-vector must be a fixed vector with matching element type")(static_cast <bool> (SubVecVT.getVectorElementType() ==
EltVT && "Sub-vector must be a fixed vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a fixed vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7870, __extension__ __PRETTY_FUNCTION__));

7871

Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,

7872

SubVecVT.getVectorNumElements());

7873

}

7874

7875

EVT IdxVT = Index.getValueType();

7876

7877

Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,

7878

DAG.getConstant(EltSize, dl, IdxVT));

7879

return DAG.getMemBasePlusOffset(VecPtr, Index, dl);

7880

}

7881

7882

//===----------------------------------------------------------------------===//

7883

// Implementation of Emulated TLS Model

7884

//===----------------------------------------------------------------------===//

7885

7886

SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,

7887

SelectionDAG &DAG) const {

7888

// Access to address of TLS varialbe xyz is lowered to a function call:

7889

// __emutls_get_address( address of global variable named "__emutls_v.xyz" )

7890

EVT PtrVT = getPointerTy(DAG.getDataLayout());

7891

PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());

7892

SDLoc dl(GA);

7893

7894

ArgListTy Args;

7895

ArgListEntry Entry;

7896

std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();

7897

Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());

7898

StringRef EmuTlsVarName(NameString);

7899

GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);

7900

assert(EmuTlsVar && "Cannot find EmuTlsVar ")(static_cast <bool> (EmuTlsVar && "Cannot find EmuTlsVar "
) ? void (0) : __assert_fail ("EmuTlsVar && \"Cannot find EmuTlsVar \""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7900, __extension__ __PRETTY_FUNCTION__));

7901

Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);

7902

Entry.Ty = VoidPtrType;

7903

Args.push_back(Entry);

7904

7905

SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);

7906

7907

TargetLowering::CallLoweringInfo CLI(DAG);

7908

CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());

7909

CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));

7910

std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);

7911

7912

// TLSADDR will be codegen'ed as call. Inform MFI that function has calls.

7913

// At last for X86 targets, maybe good for other targets too?

7914

MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();

7915

MFI.setAdjustsStack(true); // Is this only for X86 target?

7916

MFI.setHasCalls(true);

7917

7918

assert((GA->getOffset() == 0) &&(static_cast <bool> ((GA->getOffset() == 0) &&
"Emulated TLS must have zero offset in GlobalAddressSDNode")
? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7919, __extension__ __PRETTY_FUNCTION__))

7919

"Emulated TLS must have zero offset in GlobalAddressSDNode")(static_cast <bool> ((GA->getOffset() == 0) &&
"Emulated TLS must have zero offset in GlobalAddressSDNode")
? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7919, __extension__ __PRETTY_FUNCTION__));

7920

return CallResult.first;

7921

}

7922

7923

SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,

7924

SelectionDAG &DAG) const {

7925

assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.")(static_cast <bool> ((Op->getOpcode() == ISD::SETCC)
&& "Input has to be a SETCC node.") ? void (0) : __assert_fail
("(Op->getOpcode() == ISD::SETCC) && \"Input has to be a SETCC node.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7925, __extension__ __PRETTY_FUNCTION__));

7926

if (!isCtlzFast())

7927

return SDValue();

7928

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

7929

SDLoc dl(Op);

7930

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {

7931

if (C->isZero() && CC == ISD::SETEQ) {

7932

EVT VT = Op.getOperand(0).getValueType();

7933

SDValue Zext = Op.getOperand(0);

7934

if (VT.bitsLT(MVT::i32)) {

7935

VT = MVT::i32;

7936

Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));

7937

}

7938

unsigned Log2b = Log2_32(VT.getSizeInBits());

7939

SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);

7940

SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,

7941

DAG.getConstant(Log2b, dl, MVT::i32));

7942

return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);

7943

}

7944

}

7945

return SDValue();

7946

}

7947

7948

// Convert redundant addressing modes (e.g. scaling is redundant

7949

// when accessing bytes).

7950

ISD::MemIndexType

7951

TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,

7952

SDValue Offsets) const {

7953

bool IsScaledIndex =

7954

(IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);

7955

bool IsSignedIndex =

7956

(IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);

7957

7958

// Scaling is unimportant for bytes, canonicalize to unscaled.

7959

if (IsScaledIndex && MemVT.getScalarType() == MVT::i8) {

7960

IsScaledIndex = false;

7961

IndexType = IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;

7962

}

7963

7964

return IndexType;

7965

}

7966

7967

SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {

7968

SDValue Op0 = Node->getOperand(0);

7969

SDValue Op1 = Node->getOperand(1);

7970

EVT VT = Op0.getValueType();

7971

unsigned Opcode = Node->getOpcode();

7972

SDLoc DL(Node);

7973

7974

// umin(x,y) -> sub(x,usubsat(x,y))

7975

if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&

7976

isOperationLegal(ISD::USUBSAT, VT)) {

7977

return DAG.getNode(ISD::SUB, DL, VT, Op0,

7978

DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));

7979

}

7980

7981

// umax(x,y) -> add(x,usubsat(y,x))

7982

if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&

7983

isOperationLegal(ISD::USUBSAT, VT)) {

7984

return DAG.getNode(ISD::ADD, DL, VT, Op0,

7985

DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));

7986

}

7987

7988

// Expand Y = MAX(A, B) -> Y = (A > B) ? A : B

7989

ISD::CondCode CC;

7990

switch (Opcode) {

7991

default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7991);

7992

case ISD::SMAX: CC = ISD::SETGT; break;

7993

case ISD::SMIN: CC = ISD::SETLT; break;

7994

case ISD::UMAX: CC = ISD::SETUGT; break;

7995

case ISD::UMIN: CC = ISD::SETULT; break;

7996

}

7997

7998

// FIXME: Should really try to split the vector in case it's legal on a

7999

// subvector.

8000

if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))

8001

return DAG.UnrollVectorOp(Node);

8002

8003

SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);

8004

return DAG.getSelect(DL, VT, Cond, Op0, Op1);

8005

}

8006

8007

SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {

8008

unsigned Opcode = Node->getOpcode();

8009

SDValue LHS = Node->getOperand(0);

8010

SDValue RHS = Node->getOperand(1);

8011

EVT VT = LHS.getValueType();

8012

SDLoc dl(Node);

8013

8014

assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
"Expected operands to be the same type") ? void (0) : __assert_fail
("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8014, __extension__ __PRETTY_FUNCTION__));

8015

assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8015, __extension__ __PRETTY_FUNCTION__));

8016

8017

// usub.sat(a, b) -> umax(a, b) - b

8018

if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {

8019

SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);

8020

return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);

8021

}

8022

8023

// uadd.sat(a, b) -> umin(a, ~b) + b

8024

if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {

8025

SDValue InvRHS = DAG.getNOT(dl, RHS, VT);

8026

SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);

8027

return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);

8028

}

8029

8030

unsigned OverflowOp;

8031

switch (Opcode) {

8032

case ISD::SADDSAT:

8033

OverflowOp = ISD::SADDO;

8034

break;

8035

case ISD::UADDSAT:

8036

OverflowOp = ISD::UADDO;

8037

break;

8038

case ISD::SSUBSAT:

8039

OverflowOp = ISD::SSUBO;

8040

break;

8041

case ISD::USUBSAT:

8042

OverflowOp = ISD::USUBO;

8043

break;

8044

default:

8045

llvm_unreachable("Expected method to receive signed or unsigned saturation "::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
"addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8046)

8046

"addition or subtraction node.")::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
"addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8046);

8047

}

8048

8049

// FIXME: Should really try to split the vector in case it's legal on a

8050

// subvector.

8051

if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))

8052

return DAG.UnrollVectorOp(Node);

8053

8054

unsigned BitWidth = LHS.getScalarValueSizeInBits();

8055

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8056

SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8057

SDValue SumDiff = Result.getValue(0);

8058

SDValue Overflow = Result.getValue(1);

8059

SDValue Zero = DAG.getConstant(0, dl, VT);

8060

SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);

8061

8062

if (Opcode == ISD::UADDSAT) {

8063

if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {

8064

// (LHS + RHS) | OverflowMask

8065

SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);

8066

return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);

8067

}

8068

// Overflow ? 0xffff.... : (LHS + RHS)

8069

return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);

8070

}

8071

8072

if (Opcode == ISD::USUBSAT) {

8073

if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {

8074

// (LHS - RHS) & ~OverflowMask

8075

SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);

8076

SDValue Not = DAG.getNOT(dl, OverflowMask, VT);

8077

return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);

8078

}

8079

// Overflow ? 0 : (LHS - RHS)

8080

return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);

8081

}

8082

8083

// Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff

8084

APInt MinVal = APInt::getSignedMinValue(BitWidth);

8085

SDValue SatMin = DAG.getConstant(MinVal, dl, VT);

8086

SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,

8087

DAG.getConstant(BitWidth - 1, dl, VT));

8088

Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);

8089

return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);

8090

}

8091

8092

SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {

8093

unsigned Opcode = Node->getOpcode();

8094

bool IsSigned = Opcode == ISD::SSHLSAT;

8095

SDValue LHS = Node->getOperand(0);

8096

SDValue RHS = Node->getOperand(1);

8097

EVT VT = LHS.getValueType();

8098

SDLoc dl(Node);

8099

8100

assert((Node->getOpcode() == ISD::SSHLSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8102, __extension__ __PRETTY_FUNCTION__))

8101

Node->getOpcode() == ISD::USHLSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8102, __extension__ __PRETTY_FUNCTION__))

8102

"Expected a SHLSAT opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8102, __extension__ __PRETTY_FUNCTION__));

8103

assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
"Expected operands to be the same type") ? void (0) : __assert_fail
("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8103, __extension__ __PRETTY_FUNCTION__));

8104

assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8104, __extension__ __PRETTY_FUNCTION__));

8105

8106

// If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.

8107

8108

unsigned BW = VT.getScalarSizeInBits();

8109

SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);

8110

SDValue Orig =

8111

DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);

8112

8113

SDValue SatVal;

8114

if (IsSigned) {

8115

SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);

8116

SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);

8117

SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),

8118

SatMin, SatMax, ISD::SETLT);

8119

} else {

8120

SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);

8121

}

8122

Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);

8123

8124

return Result;

8125

}

8126

8127

SDValue

8128

TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {

8129

assert((Node->getOpcode() == ISD::SMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8133, __extension__ __PRETTY_FUNCTION__))

8130

Node->getOpcode() == ISD::UMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8133, __extension__ __PRETTY_FUNCTION__))

8131

Node->getOpcode() == ISD::SMULFIXSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8133, __extension__ __PRETTY_FUNCTION__))

8132

Node->getOpcode() == ISD::UMULFIXSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8133, __extension__ __PRETTY_FUNCTION__))

8133

"Expected a fixed point multiplication opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8133, __extension__ __PRETTY_FUNCTION__));

8134

8135

SDLoc dl(Node);

8136

SDValue LHS = Node->getOperand(0);

8137

SDValue RHS = Node->getOperand(1);

8138

EVT VT = LHS.getValueType();

8139

unsigned Scale = Node->getConstantOperandVal(2);

8140

bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||

8141

Node->getOpcode() == ISD::UMULFIXSAT);

8142

bool Signed = (Node->getOpcode() == ISD::SMULFIX ||

8143

Node->getOpcode() == ISD::SMULFIXSAT);

8144

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8145

unsigned VTSize = VT.getScalarSizeInBits();

8146

8147

if (!Scale) {

8148

// [us]mul.fix(a, b, 0) -> mul(a, b)

8149

if (!Saturating) {

8150

if (isOperationLegalOrCustom(ISD::MUL, VT))

8151

return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8152

} else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {

8153

SDValue Result =

8154

DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8155

SDValue Product = Result.getValue(0);

8156

SDValue Overflow = Result.getValue(1);

8157

SDValue Zero = DAG.getConstant(0, dl, VT);

8158

8159

APInt MinVal = APInt::getSignedMinValue(VTSize);

8160

APInt MaxVal = APInt::getSignedMaxValue(VTSize);

8161

SDValue SatMin = DAG.getConstant(MinVal, dl, VT);

8162

SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);

8163

// Xor the inputs, if resulting sign bit is 0 the product will be

8164

// positive, else negative.

8165

SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);

8166

SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);

8167

Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);

8168

return DAG.getSelect(dl, VT, Overflow, Result, Product);

8169

} else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {

8170

SDValue Result =

8171

DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8172

SDValue Product = Result.getValue(0);

8173

SDValue Overflow = Result.getValue(1);

8174

8175

APInt MaxVal = APInt::getMaxValue(VTSize);

8176

SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);

8177

return DAG.getSelect(dl, VT, Overflow, SatMax, Product);

8178

}

8179

}

8180

8181

assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8183, __extension__ __PRETTY_FUNCTION__))

8182

"Expected scale to be less than the number of bits if signed or at "(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8183, __extension__ __PRETTY_FUNCTION__))

8183

"most the number of bits if unsigned.")(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8183, __extension__ __PRETTY_FUNCTION__));

8184

assert(LHS.getValueType() == RHS.getValueType() &&(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8185, __extension__ __PRETTY_FUNCTION__))

8185

"Expected both operands to be the same type")(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8185, __extension__ __PRETTY_FUNCTION__));

8186

8187

// Get the upper and lower bits of the result.

8188

SDValue Lo, Hi;

8189

unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;

8190

unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;

8191

if (isOperationLegalOrCustom(LoHiOp, VT)) {

8192

SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);

8193

Lo = Result.getValue(0);

8194

Hi = Result.getValue(1);

8195

} else if (isOperationLegalOrCustom(HiOp, VT)) {

8196

Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8197

Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);

8198

} else if (VT.isVector()) {

8199

return SDValue();

8200

} else {

8201

report_fatal_error("Unable to expand fixed point multiplication.");

8202

}

8203

8204

if (Scale == VTSize)

8205

// Result is just the top half since we'd be shifting by the width of the

8206

// operand. Overflow impossible so this works for both UMULFIX and

8207

// UMULFIXSAT.

8208

return Hi;

8209

8210

// The result will need to be shifted right by the scale since both operands

8211

// are scaled. The result is given to us in 2 halves, so we only want part of

8212

// both in the result.

8213

EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());

8214

SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,

8215

DAG.getConstant(Scale, dl, ShiftTy));

8216

if (!Saturating)

8217

return Result;

8218

8219

if (!Signed) {

8220

// Unsigned overflow happened if the upper (VTSize - Scale) bits (of the

8221

// widened multiplication) aren't all zeroes.

8222

8223

// Saturate to max if ((Hi >> Scale) != 0),

8224

// which is the same as if (Hi > ((1 << Scale) - 1))

8225

APInt MaxVal = APInt::getMaxValue(VTSize);

8226

SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),

8227

dl, VT);

8228

Result = DAG.getSelectCC(dl, Hi, LowMask,

8229

DAG.getConstant(MaxVal, dl, VT), Result,

8230

ISD::SETUGT);

8231

8232

return Result;

8233

}

8234

8235

// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the

8236

// widened multiplication) aren't all ones or all zeroes.

8237

8238

SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);

8239

SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);

8240

8241

if (Scale == 0) {

8242

SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,

8243

DAG.getConstant(VTSize - 1, dl, ShiftTy));

8244

SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);

8245

// Saturated to SatMin if wide product is negative, and SatMax if wide

8246

// product is positive ...

8247

SDValue Zero = DAG.getConstant(0, dl, VT);

8248

SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,

8249

ISD::SETLT);

8250

// ... but only if we overflowed.

8251

return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);

8252

}

8253

8254

// We handled Scale==0 above so all the bits to examine is in Hi.

8255

8256

// Saturate to max if ((Hi >> (Scale - 1)) > 0),

8257

// which is the same as if (Hi > (1 << (Scale - 1)) - 1)

8258

SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),

8259

dl, VT);

8260

Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);

8261

// Saturate to min if (Hi >> (Scale - 1)) < -1),

8262

// which is the same as if (HI < (-1 << (Scale - 1))

8263

SDValue HighMask =

8264

DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),

8265

dl, VT);

8266

Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);

8267

return Result;

8268

}

8269

8270

SDValue

8271

TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,

8272

SDValue LHS, SDValue RHS,

8273

unsigned Scale, SelectionDAG &DAG) const {

8274

assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8276, __extension__ __PRETTY_FUNCTION__))

8275

Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8276, __extension__ __PRETTY_FUNCTION__))

8276

"Expected a fixed point division opcode")(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8276, __extension__ __PRETTY_FUNCTION__));

8277

8278

EVT VT = LHS.getValueType();

8279

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

8280

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

8281

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8282

8283

// If there is enough room in the type to upscale the LHS or downscale the

8284

// RHS before the division, we can perform it in this type without having to

8285

// resize. For signed operations, the LHS headroom is the number of

8286

// redundant sign bits, and for unsigned ones it is the number of zeroes.

8287

// The headroom for the RHS is the number of trailing zeroes.

8288

unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1

8289

: DAG.computeKnownBits(LHS).countMinLeadingZeros();

8290

unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();

8291

8292

// For signed saturating operations, we need to be able to detect true integer

8293

// division overflow; that is, when you have MIN / -EPS. However, this

8294

// is undefined behavior and if we emit divisions that could take such

8295

// values it may cause undesired behavior (arithmetic exceptions on x86, for

8296

// example).

8297

// Avoid this by requiring an extra bit so that we never get this case.

8298

// FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale

8299

// signed saturating division, we need to emit a whopping 32-bit division.

8300

if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))

8301

return SDValue();

8302

8303

unsigned LHSShift = std::min(LHSLead, Scale);

8304

unsigned RHSShift = Scale - LHSShift;

8305

8306

// At this point, we know that if we shift the LHS up by LHSShift and the

8307

// RHS down by RHSShift, we can emit a regular division with a final scaling

8308

// factor of Scale.

8309

8310

EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());

8311

if (LHSShift)

8312

LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,

8313

DAG.getConstant(LHSShift, dl, ShiftTy));

8314

if (RHSShift)

8315

RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,

8316

DAG.getConstant(RHSShift, dl, ShiftTy));

8317

8318

SDValue Quot;

8319

if (Signed) {

8320

// For signed operations, if the resulting quotient is negative and the

8321

// remainder is nonzero, subtract 1 from the quotient to round towards

8322

// negative infinity.

8323

SDValue Rem;

8324

// FIXME: Ideally we would always produce an SDIVREM here, but if the

8325

// type isn't legal, SDIVREM cannot be expanded. There is no reason why

8326

// we couldn't just form a libcall, but the type legalizer doesn't do it.

8327

if (isTypeLegal(VT) &&

8328

isOperationLegalOrCustom(ISD::SDIVREM, VT)) {

8329

Quot = DAG.getNode(ISD::SDIVREM, dl,

8330

DAG.getVTList(VT, VT),

8331

LHS, RHS);

8332

Rem = Quot.getValue(1);

8333

Quot = Quot.getValue(0);

8334

} else {

8335

Quot = DAG.getNode(ISD::SDIV, dl, VT,

8336

LHS, RHS);

8337

Rem = DAG.getNode(ISD::SREM, dl, VT,

8338

LHS, RHS);

8339

}

8340

SDValue Zero = DAG.getConstant(0, dl, VT);

8341

SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);

8342

SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);

8343

SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);

8344

SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);

8345

SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,

8346

DAG.getConstant(1, dl, VT));

8347

Quot = DAG.getSelect(dl, VT,

8348

DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),

8349

Sub1, Quot);

8350

} else

8351

Quot = DAG.getNode(ISD::UDIV, dl, VT,

8352

LHS, RHS);

8353

8354

return Quot;

8355

}

8356

8357

void TargetLowering::expandUADDSUBO(

8358

SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {

8359

SDLoc dl(Node);

8360

SDValue LHS = Node->getOperand(0);

8361

SDValue RHS = Node->getOperand(1);

8362

bool IsAdd = Node->getOpcode() == ISD::UADDO;

8363

8364

// If ADD/SUBCARRY is legal, use that instead.

8365

unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;

8366

if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {

8367

SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));

8368

SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),

8369

{ LHS, RHS, CarryIn });

8370

Result = SDValue(NodeCarry.getNode(), 0);

8371

Overflow = SDValue(NodeCarry.getNode(), 1);

8372

return;

8373

}

8374

8375

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,

8376

LHS.getValueType(), LHS, RHS);

8377

8378

EVT ResultType = Node->getValueType(1);

8379

EVT SetCCType = getSetCCResultType(

8380

DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

8381

ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;

8382

SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);

8383

Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);

8384

}

8385

8386

void TargetLowering::expandSADDSUBO(

8387

SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {

8388

SDLoc dl(Node);

8389

SDValue LHS = Node->getOperand(0);

8390

SDValue RHS = Node->getOperand(1);

8391

bool IsAdd = Node->getOpcode() == ISD::SADDO;

8392

8393

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,

8394

LHS.getValueType(), LHS, RHS);

8395

8396

EVT ResultType = Node->getValueType(1);

8397

EVT OType = getSetCCResultType(

8398

DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

8399

8400

// If SADDSAT/SSUBSAT is legal, compare results to detect overflow.

8401

unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;

8402

if (isOperationLegal(OpcSat, LHS.getValueType())) {

8403

SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);

8404

SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);

8405

Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);

8406

return;

8407

}

8408

8409

SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());

8410

8411

// For an addition, the result should be less than one of the operands (LHS)

8412

// if and only if the other operand (RHS) is negative, otherwise there will

8413

// be overflow.

8414

// For a subtraction, the result should be less than one of the operands

8415

// (LHS) if and only if the other operand (RHS) is (non-zero) positive,

8416

// otherwise there will be overflow.

8417

SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);

8418

SDValue ConditionRHS =

8419

DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);

8420

8421

Overflow = DAG.getBoolExtOrTrunc(

8422

DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,

8423

ResultType, ResultType);

8424

}

8425

8426

bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,

8427

SDValue &Overflow, SelectionDAG &DAG) const {

8428

SDLoc dl(Node);

8429

EVT VT = Node->getValueType(0);

8430

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8431

SDValue LHS = Node->getOperand(0);

8432

SDValue RHS = Node->getOperand(1);

8433

bool isSigned = Node->getOpcode() == ISD::SMULO;

8434

8435

// For power-of-two multiplications we can use a simpler shift expansion.

8436

if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {

8437

const APInt &C = RHSC->getAPIntValue();

8438

// mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }

8439

if (C.isPowerOf2()) {

8440

// smulo(x, signed_min) is same as umulo(x, signed_min).

8441

bool UseArithShift = isSigned && !C.isMinSignedValue();

8442

EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());

8443

SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);

8444

Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);

8445

Overflow = DAG.getSetCC(dl, SetCCVT,

8446

DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,

8447

dl, VT, Result, ShiftAmt),

8448

LHS, ISD::SETNE);

8449

return true;

8450

}

8451

}

8452

8453

EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);

8454

if (VT.isVector())

8455

WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,

8456

VT.getVectorNumElements());

8457

8458

SDValue BottomHalf;

8459

SDValue TopHalf;

8460

static const unsigned Ops[2][3] =

8461

{ { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },

8462

{ ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};

8463

if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {

8464

BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8465

TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);

8466

} else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {

8467

BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,

8468

RHS);

8469

TopHalf = BottomHalf.getValue(1);

8470

} else if (isTypeLegal(WideVT)) {

8471

LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);

8472

RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);

8473

SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);

8474

BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);

8475

SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,

8476

getShiftAmountTy(WideVT, DAG.getDataLayout()));

8477

TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,

8478

DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));

8479

} else {

8480

if (VT.isVector())

8481

return false;

8482

8483

// We can fall back to a libcall with an illegal type for the MUL if we

8484

// have a libcall big enough.

8485

// Also, we can fall back to a division in some cases, but that's a big

8486

// performance hit in the general case.

8487

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;

8488

if (WideVT == MVT::i16)

8489

LC = RTLIB::MUL_I16;

8490

else if (WideVT == MVT::i32)

8491

LC = RTLIB::MUL_I32;

8492

else if (WideVT == MVT::i64)

8493

LC = RTLIB::MUL_I64;

8494

else if (WideVT == MVT::i128)

8495

LC = RTLIB::MUL_I128;

8496

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8496, __extension__ __PRETTY_FUNCTION__));

8497

8498

SDValue HiLHS;

8499

SDValue HiRHS;

8500

if (isSigned) {

8501

// The high part is obtained by SRA'ing all but one of the bits of low

8502

// part.

8503

unsigned LoSize = VT.getFixedSizeInBits();

8504

HiLHS =

8505

DAG.getNode(ISD::SRA, dl, VT, LHS,

8506

DAG.getConstant(LoSize - 1, dl,

8507

getPointerTy(DAG.getDataLayout())));

8508

HiRHS =

8509

DAG.getNode(ISD::SRA, dl, VT, RHS,

8510

DAG.getConstant(LoSize - 1, dl,

8511

getPointerTy(DAG.getDataLayout())));

8512

} else {

8513

HiLHS = DAG.getConstant(0, dl, VT);

8514

HiRHS = DAG.getConstant(0, dl, VT);

8515

}

8516

8517

// Here we're passing the 2 arguments explicitly as 4 arguments that are

8518

// pre-lowered to the correct types. This all depends upon WideVT not

8519

// being a legal type for the architecture and thus has to be split to

8520

// two arguments.

8521

SDValue Ret;

8522

TargetLowering::MakeLibCallOptions CallOptions;

8523

CallOptions.setSExt(isSigned);

8524

CallOptions.setIsPostTypeLegalization(true);

8525

if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {

8526

// Halves of WideVT are packed into registers in different order

8527

// depending on platform endianness. This is usually handled by

8528

// the C calling convention, but we can't defer to it in

8529

// the legalizer.

8530

SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };

8531

Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;

8532

} else {

8533

SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };

8534

Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;

8535

}

8536

assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8537, __extension__ __PRETTY_FUNCTION__))

8537

"Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8537, __extension__ __PRETTY_FUNCTION__));

8538

if (DAG.getDataLayout().isLittleEndian()) {

8539

// Same as above.

8540

BottomHalf = Ret.getOperand(0);

8541

TopHalf = Ret.getOperand(1);

8542

} else {

8543

BottomHalf = Ret.getOperand(1);

8544

TopHalf = Ret.getOperand(0);

8545

}

8546

}

8547

8548

Result = BottomHalf;

8549

if (isSigned) {

8550

SDValue ShiftAmt = DAG.getConstant(

8551

VT.getScalarSizeInBits() - 1, dl,

8552

getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));

8553

SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);

8554

Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);

8555

} else {

8556

Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,

8557

DAG.getConstant(0, dl, VT), ISD::SETNE);

8558

}

8559

8560

// Truncate the result if SetCC returns a larger type than needed.

8561

EVT RType = Node->getValueType(1);

8562

if (RType.bitsLT(Overflow.getValueType()))

8563

Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);

8564

8565

assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8566, __extension__ __PRETTY_FUNCTION__))

8566

"Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8566, __extension__ __PRETTY_FUNCTION__));

8567

return true;

8568

}

8569

8570

SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {

8571

SDLoc dl(Node);

8572

unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

8573

SDValue Op = Node->getOperand(0);

8574

EVT VT = Op.getValueType();

8575

8576

if (VT.isScalableVector())

8577

report_fatal_error(

8578

"Expanding reductions for scalable vectors is undefined.");

8579

8580

// Try to use a shuffle reduction for power of two vectors.

8581

if (VT.isPow2VectorType()) {

8582

while (VT.getVectorNumElements() > 1) {

8583

EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());

8584

if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))

8585

break;

8586

8587

SDValue Lo, Hi;

8588

std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);

8589

Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);

8590

VT = HalfVT;

8591

}

8592

}

8593

8594

EVT EltVT = VT.getVectorElementType();

8595

unsigned NumElts = VT.getVectorNumElements();

8596

8597

SmallVector<SDValue, 8> Ops;

8598

DAG.ExtractVectorElements(Op, Ops, 0, NumElts);

8599

8600

SDValue Res = Ops[0];

8601

for (unsigned i = 1; i < NumElts; i++)

8602

Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());

8603

8604

// Result type may be wider than element type.

8605

if (EltVT != Node->getValueType(0))

8606

Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);

8607

return Res;

8608

}

8609

8610

SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {

8611

SDLoc dl(Node);

8612

SDValue AccOp = Node->getOperand(0);

8613

SDValue VecOp = Node->getOperand(1);

8614

SDNodeFlags Flags = Node->getFlags();

8615

8616

EVT VT = VecOp.getValueType();

8617

EVT EltVT = VT.getVectorElementType();

8618

8619

if (VT.isScalableVector())

8620

report_fatal_error(

8621

"Expanding reductions for scalable vectors is undefined.");

8622

8623

unsigned NumElts = VT.getVectorNumElements();

8624

8625

SmallVector<SDValue, 8> Ops;

8626

DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);

8627

8628

unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

8629

8630

SDValue Res = AccOp;

8631

for (unsigned i = 0; i < NumElts; i++)

8632

Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);

8633

8634

return Res;

8635

}

8636

8637

bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,

8638

SelectionDAG &DAG) const {

8639

EVT VT = Node->getValueType(0);

8640

SDLoc dl(Node);

8641

bool isSigned = Node->getOpcode() == ISD::SREM;

8642

unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;

8643

unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;

8644

SDValue Dividend = Node->getOperand(0);

8645

SDValue Divisor = Node->getOperand(1);

8646

if (isOperationLegalOrCustom(DivRemOpc, VT)) {

8647

SDVTList VTs = DAG.getVTList(VT, VT);

8648

Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);

8649

return true;

8650

}

8651

if (isOperationLegalOrCustom(DivOpc, VT)) {

8652

// X % Y -> X-X/Y*Y

8653

SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);

8654

SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);

8655

Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);

8656

return true;

8657

}

8658

return false;

8659

}

8660

8661

SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,

8662

SelectionDAG &DAG) const {

8663

bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;

8664

SDLoc dl(SDValue(Node, 0));

8665

SDValue Src = Node->getOperand(0);

8666

8667

// DstVT is the result type, while SatVT is the size to which we saturate

8668

EVT SrcVT = Src.getValueType();

8669

EVT DstVT = Node->getValueType(0);

8670

8671

EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();

8672

unsigned SatWidth = SatVT.getScalarSizeInBits();

8673

unsigned DstWidth = DstVT.getScalarSizeInBits();

8674

assert(SatWidth <= DstWidth &&(static_cast <bool> (SatWidth <= DstWidth &&
"Expected saturation width smaller than result width") ? void
(0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8675, __extension__ __PRETTY_FUNCTION__))

8675

"Expected saturation width smaller than result width")(static_cast <bool> (SatWidth <= DstWidth &&
"Expected saturation width smaller than result width") ? void
(0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8675, __extension__ __PRETTY_FUNCTION__));

8676

8677

// Determine minimum and maximum integer values and their corresponding

8678

// floating-point values.

8679

APInt MinInt, MaxInt;

8680

if (IsSigned) {

8681

MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);

8682

MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);

8683

} else {

8684

MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);

8685

MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);

8686

}

8687

8688

// We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as

8689

// libcall emission cannot handle this. Large result types will fail.

8690

if (SrcVT == MVT::f16) {

8691

Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);

8692

SrcVT = Src.getValueType();

8693

}

8694

8695

APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));

8696

APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));

8697

8698

APFloat::opStatus MinStatus =

8699

MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);

8700

APFloat::opStatus MaxStatus =

8701

MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);

8702

bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&

8703

!(MaxStatus & APFloat::opStatus::opInexact);

8704

8705

SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);

8706

SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);

8707

8708

// If the integer bounds are exactly representable as floats and min/max are

8709

// legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence

8710

// of comparisons and selects.

8711

bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&

8712

isOperationLegal(ISD::FMAXNUM, SrcVT);

8713

if (AreExactFloatBounds && MinMaxLegal) {

8714

SDValue Clamped = Src;

8715

8716

// Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.

8717

Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);

8718

// Clamp by MaxFloat from above. NaN cannot occur.

8719

Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);

8720

// Convert clamped value to integer.

8721

SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,

8722

dl, DstVT, Clamped);

8723

8724

// In the unsigned case we're done, because we mapped NaN to MinFloat,

8725

// which will cast to zero.

8726

if (!IsSigned)

8727

return FpToInt;

8728

8729

// Otherwise, select 0 if Src is NaN.

8730

SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);

8731

return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,

8732

ISD::CondCode::SETUO);

8733

}

8734

8735

SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);

8736

SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);

8737

8738

// Result of direct conversion. The assumption here is that the operation is

8739

// non-trapping and it's fine to apply it to an out-of-range value if we

8740

// select it away later.

8741

SDValue FpToInt =

8742

DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);

8743

8744

SDValue Select = FpToInt;

8745

8746

// If Src ULT MinFloat, select MinInt. In particular, this also selects

8747

// MinInt if Src is NaN.

8748

Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,

8749

ISD::CondCode::SETULT);

8750

// If Src OGT MaxFloat, select MaxInt.

8751

Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,

8752

ISD::CondCode::SETOGT);

8753

8754

// In the unsigned case we are done, because we mapped NaN to MinInt, which

8755

// is already zero.

8756

if (!IsSigned)

8757

return Select;

8758

8759

// Otherwise, select 0 if Src is NaN.

8760

SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);

8761

return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);

8762

}

8763

8764

SDValue TargetLowering::expandVectorSplice(SDNode *Node,

8765

SelectionDAG &DAG) const {

8766

assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!")(static_cast <bool> (Node->getOpcode() == ISD::VECTOR_SPLICE
&& "Unexpected opcode!") ? void (0) : __assert_fail (
"Node->getOpcode() == ISD::VECTOR_SPLICE && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8766, __extension__ __PRETTY_FUNCTION__));

8767

assert(Node->getValueType(0).isScalableVector() &&(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8768, __extension__ __PRETTY_FUNCTION__))

8768

"Fixed length vector types expected to use SHUFFLE_VECTOR!")(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8768, __extension__ __PRETTY_FUNCTION__));

8769

8770

EVT VT = Node->getValueType(0);

8771

SDValue V1 = Node->getOperand(0);

8772

SDValue V2 = Node->getOperand(1);

8773

int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();

8774

SDLoc DL(Node);

8775

8776

// Expand through memory thusly:

8777

// Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr

8778

// Store V1, Ptr

8779

// Store V2, Ptr + sizeof(V1)

8780

// If (Imm < 0)

8781

// TrailingElts = -Imm

8782

// Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))

8783

// else

8784

// Ptr = Ptr + (Imm * sizeof(VT.Elt))

8785

// Res = Load Ptr

8786

8787

Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);

8788

8789

EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),

8790

VT.getVectorElementCount() * 2);

8791

SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);

8792

EVT PtrVT = StackPtr.getValueType();

8793

auto &MF = DAG.getMachineFunction();

8794

auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

8795

auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);

8796

8797

// Store the lo part of CONCAT_VECTORS(V1, V2)

8798

SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);

8799

// Store the hi part of CONCAT_VECTORS(V1, V2)

8800

SDValue OffsetToV2 = DAG.getVScale(

8801

DL, PtrVT,

8802

APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));

8803

SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);

8804

SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);

8805

8806

if (Imm >= 0) {

8807

// Load back the required element. getVectorElementPointer takes care of

8808

// clamping the index if it's out-of-bounds.

8809

StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));

8810

// Load the spliced result

8811

return DAG.getLoad(VT, DL, StoreV2, StackPtr,

8812

MachinePointerInfo::getUnknownStack(MF));

8813

}

8814

8815

uint64_t TrailingElts = -Imm;

8816

8817

// NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.

8818

TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();

8819

SDValue TrailingBytes =

8820

DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);

8821

8822

if (TrailingElts > VT.getVectorMinNumElements()) {

8823

SDValue VLBytes = DAG.getVScale(

8824

DL, PtrVT,

8825

APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));

8826

TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);

8827

}

8828

8829

// Calculate the start address of the spliced result.

8830

StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);

8831

8832

// Load the spliced result

8833

return DAG.getLoad(VT, DL, StoreV2, StackPtr2,

8834

MachinePointerInfo::getUnknownStack(MF));

8835

}

8836

8837

bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,

8838

SDValue &LHS, SDValue &RHS,

8839

SDValue &CC, bool &NeedInvert,

8840

const SDLoc &dl, SDValue &Chain,

8841

bool IsSignaling) const {

8842

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

8843

MVT OpVT = LHS.getSimpleValueType();

8844

ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();

8845

NeedInvert = false;

8846

switch (TLI.getCondCodeAction(CCCode, OpVT)) {

8847

default:

8848

llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8848);

8849

case TargetLowering::Legal:

8850

// Nothing to do.

8851

break;

8852

case TargetLowering::Expand: {

8853

ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);

8854

if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8855

std::swap(LHS, RHS);

8856

CC = DAG.getCondCode(InvCC);

8857

return true;

8858

}

8859

// Swapping operands didn't work. Try inverting the condition.

8860

bool NeedSwap = false;

8861

InvCC = getSetCCInverse(CCCode, OpVT);

8862

if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8863

// If inverting the condition is not enough, try swapping operands

8864

// on top of it.

8865

InvCC = ISD::getSetCCSwappedOperands(InvCC);

8866

NeedSwap = true;

8867

}

8868

if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8869

CC = DAG.getCondCode(InvCC);

8870

NeedInvert = true;

8871

if (NeedSwap)

8872

std::swap(LHS, RHS);

8873

return true;

8874

}

8875

8876

ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;

8877

unsigned Opc = 0;

8878

switch (CCCode) {

8879

default:

8880

llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8880);

8881

case ISD::SETUO:

8882

if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {

8883

CC1 = ISD::SETUNE;

8884

CC2 = ISD::SETUNE;

8885

Opc = ISD::OR;

8886

break;

8887

}

8888

assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8889, __extension__ __PRETTY_FUNCTION__))

8889

"If SETUE is expanded, SETOEQ or SETUNE must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8889, __extension__ __PRETTY_FUNCTION__));

8890

NeedInvert = true;

8891

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8892

case ISD::SETO:

8893

assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8894, __extension__ __PRETTY_FUNCTION__))

8894

"If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8894, __extension__ __PRETTY_FUNCTION__));

8895

CC1 = ISD::SETOEQ;

8896

CC2 = ISD::SETOEQ;

8897

Opc = ISD::AND;

8898

break;

8899

case ISD::SETONE:

8900

case ISD::SETUEQ:

8901

// If the SETUO or SETO CC isn't legal, we might be able to use

8902

// SETOGT || SETOLT, inverting the result for SETUEQ. We only need one

8903

// of SETOGT/SETOLT to be legal, the other can be emulated by swapping

8904

// the operands.

8905

CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;

8906

if (!TLI.isCondCodeLegal(CC2, OpVT) &&

8907

(TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||

8908

TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {

8909

CC1 = ISD::SETOGT;

8910

CC2 = ISD::SETOLT;

8911

Opc = ISD::OR;

8912

NeedInvert = ((unsigned)CCCode & 0x8U);

8913

break;

8914

}

8915

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8916

case ISD::SETOEQ:

8917

case ISD::SETOGT:

8918

case ISD::SETOGE:

8919

case ISD::SETOLT:

8920

case ISD::SETOLE:

8921

case ISD::SETUNE:

8922

case ISD::SETUGT:

8923

case ISD::SETUGE:

8924

case ISD::SETULT:

8925

case ISD::SETULE:

8926

// If we are floating point, assign and break, otherwise fall through.

8927

if (!OpVT.isInteger()) {

8928

// We can use the 4th bit to tell if we are the unordered

8929

// or ordered version of the opcode.

8930

CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;

8931

Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;

8932

CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);

8933

break;

8934

}

8935

// Fallthrough if we are unsigned integer.

8936

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8937

case ISD::SETLE:

8938

case ISD::SETGT:

8939

case ISD::SETGE:

8940

case ISD::SETLT:

8941

case ISD::SETNE:

8942

case ISD::SETEQ:

8943

// If all combinations of inverting the condition and swapping operands

8944

// didn't work then we have no means to expand the condition.

8945

llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20211014061306+0fbd3aad75f9/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8945);

8946

}

8947

8948

SDValue SetCC1, SetCC2;

8949

if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {

8950

// If we aren't the ordered or unorder operation,

8951

// then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).

8952

SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);

8953

SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);

8954

} else {

8955

// Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)

8956

SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);

8957

SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);

8958

}

8959

if (Chain)

8960

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),

8961

SetCC2.getValue(1));

8962

LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);

8963

RHS = SDValue();

8964

CC = SDValue();

8965

return true;

8966

}

8967

}

8968

return false;

8969

}

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 7960, column 5 Value stored to 'IsScaledIndex' is never read

Bug Summary

Annotated Source Code