/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1

//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements the TargetLowering class.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "llvm/CodeGen/TargetLowering.h"

14

#include "llvm/ADT/STLExtras.h"

15

#include "llvm/CodeGen/CallingConvLower.h"

16

#include "llvm/CodeGen/MachineFrameInfo.h"

17

#include "llvm/CodeGen/MachineFunction.h"

18

#include "llvm/CodeGen/MachineJumpTableInfo.h"

19

#include "llvm/CodeGen/MachineRegisterInfo.h"

20

#include "llvm/CodeGen/SelectionDAG.h"

21

#include "llvm/CodeGen/TargetRegisterInfo.h"

22

#include "llvm/CodeGen/TargetSubtargetInfo.h"

23

#include "llvm/IR/DataLayout.h"

24

#include "llvm/IR/DerivedTypes.h"

25

#include "llvm/IR/GlobalVariable.h"

26

#include "llvm/IR/LLVMContext.h"

27

#include "llvm/MC/MCAsmInfo.h"

28

#include "llvm/MC/MCExpr.h"

29

#include "llvm/Support/ErrorHandling.h"

30

#include "llvm/Support/KnownBits.h"

31

#include "llvm/Support/MathExtras.h"

32

#include "llvm/Target/TargetLoweringObjectFile.h"

33

#include "llvm/Target/TargetMachine.h"

34

#include <cctype>

35

using namespace llvm;

36

37

/// NOTE: The TargetMachine owns TLOF.

38

TargetLowering::TargetLowering(const TargetMachine &tm)

39

: TargetLoweringBase(tm) {}

40

41

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {

42

return nullptr;

43

}

44

45

bool TargetLowering::isPositionIndependent() const {

46

return getTargetMachine().isPositionIndependent();

47

}

48

49

/// Check whether a given call node is in tail position within its function. If

50

/// so, it sets Chain to the input chain of the tail call.

51

bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,

52

SDValue &Chain) const {

53

const Function &F = DAG.getMachineFunction().getFunction();

54

55

// First, check if tail calls have been disabled in this function.

56

if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")

57

return false;

58

59

// Conservatively require the attributes of the call to match those of

60

// the return. Ignore NoAlias and NonNull because they don't affect the

61

// call sequence.

62

AttributeList CallerAttrs = F.getAttributes();

63

if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)

64

.removeAttribute(Attribute::NoAlias)

65

.removeAttribute(Attribute::NonNull)

66

.hasAttributes())

67

return false;

68

69

// It's not safe to eliminate the sign / zero extension of the return value.

70

if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||

71

CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))

72

return false;

73

74

// Check if the only use is a function return node.

75

return isUsedByReturnOnly(Node, Chain);

76

}

77

78

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,

79

const uint32_t *CallerPreservedMask,

80

const SmallVectorImpl<CCValAssign> &ArgLocs,

81

const SmallVectorImpl<SDValue> &OutVals) const {

82

for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {

83

const CCValAssign &ArgLoc = ArgLocs[I];

84

if (!ArgLoc.isRegLoc())

85

continue;

86

MCRegister Reg = ArgLoc.getLocReg();

87

// Only look at callee saved registers.

88

if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))

89

continue;

90

// Check that we pass the value used for the caller.

91

// (We look for a CopyFromReg reading a virtual register that is used

92

// for the function live-in value of register Reg)

93

SDValue Value = OutVals[I];

94

if (Value->getOpcode() != ISD::CopyFromReg)

95

return false;

96

Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();

97

if (MRI.getLiveInPhysReg(ArgReg) != Reg)

98

return false;

99

}

100

return true;

101

}

102

103

/// Set CallLoweringInfo attribute flags based on a call instruction

104

/// and called function attributes.

105

void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,

106

unsigned ArgIdx) {

107

IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);

108

IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);

109

IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);

110

IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);

111

IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);

112

IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);

113

IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);

114

IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);

115

IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);

116

IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);

117

IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);

118

Alignment = Call->getParamAlign(ArgIdx);

119

ByValType = nullptr;

120

if (IsByVal)

121

ByValType = Call->getParamByValType(ArgIdx);

122

PreallocatedType = nullptr;

123

if (IsPreallocated)

124

PreallocatedType = Call->getParamPreallocatedType(ArgIdx);

125

}

126

127

/// Generate a libcall taking the given operands as arguments and returning a

128

/// result of type RetVT.

129

std::pair<SDValue, SDValue>

130

TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,

131

ArrayRef<SDValue> Ops,

132

MakeLibCallOptions CallOptions,

133

const SDLoc &dl,

134

SDValue InChain) const {

135

if (!InChain)

136

InChain = DAG.getEntryNode();

137

138

TargetLowering::ArgListTy Args;

139

Args.reserve(Ops.size());

140

141

TargetLowering::ArgListEntry Entry;

142

for (unsigned i = 0; i < Ops.size(); ++i) {

143

SDValue NewOp = Ops[i];

144

Entry.Node = NewOp;

145

Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());

146

Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),

147

CallOptions.IsSExt);

148

Entry.IsZExt = !Entry.IsSExt;

149

150

if (CallOptions.IsSoften &&

151

!shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {

152

Entry.IsSExt = Entry.IsZExt = false;

153

}

154

Args.push_back(Entry);

155

}

156

157

if (LC == RTLIB::UNKNOWN_LIBCALL)

158

report_fatal_error("Unsupported library call operation!");

159

SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),

160

getPointerTy(DAG.getDataLayout()));

161

162

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());

163

TargetLowering::CallLoweringInfo CLI(DAG);

164

bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);

165

bool zeroExtend = !signExtend;

166

167

if (CallOptions.IsSoften &&

168

!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {

169

signExtend = zeroExtend = false;

170

}

171

172

CLI.setDebugLoc(dl)

173

.setChain(InChain)

174

.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))

175

.setNoReturn(CallOptions.DoesNotReturn)

176

.setDiscardResult(!CallOptions.IsReturnValueUsed)

177

.setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)

178

.setSExtResult(signExtend)

179

.setZExtResult(zeroExtend);

180

return LowerCallTo(CLI);

181

}

182

183

bool TargetLowering::findOptimalMemOpLowering(

184

std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,

185

unsigned SrcAS, const AttributeList &FuncAttributes) const {

186

if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())

187

return false;

188

189

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

190

191

if (VT == MVT::Other) {

192

// Use the largest integer type whose alignment constraints are satisfied.

193

// We only need to check DstAlign here as SrcAlign is always greater or

194

// equal to DstAlign (or zero).

195

VT = MVT::i64;

196

if (Op.isFixedDstAlign())

197

while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

198

!allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))

199

VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);

200

assert(VT.isInteger())((VT.isInteger()) ? static_cast<void> (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 200, __PRETTY_FUNCTION__));

201

202

// Find the largest legal integer type.

203

MVT LVT = MVT::i64;

204

while (!isTypeLegal(LVT))

205

LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);

206

assert(LVT.isInteger())((LVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 206, __PRETTY_FUNCTION__));

207

208

// If the type we've chosen is larger than the largest legal integer type

209

// then use that instead.

210

if (VT.bitsGT(LVT))

211

VT = LVT;

212

}

213

214

unsigned NumMemOps = 0;

215

uint64_t Size = Op.size();

216

while (Size) {

217

unsigned VTSize = VT.getSizeInBits() / 8;

218

while (VTSize > Size) {

219

// For now, only use non-vector load / store's for the left-over pieces.

220

EVT NewVT = VT;

221

unsigned NewVTSize;

222

223

bool Found = false;

224

if (VT.isVector() || VT.isFloatingPoint()) {

225

NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;

226

if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&

227

isSafeMemOpType(NewVT.getSimpleVT()))

228

Found = true;

229

else if (NewVT == MVT::i64 &&

230

isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&

231

isSafeMemOpType(MVT::f64)) {

232

// i64 is usually not legal on 32-bit targets, but f64 may be.

233

NewVT = MVT::f64;

234

Found = true;

235

}

236

}

237

238

if (!Found) {

239

do {

240

NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);

241

if (NewVT == MVT::i8)

242

break;

243

} while (!isSafeMemOpType(NewVT.getSimpleVT()));

244

}

245

NewVTSize = NewVT.getSizeInBits() / 8;

246

247

// If the new VT cannot cover all of the remaining bits, then consider

248

// issuing a (or a pair of) unaligned and overlapping load / store.

249

bool Fast;

250

if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&

251

allowsMisalignedMemoryAccesses(

252

VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),

253

MachineMemOperand::MONone, &Fast) &&

254

Fast)

255

VTSize = Size;

256

else {

257

VT = NewVT;

258

VTSize = NewVTSize;

259

}

260

}

261

262

if (++NumMemOps > Limit)

263

return false;

264

265

MemOps.push_back(VT);

266

Size -= VTSize;

267

}

268

269

return true;

270

}

271

272

/// Soften the operands of a comparison. This code is shared among BR_CC,

273

/// SELECT_CC, and SETCC handlers.

274

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

275

SDValue &NewLHS, SDValue &NewRHS,

276

ISD::CondCode &CCCode,

277

const SDLoc &dl, const SDValue OldLHS,

278

const SDValue OldRHS) const {

279

SDValue Chain;

280

return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,

281

OldRHS, Chain);

282

}

283

284

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

285

SDValue &NewLHS, SDValue &NewRHS,

286

ISD::CondCode &CCCode,

287

const SDLoc &dl, const SDValue OldLHS,

288

const SDValue OldRHS,

289

SDValue &Chain,

290

bool IsSignaling) const {

291

// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc

292

// not supporting it. We can update this code when libgcc provides such

293

// functions.

294

295

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 296, __PRETTY_FUNCTION__))

296

&& "Unsupported setcc type!")(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 296, __PRETTY_FUNCTION__));

297

298

// Expand into one or more soft-fp libcall(s).

299

RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;

300

bool ShouldInvertCC = false;

301

switch (CCCode) {

302

case ISD::SETEQ:

303

case ISD::SETOEQ:

304

LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

305

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

306

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

307

break;

308

case ISD::SETNE:

309

case ISD::SETUNE:

310

LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :

311

(VT == MVT::f64) ? RTLIB::UNE_F64 :

312

(VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;

313

break;

314

case ISD::SETGE:

315

case ISD::SETOGE:

316

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

317

(VT == MVT::f64) ? RTLIB::OGE_F64 :

318

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

319

break;

320

case ISD::SETLT:

321

case ISD::SETOLT:

322

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

323

(VT == MVT::f64) ? RTLIB::OLT_F64 :

324

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

325

break;

326

case ISD::SETLE:

327

case ISD::SETOLE:

328

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

329

(VT == MVT::f64) ? RTLIB::OLE_F64 :

330

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

331

break;

332

case ISD::SETGT:

333

case ISD::SETOGT:

334

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

335

(VT == MVT::f64) ? RTLIB::OGT_F64 :

336

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

337

break;

338

case ISD::SETO:

339

ShouldInvertCC = true;

340

LLVM_FALLTHROUGH[[gnu::fallthrough]];

341

case ISD::SETUO:

342

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

343

(VT == MVT::f64) ? RTLIB::UO_F64 :

344

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

345

break;

346

case ISD::SETONE:

347

// SETONE = O && UNE

348

ShouldInvertCC = true;

349

LLVM_FALLTHROUGH[[gnu::fallthrough]];

350

case ISD::SETUEQ:

351

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

352

(VT == MVT::f64) ? RTLIB::UO_F64 :

353

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

354

LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

355

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

356

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

357

break;

358

default:

359

// Invert CC for unordered comparisons

360

ShouldInvertCC = true;

361

switch (CCCode) {

362

case ISD::SETULT:

363

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

364

(VT == MVT::f64) ? RTLIB::OGE_F64 :

365

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

366

break;

367

case ISD::SETULE:

368

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

369

(VT == MVT::f64) ? RTLIB::OGT_F64 :

370

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

371

break;

372

case ISD::SETUGT:

373

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

374

(VT == MVT::f64) ? RTLIB::OLE_F64 :

375

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

376

break;

377

case ISD::SETUGE:

378

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

379

(VT == MVT::f64) ? RTLIB::OLT_F64 :

380

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

381

break;

382

default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 382);

383

}

384

}

385

386

// Use the target specific return value for comparions lib calls.

387

EVT RetVT = getCmpLibcallReturnType();

388

SDValue Ops[2] = {NewLHS, NewRHS};

389

TargetLowering::MakeLibCallOptions CallOptions;

390

EVT OpsVT[2] = { OldLHS.getValueType(),

391

OldRHS.getValueType() };

392

CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);

393

auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);

394

NewLHS = Call.first;

395

NewRHS = DAG.getConstant(0, dl, RetVT);

396

397

CCCode = getCmpLibcallCC(LC1);

398

if (ShouldInvertCC) {

399

assert(RetVT.isInteger())((RetVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 399, __PRETTY_FUNCTION__));

400

CCCode = getSetCCInverse(CCCode, RetVT);

401

}

402

403

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {

404

// Update Chain.

405

Chain = Call.second;

406

} else {

407

EVT SetCCVT =

408

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);

409

SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);

410

auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);

411

CCCode = getCmpLibcallCC(LC2);

412

if (ShouldInvertCC)

413

CCCode = getSetCCInverse(CCCode, RetVT);

414

NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);

415

if (Chain)

416

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,

417

Call2.second);

418

NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,

419

Tmp.getValueType(), Tmp, NewLHS);

420

NewRHS = SDValue();

421

}

422

}

423

424

/// Return the entry encoding for a jump table in the current function. The

425

/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.

426

unsigned TargetLowering::getJumpTableEncoding() const {

427

// In non-pic modes, just use the address of a block.

428

if (!isPositionIndependent())

429

return MachineJumpTableInfo::EK_BlockAddress;

430

431

// In PIC mode, if the target supports a GPRel32 directive, use it.

432

if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)

433

return MachineJumpTableInfo::EK_GPRel32BlockAddress;

434

435

// Otherwise, use a label difference.

436

return MachineJumpTableInfo::EK_LabelDifference32;

437

}

438

439

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,

440

SelectionDAG &DAG) const {

441

// If our PIC model is GP relative, use the global offset table as the base.

442

unsigned JTEncoding = getJumpTableEncoding();

443

444

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||

445

(JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))

446

return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

447

448

return Table;

449

}

450

451

/// This returns the relocation base for the given PIC jumptable, the same as

452

/// getPICJumpTableRelocBase, but as an MCExpr.

453

const MCExpr *

454

TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,

455

unsigned JTI,MCContext &Ctx) const{

456

// The normal PIC reloc base is the label at the start of the jump table.

457

return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);

458

}

459

460

bool

461

TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

462

const TargetMachine &TM = getTargetMachine();

463

const GlobalValue *GV = GA->getGlobal();

464

465

// If the address is not even local to this DSO we will have to load it from

466

// a got and then add the offset.

467

if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))

468

return false;

469

470

// If the code is position independent we will have to add a base register.

471

if (isPositionIndependent())

472

return false;

473

474

// Otherwise we can do it.

475

return true;

476

}

477

478

//===----------------------------------------------------------------------===//

479

// Optimization Methods

480

//===----------------------------------------------------------------------===//

481

482

/// If the specified instruction has a constant integer operand and there are

483

/// bits set in that constant that are not demanded, then clear those bits and

484

/// return true.

485

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

486

const APInt &DemandedBits,

487

const APInt &DemandedElts,

488

TargetLoweringOpt &TLO) const {

489

SDLoc DL(Op);

490

unsigned Opcode = Op.getOpcode();

491

492

// Do target-specific constant optimization.

493

if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

494

return TLO.New.getNode();

495

496

// FIXME: ISD::SELECT, ISD::SELECT_CC

497

switch (Opcode) {

498

default:

499

break;

500

case ISD::XOR:

501

case ISD::AND:

502

case ISD::OR: {

503

auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));

504

if (!Op1C)

505

return false;

506

507

// If this is a 'not' op, don't touch it because that's a canonical form.

508

const APInt &C = Op1C->getAPIntValue();

509

if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))

510

return false;

511

512

if (!C.isSubsetOf(DemandedBits)) {

513

EVT VT = Op.getValueType();

514

SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);

515

SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);

516

return TLO.CombineTo(Op, NewOp);

517

}

518

519

break;

520

}

521

}

522

523

return false;

524

}

525

526

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

527

const APInt &DemandedBits,

528

TargetLoweringOpt &TLO) const {

529

EVT VT = Op.getValueType();

530

APInt DemandedElts = VT.isVector()

531

? APInt::getAllOnesValue(VT.getVectorNumElements())

532

: APInt(1, 1);

533

return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);

534

}

535

536

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.

537

/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be

538

/// generalized for targets with other types of implicit widening casts.

539

bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,

540

const APInt &Demanded,

541

TargetLoweringOpt &TLO) const {

542

assert(Op.getNumOperands() == 2 &&((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 543, __PRETTY_FUNCTION__))

543

"ShrinkDemandedOp only supports binary operators!")((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 543, __PRETTY_FUNCTION__));

544

assert(Op.getNode()->getNumValues() == 1 &&((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 545, __PRETTY_FUNCTION__))

545

"ShrinkDemandedOp only supports nodes with one result!")((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 545, __PRETTY_FUNCTION__));

546

547

SelectionDAG &DAG = TLO.DAG;

548

SDLoc dl(Op);

549

550

// Early return, as this function cannot handle vector types.

551

if (Op.getValueType().isVector())

552

return false;

553

554

// Don't do this if the node has another user, which may require the

555

// full value.

556

if (!Op.getNode()->hasOneUse())

557

return false;

558

559

// Search for the smallest integer type with free casts to and from

560

// Op's type. For expedience, just check power-of-2 integer types.

561

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

562

unsigned DemandedSize = Demanded.getActiveBits();

563

unsigned SmallVTBits = DemandedSize;

564

if (!isPowerOf2_32(SmallVTBits))

565

SmallVTBits = NextPowerOf2(SmallVTBits);

566

for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {

567

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);

568

if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&

569

TLI.isZExtFree(SmallVT, Op.getValueType())) {

570

// We found a type with free casts.

571

SDValue X = DAG.getNode(

572

Op.getOpcode(), dl, SmallVT,

573

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),

574

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));

575

assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((DemandedSize <= SmallVTBits && "Narrowed below demanded bits?"
) ? static_cast<void> (0) : __assert_fail ("DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 575, __PRETTY_FUNCTION__));

576

SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);

577

return TLO.CombineTo(Op, Z);

578

}

579

}

580

return false;

581

}

582

583

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

584

DAGCombinerInfo &DCI) const {

585

SelectionDAG &DAG = DCI.DAG;

586

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

587

!DCI.isBeforeLegalizeOps());

588

KnownBits Known;

589

590

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);

591

if (Simplified) {

592

DCI.AddToWorklist(Op.getNode());

593

DCI.CommitTargetLoweringOpt(TLO);

594

}

595

return Simplified;

596

}

597

598

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

599

KnownBits &Known,

600

TargetLoweringOpt &TLO,

601

unsigned Depth,

602

bool AssumeSingleUse) const {

603

EVT VT = Op.getValueType();

604

605

// TODO: We can probably do more work on calculating the known bits and

606

// simplifying the operations for scalable vectors, but for now we just

607

// bail out.

608

if (VT.isScalableVector()) {

609

// Pretend we don't know anything for now.

610

Known = KnownBits(DemandedBits.getBitWidth());

611

return false;

612

}

613

614

APInt DemandedElts = VT.isVector()

615

? APInt::getAllOnesValue(VT.getVectorNumElements())

616

: APInt(1, 1);

617

return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,

618

AssumeSingleUse);

619

}

620

621

// TODO: Can we merge SelectionDAG::GetDemandedBits into this?

622

// TODO: Under what circumstances can we create nodes? Constant folding?

623

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

624

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

625

SelectionDAG &DAG, unsigned Depth) const {

626

// Limit search depth.

627

if (Depth >= SelectionDAG::MaxRecursionDepth)

628

return SDValue();

629

630

// Ignore UNDEFs.

631

if (Op.isUndef())

632

return SDValue();

633

634

// Not demanding any bits/elts from Op.

635

if (DemandedBits == 0 || DemandedElts == 0)

636

return DAG.getUNDEF(Op.getValueType());

637

638

unsigned NumElts = DemandedElts.getBitWidth();

639

unsigned BitWidth = DemandedBits.getBitWidth();

640

KnownBits LHSKnown, RHSKnown;

641

switch (Op.getOpcode()) {

642

case ISD::BITCAST: {

643

SDValue Src = peekThroughBitcasts(Op.getOperand(0));

644

EVT SrcVT = Src.getValueType();

645

EVT DstVT = Op.getValueType();

646

if (SrcVT == DstVT)

647

return Src;

648

649

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

650

unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

651

if (NumSrcEltBits == NumDstEltBits)

652

if (SDValue V = SimplifyMultipleUseDemandedBits(

653

Src, DemandedBits, DemandedElts, DAG, Depth + 1))

654

return DAG.getBitcast(DstVT, V);

655

656

// TODO - bigendian once we have test coverage.

657

if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&

658

DAG.getDataLayout().isLittleEndian()) {

659

unsigned Scale = NumDstEltBits / NumSrcEltBits;

660

unsigned NumSrcElts = SrcVT.getVectorNumElements();

661

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

662

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

663

for (unsigned i = 0; i != Scale; ++i) {

664

unsigned Offset = i * NumSrcEltBits;

665

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

666

if (!Sub.isNullValue()) {

667

DemandedSrcBits |= Sub;

668

for (unsigned j = 0; j != NumElts; ++j)

669

if (DemandedElts[j])

670

DemandedSrcElts.setBit((j * Scale) + i);

671

}

672

}

673

674

if (SDValue V = SimplifyMultipleUseDemandedBits(

675

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

676

return DAG.getBitcast(DstVT, V);

677

}

678

679

// TODO - bigendian once we have test coverage.

680

if ((NumSrcEltBits % NumDstEltBits) == 0 &&

681

DAG.getDataLayout().isLittleEndian()) {

682

unsigned Scale = NumSrcEltBits / NumDstEltBits;

683

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

684

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

685

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

686

for (unsigned i = 0; i != NumElts; ++i)

687

if (DemandedElts[i]) {

688

unsigned Offset = (i % Scale) * NumDstEltBits;

689

DemandedSrcBits.insertBits(DemandedBits, Offset);

690

DemandedSrcElts.setBit(i / Scale);

691

}

692

693

if (SDValue V = SimplifyMultipleUseDemandedBits(

694

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

695

return DAG.getBitcast(DstVT, V);

696

}

697

698

break;

699

}

700

case ISD::AND: {

701

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

702

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

703

704

// If all of the demanded bits are known 1 on one side, return the other.

705

// These bits cannot contribute to the result of the 'and' in this

706

// context.

707

if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))

708

return Op.getOperand(0);

709

if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))

710

return Op.getOperand(1);

711

break;

712

}

713

case ISD::OR: {

714

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

715

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

716

717

// If all of the demanded bits are known zero on one side, return the

718

// other. These bits cannot contribute to the result of the 'or' in this

719

// context.

720

if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))

721

return Op.getOperand(0);

722

if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))

723

return Op.getOperand(1);

724

break;

725

}

726

case ISD::XOR: {

727

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

728

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

729

730

// If all of the demanded bits are known zero on one side, return the

731

// other.

732

if (DemandedBits.isSubsetOf(RHSKnown.Zero))

733

return Op.getOperand(0);

734

if (DemandedBits.isSubsetOf(LHSKnown.Zero))

735

return Op.getOperand(1);

736

break;

737

}

738

case ISD::SHL: {

739

// If we are only demanding sign bits then we can use the shift source

740

// directly.

741

if (const APInt *MaxSA =

742

DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

743

SDValue Op0 = Op.getOperand(0);

744

unsigned ShAmt = MaxSA->getZExtValue();

745

unsigned NumSignBits =

746

DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

747

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

748

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

749

return Op0;

750

}

751

break;

752

}

753

case ISD::SETCC: {

754

SDValue Op0 = Op.getOperand(0);

755

SDValue Op1 = Op.getOperand(1);

756

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

757

// If (1) we only need the sign-bit, (2) the setcc operands are the same

758

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

759

// -1, we may be able to bypass the setcc.

760

if (DemandedBits.isSignMask() &&

761

Op0.getScalarValueSizeInBits() == BitWidth &&

762

getBooleanContents(Op0.getValueType()) ==

763

BooleanContent::ZeroOrNegativeOneBooleanContent) {

764

// If we're testing X < 0, then this compare isn't needed - just use X!

765

// FIXME: We're limiting to integer types here, but this should also work

766

// if we don't care about FP signed-zero. The use of SETLT with FP means

767

// that we don't care about NaNs.

768

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

769

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

770

return Op0;

771

}

772

break;

773

}

774

case ISD::SIGN_EXTEND_INREG: {

775

// If none of the extended bits are demanded, eliminate the sextinreg.

776

SDValue Op0 = Op.getOperand(0);

777

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

778

unsigned ExBits = ExVT.getScalarSizeInBits();

779

if (DemandedBits.getActiveBits() <= ExBits)

780

return Op0;

781

// If the input is already sign extended, just drop the extension.

782

unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

783

if (NumSignBits >= (BitWidth - ExBits + 1))

784

return Op0;

785

break;

786

}

787

case ISD::ANY_EXTEND_VECTOR_INREG:

788

case ISD::SIGN_EXTEND_VECTOR_INREG:

789

case ISD::ZERO_EXTEND_VECTOR_INREG: {

790

// If we only want the lowest element and none of extended bits, then we can

791

// return the bitcasted source vector.

792

SDValue Src = Op.getOperand(0);

793

EVT SrcVT = Src.getValueType();

794

EVT DstVT = Op.getValueType();

795

if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&

796

DAG.getDataLayout().isLittleEndian() &&

797

DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {

798

return DAG.getBitcast(DstVT, Src);

799

}

800

break;

801

}

802

case ISD::INSERT_VECTOR_ELT: {

803

// If we don't demand the inserted element, return the base vector.

804

SDValue Vec = Op.getOperand(0);

805

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

806

EVT VecVT = Vec.getValueType();

807

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&

808

!DemandedElts[CIdx->getZExtValue()])

809

return Vec;

810

break;

811

}

812

case ISD::INSERT_SUBVECTOR: {

813

// If we don't demand the inserted subvector, return the base vector.

814

SDValue Vec = Op.getOperand(0);

815

SDValue Sub = Op.getOperand(1);

816

uint64_t Idx = Op.getConstantOperandVal(2);

817

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

818

if (DemandedElts.extractBits(NumSubElts, Idx) == 0)

819

return Vec;

820

break;

821

}

822

case ISD::VECTOR_SHUFFLE: {

823

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

824

825

// If all the demanded elts are from one operand and are inline,

826

// then we can use the operand directly.

827

bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;

828

for (unsigned i = 0; i != NumElts; ++i) {

829

int M = ShuffleMask[i];

830

if (M < 0 || !DemandedElts[i])

831

continue;

832

AllUndef = false;

833

IdentityLHS &= (M == (int)i);

834

IdentityRHS &= ((M - NumElts) == i);

835

}

836

837

if (AllUndef)

838

return DAG.getUNDEF(Op.getValueType());

839

if (IdentityLHS)

840

return Op.getOperand(0);

841

if (IdentityRHS)

842

return Op.getOperand(1);

843

break;

844

}

845

default:

846

if (Op.getOpcode() >= ISD::BUILTIN_OP_END)

847

if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(

848

Op, DemandedBits, DemandedElts, DAG, Depth))

849

return V;

850

break;

851

}

852

return SDValue();

853

}

854

855

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

856

SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,

857

unsigned Depth) const {

858

EVT VT = Op.getValueType();

859

APInt DemandedElts = VT.isVector()

860

? APInt::getAllOnesValue(VT.getVectorNumElements())

861

: APInt(1, 1);

862

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

863

Depth);

864

}

865

866

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(

867

SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,

868

unsigned Depth) const {

869

APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());

870

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

871

Depth);

872

}

873

874

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the

875

/// result of Op are ever used downstream. If we can use this information to

876

/// simplify Op, create a new simplified DAG node and return true, returning the

877

/// original and new nodes in Old and New. Otherwise, analyze the expression and

878

/// return a mask of Known bits for the expression (used to simplify the

879

/// caller). The Known bits may only be accurate for those bits in the

880

/// OriginalDemandedBits and OriginalDemandedElts.

881

bool TargetLowering::SimplifyDemandedBits(

882

SDValue Op, const APInt &OriginalDemandedBits,

883

const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,

884

unsigned Depth, bool AssumeSingleUse) const {

885

unsigned BitWidth = OriginalDemandedBits.getBitWidth();

886

assert(Op.getScalarValueSizeInBits() == BitWidth &&((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 887, __PRETTY_FUNCTION__))

887

"Mask size mismatches value type size!")((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 887, __PRETTY_FUNCTION__));

888

889

// Don't know anything.

890

Known = KnownBits(BitWidth);

891

892

// TODO: We can probably do more work on calculating the known bits and

893

// simplifying the operations for scalable vectors, but for now we just

894

// bail out.

895

if (Op.getValueType().isScalableVector())

896

return false;

897

898

unsigned NumElts = OriginalDemandedElts.getBitWidth();

899

assert((!Op.getValueType().isVector() ||(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 901, __PRETTY_FUNCTION__))

900

NumElts == Op.getValueType().getVectorNumElements()) &&(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 901, __PRETTY_FUNCTION__))

901

"Unexpected vector size")(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 901, __PRETTY_FUNCTION__));

902

903

APInt DemandedBits = OriginalDemandedBits;

904

APInt DemandedElts = OriginalDemandedElts;

905

SDLoc dl(Op);

906

auto &DL = TLO.DAG.getDataLayout();

907

908

// Undef operand.

909

if (Op.isUndef())

910

return false;

911

912

if (Op.getOpcode() == ISD::Constant) {

913

// We know all of the bits for a constant!

914

Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());

915

return false;

916

}

917

918

if (Op.getOpcode() == ISD::ConstantFP) {

919

// We know all of the bits for a floating point constant!

920

Known = KnownBits::makeConstant(

921

cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());

922

return false;

923

}

924

925

// Other users may use these bits.

926

EVT VT = Op.getValueType();

927

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {

928

if (Depth != 0) {

929

// If not at the root, Just compute the Known bits to

930

// simplify things downstream.

931

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

932

return false;

933

}

934

// If this is the root being simplified, allow it to have multiple uses,

935

// just set the DemandedBits/Elts to all bits.

936

DemandedBits = APInt::getAllOnesValue(BitWidth);

937

DemandedElts = APInt::getAllOnesValue(NumElts);

938

} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {

939

// Not demanding any bits/elts from Op.

940

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

941

} else if (Depth >= SelectionDAG::MaxRecursionDepth) {

942

// Limit search depth.

943

return false;

944

}

945

946

KnownBits Known2;

947

switch (Op.getOpcode()) {

948

case ISD::TargetConstant:

949

llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 949);

950

case ISD::SCALAR_TO_VECTOR: {

951

if (!DemandedElts[0])

952

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

953

954

KnownBits SrcKnown;

955

SDValue Src = Op.getOperand(0);

956

unsigned SrcBitWidth = Src.getScalarValueSizeInBits();

957

APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);

958

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))

959

return true;

960

961

// Upper elements are undef, so only get the knownbits if we just demand

962

// the bottom element.

963

if (DemandedElts == 1)

964

Known = SrcKnown.anyextOrTrunc(BitWidth);

965

break;

966

}

967

case ISD::BUILD_VECTOR:

968

// Collect the known bits that are shared by every demanded element.

969

// TODO: Call SimplifyDemandedBits for non-constant demanded elements.

970

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

971

return false; // Don't fall through, will infinitely loop.

972

case ISD::LOAD: {

973

LoadSDNode *LD = cast<LoadSDNode>(Op);

974

if (getTargetConstantFromLoad(LD)) {

975

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

976

return false; // Don't fall through, will infinitely loop.

977

} else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {

978

// If this is a ZEXTLoad and we are looking at the loaded value.

979

EVT MemVT = LD->getMemoryVT();

980

unsigned MemBits = MemVT.getScalarSizeInBits();

981

Known.Zero.setBitsFrom(MemBits);

982

return false; // Don't fall through, will infinitely loop.

983

}

984

break;

985

}

986

case ISD::INSERT_VECTOR_ELT: {

987

SDValue Vec = Op.getOperand(0);

988

SDValue Scl = Op.getOperand(1);

989

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

990

EVT VecVT = Vec.getValueType();

991

992

// If index isn't constant, assume we need all vector elements AND the

993

// inserted element.

994

APInt DemandedVecElts(DemandedElts);

995

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {

996

unsigned Idx = CIdx->getZExtValue();

997

DemandedVecElts.clearBit(Idx);

998

999

// Inserted element is not required.

1000

if (!DemandedElts[Idx])

1001

return TLO.CombineTo(Op, Vec);

1002

}

1003

1004

KnownBits KnownScl;

1005

unsigned NumSclBits = Scl.getScalarValueSizeInBits();

1006

APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);

1007

if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))

1008

return true;

1009

1010

Known = KnownScl.anyextOrTrunc(BitWidth);

1011

1012

KnownBits KnownVec;

1013

if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,

1014

Depth + 1))

1015

return true;

1016

1017

if (!!DemandedVecElts)

1018

Known = KnownBits::commonBits(Known, KnownVec);

1019

1020

return false;

1021

}

1022

case ISD::INSERT_SUBVECTOR: {

1023

// Demand any elements from the subvector and the remainder from the src its

1024

// inserted into.

1025

SDValue Src = Op.getOperand(0);

1026

SDValue Sub = Op.getOperand(1);

1027

uint64_t Idx = Op.getConstantOperandVal(2);

1028

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

1029

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

1030

APInt DemandedSrcElts = DemandedElts;

1031

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

1032

1033

KnownBits KnownSub, KnownSrc;

1034

if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,

1035

Depth + 1))

1036

return true;

1037

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,

1038

Depth + 1))

1039

return true;

1040

1041

Known.Zero.setAllBits();

1042

Known.One.setAllBits();

1043

if (!!DemandedSubElts)

1044

Known = KnownBits::commonBits(Known, KnownSub);

1045

if (!!DemandedSrcElts)

1046

Known = KnownBits::commonBits(Known, KnownSrc);

1047

1048

// Attempt to avoid multi-use src if we don't need anything from it.

1049

if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||

1050

!DemandedSrcElts.isAllOnesValue()) {

1051

SDValue NewSub = SimplifyMultipleUseDemandedBits(

1052

Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);

1053

SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1054

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1055

if (NewSub || NewSrc) {

1056

NewSub = NewSub ? NewSub : Sub;

1057

NewSrc = NewSrc ? NewSrc : Src;

1058

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,

1059

Op.getOperand(2));

1060

return TLO.CombineTo(Op, NewOp);

1061

}

1062

}

1063

break;

1064

}

1065

case ISD::EXTRACT_SUBVECTOR: {

1066

// Offset the demanded elts by the subvector index.

1067

SDValue Src = Op.getOperand(0);

1068

if (Src.getValueType().isScalableVector())

1069

break;

1070

uint64_t Idx = Op.getConstantOperandVal(1);

1071

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

1072

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

1073

1074

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,

1075

Depth + 1))

1076

return true;

1077

1078

// Attempt to avoid multi-use src if we don't need anything from it.

1079

if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {

1080

SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

1081

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1082

if (DemandedSrc) {

1083

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,

1084

Op.getOperand(1));

1085

return TLO.CombineTo(Op, NewOp);

1086

}

1087

}

1088

break;

1089

}

1090

case ISD::CONCAT_VECTORS: {

1091

Known.Zero.setAllBits();

1092

Known.One.setAllBits();

1093

EVT SubVT = Op.getOperand(0).getValueType();

1094

unsigned NumSubVecs = Op.getNumOperands();

1095

unsigned NumSubElts = SubVT.getVectorNumElements();

1096

for (unsigned i = 0; i != NumSubVecs; ++i) {

1097

APInt DemandedSubElts =

1098

DemandedElts.extractBits(NumSubElts, i * NumSubElts);

1099

if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,

1100

Known2, TLO, Depth + 1))

1101

return true;

1102

// Known bits are shared by every demanded subvector element.

1103

if (!!DemandedSubElts)

1104

Known = KnownBits::commonBits(Known, Known2);

1105

}

1106

break;

1107

}

1108

case ISD::VECTOR_SHUFFLE: {

1109

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

1110

1111

// Collect demanded elements from shuffle operands..

1112

APInt DemandedLHS(NumElts, 0);

1113

APInt DemandedRHS(NumElts, 0);

1114

for (unsigned i = 0; i != NumElts; ++i) {

1115

if (!DemandedElts[i])

1116

continue;

1117

int M = ShuffleMask[i];

1118

if (M < 0) {

1119

// For UNDEF elements, we don't know anything about the common state of

1120

// the shuffle result.

1121

DemandedLHS.clearAllBits();

1122

DemandedRHS.clearAllBits();

1123

break;

1124

}

1125

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1125, __PRETTY_FUNCTION__));

1126

if (M < (int)NumElts)

1127

DemandedLHS.setBit(M);

1128

else

1129

DemandedRHS.setBit(M - NumElts);

1130

}

1131

1132

if (!!DemandedLHS || !!DemandedRHS) {

1133

SDValue Op0 = Op.getOperand(0);

1134

SDValue Op1 = Op.getOperand(1);

1135

1136

Known.Zero.setAllBits();

1137

Known.One.setAllBits();

1138

if (!!DemandedLHS) {

1139

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,

1140

Depth + 1))

1141

return true;

1142

Known = KnownBits::commonBits(Known, Known2);

1143

}

1144

if (!!DemandedRHS) {

1145

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,

1146

Depth + 1))

1147

return true;

1148

Known = KnownBits::commonBits(Known, Known2);

1149

}

1150

1151

// Attempt to avoid multi-use ops if we don't need anything from them.

1152

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1153

Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);

1154

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1155

Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);

1156

if (DemandedOp0 || DemandedOp1) {

1157

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1158

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1159

SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);

1160

return TLO.CombineTo(Op, NewOp);

1161

}

1162

}

1163

break;

1164

}

1165

case ISD::AND: {

1166

SDValue Op0 = Op.getOperand(0);

1167

SDValue Op1 = Op.getOperand(1);

1168

1169

// If the RHS is a constant, check to see if the LHS would be zero without

1170

// using the bits from the RHS. Below, we use knowledge about the RHS to

1171

// simplify the LHS, here we're using information from the LHS to simplify

1172

// the RHS.

1173

if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {

1174

// Do not increment Depth here; that can cause an infinite loop.

1175

KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);

1176

// If the LHS already has zeros where RHSC does, this 'and' is dead.

1177

if ((LHSKnown.Zero & DemandedBits) ==

1178

(~RHSC->getAPIntValue() & DemandedBits))

1179

return TLO.CombineTo(Op, Op0);

1180

1181

// If any of the set bits in the RHS are known zero on the LHS, shrink

1182

// the constant.

1183

if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,

1184

DemandedElts, TLO))

1185

return true;

1186

1187

// Bitwise-not (xor X, -1) is a special case: we don't usually shrink its

1188

// constant, but if this 'and' is only clearing bits that were just set by

1189

// the xor, then this 'and' can be eliminated by shrinking the mask of

1190

// the xor. For example, for a 32-bit X:

1191

// and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1

1192

if (isBitwiseNot(Op0) && Op0.hasOneUse() &&

1193

LHSKnown.One == ~RHSC->getAPIntValue()) {

1194

SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);

1195

return TLO.CombineTo(Op, Xor);

1196

}

1197

}

1198

1199

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1200

Depth + 1))

1201

return true;

1202

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1202, __PRETTY_FUNCTION__));

1203

if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,

1204

Known2, TLO, Depth + 1))

1205

return true;

1206

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1206, __PRETTY_FUNCTION__));

1207

1208

// Attempt to avoid multi-use ops if we don't need anything from them.

1209

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1210

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1211

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1212

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1213

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1214

if (DemandedOp0 || DemandedOp1) {

1215

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1216

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1217

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1218

return TLO.CombineTo(Op, NewOp);

1219

}

1220

}

1221

1222

// If all of the demanded bits are known one on one side, return the other.

1223

// These bits cannot contribute to the result of the 'and'.

1224

if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))

1225

return TLO.CombineTo(Op, Op0);

1226

if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))

1227

return TLO.CombineTo(Op, Op1);

1228

// If all of the demanded bits in the inputs are known zeros, return zero.

1229

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1230

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));

1231

// If the RHS is a constant, see if we can simplify it.

1232

if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,

1233

TLO))

1234

return true;

1235

// If the operation can be done in a smaller type, do so.

1236

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1237

return true;

1238

1239

Known &= Known2;

1240

break;

1241

}

1242

case ISD::OR: {

1243

SDValue Op0 = Op.getOperand(0);

1244

SDValue Op1 = Op.getOperand(1);

1245

1246

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1247

Depth + 1))

1248

return true;

1249

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1249, __PRETTY_FUNCTION__));

1250

if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,

1251

Known2, TLO, Depth + 1))

1252

return true;

1253

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1253, __PRETTY_FUNCTION__));

1254

1255

// Attempt to avoid multi-use ops if we don't need anything from them.

1256

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1257

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1258

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1259

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1260

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1261

if (DemandedOp0 || DemandedOp1) {

1262

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1263

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1264

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1265

return TLO.CombineTo(Op, NewOp);

1266

}

1267

}

1268

1269

// If all of the demanded bits are known zero on one side, return the other.

1270

// These bits cannot contribute to the result of the 'or'.

1271

if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))

1272

return TLO.CombineTo(Op, Op0);

1273

if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))

1274

return TLO.CombineTo(Op, Op1);

1275

// If the RHS is a constant, see if we can simplify it.

1276

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1277

return true;

1278

// If the operation can be done in a smaller type, do so.

1279

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1280

return true;

1281

1282

Known |= Known2;

1283

break;

1284

}

1285

case ISD::XOR: {

1286

SDValue Op0 = Op.getOperand(0);

1287

SDValue Op1 = Op.getOperand(1);

1288

1289

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1290

Depth + 1))

1291

return true;

1292

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1292, __PRETTY_FUNCTION__));

1293

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,

1294

Depth + 1))

1295

return true;

1296

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1296, __PRETTY_FUNCTION__));

1297

1298

// Attempt to avoid multi-use ops if we don't need anything from them.

1299

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1300

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1301

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1302

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1303

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1304

if (DemandedOp0 || DemandedOp1) {

1305

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1306

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1307

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1308

return TLO.CombineTo(Op, NewOp);

1309

}

1310

}

1311

1312

// If all of the demanded bits are known zero on one side, return the other.

1313

// These bits cannot contribute to the result of the 'xor'.

1314

if (DemandedBits.isSubsetOf(Known.Zero))

1315

return TLO.CombineTo(Op, Op0);

1316

if (DemandedBits.isSubsetOf(Known2.Zero))

1317

return TLO.CombineTo(Op, Op1);

1318

// If the operation can be done in a smaller type, do so.

1319

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1320

return true;

1321

1322

// If all of the unknown bits are known to be zero on one side or the other

1323

// turn this into an *inclusive* or.

1324

// e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0

1325

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1326

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

1327

1328

ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);

1329

if (C) {

1330

// If one side is a constant, and all of the set bits in the constant are

1331

// also known set on the other side, turn this into an AND, as we know

1332

// the bits will be cleared.

1333

// e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2

1334

// NB: it is okay if more bits are known than are requested

1335

if (C->getAPIntValue() == Known2.One) {

1336

SDValue ANDC =

1337

TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);

1338

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));

1339

}

1340

1341

// If the RHS is a constant, see if we can change it. Don't alter a -1

1342

// constant because that's a 'not' op, and that is better for combining

1343

// and codegen.

1344

if (!C->isAllOnesValue() &&

1345

DemandedBits.isSubsetOf(C->getAPIntValue())) {

1346

// We're flipping all demanded bits. Flip the undemanded bits too.

1347

SDValue New = TLO.DAG.getNOT(dl, Op0, VT);

1348

return TLO.CombineTo(Op, New);

1349

}

1350

}

1351

1352

// If we can't turn this into a 'not', try to shrink the constant.

1353

if (!C || !C->isAllOnesValue())

1354

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1355

return true;

1356

1357

Known ^= Known2;

1358

break;

1359

}

1360

case ISD::SELECT:

1361

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,

1362

Depth + 1))

1363

return true;

1364

if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,

1365

Depth + 1))

1366

return true;

1367

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1367, __PRETTY_FUNCTION__));

1368

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1368, __PRETTY_FUNCTION__));

1369

1370

// If the operands are constants, see if we can simplify them.

1371

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1372

return true;

1373

1374

// Only known if known in both the LHS and RHS.

1375

Known = KnownBits::commonBits(Known, Known2);

1376

break;

1377

case ISD::SELECT_CC:

1378

if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,

1379

Depth + 1))

1380

return true;

1381

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,

1382

Depth + 1))

1383

return true;

1384

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1384, __PRETTY_FUNCTION__));

1385

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1385, __PRETTY_FUNCTION__));

1386

1387

// If the operands are constants, see if we can simplify them.

1388

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1389

return true;

1390

1391

// Only known if known in both the LHS and RHS.

1392

Known = KnownBits::commonBits(Known, Known2);

1393

break;

1394

case ISD::SETCC: {

1395

SDValue Op0 = Op.getOperand(0);

1396

SDValue Op1 = Op.getOperand(1);

1397

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

1398

// If (1) we only need the sign-bit, (2) the setcc operands are the same

1399

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

1400

// -1, we may be able to bypass the setcc.

1401

if (DemandedBits.isSignMask() &&

1402

Op0.getScalarValueSizeInBits() == BitWidth &&

1403

getBooleanContents(Op0.getValueType()) ==

1404

BooleanContent::ZeroOrNegativeOneBooleanContent) {

1405

// If we're testing X < 0, then this compare isn't needed - just use X!

1406

// FIXME: We're limiting to integer types here, but this should also work

1407

// if we don't care about FP signed-zero. The use of SETLT with FP means

1408

// that we don't care about NaNs.

1409

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

1410

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

1411

return TLO.CombineTo(Op, Op0);

1412

1413

// TODO: Should we check for other forms of sign-bit comparisons?

1414

// Examples: X <= -1, X >= 0

1415

}

1416

if (getBooleanContents(Op0.getValueType()) ==

1417

TargetLowering::ZeroOrOneBooleanContent &&

1418

BitWidth > 1)

1419

Known.Zero.setBitsFrom(1);

1420

break;

1421

}

1422

case ISD::SHL: {

1423

SDValue Op0 = Op.getOperand(0);

1424

SDValue Op1 = Op.getOperand(1);

1425

EVT ShiftVT = Op1.getValueType();

1426

1427

if (const APInt *SA =

1428

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1429

unsigned ShAmt = SA->getZExtValue();

1430

if (ShAmt == 0)

1431

return TLO.CombineTo(Op, Op0);

1432

1433

// If this is ((X >>u C1) << ShAmt), see if we can simplify this into a

1434

// single shift. We can do this if the bottom bits (which are shifted

1435

// out) are never demanded.

1436

// TODO - support non-uniform vector amounts.

1437

if (Op0.getOpcode() == ISD::SRL) {

1438

if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {

1439

if (const APInt *SA2 =

1440

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1441

unsigned C1 = SA2->getZExtValue();

1442

unsigned Opc = ISD::SHL;

1443

int Diff = ShAmt - C1;

1444

if (Diff < 0) {

1445

Diff = -Diff;

1446

Opc = ISD::SRL;

1447

}

1448

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1449

return TLO.CombineTo(

1450

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1451

}

1452

}

1453

}

1454

1455

// Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits

1456

// are not demanded. This will likely allow the anyext to be folded away.

1457

// TODO - support non-uniform vector amounts.

1458

if (Op0.getOpcode() == ISD::ANY_EXTEND) {

1459

SDValue InnerOp = Op0.getOperand(0);

1460

EVT InnerVT = InnerOp.getValueType();

1461

unsigned InnerBits = InnerVT.getScalarSizeInBits();

1462

if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&

1463

isTypeDesirableForOp(ISD::SHL, InnerVT)) {

1464

EVT ShTy = getShiftAmountTy(InnerVT, DL);

1465

if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))

1466

ShTy = InnerVT;

1467

SDValue NarrowShl =

1468

TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,

1469

TLO.DAG.getConstant(ShAmt, dl, ShTy));

1470

return TLO.CombineTo(

1471

Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));

1472

}

1473

1474

// Repeat the SHL optimization above in cases where an extension

1475

// intervenes: (shl (anyext (shr x, c1)), c2) to

1476

// (shl (anyext x), c2-c1). This requires that the bottom c1 bits

1477

// aren't demanded (as above) and that the shifted upper c1 bits of

1478

// x aren't demanded.

1479

// TODO - support non-uniform vector amounts.

1480

if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&

1481

InnerOp.hasOneUse()) {

1482

if (const APInt *SA2 =

1483

TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {

1484

unsigned InnerShAmt = SA2->getZExtValue();

1485

if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&

1486

DemandedBits.getActiveBits() <=

1487

(InnerBits - InnerShAmt + ShAmt) &&

1488

DemandedBits.countTrailingZeros() >= ShAmt) {

1489

SDValue NewSA =

1490

TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);

1491

SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,

1492

InnerOp.getOperand(0));

1493

return TLO.CombineTo(

1494

Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));

1495

}

1496

}

1497

}

1498

}

1499

1500

APInt InDemandedMask = DemandedBits.lshr(ShAmt);

1501

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1502

Depth + 1))

1503

return true;

1504

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1504, __PRETTY_FUNCTION__));

1505

Known.Zero <<= ShAmt;

1506

Known.One <<= ShAmt;

1507

// low bits known zero.

1508

Known.Zero.setLowBits(ShAmt);

1509

1510

// Try shrinking the operation as long as the shift amount will still be

1511

// in range.

1512

if ((ShAmt < DemandedBits.getActiveBits()) &&

1513

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1514

return true;

1515

}

1516

1517

// If we are only demanding sign bits then we can use the shift source

1518

// directly.

1519

if (const APInt *MaxSA =

1520

TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

1521

unsigned ShAmt = MaxSA->getZExtValue();

1522

unsigned NumSignBits =

1523

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1524

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1525

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

1526

return TLO.CombineTo(Op, Op0);

1527

}

1528

break;

1529

}

1530

case ISD::SRL: {

1531

SDValue Op0 = Op.getOperand(0);

1532

SDValue Op1 = Op.getOperand(1);

1533

EVT ShiftVT = Op1.getValueType();

1534

1535

if (const APInt *SA =

1536

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1537

unsigned ShAmt = SA->getZExtValue();

1538

if (ShAmt == 0)

1539

return TLO.CombineTo(Op, Op0);

1540

1541

// If this is ((X << C1) >>u ShAmt), see if we can simplify this into a

1542

// single shift. We can do this if the top bits (which are shifted out)

1543

// are never demanded.

1544

// TODO - support non-uniform vector amounts.

1545

if (Op0.getOpcode() == ISD::SHL) {

1546

if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {

1547

if (const APInt *SA2 =

1548

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1549

unsigned C1 = SA2->getZExtValue();

1550

unsigned Opc = ISD::SRL;

1551

int Diff = ShAmt - C1;

1552

if (Diff < 0) {

1553

Diff = -Diff;

1554

Opc = ISD::SHL;

1555

}

1556

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1557

return TLO.CombineTo(

1558

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1559

}

1560

}

1561

}

1562

1563

APInt InDemandedMask = (DemandedBits << ShAmt);

1564

1565

// If the shift is exact, then it does demand the low bits (and knows that

1566

// they are zero).

1567

if (Op->getFlags().hasExact())

1568

InDemandedMask.setLowBits(ShAmt);

1569

1570

// Compute the new bits that are at the top now.

1571

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1572

Depth + 1))

1573

return true;

1574

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1574, __PRETTY_FUNCTION__));

1575

Known.Zero.lshrInPlace(ShAmt);

1576

Known.One.lshrInPlace(ShAmt);

1577

// High bits known zero.

1578

Known.Zero.setHighBits(ShAmt);

1579

}

1580

break;

1581

}

1582

case ISD::SRA: {

1583

SDValue Op0 = Op.getOperand(0);

1584

SDValue Op1 = Op.getOperand(1);

1585

EVT ShiftVT = Op1.getValueType();

1586

1587

// If we only want bits that already match the signbit then we don't need

1588

// to shift.

1589

unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1590

if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=

1591

NumHiDemandedBits)

1592

return TLO.CombineTo(Op, Op0);

1593

1594

// If this is an arithmetic shift right and only the low-bit is set, we can

1595

// always convert this into a logical shr, even if the shift amount is

1596

// variable. The low bit of the shift cannot be an input sign bit unless

1597

// the shift amount is >= the size of the datatype, which is undefined.

1598

if (DemandedBits.isOneValue())

1599

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

1600

1601

if (const APInt *SA =

1602

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1603

unsigned ShAmt = SA->getZExtValue();

1604

if (ShAmt == 0)

1605

return TLO.CombineTo(Op, Op0);

1606

1607

APInt InDemandedMask = (DemandedBits << ShAmt);

1608

1609

// If the shift is exact, then it does demand the low bits (and knows that

1610

// they are zero).

1611

if (Op->getFlags().hasExact())

1612

InDemandedMask.setLowBits(ShAmt);

1613

1614

// If any of the demanded bits are produced by the sign extension, we also

1615

// demand the input sign bit.

1616

if (DemandedBits.countLeadingZeros() < ShAmt)

1617

InDemandedMask.setSignBit();

1618

1619

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1620

Depth + 1))

1621

return true;

1622

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1622, __PRETTY_FUNCTION__));

1623

Known.Zero.lshrInPlace(ShAmt);

1624

Known.One.lshrInPlace(ShAmt);

1625

1626

// If the input sign bit is known to be zero, or if none of the top bits

1627

// are demanded, turn this into an unsigned shift right.

1628

if (Known.Zero[BitWidth - ShAmt - 1] ||

1629

DemandedBits.countLeadingZeros() >= ShAmt) {

1630

SDNodeFlags Flags;

1631

Flags.setExact(Op->getFlags().hasExact());

1632

return TLO.CombineTo(

1633

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));

1634

}

1635

1636

int Log2 = DemandedBits.exactLogBase2();

1637

if (Log2 >= 0) {

1638

// The bit must come from the sign.

1639

SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);

1640

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));

1641

}

1642

1643

if (Known.One[BitWidth - ShAmt - 1])

1644

// New bits are known one.

1645

Known.One.setHighBits(ShAmt);

1646

1647

// Attempt to avoid multi-use ops if we don't need anything from them.

1648

if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1649

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1650

Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);

1651

if (DemandedOp0) {

1652

SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);

1653

return TLO.CombineTo(Op, NewOp);

1654

}

1655

}

1656

}

1657

break;

1658

}

1659

case ISD::FSHL:

1660

case ISD::FSHR: {

1661

SDValue Op0 = Op.getOperand(0);

1662

SDValue Op1 = Op.getOperand(1);

1663

SDValue Op2 = Op.getOperand(2);

1664

bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

1665

1666

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {

1667

unsigned Amt = SA->getAPIntValue().urem(BitWidth);

1668

1669

// For fshl, 0-shift returns the 1st arg.

1670

// For fshr, 0-shift returns the 2nd arg.

1671

if (Amt == 0) {

1672

if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,

1673

Known, TLO, Depth + 1))

1674

return true;

1675

break;

1676

}

1677

1678

// fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))

1679

// fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)

1680

APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));

1681

APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);

1682

if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,

1683

Depth + 1))

1684

return true;

1685

if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,

1686

Depth + 1))

1687

return true;

1688

1689

Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));

1690

Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));

1691

Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1692

Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1693

Known.One |= Known2.One;

1694

Known.Zero |= Known2.Zero;

1695

}

1696

1697

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1698

if (isPowerOf2_32(BitWidth)) {

1699

APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);

1700

if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,

1701

Known2, TLO, Depth + 1))

1702

return true;

1703

}

1704

break;

1705

}

1706

case ISD::ROTL:

1707

case ISD::ROTR: {

1708

SDValue Op0 = Op.getOperand(0);

1709

SDValue Op1 = Op.getOperand(1);

1710

1711

// If we're rotating an 0/-1 value, then it stays an 0/-1 value.

1712

if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))

1713

return TLO.CombineTo(Op, Op0);

1714

1715

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1716

if (isPowerOf2_32(BitWidth)) {

1717

APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);

1718

if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,

1719

Depth + 1))

1720

return true;

1721

}

1722

break;

1723

}

1724

case ISD::UMIN: {

1725

// Check if one arg is always less than (or equal) to the other arg.

1726

SDValue Op0 = Op.getOperand(0);

1727

SDValue Op1 = Op.getOperand(1);

1728

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1729

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1730

Known = KnownBits::umin(Known0, Known1);

1731

if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))

1732

return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);

1733

if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))

1734

return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);

1735

break;

1736

}

1737

case ISD::UMAX: {

1738

// Check if one arg is always greater than (or equal) to the other arg.

1739

SDValue Op0 = Op.getOperand(0);

1740

SDValue Op1 = Op.getOperand(1);

1741

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1742

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1743

Known = KnownBits::umax(Known0, Known1);

1744

if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))

1745

return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);

1746

if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))

1747

return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);

1748

break;

1749

}

1750

case ISD::BITREVERSE: {

1751

SDValue Src = Op.getOperand(0);

1752

APInt DemandedSrcBits = DemandedBits.reverseBits();

1753

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1754

Depth + 1))

1755

return true;

1756

Known.One = Known2.One.reverseBits();

1757

Known.Zero = Known2.Zero.reverseBits();

1758

break;

1759

}

1760

case ISD::BSWAP: {

1761

SDValue Src = Op.getOperand(0);

1762

APInt DemandedSrcBits = DemandedBits.byteSwap();

1763

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1764

Depth + 1))

1765

return true;

1766

Known.One = Known2.One.byteSwap();

1767

Known.Zero = Known2.Zero.byteSwap();

1768

break;

1769

}

1770

case ISD::CTPOP: {

1771

// If only 1 bit is demanded, replace with PARITY as long as we're before

1772

// op legalization.

1773

// FIXME: Limit to scalars for now.

1774

if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())

1775

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,

1776

Op.getOperand(0)));

1777

1778

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

1779

break;

1780

}

1781

case ISD::SIGN_EXTEND_INREG: {

1782

SDValue Op0 = Op.getOperand(0);

1783

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

1784

unsigned ExVTBits = ExVT.getScalarSizeInBits();

1785

1786

// If we only care about the highest bit, don't bother shifting right.

1787

if (DemandedBits.isSignMask()) {

1788

unsigned NumSignBits =

1789

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1790

bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;

1791

// However if the input is already sign extended we expect the sign

1792

// extension to be dropped altogether later and do not simplify.

1793

if (!AlreadySignExtended) {

1794

// Compute the correct shift amount type, which must be getShiftAmountTy

1795

// for scalar types after legalization.

1796

EVT ShiftAmtTy = VT;

1797

if (TLO.LegalTypes() && !ShiftAmtTy.isVector())

1798

ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

1799

1800

SDValue ShiftAmt =

1801

TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);

1802

return TLO.CombineTo(Op,

1803

TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));

1804

}

1805

}

1806

1807

// If none of the extended bits are demanded, eliminate the sextinreg.

1808

if (DemandedBits.getActiveBits() <= ExVTBits)

1809

return TLO.CombineTo(Op, Op0);

1810

1811

APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

1812

1813

// Since the sign extended bits are demanded, we know that the sign

1814

// bit is demanded.

1815

InputDemandedBits.setBit(ExVTBits - 1);

1816

1817

if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))

1818

return true;

1819

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1819, __PRETTY_FUNCTION__));

1820

1821

// If the sign bit of the input is known set or clear, then we know the

1822

// top bits of the result.

1823

1824

// If the input sign bit is known zero, convert this into a zero extension.

1825

if (Known.Zero[ExVTBits - 1])

1826

return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

1827

1828

APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);

1829

if (Known.One[ExVTBits - 1]) { // Input sign bit known set

1830

Known.One.setBitsFrom(ExVTBits);

1831

Known.Zero &= Mask;

1832

} else { // Input sign bit unknown

1833

Known.Zero &= Mask;

1834

Known.One &= Mask;

1835

}

1836

break;

1837

}

1838

case ISD::BUILD_PAIR: {

1839

EVT HalfVT = Op.getOperand(0).getValueType();

1840

unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

1841

1842

APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);

1843

APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

1844

1845

KnownBits KnownLo, KnownHi;

1846

1847

if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))

1848

return true;

1849

1850

if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))

1851

return true;

1852

1853

Known.Zero = KnownLo.Zero.zext(BitWidth) |

1854

KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

1855

1856

Known.One = KnownLo.One.zext(BitWidth) |

1857

KnownHi.One.zext(BitWidth).shl(HalfBitWidth);

1858

break;

1859

}

1860

case ISD::ZERO_EXTEND:

1861

case ISD::ZERO_EXTEND_VECTOR_INREG: {

1862

SDValue Src = Op.getOperand(0);

1863

EVT SrcVT = Src.getValueType();

1864

unsigned InBits = SrcVT.getScalarSizeInBits();

1865

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1866

bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

1867

1868

// If none of the top bits are demanded, convert this into an any_extend.

1869

if (DemandedBits.getActiveBits() <= InBits) {

1870

// If we only need the non-extended bits of the bottom element

1871

// then we can just bitcast to the result.

1872

if (IsVecInReg && DemandedElts == 1 &&

1873

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1874

TLO.DAG.getDataLayout().isLittleEndian())

1875

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1876

1877

unsigned Opc =

1878

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1879

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1880

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1881

}

1882

1883

APInt InDemandedBits = DemandedBits.trunc(InBits);

1884

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1885

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1886

Depth + 1))

1887

return true;

1888

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1888, __PRETTY_FUNCTION__));

1889

assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1889, __PRETTY_FUNCTION__));

1890

Known = Known.zext(BitWidth);

1891

1892

// Attempt to avoid multi-use ops if we don't need anything from them.

1893

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1894

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1895

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1896

break;

1897

}

1898

case ISD::SIGN_EXTEND:

1899

case ISD::SIGN_EXTEND_VECTOR_INREG: {

1900

SDValue Src = Op.getOperand(0);

1901

EVT SrcVT = Src.getValueType();

1902

unsigned InBits = SrcVT.getScalarSizeInBits();

1903

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1904

bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

1905

1906

// If none of the top bits are demanded, convert this into an any_extend.

1907

if (DemandedBits.getActiveBits() <= InBits) {

1908

// If we only need the non-extended bits of the bottom element

1909

// then we can just bitcast to the result.

1910

if (IsVecInReg && DemandedElts == 1 &&

1911

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1912

TLO.DAG.getDataLayout().isLittleEndian())

1913

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1914

1915

unsigned Opc =

1916

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1917

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1918

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1919

}

1920

1921

APInt InDemandedBits = DemandedBits.trunc(InBits);

1922

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1923

1924

// Since some of the sign extended bits are demanded, we know that the sign

1925

// bit is demanded.

1926

InDemandedBits.setBit(InBits - 1);

1927

1928

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1929

Depth + 1))

1930

return true;

1931

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1931, __PRETTY_FUNCTION__));

1932

assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1932, __PRETTY_FUNCTION__));

1933

1934

// If the sign bit is known one, the top bits match.

1935

Known = Known.sext(BitWidth);

1936

1937

// If the sign bit is known zero, convert this to a zero extend.

1938

if (Known.isNonNegative()) {

1939

unsigned Opc =

1940

IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;

1941

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1942

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1943

}

1944

1945

// Attempt to avoid multi-use ops if we don't need anything from them.

1946

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1947

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1948

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1949

break;

1950

}

1951

case ISD::ANY_EXTEND:

1952

case ISD::ANY_EXTEND_VECTOR_INREG: {

1953

SDValue Src = Op.getOperand(0);

1954

EVT SrcVT = Src.getValueType();

1955

unsigned InBits = SrcVT.getScalarSizeInBits();

1956

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1957

bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

1958

1959

// If we only need the bottom element then we can just bitcast.

1960

// TODO: Handle ANY_EXTEND?

1961

if (IsVecInReg && DemandedElts == 1 &&

1962

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1963

TLO.DAG.getDataLayout().isLittleEndian())

1964

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1965

1966

APInt InDemandedBits = DemandedBits.trunc(InBits);

1967

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1968

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1969

Depth + 1))

1970

return true;

1971

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1971, __PRETTY_FUNCTION__));

1972

assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1972, __PRETTY_FUNCTION__));

1973

Known = Known.anyext(BitWidth);

1974

1975

// Attempt to avoid multi-use ops if we don't need anything from them.

1976

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1977

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1978

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1979

break;

1980

}

1981

case ISD::TRUNCATE: {

1982

SDValue Src = Op.getOperand(0);

1983

1984

// Simplify the input, using demanded bit information, and compute the known

1985

// zero/one bits live out.

1986

unsigned OperandBitWidth = Src.getScalarValueSizeInBits();

1987

APInt TruncMask = DemandedBits.zext(OperandBitWidth);

1988

if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,

1989

Depth + 1))

1990

return true;

1991

Known = Known.trunc(BitWidth);

1992

1993

// Attempt to avoid multi-use ops if we don't need anything from them.

1994

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1995

Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))

1996

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

1997

1998

// If the input is only used by this truncate, see if we can shrink it based

1999

// on the known demanded bits.

2000

if (Src.getNode()->hasOneUse()) {

2001

switch (Src.getOpcode()) {

2002

default:

2003

break;

2004

case ISD::SRL:

2005

// Shrink SRL by a constant if none of the high bits shifted in are

2006

// demanded.

2007

if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))

2008

// Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is

2009

// undesirable.

2010

break;

2011

2012

const APInt *ShAmtC =

2013

TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);

2014

if (!ShAmtC || ShAmtC->uge(BitWidth))

2015

break;

2016

uint64_t ShVal = ShAmtC->getZExtValue();

2017

2018

APInt HighBits =

2019

APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);

2020

HighBits.lshrInPlace(ShVal);

2021

HighBits = HighBits.trunc(BitWidth);

2022

2023

if (!(HighBits & DemandedBits)) {

2024

// None of the shifted in bits are needed. Add a truncate of the

2025

// shift input, then shift it.

2026

SDValue NewShAmt = TLO.DAG.getConstant(

2027

ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));

2028

SDValue NewTrunc =

2029

TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));

2030

return TLO.CombineTo(

2031

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));

2032

}

2033

break;

2034

}

2035

}

2036

2037

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2037, __PRETTY_FUNCTION__));

2038

break;

2039

}

2040

case ISD::AssertZext: {

2041

// AssertZext demands all of the high bits, plus any of the low bits

2042

// demanded by its users.

2043

EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

2044

APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());

2045

if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,

2046

TLO, Depth + 1))

2047

return true;

2048

assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2048, __PRETTY_FUNCTION__));

2049

2050

Known.Zero |= ~InMask;

2051

break;

2052

}

2053

case ISD::EXTRACT_VECTOR_ELT: {

2054

SDValue Src = Op.getOperand(0);

2055

SDValue Idx = Op.getOperand(1);

2056

ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();

2057

unsigned EltBitWidth = Src.getScalarValueSizeInBits();

2058

2059

if (SrcEltCnt.isScalable())

2060

return false;

2061

2062

// Demand the bits from every vector element without a constant index.

2063

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2064

APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);

2065

if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))

2066

if (CIdx->getAPIntValue().ult(NumSrcElts))

2067

DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

2068

2069

// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know

2070

// anything about the extended bits.

2071

APInt DemandedSrcBits = DemandedBits;

2072

if (BitWidth > EltBitWidth)

2073

DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

2074

2075

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,

2076

Depth + 1))

2077

return true;

2078

2079

// Attempt to avoid multi-use ops if we don't need anything from them.

2080

if (!DemandedSrcBits.isAllOnesValue() ||

2081

!DemandedSrcElts.isAllOnesValue()) {

2082

if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

2083

Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {

2084

SDValue NewOp =

2085

TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);

2086

return TLO.CombineTo(Op, NewOp);

2087

}

2088

}

2089

2090

Known = Known2;

2091

if (BitWidth > EltBitWidth)

2092

Known = Known.anyext(BitWidth);

2093

break;

2094

}

2095

case ISD::BITCAST: {

2096

SDValue Src = Op.getOperand(0);

2097

EVT SrcVT = Src.getValueType();

2098

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

2099

2100

// If this is an FP->Int bitcast and if the sign bit is the only

2101

// thing demanded, turn this into a FGETSIGN.

2102

if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&

2103

DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&

2104

SrcVT.isFloatingPoint()) {

2105

bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);

2106

bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);

2107

if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&

2108

SrcVT != MVT::f128) {

2109

// Cannot eliminate/lower SHL for f128 yet.

2110

EVT Ty = OpVTLegal ? VT : MVT::i32;

2111

// Make a FGETSIGN + SHL to move the sign bit into the appropriate

2112

// place. We expect the SHL to be eliminated by other optimizations.

2113

SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);

2114

unsigned OpVTSizeInBits = Op.getValueSizeInBits();

2115

if (!OpVTLegal && OpVTSizeInBits > 32)

2116

Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);

2117

unsigned ShVal = Op.getValueSizeInBits() - 1;

2118

SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);

2119

return TLO.CombineTo(Op,

2120

TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));

2121

}

2122

}

2123

2124

// Bitcast from a vector using SimplifyDemanded Bits/VectorElts.

2125

// Demand the elt/bit if any of the original elts/bits are demanded.

2126

// TODO - bigendian once we have test coverage.

2127

if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&

2128

TLO.DAG.getDataLayout().isLittleEndian()) {

2129

unsigned Scale = BitWidth / NumSrcEltBits;

2130

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2131

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2132

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2133

for (unsigned i = 0; i != Scale; ++i) {

2134

unsigned Offset = i * NumSrcEltBits;

2135

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

2136

if (!Sub.isNullValue()) {

2137

DemandedSrcBits |= Sub;

2138

for (unsigned j = 0; j != NumElts; ++j)

2139

if (DemandedElts[j])

2140

DemandedSrcElts.setBit((j * Scale) + i);

2141

}

2142

}

2143

2144

APInt KnownSrcUndef, KnownSrcZero;

2145

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2146

KnownSrcZero, TLO, Depth + 1))

2147

return true;

2148

2149

KnownBits KnownSrcBits;

2150

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2151

KnownSrcBits, TLO, Depth + 1))

2152

return true;

2153

} else if ((NumSrcEltBits % BitWidth) == 0 &&

2154

TLO.DAG.getDataLayout().isLittleEndian()) {

2155

unsigned Scale = NumSrcEltBits / BitWidth;

2156

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

2157

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2158

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2159

for (unsigned i = 0; i != NumElts; ++i)

2160

if (DemandedElts[i]) {

2161

unsigned Offset = (i % Scale) * BitWidth;

2162

DemandedSrcBits.insertBits(DemandedBits, Offset);

2163

DemandedSrcElts.setBit(i / Scale);

2164

}

2165

2166

if (SrcVT.isVector()) {

2167

APInt KnownSrcUndef, KnownSrcZero;

2168

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2169

KnownSrcZero, TLO, Depth + 1))

2170

return true;

2171

}

2172

2173

KnownBits KnownSrcBits;

2174

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2175

KnownSrcBits, TLO, Depth + 1))

2176

return true;

2177

}

2178

2179

// If this is a bitcast, let computeKnownBits handle it. Only do this on a

2180

// recursive call where Known may be useful to the caller.

2181

if (Depth > 0) {

2182

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2183

return false;

2184

}

2185

break;

2186

}

2187

case ISD::ADD:

2188

case ISD::MUL:

2189

case ISD::SUB: {

2190

// Add, Sub, and Mul don't demand any bits in positions beyond that

2191

// of the highest bit demanded of them.

2192

SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);

2193

SDNodeFlags Flags = Op.getNode()->getFlags();

2194

unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();

2195

APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

2196

if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,

2197

Depth + 1) ||

2198

SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,

2199

Depth + 1) ||

2200

// See if the operation should be performed at a smaller bit width.

2201

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {

2202

if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {

2203

// Disable the nsw and nuw flags. We can no longer guarantee that we

2204

// won't wrap after simplification.

2205

Flags.setNoSignedWrap(false);

2206

Flags.setNoUnsignedWrap(false);

2207

SDValue NewOp =

2208

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2209

return TLO.CombineTo(Op, NewOp);

2210

}

2211

return true;

2212

}

2213

2214

// Attempt to avoid multi-use ops if we don't need anything from them.

2215

if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

2216

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

2217

Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2218

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

2219

Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2220

if (DemandedOp0 || DemandedOp1) {

2221

Flags.setNoSignedWrap(false);

2222

Flags.setNoUnsignedWrap(false);

2223

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

2224

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

2225

SDValue NewOp =

2226

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2227

return TLO.CombineTo(Op, NewOp);

2228

}

2229

}

2230

2231

// If we have a constant operand, we may be able to turn it into -1 if we

2232

// do not demand the high bits. This can make the constant smaller to

2233

// encode, allow more general folding, or match specialized instruction

2234

// patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that

2235

// is probably not useful (and could be detrimental).

2236

ConstantSDNode *C = isConstOrConstSplat(Op1);

2237

APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);

2238

if (C && !C->isAllOnesValue() && !C->isOne() &&

2239

(C->getAPIntValue() | HighMask).isAllOnesValue()) {

2240

SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);

2241

// Disable the nsw and nuw flags. We can no longer guarantee that we

2242

// won't wrap after simplification.

2243

Flags.setNoSignedWrap(false);

2244

Flags.setNoUnsignedWrap(false);

2245

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);

2246

return TLO.CombineTo(Op, NewOp);

2247

}

2248

2249

LLVM_FALLTHROUGH[[gnu::fallthrough]];

2250

}

2251

default:

2252

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2253

if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,

2254

Known, TLO, Depth))

2255

return true;

2256

break;

2257

}

2258

2259

// Just use computeKnownBits to compute output bits.

2260

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2261

break;

2262

}

2263

2264

// If we know the value of all of the demanded bits, return this as a

2265

// constant.

2266

if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {

2267

// Avoid folding to a constant if any OpaqueConstant is involved.

2268

const SDNode *N = Op.getNode();

2269

for (SDNode *Op :

2270

llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {

2271

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))

2272

if (C->isOpaque())

2273

return false;

2274

}

2275

if (VT.isInteger())

2276

return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));

2277

if (VT.isFloatingPoint())

2278

return TLO.CombineTo(

2279

Op,

2280

TLO.DAG.getConstantFP(

2281

APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));

2282

}

2283

2284

return false;

2285

}

2286

2287

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,

2288

const APInt &DemandedElts,

2289

APInt &KnownUndef,

2290

APInt &KnownZero,

2291

DAGCombinerInfo &DCI) const {

2292

SelectionDAG &DAG = DCI.DAG;

2293

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

2294

!DCI.isBeforeLegalizeOps());

2295

2296

bool Simplified =

2297

SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);

2298

if (Simplified) {

2299

DCI.AddToWorklist(Op.getNode());

2300

DCI.CommitTargetLoweringOpt(TLO);

2301

}

2302

2303

return Simplified;

2304

}

2305

2306

/// Given a vector binary operation and known undefined elements for each input

2307

/// operand, compute whether each element of the output is undefined.

2308

static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,

2309

const APInt &UndefOp0,

2310

const APInt &UndefOp1) {

2311

EVT VT = BO.getValueType();

2312

assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2313, __PRETTY_FUNCTION__))

2313

"Vector binop only")((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2313, __PRETTY_FUNCTION__));

2314

2315

EVT EltVT = VT.getVectorElementType();

2316

unsigned NumElts = VT.getVectorNumElements();

2317

assert(UndefOp0.getBitWidth() == NumElts &&((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2318, __PRETTY_FUNCTION__))

2318

UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2318, __PRETTY_FUNCTION__));

2319

2320

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,

2321

const APInt &UndefVals) {

2322

if (UndefVals[Index])

2323

return DAG.getUNDEF(EltVT);

2324

2325

if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {

2326

// Try hard to make sure that the getNode() call is not creating temporary

2327

// nodes. Ignore opaque integers because they do not constant fold.

2328

SDValue Elt = BV->getOperand(Index);

2329

auto *C = dyn_cast<ConstantSDNode>(Elt);

2330

if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))

2331

return Elt;

2332

}

2333

2334

return SDValue();

2335

};

2336

2337

APInt KnownUndef = APInt::getNullValue(NumElts);

2338

for (unsigned i = 0; i != NumElts; ++i) {

2339

// If both inputs for this element are either constant or undef and match

2340

// the element type, compute the constant/undef result for this element of

2341

// the vector.

2342

// TODO: Ideally we would use FoldConstantArithmetic() here, but that does

2343

// not handle FP constants. The code within getNode() should be refactored

2344

// to avoid the danger of creating a bogus temporary node here.

2345

SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);

2346

SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);

2347

if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)

2348

if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())

2349

KnownUndef.setBit(i);

2350

}

2351

return KnownUndef;

2352

}

2353

2354

bool TargetLowering::SimplifyDemandedVectorElts(

2355

SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,

2356

APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,

2357

bool AssumeSingleUse) const {

2358

EVT VT = Op.getValueType();

2359

unsigned Opcode = Op.getOpcode();

2360

APInt DemandedElts = OriginalDemandedElts;

2361

unsigned NumElts = DemandedElts.getBitWidth();

2362

assert(VT.isVector() && "Expected vector op")((VT.isVector() && "Expected vector op") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2362, __PRETTY_FUNCTION__));

2363

2364

KnownUndef = KnownZero = APInt::getNullValue(NumElts);

2365

2366

// TODO: For now we assume we know nothing about scalable vectors.

2367

if (VT.isScalableVector())

2368

return false;

2369

2370

assert(VT.getVectorNumElements() == NumElts &&((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2371, __PRETTY_FUNCTION__))

2371

"Mask size mismatches value type element count!")((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2371, __PRETTY_FUNCTION__));

2372

2373

// Undef operand.

2374

if (Op.isUndef()) {

2375

KnownUndef.setAllBits();

2376

return false;

2377

}

2378

2379

// If Op has other users, assume that all elements are needed.

2380

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)

2381

DemandedElts.setAllBits();

2382

2383

// Not demanding any elements from Op.

2384

if (DemandedElts == 0) {

2385

KnownUndef.setAllBits();

2386

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2387

}

2388

2389

// Limit search depth.

2390

if (Depth >= SelectionDAG::MaxRecursionDepth)

2391

return false;

2392

2393

SDLoc DL(Op);

2394

unsigned EltSizeInBits = VT.getScalarSizeInBits();

2395

2396

// Helper for demanding the specified elements and all the bits of both binary

2397

// operands.

2398

auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {

2399

SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,

2400

TLO.DAG, Depth + 1);

2401

SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,

2402

TLO.DAG, Depth + 1);

2403

if (NewOp0 || NewOp1) {

2404

SDValue NewOp = TLO.DAG.getNode(

2405

Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);

2406

return TLO.CombineTo(Op, NewOp);

2407

}

2408

return false;

2409

};

2410

2411

switch (Opcode) {

2412

case ISD::SCALAR_TO_VECTOR: {

2413

if (!DemandedElts[0]) {

2414

KnownUndef.setAllBits();

2415

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2416

}

2417

KnownUndef.setHighBits(NumElts - 1);

2418

break;

2419

}

2420

case ISD::BITCAST: {

2421

SDValue Src = Op.getOperand(0);

2422

EVT SrcVT = Src.getValueType();

2423

2424

// We only handle vectors here.

2425

// TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?

2426

if (!SrcVT.isVector())

2427

break;

2428

2429

// Fast handling of 'identity' bitcasts.

2430

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2431

if (NumSrcElts == NumElts)

2432

return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,

2433

KnownZero, TLO, Depth + 1);

2434

2435

APInt SrcZero, SrcUndef;

2436

APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);

2437

2438

// Bitcast from 'large element' src vector to 'small element' vector, we

2439

// must demand a source element if any DemandedElt maps to it.

2440

if ((NumElts % NumSrcElts) == 0) {

2441

unsigned Scale = NumElts / NumSrcElts;

2442

for (unsigned i = 0; i != NumElts; ++i)

2443

if (DemandedElts[i])

2444

SrcDemandedElts.setBit(i / Scale);

2445

2446

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2447

TLO, Depth + 1))

2448

return true;

2449

2450

// Try calling SimplifyDemandedBits, converting demanded elts to the bits

2451

// of the large element.

2452

// TODO - bigendian once we have test coverage.

2453

if (TLO.DAG.getDataLayout().isLittleEndian()) {

2454

unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();

2455

APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);

2456

for (unsigned i = 0; i != NumElts; ++i)

2457

if (DemandedElts[i]) {

2458

unsigned Ofs = (i % Scale) * EltSizeInBits;

2459

SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);

2460

}

2461

2462

KnownBits Known;

2463

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,

2464

TLO, Depth + 1))

2465

return true;

2466

}

2467

2468

// If the src element is zero/undef then all the output elements will be -

2469

// only demanded elements are guaranteed to be correct.

2470

for (unsigned i = 0; i != NumSrcElts; ++i) {

2471

if (SrcDemandedElts[i]) {

2472

if (SrcZero[i])

2473

KnownZero.setBits(i * Scale, (i + 1) * Scale);

2474

if (SrcUndef[i])

2475

KnownUndef.setBits(i * Scale, (i + 1) * Scale);

2476

}

2477

}

2478

}

2479

2480

// Bitcast from 'small element' src vector to 'large element' vector, we

2481

// demand all smaller source elements covered by the larger demanded element

2482

// of this vector.

2483

if ((NumSrcElts % NumElts) == 0) {

2484

unsigned Scale = NumSrcElts / NumElts;

2485

for (unsigned i = 0; i != NumElts; ++i)

2486

if (DemandedElts[i])

2487

SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);

2488

2489

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2490

TLO, Depth + 1))

2491

return true;

2492

2493

// If all the src elements covering an output element are zero/undef, then

2494

// the output element will be as well, assuming it was demanded.

2495

for (unsigned i = 0; i != NumElts; ++i) {

2496

if (DemandedElts[i]) {

2497

if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())

2498

KnownZero.setBit(i);

2499

if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())

2500

KnownUndef.setBit(i);

2501

}

2502

}

2503

}

2504

break;

2505

}

2506

case ISD::BUILD_VECTOR: {

2507

// Check all elements and simplify any unused elements with UNDEF.

2508

if (!DemandedElts.isAllOnesValue()) {

2509

// Don't simplify BROADCASTS.

2510

if (llvm::any_of(Op->op_values(),

2511

[&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {

2512

SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());

2513

bool Updated = false;

2514

for (unsigned i = 0; i != NumElts; ++i) {

2515

if (!DemandedElts[i] && !Ops[i].isUndef()) {

2516

Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());

2517

KnownUndef.setBit(i);

2518

Updated = true;

2519

}

2520

}

2521

if (Updated)

2522

return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));

2523

}

2524

}

2525

for (unsigned i = 0; i != NumElts; ++i) {

2526

SDValue SrcOp = Op.getOperand(i);

2527

if (SrcOp.isUndef()) {

2528

KnownUndef.setBit(i);

2529

} else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&

2530

(isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {

2531

KnownZero.setBit(i);

2532

}

2533

}

2534

break;

2535

}

2536

case ISD::CONCAT_VECTORS: {

2537

EVT SubVT = Op.getOperand(0).getValueType();

2538

unsigned NumSubVecs = Op.getNumOperands();

2539

unsigned NumSubElts = SubVT.getVectorNumElements();

2540

for (unsigned i = 0; i != NumSubVecs; ++i) {

2541

SDValue SubOp = Op.getOperand(i);

2542

APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);

2543

APInt SubUndef, SubZero;

2544

if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,

2545

Depth + 1))

2546

return true;

2547

KnownUndef.insertBits(SubUndef, i * NumSubElts);

2548

KnownZero.insertBits(SubZero, i * NumSubElts);

2549

}

2550

break;

2551

}

2552

case ISD::INSERT_SUBVECTOR: {

2553

// Demand any elements from the subvector and the remainder from the src its

2554

// inserted into.

2555

SDValue Src = Op.getOperand(0);

2556

SDValue Sub = Op.getOperand(1);

2557

uint64_t Idx = Op.getConstantOperandVal(2);

2558

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

2559

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

2560

APInt DemandedSrcElts = DemandedElts;

2561

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

2562

2563

APInt SubUndef, SubZero;

2564

if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,

2565

Depth + 1))

2566

return true;

2567

2568

// If none of the src operand elements are demanded, replace it with undef.

2569

if (!DemandedSrcElts && !Src.isUndef())

2570

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,

2571

TLO.DAG.getUNDEF(VT), Sub,

2572

Op.getOperand(2)));

2573

2574

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,

2575

TLO, Depth + 1))

2576

return true;

2577

KnownUndef.insertBits(SubUndef, Idx);

2578

KnownZero.insertBits(SubZero, Idx);

2579

2580

// Attempt to avoid multi-use ops if we don't need anything from them.

2581

if (!DemandedSrcElts.isAllOnesValue() ||

2582

!DemandedSubElts.isAllOnesValue()) {

2583

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2584

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2585

SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(

2586

Sub, DemandedSubElts, TLO.DAG, Depth + 1);

2587

if (NewSrc || NewSub) {

2588

NewSrc = NewSrc ? NewSrc : Src;

2589

NewSub = NewSub ? NewSub : Sub;

2590

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2591

NewSub, Op.getOperand(2));

2592

return TLO.CombineTo(Op, NewOp);

2593

}

2594

}

2595

break;

2596

}

2597

case ISD::EXTRACT_SUBVECTOR: {

2598

// Offset the demanded elts by the subvector index.

2599

SDValue Src = Op.getOperand(0);

2600

if (Src.getValueType().isScalableVector())

2601

break;

2602

uint64_t Idx = Op.getConstantOperandVal(1);

2603

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2604

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

2605

2606

APInt SrcUndef, SrcZero;

2607

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2608

Depth + 1))

2609

return true;

2610

KnownUndef = SrcUndef.extractBits(NumElts, Idx);

2611

KnownZero = SrcZero.extractBits(NumElts, Idx);

2612

2613

// Attempt to avoid multi-use ops if we don't need anything from them.

2614

if (!DemandedElts.isAllOnesValue()) {

2615

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2616

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2617

if (NewSrc) {

2618

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2619

Op.getOperand(1));

2620

return TLO.CombineTo(Op, NewOp);

2621

}

2622

}

2623

break;

2624

}

2625

case ISD::INSERT_VECTOR_ELT: {

2626

SDValue Vec = Op.getOperand(0);

2627

SDValue Scl = Op.getOperand(1);

2628

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

2629

2630

// For a legal, constant insertion index, if we don't need this insertion

2631

// then strip it, else remove it from the demanded elts.

2632

if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {

2633

unsigned Idx = CIdx->getZExtValue();

2634

if (!DemandedElts[Idx])

2635

return TLO.CombineTo(Op, Vec);

2636

2637

APInt DemandedVecElts(DemandedElts);

2638

DemandedVecElts.clearBit(Idx);

2639

if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,

2640

KnownZero, TLO, Depth + 1))

2641

return true;

2642

2643

KnownUndef.setBitVal(Idx, Scl.isUndef());

2644

2645

KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));

2646

break;

2647

}

2648

2649

APInt VecUndef, VecZero;

2650

if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,

2651

Depth + 1))

2652

return true;

2653

// Without knowing the insertion index we can't set KnownUndef/KnownZero.

2654

break;

2655

}

2656

case ISD::VSELECT: {

2657

// Try to transform the select condition based on the current demanded

2658

// elements.

2659

// TODO: If a condition element is undef, we can choose from one arm of the

2660

// select (and if one arm is undef, then we can propagate that to the

2661

// result).

2662

// TODO - add support for constant vselect masks (see IR version of this).

2663

APInt UnusedUndef, UnusedZero;

2664

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,

2665

UnusedZero, TLO, Depth + 1))

2666

return true;

2667

2668

// See if we can simplify either vselect operand.

2669

APInt DemandedLHS(DemandedElts);

2670

APInt DemandedRHS(DemandedElts);

2671

APInt UndefLHS, ZeroLHS;

2672

APInt UndefRHS, ZeroRHS;

2673

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,

2674

ZeroLHS, TLO, Depth + 1))

2675

return true;

2676

if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,

2677

ZeroRHS, TLO, Depth + 1))

2678

return true;

2679

2680

KnownUndef = UndefLHS & UndefRHS;

2681

KnownZero = ZeroLHS & ZeroRHS;

2682

break;

2683

}

2684

case ISD::VECTOR_SHUFFLE: {

2685

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

2686

2687

// Collect demanded elements from shuffle operands..

2688

APInt DemandedLHS(NumElts, 0);

2689

APInt DemandedRHS(NumElts, 0);

2690

for (unsigned i = 0; i != NumElts; ++i) {

2691

int M = ShuffleMask[i];

2692

if (M < 0 || !DemandedElts[i])

2693

continue;

2694

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__));

2695

if (M < (int)NumElts)

2696

DemandedLHS.setBit(M);

2697

else

2698

DemandedRHS.setBit(M - NumElts);

2699

}

2700

2701

// See if we can simplify either shuffle operand.

2702

APInt UndefLHS, ZeroLHS;

2703

APInt UndefRHS, ZeroRHS;

2704

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,

2705

ZeroLHS, TLO, Depth + 1))

2706

return true;

2707

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,

2708

ZeroRHS, TLO, Depth + 1))

2709

return true;

2710

2711

// Simplify mask using undef elements from LHS/RHS.

2712

bool Updated = false;

2713

bool IdentityLHS = true, IdentityRHS = true;

2714

SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());

2715

for (unsigned i = 0; i != NumElts; ++i) {

2716

int &M = NewMask[i];

2717

if (M < 0)

2718

continue;

2719

if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||

2720

(M >= (int)NumElts && UndefRHS[M - NumElts])) {

2721

Updated = true;

2722

M = -1;

2723

}

2724

IdentityLHS &= (M < 0) || (M == (int)i);

2725

IdentityRHS &= (M < 0) || ((M - NumElts) == i);

2726

}

2727

2728

// Update legal shuffle masks based on demanded elements if it won't reduce

2729

// to Identity which can cause premature removal of the shuffle mask.

2730

if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {

2731

SDValue LegalShuffle =

2732

buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),

2733

NewMask, TLO.DAG);

2734

if (LegalShuffle)

2735

return TLO.CombineTo(Op, LegalShuffle);

2736

}

2737

2738

// Propagate undef/zero elements from LHS/RHS.

2739

for (unsigned i = 0; i != NumElts; ++i) {

2740

int M = ShuffleMask[i];

2741

if (M < 0) {

2742

KnownUndef.setBit(i);

2743

} else if (M < (int)NumElts) {

2744

if (UndefLHS[M])

2745

KnownUndef.setBit(i);

2746

if (ZeroLHS[M])

2747

KnownZero.setBit(i);

2748

} else {

2749

if (UndefRHS[M - NumElts])

2750

KnownUndef.setBit(i);

2751

if (ZeroRHS[M - NumElts])

2752

KnownZero.setBit(i);

2753

}

2754

}

2755

break;

2756

}

2757

case ISD::ANY_EXTEND_VECTOR_INREG:

2758

case ISD::SIGN_EXTEND_VECTOR_INREG:

2759

case ISD::ZERO_EXTEND_VECTOR_INREG: {

2760

APInt SrcUndef, SrcZero;

2761

SDValue Src = Op.getOperand(0);

2762

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2763

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);

2764

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2765

Depth + 1))

2766

return true;

2767

KnownZero = SrcZero.zextOrTrunc(NumElts);

2768

KnownUndef = SrcUndef.zextOrTrunc(NumElts);

2769

2770

if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&

2771

Op.getValueSizeInBits() == Src.getValueSizeInBits() &&

2772

DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {

2773

// aext - if we just need the bottom element then we can bitcast.

2774

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

2775

}

2776

2777

if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {

2778

// zext(undef) upper bits are guaranteed to be zero.

2779

if (DemandedElts.isSubsetOf(KnownUndef))

2780

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2781

KnownUndef.clearAllBits();

2782

}

2783

break;

2784

}

2785

2786

// TODO: There are more binop opcodes that could be handled here - MIN,

2787

// MAX, saturated math, etc.

2788

case ISD::OR:

2789

case ISD::XOR:

2790

case ISD::ADD:

2791

case ISD::SUB:

2792

case ISD::FADD:

2793

case ISD::FSUB:

2794

case ISD::FMUL:

2795

case ISD::FDIV:

2796

case ISD::FREM: {

2797

SDValue Op0 = Op.getOperand(0);

2798

SDValue Op1 = Op.getOperand(1);

2799

2800

APInt UndefRHS, ZeroRHS;

2801

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2802

Depth + 1))

2803

return true;

2804

APInt UndefLHS, ZeroLHS;

2805

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2806

Depth + 1))

2807

return true;

2808

2809

KnownZero = ZeroLHS & ZeroRHS;

2810

KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

2811

2812

// Attempt to avoid multi-use ops if we don't need anything from them.

2813

// TODO - use KnownUndef to relax the demandedelts?

2814

if (!DemandedElts.isAllOnesValue())

2815

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2816

return true;

2817

break;

2818

}

2819

case ISD::SHL:

2820

case ISD::SRL:

2821

case ISD::SRA:

2822

case ISD::ROTL:

2823

case ISD::ROTR: {

2824

SDValue Op0 = Op.getOperand(0);

2825

SDValue Op1 = Op.getOperand(1);

2826

2827

APInt UndefRHS, ZeroRHS;

2828

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2829

Depth + 1))

2830

return true;

2831

APInt UndefLHS, ZeroLHS;

2832

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2833

Depth + 1))

2834

return true;

2835

2836

KnownZero = ZeroLHS;

2837

KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

2838

2839

// Attempt to avoid multi-use ops if we don't need anything from them.

2840

// TODO - use KnownUndef to relax the demandedelts?

2841

if (!DemandedElts.isAllOnesValue())

2842

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2843

return true;

2844

break;

2845

}

2846

case ISD::MUL:

2847

case ISD::AND: {

2848

SDValue Op0 = Op.getOperand(0);

2849

SDValue Op1 = Op.getOperand(1);

2850

2851

APInt SrcUndef, SrcZero;

2852

if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,

2853

Depth + 1))

2854

return true;

2855

if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,

2856

TLO, Depth + 1))

2857

return true;

2858

2859

// If either side has a zero element, then the result element is zero, even

2860

// if the other is an UNDEF.

2861

// TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros

2862

// and then handle 'and' nodes with the rest of the binop opcodes.

2863

KnownZero |= SrcZero;

2864

KnownUndef &= SrcUndef;

2865

KnownUndef &= ~KnownZero;

2866

2867

// Attempt to avoid multi-use ops if we don't need anything from them.

2868

// TODO - use KnownUndef to relax the demandedelts?

2869

if (!DemandedElts.isAllOnesValue())

2870

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2871

return true;

2872

break;

2873

}

2874

case ISD::TRUNCATE:

2875

case ISD::SIGN_EXTEND:

2876

case ISD::ZERO_EXTEND:

2877

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,

2878

KnownZero, TLO, Depth + 1))

2879

return true;

2880

2881

if (Op.getOpcode() == ISD::ZERO_EXTEND) {

2882

// zext(undef) upper bits are guaranteed to be zero.

2883

if (DemandedElts.isSubsetOf(KnownUndef))

2884

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2885

KnownUndef.clearAllBits();

2886

}

2887

break;

2888

default: {

2889

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2890

if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,

2891

KnownZero, TLO, Depth))

2892

return true;

2893

} else {

2894

KnownBits Known;

2895

APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);

2896

if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,

2897

TLO, Depth, AssumeSingleUse))

2898

return true;

2899

}

2900

break;

2901

}

2902

}

2903

assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero"
) ? static_cast<void> (0) : __assert_fail ("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2903, __PRETTY_FUNCTION__));

2904

2905

// Constant fold all undef cases.

2906

// TODO: Handle zero cases as well.

2907

if (DemandedElts.isSubsetOf(KnownUndef))

2908

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2909

2910

return false;

2911

}

2912

2913

/// Determine which of the bits specified in Mask are known to be either zero or

2914

/// one and return them in the Known.

2915

void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,

2916

KnownBits &Known,

2917

const APInt &DemandedElts,

2918

const SelectionDAG &DAG,

2919

unsigned Depth) const {

2920

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__))

2921

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__))

2922

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__))

2923

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__))

2924

"Should use MaskedValueIsZero if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__))

2925

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2925, __PRETTY_FUNCTION__));

2926

Known.resetAll();

2927

}

2928

2929

void TargetLowering::computeKnownBitsForTargetInstr(

2930

GISelKnownBits &Analysis, Register R, KnownBits &Known,

2931

const APInt &DemandedElts, const MachineRegisterInfo &MRI,

2932

unsigned Depth) const {

2933

Known.resetAll();

2934

}

2935

2936

void TargetLowering::computeKnownBitsForFrameIndex(

2937

const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {

2938

// The low bits are known zero if the pointer is aligned.

2939

Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));

2940

}

2941

2942

Align TargetLowering::computeKnownAlignForTargetInstr(

2943

GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,

2944

unsigned Depth) const {

2945

return Align(1);

2946

}

2947

2948

/// This method can be implemented by targets that want to expose additional

2949

/// information about sign bits to the DAG Combiner.

2950

unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,

2951

const APInt &,

2952

const SelectionDAG &,

2953

unsigned Depth) const {

2954

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__))

2955

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__))

2956

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__))

2957

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__))

2958

"Should use ComputeNumSignBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__))

2959

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2959, __PRETTY_FUNCTION__));

2960

return 1;

2961

}

2962

2963

unsigned TargetLowering::computeNumSignBitsForTargetInstr(

2964

GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,

2965

const MachineRegisterInfo &MRI, unsigned Depth) const {

2966

return 1;

2967

}

2968

2969

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(

2970

SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,

2971

TargetLoweringOpt &TLO, unsigned Depth) const {

2972

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__))

2973

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__))

2974

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__))

2975

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__))

2976

"Should use SimplifyDemandedVectorElts if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__))

2977

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2977, __PRETTY_FUNCTION__));

2978

return false;

2979

}

2980

2981

bool TargetLowering::SimplifyDemandedBitsForTargetNode(

2982

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

2983

KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {

2984

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__))

2985

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__))

2986

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__))

2987

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__))

2988

"Should use SimplifyDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__))

2989

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __PRETTY_FUNCTION__));

2990

computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);

2991

return false;

2992

}

2993

2994

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(

2995

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

2996

SelectionDAG &DAG, unsigned Depth) const {

2997

assert((((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

2998

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

2999

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

3000

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

3001

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

3002

"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__))

3003

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3003, __PRETTY_FUNCTION__));

3004

return SDValue();

3005

}

3006

3007

SDValue

3008

TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,

3009

SDValue N1, MutableArrayRef<int> Mask,

3010

SelectionDAG &DAG) const {

3011

bool LegalMask = isShuffleMaskLegal(Mask, VT);

3012

if (!LegalMask) {

3013

std::swap(N0, N1);

3014

ShuffleVectorSDNode::commuteMask(Mask);

3015

LegalMask = isShuffleMaskLegal(Mask, VT);

3016

}

3017

3018

if (!LegalMask)

3019

return SDValue();

3020

3021

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);

3022

}

3023

3024

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {

3025

return nullptr;

3026

}

3027

3028

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,

3029

const SelectionDAG &DAG,

3030

bool SNaN,

3031

unsigned Depth) const {

3032

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__))

3033

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__))

3034

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__))

3035

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__))

3036

"Should use isKnownNeverNaN if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__))

3037

" is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3037, __PRETTY_FUNCTION__));

3038

return false;

3039

}

3040

3041

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must

3042

// work with truncating build vectors and vectors with elements of less than

3043

// 8 bits.

3044

bool TargetLowering::isConstTrueVal(const SDNode *N) const {

3045

if (!N)

3046

return false;

3047

3048

APInt CVal;

3049

if (auto *CN = dyn_cast<ConstantSDNode>(N)) {

3050

CVal = CN->getAPIntValue();

3051

} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {

3052

auto *CN = BV->getConstantSplatNode();

3053

if (!CN)

3054

return false;

3055

3056

// If this is a truncating build vector, truncate the splat value.

3057

// Otherwise, we may fail to match the expected values below.

3058

unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();

3059

CVal = CN->getAPIntValue();

3060

if (BVEltWidth < CVal.getBitWidth())

3061

CVal = CVal.trunc(BVEltWidth);

3062

} else {

3063

return false;

3064

}

3065

3066

switch (getBooleanContents(N->getValueType(0))) {

3067

case UndefinedBooleanContent:

3068

return CVal[0];

3069

case ZeroOrOneBooleanContent:

3070

return CVal.isOneValue();

3071

case ZeroOrNegativeOneBooleanContent:

3072

return CVal.isAllOnesValue();

3073

}

3074

3075

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3075);

3076

}

3077

3078

bool TargetLowering::isConstFalseVal(const SDNode *N) const {

3079

if (!N)

3080

return false;

3081

3082

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);

3083

if (!CN) {

3084

const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);

3085

if (!BV)

3086

return false;

3087

3088

// Only interested in constant splats, we don't care about undef

3089

// elements in identifying boolean constants and getConstantSplatNode

3090

// returns NULL if all ops are undef;

3091

CN = BV->getConstantSplatNode();

3092

if (!CN)

3093

return false;

3094

}

3095

3096

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)

3097

return !CN->getAPIntValue()[0];

3098

3099

return CN->isNullValue();

3100

}

3101

3102

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,

3103

bool SExt) const {

3104

if (VT == MVT::i1)

3105

return N->isOne();

3106

3107

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);

3108

switch (Cnt) {

3109

case TargetLowering::ZeroOrOneBooleanContent:

3110

// An extended value of 1 is always true, unless its original type is i1,

3111

// in which case it will be sign extended to -1.

3112

return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));

3113

case TargetLowering::UndefinedBooleanContent:

3114

case TargetLowering::ZeroOrNegativeOneBooleanContent:

3115

return N->isAllOnesValue() && SExt;

3116

}

3117

llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3117);

3118

}

3119

3120

/// This helper function of SimplifySetCC tries to optimize the comparison when

3121

/// either operand of the SetCC node is a bitwise-and instruction.

3122

SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,

3123

ISD::CondCode Cond, const SDLoc &DL,

3124

DAGCombinerInfo &DCI) const {

3125

// Match these patterns in any of their permutations:

3126

// (X & Y) == Y

3127

// (X & Y) != Y

3128

if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)

3129

std::swap(N0, N1);

3130

3131

EVT OpVT = N0.getValueType();

3132

if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||

3133

(Cond != ISD::SETEQ && Cond != ISD::SETNE))

3134

return SDValue();

3135

3136

SDValue X, Y;

3137

if (N0.getOperand(0) == N1) {

3138

X = N0.getOperand(1);

3139

Y = N0.getOperand(0);

3140

} else if (N0.getOperand(1) == N1) {

3141

X = N0.getOperand(0);

3142

Y = N0.getOperand(1);

3143

} else {

3144

return SDValue();

3145

}

3146

3147

SelectionDAG &DAG = DCI.DAG;

3148

SDValue Zero = DAG.getConstant(0, DL, OpVT);

3149

if (DAG.isKnownToBeAPowerOfTwo(Y)) {

3150

// Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.

3151

// Note that where Y is variable and is known to have at most one bit set

3152

// (for example, if it is Z & 1) we cannot do this; the expressions are not

3153

// equivalent when Y == 0.

3154

assert(OpVT.isInteger())((OpVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3154, __PRETTY_FUNCTION__));

3155

Cond = ISD::getSetCCInverse(Cond, OpVT);

3156

if (DCI.isBeforeLegalizeOps() ||

3157

isCondCodeLegal(Cond, N0.getSimpleValueType()))

3158

return DAG.getSetCC(DL, VT, N0, Zero, Cond);

3159

} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {

3160

// If the target supports an 'and-not' or 'and-complement' logic operation,

3161

// try to use that to make a comparison operation more efficient.

3162

// But don't do this transform if the mask is a single bit because there are

3163

// more efficient ways to deal with that case (for example, 'bt' on x86 or

3164

// 'rlwinm' on PPC).

3165

3166

// Bail out if the compare operand that we want to turn into a zero is

3167

// already a zero (otherwise, infinite loop).

3168

auto *YConst = dyn_cast<ConstantSDNode>(Y);

3169

if (YConst && YConst->isNullValue())

3170

return SDValue();

3171

3172

// Transform this into: ~X & Y == 0.

3173

SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);

3174

SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);

3175

return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);

3176

}

3177

3178

return SDValue();

3179

}

3180

3181

/// There are multiple IR patterns that could be checking whether certain

3182

/// truncation of a signed number would be lossy or not. The pattern which is

3183

/// best at IR level, may not lower optimally. Thus, we want to unfold it.

3184

/// We are looking for the following pattern: (KeptBits is a constant)

3185

/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)

3186

/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.

3187

/// KeptBits also can't be 1, that would have been folded to %x dstcond 0

3188

/// We will unfold it into the natural trunc+sext pattern:

3189

/// ((%x << C) a>> C) dstcond %x

3190

/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)

3191

SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(

3192

EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,

3193

const SDLoc &DL) const {

3194

// We must be comparing with a constant.

3195

ConstantSDNode *C1;

3196

if (!(C1 = dyn_cast<ConstantSDNode>(N1)))

3197

return SDValue();

3198

3199

// N0 should be: add %x, (1 << (KeptBits-1))

3200

if (N0->getOpcode() != ISD::ADD)

3201

return SDValue();

3202

3203

// And we must be 'add'ing a constant.

3204

ConstantSDNode *C01;

3205

if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))

3206

return SDValue();

3207

3208

SDValue X = N0->getOperand(0);

3209

EVT XVT = X.getValueType();

3210

3211

// Validate constants ...

3212

3213

APInt I1 = C1->getAPIntValue();

3214

3215

ISD::CondCode NewCond;

3216

if (Cond == ISD::CondCode::SETULT) {

3217

NewCond = ISD::CondCode::SETEQ;

3218

} else if (Cond == ISD::CondCode::SETULE) {

3219

NewCond = ISD::CondCode::SETEQ;

3220

// But need to 'canonicalize' the constant.

3221

I1 += 1;

3222

} else if (Cond == ISD::CondCode::SETUGT) {

3223

NewCond = ISD::CondCode::SETNE;

3224

// But need to 'canonicalize' the constant.

3225

I1 += 1;

3226

} else if (Cond == ISD::CondCode::SETUGE) {

3227

NewCond = ISD::CondCode::SETNE;

3228

} else

3229

return SDValue();

3230

3231

APInt I01 = C01->getAPIntValue();

3232

3233

auto checkConstants = [&I1, &I01]() -> bool {

3234

// Both of them must be power-of-two, and the constant from setcc is bigger.

3235

return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();

3236

};

3237

3238

if (checkConstants()) {

3239

// Great, e.g. got icmp ult i16 (add i16 %x, 128), 256

3240

} else {

3241

// What if we invert constants? (and the target predicate)

3242

I1.negate();

3243

I01.negate();

3244

assert(XVT.isInteger())((XVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3244, __PRETTY_FUNCTION__));

3245

NewCond = getSetCCInverse(NewCond, XVT);

3246

if (!checkConstants())

3247

return SDValue();

3248

// Great, e.g. got icmp uge i16 (add i16 %x, -128), -256

3249

}

3250

3251

// They are power-of-two, so which bit is set?

3252

const unsigned KeptBits = I1.logBase2();

3253

const unsigned KeptBitsMinusOne = I01.logBase2();

3254

3255

// Magic!

3256

if (KeptBits != (KeptBitsMinusOne + 1))

3257

return SDValue();

3258

assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((KeptBits > 0 && KeptBits < XVT.getSizeInBits(
) && "unreachable") ? static_cast<void> (0) : __assert_fail
("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3258, __PRETTY_FUNCTION__));

3259

3260

// We don't want to do this in every single case.

3261

SelectionDAG &DAG = DCI.DAG;

3262

if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(

3263

XVT, KeptBits))

3264

return SDValue();

3265

3266

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;

3267

assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((MaskedBits > 0 && MaskedBits < XVT.getSizeInBits
() && "unreachable") ? static_cast<void> (0) : __assert_fail
("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3267, __PRETTY_FUNCTION__));

3268

3269

// Unfold into: ((%x << C) a>> C) cond %x

3270

// Where 'cond' will be either 'eq' or 'ne'.

3271

SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);

3272

SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);

3273

SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);

3274

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

3275

3276

return T2;

3277

}

3278

3279

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3280

SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(

3281

EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,

3282

DAGCombinerInfo &DCI, const SDLoc &DL) const {

3283

assert(isConstOrConstSplat(N1C) &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3285, __PRETTY_FUNCTION__))

3284

isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3285, __PRETTY_FUNCTION__))

3285

"Should be a comparison with 0.")((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3285, __PRETTY_FUNCTION__));

3286

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3287, __PRETTY_FUNCTION__))

3287

"Valid only for [in]equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3287, __PRETTY_FUNCTION__));

3288

3289

unsigned NewShiftOpcode;

3290

SDValue X, C, Y;

3291

3292

SelectionDAG &DAG = DCI.DAG;

3293

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3294

3295

// Look for '(C l>>/<< Y)'.

3296

auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {

3297

// The shift should be one-use.

3298

if (!V.hasOneUse())

3299

return false;

3300

unsigned OldShiftOpcode = V.getOpcode();

3301

switch (OldShiftOpcode) {

3302

case ISD::SHL:

3303

NewShiftOpcode = ISD::SRL;

3304

break;

3305

case ISD::SRL:

3306

NewShiftOpcode = ISD::SHL;

3307

break;

3308

default:

3309

return false; // must be a logical shift.

3310

}

3311

// We should be shifting a constant.

3312

// FIXME: best to use isConstantOrConstantVector().

3313

C = V.getOperand(0);

3314

ConstantSDNode *CC =

3315

isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3316

if (!CC)

3317

return false;

3318

Y = V.getOperand(1);

3319

3320

ConstantSDNode *XC =

3321

isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3322

return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(

3323

X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);

3324

};

3325

3326

// LHS of comparison should be an one-use 'and'.

3327

if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())

3328

return SDValue();

3329

3330

X = N0.getOperand(0);

3331

SDValue Mask = N0.getOperand(1);

3332

3333

// 'and' is commutative!

3334

if (!Match(Mask)) {

3335

std::swap(X, Mask);

3336

if (!Match(Mask))

3337

return SDValue();

3338

}

3339

3340

EVT VT = X.getValueType();

3341

3342

// Produce:

3343

// ((X 'OppositeShiftOpcode' Y) & C) Cond 0

3344

SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);

3345

SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);

3346

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);

3347

return T2;

3348

}

3349

3350

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as

3351

/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to

3352

/// handle the commuted versions of these patterns.

3353

SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,

3354

ISD::CondCode Cond, const SDLoc &DL,

3355

DAGCombinerInfo &DCI) const {

3356

unsigned BOpcode = N0.getOpcode();

3357

assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3358, __PRETTY_FUNCTION__))

3358

"Unexpected binop")(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3358, __PRETTY_FUNCTION__));

3359

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3359, __PRETTY_FUNCTION__));

3360

3361

// (X + Y) == X --> Y == 0

3362

// (X - Y) == X --> Y == 0

3363

// (X ^ Y) == X --> Y == 0

3364

SelectionDAG &DAG = DCI.DAG;

3365

EVT OpVT = N0.getValueType();

3366

SDValue X = N0.getOperand(0);

3367

SDValue Y = N0.getOperand(1);

3368

if (X == N1)

3369

return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

3370

3371

if (Y != N1)

3372

return SDValue();

3373

3374

// (X + Y) == Y --> X == 0

3375

// (X ^ Y) == Y --> X == 0

3376

if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)

3377

return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

3378

3379

// The shift would not be valid if the operands are boolean (i1).

3380

if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)

3381

return SDValue();

3382

3383

// (X - Y) == Y --> X == Y << 1

3384

EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),

3385

!DCI.isBeforeLegalize());

3386

SDValue One = DAG.getConstant(1, DL, ShiftVT);

3387

SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);

3388

if (!DCI.isCalledByLegalizer())

3389

DCI.AddToWorklist(YShl1.getNode());

3390

return DAG.getSetCC(DL, VT, X, YShl1, Cond);

3391

}

3392

3393

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,

3394

SDValue N0, const APInt &C1,

3395

ISD::CondCode Cond, const SDLoc &dl,

3396

SelectionDAG &DAG) {

3397

// Look through truncs that don't change the value of a ctpop.

3398

// FIXME: Add vector support? Need to be careful with setcc result type below.

3399

SDValue CTPOP = N0;

3400

if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&

3401

N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))

3402

CTPOP = N0.getOperand(0);

3403

3404

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())

3405

return SDValue();

3406

3407

EVT CTVT = CTPOP.getValueType();

3408

SDValue CTOp = CTPOP.getOperand(0);

3409

3410

// If this is a vector CTPOP, keep the CTPOP if it is legal.

3411

// TODO: Should we check if CTPOP is legal(or custom) for scalars?

3412

if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))

3413

return SDValue();

3414

3415

// (ctpop x) u< 2 -> (x & x-1) == 0

3416

// (ctpop x) u> 1 -> (x & x-1) != 0

3417

if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {

3418

unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);

3419

if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))

3420

return SDValue();

3421

if (C1 == 0 && (Cond == ISD::SETULT))

3422

return SDValue(); // This is handled elsewhere.

3423

3424

unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

3425

3426

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3427

SDValue Result = CTOp;

3428

for (unsigned i = 0; i < Passes; i++) {

3429

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);

3430

Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);

3431

}

3432

ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;

3433

return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);

3434

}

3435

3436

// If ctpop is not supported, expand a power-of-2 comparison based on it.

3437

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {

3438

// For scalars, keep CTPOP if it is legal or custom.

3439

if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))

3440

return SDValue();

3441

// This is based on X86's custom lowering for CTPOP which produces more

3442

// instructions than the expansion here.

3443

3444

// (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)

3445

// (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)

3446

SDValue Zero = DAG.getConstant(0, dl, CTVT);

3447

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3448

assert(CTVT.isInteger())((CTVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3448, __PRETTY_FUNCTION__));

3449

ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);

3450

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

3451

SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

3452

SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);

3453

SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);

3454

unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;

3455

return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);

3456

}

3457

3458

return SDValue();

3459

}

3460

3461

/// Try to simplify a setcc built with the specified operands and cc. If it is

3462

/// unable to simplify it, return a null SDValue.

3463

SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,

3464

ISD::CondCode Cond, bool foldBooleans,

3465

DAGCombinerInfo &DCI,

3466

const SDLoc &dl) const {

3467

SelectionDAG &DAG = DCI.DAG;

3468

const DataLayout &Layout = DAG.getDataLayout();

3469

EVT OpVT = N0.getValueType();

3470

3471

// Constant fold or commute setcc.

3472

if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))

3473

return Fold;

3474

3475

// Ensure that the constant occurs on the RHS and fold constant comparisons.

3476

// TODO: Handle non-splat vector constants. All undef causes trouble.

3477

// FIXME: We can't yet fold constant scalable vector splats, so avoid an

3478

// infinite loop here when we encounter one.

3479

ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);

3480

if (isConstOrConstSplat(N0) &&

3481

(!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&

3482

(DCI.isBeforeLegalizeOps() ||

3483

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))

3484

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3485

3486

// If we have a subtract with the same 2 non-constant operands as this setcc

3487

// -- but in reverse order -- then try to commute the operands of this setcc

3488

// to match. A matching pair of setcc (cmp) and sub may be combined into 1

3489

// instruction on some targets.

3490

if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&

3491

(DCI.isBeforeLegalizeOps() ||

3492

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&

3493

DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&

3494

!DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))

3495

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3496

3497

if (auto *N1C = isConstOrConstSplat(N1)) {

3498

const APInt &C1 = N1C->getAPIntValue();

3499

3500

// Optimize some CTPOP cases.

3501

if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))

3502

return V;

3503

3504

// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an

3505

// equality comparison, then we're just comparing whether X itself is

3506

// zero.

3507

if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&

3508

N0.getOperand(0).getOpcode() == ISD::CTLZ &&

3509

isPowerOf2_32(N0.getScalarValueSizeInBits())) {

3510

if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {

3511

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3512

ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {

3513

if ((C1 == 0) == (Cond == ISD::SETEQ)) {

3514

// (srl (ctlz x), 5) == 0 -> X != 0

3515

// (srl (ctlz x), 5) != 1 -> X != 0

3516

Cond = ISD::SETNE;

3517

} else {

3518

// (srl (ctlz x), 5) != 0 -> X == 0

3519

// (srl (ctlz x), 5) == 1 -> X == 0

3520

Cond = ISD::SETEQ;

3521

}

3522

SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());

3523

return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,

3524

Cond);

3525

}

3526

}

3527

}

3528

}

3529

3530

// FIXME: Support vectors.

3531

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

3532

const APInt &C1 = N1C->getAPIntValue();

3533

3534

// (zext x) == C --> x == (trunc C)

3535

// (sext x) == C --> x == (trunc C)

3536

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3537

DCI.isBeforeLegalize() && N0->hasOneUse()) {

3538

unsigned MinBits = N0.getValueSizeInBits();

3539

SDValue PreExt;

3540

bool Signed = false;

3541

if (N0->getOpcode() == ISD::ZERO_EXTEND) {

3542

// ZExt

3543

MinBits = N0->getOperand(0).getValueSizeInBits();

3544

PreExt = N0->getOperand(0);

3545

} else if (N0->getOpcode() == ISD::AND) {

3546

// DAGCombine turns costly ZExts into ANDs

3547

if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))

3548

if ((C->getAPIntValue()+1).isPowerOf2()) {

3549

MinBits = C->getAPIntValue().countTrailingOnes();

3550

PreExt = N0->getOperand(0);

3551

}

3552

} else if (N0->getOpcode() == ISD::SIGN_EXTEND) {

3553

// SExt

3554

MinBits = N0->getOperand(0).getValueSizeInBits();

3555

PreExt = N0->getOperand(0);

3556

Signed = true;

3557

} else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {

3558

// ZEXTLOAD / SEXTLOAD

3559

if (LN0->getExtensionType() == ISD::ZEXTLOAD) {

3560

MinBits = LN0->getMemoryVT().getSizeInBits();

3561

PreExt = N0;

3562

} else if (LN0->getExtensionType() == ISD::SEXTLOAD) {

3563

Signed = true;

3564

MinBits = LN0->getMemoryVT().getSizeInBits();

3565

PreExt = N0;

3566

}

3567

}

3568

3569

// Figure out how many bits we need to preserve this constant.

3570

unsigned ReqdBits = Signed ?

3571

C1.getBitWidth() - C1.getNumSignBits() + 1 :

3572

C1.getActiveBits();

3573

3574

// Make sure we're not losing bits from the constant.

3575

if (MinBits > 0 &&

3576

MinBits < C1.getBitWidth() &&

3577

MinBits >= ReqdBits) {

3578

EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);

3579

if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {

3580

// Will get folded away.

3581

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);

3582

if (MinBits == 1 && C1 == 1)

3583

// Invert the condition.

3584

return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),

3585

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3586

SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);

3587

return DAG.getSetCC(dl, VT, Trunc, C, Cond);

3588

}

3589

3590

// If truncating the setcc operands is not desirable, we can still

3591

// simplify the expression in some cases:

3592

// setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)

3593

// setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))

3594

// setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))

3595

// setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)

3596

// setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))

3597

// setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)

3598

SDValue TopSetCC = N0->getOperand(0);

3599

unsigned N0Opc = N0->getOpcode();

3600

bool SExt = (N0Opc == ISD::SIGN_EXTEND);

3601

if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&

3602

TopSetCC.getOpcode() == ISD::SETCC &&

3603

(N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&

3604

(isConstFalseVal(N1C) ||

3605

isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

3606

3607

bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||

3608

(!N1C->isNullValue() && Cond == ISD::SETNE);

3609

3610

if (!Inverse)

3611

return TopSetCC;

3612

3613

ISD::CondCode InvCond = ISD::getSetCCInverse(

3614

cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),

3615

TopSetCC.getOperand(0).getValueType());

3616

return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),

3617

TopSetCC.getOperand(1),

3618

InvCond);

3619

}

3620

}

3621

}

3622

3623

// If the LHS is '(and load, const)', the RHS is 0, the test is for

3624

// equality or unsigned, and all 1 bits of the const are in the same

3625

// partial word, see if we can shorten the load.

3626

if (DCI.isBeforeLegalize() &&

3627

!ISD::isSignedIntSetCC(Cond) &&

3628

N0.getOpcode() == ISD::AND && C1 == 0 &&

3629

N0.getNode()->hasOneUse() &&

3630

isa<LoadSDNode>(N0.getOperand(0)) &&

3631

N0.getOperand(0).getNode()->hasOneUse() &&

3632

isa<ConstantSDNode>(N0.getOperand(1))) {

3633

LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));

3634

APInt bestMask;

3635

unsigned bestWidth = 0, bestOffset = 0;

3636

if (Lod->isSimple() && Lod->isUnindexed()) {

3637

unsigned origWidth = N0.getValueSizeInBits();

3638

unsigned maskWidth = origWidth;

3639

// We can narrow (e.g.) 16-bit extending loads on 32-bit target to

3640

// 8 bits, but have to be careful...

3641

if (Lod->getExtensionType() != ISD::NON_EXTLOAD)

3642

origWidth = Lod->getMemoryVT().getSizeInBits();

3643

const APInt &Mask = N0.getConstantOperandAPInt(1);

3644

for (unsigned width = origWidth / 2; width>=8; width /= 2) {

3645

APInt newMask = APInt::getLowBitsSet(maskWidth, width);

3646

for (unsigned offset=0; offset<origWidth/width; offset++) {

3647

if (Mask.isSubsetOf(newMask)) {

3648

if (Layout.isLittleEndian())

3649

bestOffset = (uint64_t)offset * (width/8);

3650

else

3651

bestOffset = (origWidth/width - offset - 1) * (width/8);

3652

bestMask = Mask.lshr(offset * (width/8) * 8);

3653

bestWidth = width;

3654

break;

3655

}

3656

newMask <<= width;

3657

}

3658

}

3659

}

3660

if (bestWidth) {

3661

EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);

3662

if (newVT.isRound() &&

3663

shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {

3664

SDValue Ptr = Lod->getBasePtr();

3665

if (bestOffset != 0)

3666

Ptr =

3667

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);

3668

SDValue NewLoad =

3669

DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,

3670

Lod->getPointerInfo().getWithOffset(bestOffset),

3671

Lod->getOriginalAlign());

3672

return DAG.getSetCC(dl, VT,

3673

DAG.getNode(ISD::AND, dl, newVT, NewLoad,

3674

DAG.getConstant(bestMask.trunc(bestWidth),

3675

dl, newVT)),

3676

DAG.getConstant(0LL, dl, newVT), Cond);

3677

}

3678

}

3679

}

3680

3681

// If the LHS is a ZERO_EXTEND, perform the comparison on the input.

3682

if (N0.getOpcode() == ISD::ZERO_EXTEND) {

3683

unsigned InSize = N0.getOperand(0).getValueSizeInBits();

3684

3685

// If the comparison constant has bits in the upper part, the

3686

// zero-extended value could never match.

3687

if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),

3688

C1.getBitWidth() - InSize))) {

3689

switch (Cond) {

3690

case ISD::SETUGT:

3691

case ISD::SETUGE:

3692

case ISD::SETEQ:

3693

return DAG.getConstant(0, dl, VT);

3694

case ISD::SETULT:

3695

case ISD::SETULE:

3696

case ISD::SETNE:

3697

return DAG.getConstant(1, dl, VT);

3698

case ISD::SETGT:

3699

case ISD::SETGE:

3700

// True if the sign bit of C1 is set.

3701

return DAG.getConstant(C1.isNegative(), dl, VT);

3702

case ISD::SETLT:

3703

case ISD::SETLE:

3704

// True if the sign bit of C1 isn't set.

3705

return DAG.getConstant(C1.isNonNegative(), dl, VT);

3706

default:

3707

break;

3708

}

3709

}

3710

3711

// Otherwise, we can perform the comparison with the low bits.

3712

switch (Cond) {

3713

case ISD::SETEQ:

3714

case ISD::SETNE:

3715

case ISD::SETUGT:

3716

case ISD::SETUGE:

3717

case ISD::SETULT:

3718

case ISD::SETULE: {

3719

EVT newVT = N0.getOperand(0).getValueType();

3720

if (DCI.isBeforeLegalizeOps() ||

3721

(isOperationLegal(ISD::SETCC, newVT) &&

3722

isCondCodeLegal(Cond, newVT.getSimpleVT()))) {

3723

EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);

3724

SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

3725

3726

SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),

3727

NewConst, Cond);

3728

return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());

3729

}

3730

break;

3731

}

3732

default:

3733

break; // todo, be more careful with signed comparisons

3734

}

3735

} else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&

3736

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3737

!isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),

3738

OpVT)) {

3739

EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();

3740

unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();

3741

EVT ExtDstTy = N0.getValueType();

3742

unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

3743

3744

// If the constant doesn't fit into the number of bits for the source of

3745

// the sign extension, it is impossible for both sides to be equal.

3746

if (C1.getMinSignedBits() > ExtSrcTyBits)

3747

return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

3748

3749

assert(ExtDstTy == N0.getOperand(0).getValueType() &&((ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy
!= ExtSrcTy && "Unexpected types!") ? static_cast<
void> (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3750, __PRETTY_FUNCTION__))

3750

ExtDstTy != ExtSrcTy && "Unexpected types!")((ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy
!= ExtSrcTy && "Unexpected types!") ? static_cast<
void> (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3750, __PRETTY_FUNCTION__));

3751

APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);

3752

SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),

3753

DAG.getConstant(Imm, dl, ExtDstTy));

3754

if (!DCI.isCalledByLegalizer())

3755

DCI.AddToWorklist(ZextOp.getNode());

3756

// Otherwise, make this a use of a zext.

3757

return DAG.getSetCC(dl, VT, ZextOp,

3758

DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);

3759

} else if ((N1C->isNullValue() || N1C->isOne()) &&

3760

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3761

// SETCC (SETCC), [0|1], [EQ|NE] -> SETCC

3762

if (N0.getOpcode() == ISD::SETCC &&

3763

isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&

3764

(N0.getValueType() == MVT::i1 ||

3765

getBooleanContents(N0.getOperand(0).getValueType()) ==

3766

ZeroOrOneBooleanContent)) {

3767

bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());

3768

if (TrueWhenTrue)

3769

return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);

3770

// Invert the condition.

3771

ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();

3772

CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());

3773

if (DCI.isBeforeLegalizeOps() ||

3774

isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))

3775

return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);

3776

}

3777

3778

if ((N0.getOpcode() == ISD::XOR ||

3779

(N0.getOpcode() == ISD::AND &&

3780

N0.getOperand(0).getOpcode() == ISD::XOR &&

3781

N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&

3782

isOneConstant(N0.getOperand(1))) {

3783

// If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We

3784

// can only do this if the top bits are known zero.

3785

unsigned BitWidth = N0.getValueSizeInBits();

3786

if (DAG.MaskedValueIsZero(N0,

3787

APInt::getHighBitsSet(BitWidth,

3788

BitWidth-1))) {

3789

// Okay, get the un-inverted input value.

3790

SDValue Val;

3791

if (N0.getOpcode() == ISD::XOR) {

3792

Val = N0.getOperand(0);

3793

} else {

3794

assert(N0.getOpcode() == ISD::AND &&((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3795, __PRETTY_FUNCTION__))

3795

N0.getOperand(0).getOpcode() == ISD::XOR)((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3795, __PRETTY_FUNCTION__));

3796

// ((X^1)&1)^1 -> X & 1

3797

Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),

3798

N0.getOperand(0).getOperand(0),

3799

N0.getOperand(1));

3800

}

3801

3802

return DAG.getSetCC(dl, VT, Val, N1,

3803

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3804

}

3805

} else if (N1C->isOne()) {

3806

SDValue Op0 = N0;

3807

if (Op0.getOpcode() == ISD::TRUNCATE)

3808

Op0 = Op0.getOperand(0);

3809

3810

if ((Op0.getOpcode() == ISD::XOR) &&

3811

Op0.getOperand(0).getOpcode() == ISD::SETCC &&

3812

Op0.getOperand(1).getOpcode() == ISD::SETCC) {

3813

SDValue XorLHS = Op0.getOperand(0);

3814

SDValue XorRHS = Op0.getOperand(1);

3815

// Ensure that the input setccs return an i1 type or 0/1 value.

3816

if (Op0.getValueType() == MVT::i1 ||

3817

(getBooleanContents(XorLHS.getOperand(0).getValueType()) ==

3818

ZeroOrOneBooleanContent &&

3819

getBooleanContents(XorRHS.getOperand(0).getValueType()) ==

3820

ZeroOrOneBooleanContent)) {

3821

// (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)

3822

Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;

3823

return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);

3824

}

3825

}

3826

if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {

3827

// If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.

3828

if (Op0.getValueType().bitsGT(VT))

3829

Op0 = DAG.getNode(ISD::AND, dl, VT,

3830

DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),

3831

DAG.getConstant(1, dl, VT));

3832

else if (Op0.getValueType().bitsLT(VT))

3833

Op0 = DAG.getNode(ISD::AND, dl, VT,

3834

DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),

3835

DAG.getConstant(1, dl, VT));

3836

3837

return DAG.getSetCC(dl, VT, Op0,

3838

DAG.getConstant(0, dl, Op0.getValueType()),

3839

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3840

}

3841

if (Op0.getOpcode() == ISD::AssertZext &&

3842

cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)

3843

return DAG.getSetCC(dl, VT, Op0,

3844

DAG.getConstant(0, dl, Op0.getValueType()),

3845

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3846

}

3847

}

3848

3849

// Given:

3850

// icmp eq/ne (urem %x, %y), 0

3851

// Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':

3852

// icmp eq/ne %x, 0

3853

if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&

3854

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3855

KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));

3856

KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));

3857

if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)

3858

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);

3859

}

3860

3861

if (SDValue V =

3862

optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))

3863

return V;

3864

}

3865

3866

// These simplifications apply to splat vectors as well.

3867

// TODO: Handle more splat vector cases.

3868

if (auto *N1C = isConstOrConstSplat(N1)) {

3869

const APInt &C1 = N1C->getAPIntValue();

3870

3871

APInt MinVal, MaxVal;

3872

unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();

3873

if (ISD::isSignedIntSetCC(Cond)) {

3874

MinVal = APInt::getSignedMinValue(OperandBitSize);

3875

MaxVal = APInt::getSignedMaxValue(OperandBitSize);

3876

} else {

3877

MinVal = APInt::getMinValue(OperandBitSize);

3878

MaxVal = APInt::getMaxValue(OperandBitSize);

3879

}

3880

3881

// Canonicalize GE/LE comparisons to use GT/LT comparisons.

3882

if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {

3883

// X >= MIN --> true

3884

if (C1 == MinVal)

3885

return DAG.getBoolConstant(true, dl, VT, OpVT);

3886

3887

if (!VT.isVector()) { // TODO: Support this for vectors.

3888

// X >= C0 --> X > (C0 - 1)

3889

APInt C = C1 - 1;

3890

ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;

3891

if ((DCI.isBeforeLegalizeOps() ||

3892

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3893

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3894

isLegalICmpImmediate(C.getSExtValue())))) {

3895

return DAG.getSetCC(dl, VT, N0,

3896

DAG.getConstant(C, dl, N1.getValueType()),

3897

NewCC);

3898

}

3899

}

3900

}

3901

3902

if (Cond == ISD::SETLE || Cond == ISD::SETULE) {

3903

// X <= MAX --> true

3904

if (C1 == MaxVal)

3905

return DAG.getBoolConstant(true, dl, VT, OpVT);

3906

3907

// X <= C0 --> X < (C0 + 1)

3908

if (!VT.isVector()) { // TODO: Support this for vectors.

3909

APInt C = C1 + 1;

3910

ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;

3911

if ((DCI.isBeforeLegalizeOps() ||

3912

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3913

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3914

isLegalICmpImmediate(C.getSExtValue())))) {

3915

return DAG.getSetCC(dl, VT, N0,

3916

DAG.getConstant(C, dl, N1.getValueType()),

3917

NewCC);

3918

}

3919

}

3920

}

3921

3922

if (Cond == ISD::SETLT || Cond == ISD::SETULT) {

3923

if (C1 == MinVal)

3924

return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

3925

3926

// TODO: Support this for vectors after legalize ops.

3927

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3928

// Canonicalize setlt X, Max --> setne X, Max

3929

if (C1 == MaxVal)

3930

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3931

3932

// If we have setult X, 1, turn it into seteq X, 0

3933

if (C1 == MinVal+1)

3934

return DAG.getSetCC(dl, VT, N0,

3935

DAG.getConstant(MinVal, dl, N0.getValueType()),

3936

ISD::SETEQ);

3937

}

3938

}

3939

3940

if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {

3941

if (C1 == MaxVal)

3942

return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

3943

3944

// TODO: Support this for vectors after legalize ops.

3945

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3946

// Canonicalize setgt X, Min --> setne X, Min

3947

if (C1 == MinVal)

3948

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3949

3950

// If we have setugt X, Max-1, turn it into seteq X, Max

3951

if (C1 == MaxVal-1)

3952

return DAG.getSetCC(dl, VT, N0,

3953

DAG.getConstant(MaxVal, dl, N0.getValueType()),

3954

ISD::SETEQ);

3955

}

3956

}

3957

3958

if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {

3959

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3960

if (C1.isNullValue())

3961

if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(

3962

VT, N0, N1, Cond, DCI, dl))

3963

return CC;

3964

3965

// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).

3966

// For example, when high 32-bits of i64 X are known clear:

3967

// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0

3968

// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1

3969

bool CmpZero = N1C->getAPIntValue().isNullValue();

3970

bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();

3971

if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {

3972

// Match or(lo,shl(hi,bw/2)) pattern.

3973

auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {

3974

unsigned EltBits = V.getScalarValueSizeInBits();

3975

if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)

3976

return false;

3977

SDValue LHS = V.getOperand(0);

3978

SDValue RHS = V.getOperand(1);

3979

APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);

3980

// Unshifted element must have zero upperbits.

3981

if (RHS.getOpcode() == ISD::SHL &&

3982

isa<ConstantSDNode>(RHS.getOperand(1)) &&

3983

RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

3984

DAG.MaskedValueIsZero(LHS, HiBits)) {

3985

Lo = LHS;

3986

Hi = RHS.getOperand(0);

3987

return true;

3988

}

3989

if (LHS.getOpcode() == ISD::SHL &&

3990

isa<ConstantSDNode>(LHS.getOperand(1)) &&

3991

LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

3992

DAG.MaskedValueIsZero(RHS, HiBits)) {

3993

Lo = RHS;

3994

Hi = LHS.getOperand(0);

3995

return true;

3996

}

3997

return false;

3998

};

3999

4000

auto MergeConcat = [&](SDValue Lo, SDValue Hi) {

4001

unsigned EltBits = N0.getScalarValueSizeInBits();

4002

unsigned HalfBits = EltBits / 2;

4003

APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);

4004

SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);

4005

SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);

4006

SDValue NewN0 =

4007

DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);

4008

SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;

4009

return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);

4010

};

4011

4012

SDValue Lo, Hi;

4013

if (IsConcat(N0, Lo, Hi))

4014

return MergeConcat(Lo, Hi);

4015

4016

if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {

4017

SDValue Lo0, Lo1, Hi0, Hi1;

4018

if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&

4019

IsConcat(N0.getOperand(1), Lo1, Hi1)) {

4020

return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),

4021

DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));

4022

}

4023

}

4024

}

4025

}

4026

4027

// If we have "setcc X, C0", check to see if we can shrink the immediate

4028

// by changing cc.

4029

// TODO: Support this for vectors after legalize ops.

4030

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4031

// SETUGT X, SINTMAX -> SETLT X, 0

4032

// SETUGE X, SINTMIN -> SETLT X, 0

4033

if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||

4034

(Cond == ISD::SETUGE && C1.isMinSignedValue()))

4035

return DAG.getSetCC(dl, VT, N0,

4036

DAG.getConstant(0, dl, N1.getValueType()),

4037

ISD::SETLT);

4038

4039

// SETULT X, SINTMIN -> SETGT X, -1

4040

// SETULE X, SINTMAX -> SETGT X, -1

4041

if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||

4042

(Cond == ISD::SETULE && C1.isMaxSignedValue()))

4043

return DAG.getSetCC(dl, VT, N0,

4044

DAG.getAllOnesConstant(dl, N1.getValueType()),

4045

ISD::SETGT);

4046

}

4047

}

4048

4049

// Back to non-vector simplifications.

4050

// TODO: Can we do these for vector splats?

4051

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

4052

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4053

const APInt &C1 = N1C->getAPIntValue();

4054

EVT ShValTy = N0.getValueType();

4055

4056

// Fold bit comparisons when we can. This will result in an

4057

// incorrect value when boolean false is negative one, unless

4058

// the bitsize is 1 in which case the false value is the same

4059

// in practice regardless of the representation.

4060

if ((VT.getSizeInBits() == 1 ||

4061

getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&

4062

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4063

(VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&

4064

N0.getOpcode() == ISD::AND) {

4065

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4066

EVT ShiftTy =

4067

getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4068

if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3

4069

// Perform the xform if the AND RHS is a single bit.

4070

unsigned ShCt = AndRHS->getAPIntValue().logBase2();

4071

if (AndRHS->getAPIntValue().isPowerOf2() &&

4072

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4073

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4074

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4075

DAG.getConstant(ShCt, dl, ShiftTy)));

4076

}

4077

} else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {

4078

// (X & 8) == 8 --> (X & 8) >> 3

4079

// Perform the xform if C1 is a single bit.

4080

unsigned ShCt = C1.logBase2();

4081

if (C1.isPowerOf2() &&

4082

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4083

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4084

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4085

DAG.getConstant(ShCt, dl, ShiftTy)));

4086

}

4087

}

4088

}

4089

}

4090

4091

if (C1.getMinSignedBits() <= 64 &&

4092

!isLegalICmpImmediate(C1.getSExtValue())) {

4093

EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4094

// (X & -256) == 256 -> (X >> 8) == 1

4095

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4096

N0.getOpcode() == ISD::AND && N0.hasOneUse()) {

4097

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4098

const APInt &AndRHSC = AndRHS->getAPIntValue();

4099

if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {

4100

unsigned ShiftBits = AndRHSC.countTrailingZeros();

4101

if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4102

SDValue Shift =

4103

DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),

4104

DAG.getConstant(ShiftBits, dl, ShiftTy));

4105

SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);

4106

return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);

4107

}

4108

}

4109

}

4110

} else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||

4111

Cond == ISD::SETULE || Cond == ISD::SETUGT) {

4112

bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);

4113

// X < 0x100000000 -> (X >> 32) < 1

4114

// X >= 0x100000000 -> (X >> 32) >= 1

4115

// X <= 0x0ffffffff -> (X >> 32) < 1

4116

// X > 0x0ffffffff -> (X >> 32) >= 1

4117

unsigned ShiftBits;

4118

APInt NewC = C1;

4119

ISD::CondCode NewCond = Cond;

4120

if (AdjOne) {

4121

ShiftBits = C1.countTrailingOnes();

4122

NewC = NewC + 1;

4123

NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;

4124

} else {

4125

ShiftBits = C1.countTrailingZeros();

4126

}

4127

NewC.lshrInPlace(ShiftBits);

4128

if (ShiftBits && NewC.getMinSignedBits() <= 64 &&

4129

isLegalICmpImmediate(NewC.getSExtValue()) &&

4130

!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4131

SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4132

DAG.getConstant(ShiftBits, dl, ShiftTy));

4133

SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);

4134

return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);

4135

}

4136

}

4137

}

4138

}

4139

4140

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {

4141

auto *CFP = cast<ConstantFPSDNode>(N1);

4142

assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((!CFP->getValueAPF().isNaN() && "Unexpected NaN value"
) ? static_cast<void> (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4142, __PRETTY_FUNCTION__));

4143

4144

// Otherwise, we know the RHS is not a NaN. Simplify the node to drop the

4145

// constant if knowing that the operand is non-nan is enough. We prefer to

4146

// have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to

4147

// materialize 0.0.

4148

if (Cond == ISD::SETO || Cond == ISD::SETUO)

4149

return DAG.getSetCC(dl, VT, N0, N0, Cond);

4150

4151

// setcc (fneg x), C -> setcc swap(pred) x, -C

4152

if (N0.getOpcode() == ISD::FNEG) {

4153

ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);

4154

if (DCI.isBeforeLegalizeOps() ||

4155

isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {

4156

SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);

4157

return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);

4158

}

4159

}

4160

4161

// If the condition is not legal, see if we can find an equivalent one

4162

// which is legal.

4163

if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {

4164

// If the comparison was an awkward floating-point == or != and one of

4165

// the comparison operands is infinity or negative infinity, convert the

4166

// condition to a less-awkward <= or >=.

4167

if (CFP->getValueAPF().isInfinity()) {

4168

bool IsNegInf = CFP->getValueAPF().isNegative();

4169

ISD::CondCode NewCond = ISD::SETCC_INVALID;

4170

switch (Cond) {

4171

case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;

4172

case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;

4173

case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;

4174

case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;

4175

default: break;

4176

}

4177

if (NewCond != ISD::SETCC_INVALID &&

4178

isCondCodeLegal(NewCond, N0.getSimpleValueType()))

4179

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4180

}

4181

}

4182

}

4183

4184

if (N0 == N1) {

4185

// The sext(setcc()) => setcc() optimization relies on the appropriate

4186

// constant being emitted.

4187

assert(!N0.getValueType().isInteger() &&((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4188, __PRETTY_FUNCTION__))

4188

"Integer types should be handled by FoldSetCC")((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4188, __PRETTY_FUNCTION__));

4189

4190

bool EqTrue = ISD::isTrueWhenEqual(Cond);

4191

unsigned UOF = ISD::getUnorderedFlavor(Cond);

4192

if (UOF == 2) // FP operators that are undefined on NaNs.

4193

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4194

if (UOF == unsigned(EqTrue))

4195

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4196

// Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO

4197

// if it is not already.

4198

ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;

4199

if (NewCond != Cond &&

4200

(DCI.isBeforeLegalizeOps() ||

4201

isCondCodeLegal(NewCond, N0.getSimpleValueType())))

4202

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4203

}

4204

4205

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4206

N0.getValueType().isInteger()) {

4207

if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||

4208

N0.getOpcode() == ISD::XOR) {

4209

// Simplify (X+Y) == (X+Z) --> Y == Z

4210

if (N0.getOpcode() == N1.getOpcode()) {

4211

if (N0.getOperand(0) == N1.getOperand(0))

4212

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);

4213

if (N0.getOperand(1) == N1.getOperand(1))

4214

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);

4215

if (isCommutativeBinOp(N0.getOpcode())) {

4216

// If X op Y == Y op X, try other combinations.

4217

if (N0.getOperand(0) == N1.getOperand(1))

4218

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),

4219

Cond);

4220

if (N0.getOperand(1) == N1.getOperand(0))

4221

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),

4222

Cond);

4223

}

4224

}

4225

4226

// If RHS is a legal immediate value for a compare instruction, we need

4227

// to be careful about increasing register pressure needlessly.

4228

bool LegalRHSImm = false;

4229

4230

if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {

4231

if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4232

// Turn (X+C1) == C2 --> X == C2-C1

4233

if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {

4234

return DAG.getSetCC(dl, VT, N0.getOperand(0),

4235

DAG.getConstant(RHSC->getAPIntValue()-

4236

LHSR->getAPIntValue(),

4237

dl, N0.getValueType()), Cond);

4238

}

4239

4240

// Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.

4241

if (N0.getOpcode() == ISD::XOR)

4242

// If we know that all of the inverted bits are zero, don't bother

4243

// performing the inversion.

4244

if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))

4245

return

4246

DAG.getSetCC(dl, VT, N0.getOperand(0),

4247

DAG.getConstant(LHSR->getAPIntValue() ^

4248

RHSC->getAPIntValue(),

4249

dl, N0.getValueType()),

4250

Cond);

4251

}

4252

4253

// Turn (C1-X) == C2 --> X == C1-C2

4254

if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {

4255

if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {

4256

return

4257

DAG.getSetCC(dl, VT, N0.getOperand(1),

4258

DAG.getConstant(SUBC->getAPIntValue() -

4259

RHSC->getAPIntValue(),

4260

dl, N0.getValueType()),

4261

Cond);

4262

}

4263

}

4264

4265

// Could RHSC fold directly into a compare?

4266

if (RHSC->getValueType(0).getSizeInBits() <= 64)

4267

LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());

4268

}

4269

4270

// (X+Y) == X --> Y == 0 and similar folds.

4271

// Don't do this if X is an immediate that can fold into a cmp

4272

// instruction and X+Y has other uses. It could be an induction variable

4273

// chain, and the transform would increase register pressure.

4274

if (!LegalRHSImm || N0.hasOneUse())

4275

if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))

4276

return V;

4277

}

4278

4279

if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||

4280

N1.getOpcode() == ISD::XOR)

4281

if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))

4282

return V;

4283

4284

if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))

4285

return V;

4286

}

4287

4288

// Fold remainder of division by a constant.

4289

if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&

4290

N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

4291

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

4292

4293

// When division is cheap or optimizing for minimum size,

4294

// fall through to DIVREM creation by skipping this fold.

4295

if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {

4296

if (N0.getOpcode() == ISD::UREM) {

4297

if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))

4298

return Folded;

4299

} else if (N0.getOpcode() == ISD::SREM) {

4300

if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))

4301

return Folded;

4302

}

4303

}

4304

}

4305

4306

// Fold away ALL boolean setcc's.

4307

if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {

4308

SDValue Temp;

4309

switch (Cond) {

4310

default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4310);

4311

case ISD::SETEQ: // X == Y -> ~(X^Y)

4312

Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4313

N0 = DAG.getNOT(dl, Temp, OpVT);

4314

if (!DCI.isCalledByLegalizer())

4315

DCI.AddToWorklist(Temp.getNode());

4316

break;

4317

case ISD::SETNE: // X != Y --> (X^Y)

4318

N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4319

break;

4320

case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y

4321

case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y

4322

Temp = DAG.getNOT(dl, N0, OpVT);

4323

N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);

4324

if (!DCI.isCalledByLegalizer())

4325

DCI.AddToWorklist(Temp.getNode());

4326

break;

4327

case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X

4328

case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X

4329

Temp = DAG.getNOT(dl, N1, OpVT);

4330

N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);

4331

if (!DCI.isCalledByLegalizer())

4332

DCI.AddToWorklist(Temp.getNode());

4333

break;

4334

case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y

4335

case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y

4336

Temp = DAG.getNOT(dl, N0, OpVT);

4337

N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);

4338

if (!DCI.isCalledByLegalizer())

4339

DCI.AddToWorklist(Temp.getNode());

4340

break;

4341

case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X

4342

case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X

4343

Temp = DAG.getNOT(dl, N1, OpVT);

4344

N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);

4345

break;

4346

}

4347

if (VT.getScalarType() != MVT::i1) {

4348

if (!DCI.isCalledByLegalizer())

4349

DCI.AddToWorklist(N0.getNode());

4350

// FIXME: If running after legalize, we probably can't do this.

4351

ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));

4352

N0 = DAG.getNode(ExtendCode, dl, VT, N0);

4353

}

4354

return N0;

4355

}

4356

4357

// Could not fold it.

4358

return SDValue();

4359

}

4360

4361

/// Returns true (and the GlobalValue and the offset) if the node is a

4362

/// GlobalAddress + offset.

4363

bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,

4364

int64_t &Offset) const {

4365

4366

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

4367

4368

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {

4369

GA = GASD->getGlobal();

4370

Offset += GASD->getOffset();

4371

return true;

4372

}

4373

4374

if (N->getOpcode() == ISD::ADD) {

4375

SDValue N1 = N->getOperand(0);

4376

SDValue N2 = N->getOperand(1);

4377

if (isGAPlusOffset(N1.getNode(), GA, Offset)) {

4378

if (auto *V = dyn_cast<ConstantSDNode>(N2)) {

4379

Offset += V->getSExtValue();

4380

return true;

4381

}

4382

} else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {

4383

if (auto *V = dyn_cast<ConstantSDNode>(N1)) {

4384

Offset += V->getSExtValue();

4385

return true;

4386

}

4387

}

4388

}

4389

4390

return false;

4391

}

4392

4393

SDValue TargetLowering::PerformDAGCombine(SDNode *N,

4394

DAGCombinerInfo &DCI) const {

4395

// Default implementation: no optimization.

4396

return SDValue();

4397

}

4398

4399

//===----------------------------------------------------------------------===//

4400

// Inline Assembler Implementation Methods

4401

//===----------------------------------------------------------------------===//

4402

4403

TargetLowering::ConstraintType

4404

TargetLowering::getConstraintType(StringRef Constraint) const {

4405

unsigned S = Constraint.size();

4406

4407

if (S == 1) {

4408

switch (Constraint[0]) {

4409

default: break;

4410

case 'r':

4411

return C_RegisterClass;

4412

case 'm': // memory

4413

case 'o': // offsetable

4414

case 'V': // not offsetable

4415

return C_Memory;

4416

case 'n': // Simple Integer

4417

case 'E': // Floating Point Constant

4418

case 'F': // Floating Point Constant

4419

return C_Immediate;

4420

case 'i': // Simple Integer or Relocatable Constant

4421

case 's': // Relocatable Constant

4422

case 'p': // Address.

4423

case 'X': // Allow ANY value.

4424

case 'I': // Target registers.

4425

case 'J':

4426

case 'K':

4427

case 'L':

4428

case 'M':

4429

case 'N':

4430

case 'O':

4431

case 'P':

4432

case '<':

4433

case '>':

4434

return C_Other;

4435

}

4436

}

4437

4438

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {

4439

if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"

4440

return C_Memory;

4441

return C_Register;

4442

}

4443

return C_Unknown;

4444

}

4445

4446

/// Try to replace an X constraint, which matches anything, with another that

4447

/// has more specific requirements based on the type of the corresponding

4448

/// operand.

4449

const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {

4450

if (ConstraintVT.isInteger())

4451

return "r";

4452

if (ConstraintVT.isFloatingPoint())

4453

return "f"; // works for many targets

4454

return nullptr;

4455

}

4456

4457

SDValue TargetLowering::LowerAsmOutputForConstraint(

4458

SDValue &Chain, SDValue &Flag, const SDLoc &DL,

4459

const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {

4460

return SDValue();

4461

}

4462

4463

/// Lower the specified operand into the Ops vector.

4464

/// If it is invalid, don't add anything to Ops.

4465

void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,

4466

std::string &Constraint,

4467

std::vector<SDValue> &Ops,

4468

SelectionDAG &DAG) const {

4469

4470

if (Constraint.length() > 1) return;

4471

4472

char ConstraintLetter = Constraint[0];

4473

switch (ConstraintLetter) {

4474

default: break;

4475

case 'X': // Allows any operand; labels (basic block) use this.

4476

if (Op.getOpcode() == ISD::BasicBlock ||

4477

Op.getOpcode() == ISD::TargetBlockAddress) {

4478

Ops.push_back(Op);

4479

return;

4480

}

4481

LLVM_FALLTHROUGH[[gnu::fallthrough]];

4482

case 'i': // Simple Integer or Relocatable Constant

4483

case 'n': // Simple Integer

4484

case 's': { // Relocatable Constant

4485

4486

GlobalAddressSDNode *GA;

4487

ConstantSDNode *C;

4488

BlockAddressSDNode *BA;

4489

uint64_t Offset = 0;

4490

4491

// Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),

4492

// etc., since getelementpointer is variadic. We can't use

4493

// SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible

4494

// while in this case the GA may be furthest from the root node which is

4495

// likely an ISD::ADD.

4496

while (1) {

4497

if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4498

Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),

4499

GA->getValueType(0),

4500

Offset + GA->getOffset()));

4501

return;

4502

} else if ((C = dyn_cast<ConstantSDNode>(Op)) &&

4503

ConstraintLetter != 's') {

4504

// gcc prints these as sign extended. Sign extend value to 64 bits

4505

// now; without this it would get ZExt'd later in

4506

// ScheduleDAGSDNodes::EmitNode, which is very generic.

4507

bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;

4508

BooleanContent BCont = getBooleanContents(MVT::i64);

4509

ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)

4510

: ISD::SIGN_EXTEND;

4511

int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue()

4512

: C->getSExtValue();

4513

Ops.push_back(DAG.getTargetConstant(Offset + ExtVal,

4514

SDLoc(C), MVT::i64));

4515

return;

4516

} else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) &&

4517

ConstraintLetter != 'n') {

4518

Ops.push_back(DAG.getTargetBlockAddress(

4519

BA->getBlockAddress(), BA->getValueType(0),

4520

Offset + BA->getOffset(), BA->getTargetFlags()));

4521

return;

4522

} else {

4523

const unsigned OpCode = Op.getOpcode();

4524

if (OpCode == ISD::ADD || OpCode == ISD::SUB) {

4525

if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))

4526

Op = Op.getOperand(1);

4527

// Subtraction is not commutative.

4528

else if (OpCode == ISD::ADD &&

4529

(C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))

4530

Op = Op.getOperand(0);

4531

else

4532

return;

4533

Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();

4534

continue;

4535

}

4536

}

4537

return;

4538

}

4539

break;

4540

}

4541

}

4542

}

4543

4544

std::pair<unsigned, const TargetRegisterClass *>

4545

TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,

4546

StringRef Constraint,

4547

MVT VT) const {

4548

if (Constraint.empty() || Constraint[0] != '{')

4549

return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));

4550

assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?"
) ? static_cast<void> (0) : __assert_fail ("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4550, __PRETTY_FUNCTION__));

4551

4552

// Remove the braces from around the name.

4553

StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

4554

4555

std::pair<unsigned, const TargetRegisterClass *> R =

4556

std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

4557

4558

// Figure out which register class contains this reg.

4559

for (const TargetRegisterClass *RC : RI->regclasses()) {

4560

// If none of the value types for this register class are valid, we

4561

// can't use it. For example, 64-bit reg classes on 32-bit targets.

4562

if (!isLegalRC(*RI, *RC))

4563

continue;

4564

4565

for (const MCPhysReg &PR : *RC) {

4566

if (RegName.equals_lower(RI->getRegAsmName(PR))) {

4567

std::pair<unsigned, const TargetRegisterClass *> S =

4568

std::make_pair(PR, RC);

4569

4570

// If this register class has the requested value type, return it,

4571

// otherwise keep searching and return the first class found

4572

// if no other is found which explicitly has the requested type.

4573

if (RI->isTypeLegalForClass(*RC, VT))

4574

return S;

4575

if (!R.second)

4576

R = S;

4577

}

4578

}

4579

}

4580

4581

return R;

4582

}

4583

4584

//===----------------------------------------------------------------------===//

4585

// Constraint Selection.

4586

4587

/// Return true of this is an input operand that is a matching constraint like

4588

/// "4".

4589

bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {

4590

assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4590, __PRETTY_FUNCTION__));

4591

return isdigit(static_cast<unsigned char>(ConstraintCode[0]));

4592

}

4593

4594

/// If this is an input matching constraint, this method returns the output

4595

/// operand it matches.

4596

unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {

4597

assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4597, __PRETTY_FUNCTION__));

4598

return atoi(ConstraintCode.c_str());

4599

}

4600

4601

/// Split up the constraint string from the inline assembly value into the

4602

/// specific constraints and their prefixes, and also tie in the associated

4603

/// operand values.

4604

/// If this returns an empty vector, and if the constraint string itself

4605

/// isn't empty, there was an error parsing.

4606

TargetLowering::AsmOperandInfoVector

4607

TargetLowering::ParseConstraints(const DataLayout &DL,

4608

const TargetRegisterInfo *TRI,

4609

const CallBase &Call) const {

4610

/// Information about all of the constraints.

4611

AsmOperandInfoVector ConstraintOperands;

4612

const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());

4613

unsigned maCount = 0; // Largest number of multiple alternative constraints.

4614

4615

// Do a prepass over the constraints, canonicalizing them, and building up the

4616

// ConstraintOperands list.

4617

unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.

4618

unsigned ResNo = 0; // ResNo - The result number of the next output.

4619

4620

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {

4621

ConstraintOperands.emplace_back(std::move(CI));

4622

AsmOperandInfo &OpInfo = ConstraintOperands.back();

4623

4624

// Update multiple alternative constraint count.

4625

if (OpInfo.multipleAlternatives.size() > maCount)

4626

maCount = OpInfo.multipleAlternatives.size();

4627

4628

OpInfo.ConstraintVT = MVT::Other;

4629

4630

// Compute the value type for each operand.

4631

switch (OpInfo.Type) {

4632

case InlineAsm::isOutput:

4633

// Indirect outputs just consume an argument.

4634

if (OpInfo.isIndirect) {

4635

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4636

break;

4637

}

4638

4639

// The return value of the call is this value. As such, there is no

4640

// corresponding argument.

4641

assert(!Call.getType()->isVoidTy() && "Bad inline asm!")((!Call.getType()->isVoidTy() && "Bad inline asm!"
) ? static_cast<void> (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4641, __PRETTY_FUNCTION__));

4642

if (StructType *STy = dyn_cast<StructType>(Call.getType())) {

4643

OpInfo.ConstraintVT =

4644

getSimpleValueType(DL, STy->getElementType(ResNo));

4645

} else {

4646

assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4646, __PRETTY_FUNCTION__));

4647

OpInfo.ConstraintVT = getSimpleValueType(DL, Call.getType());

4648

}

4649

++ResNo;

4650

break;

4651

case InlineAsm::isInput:

4652

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4653

break;

4654

case InlineAsm::isClobber:

4655

// Nothing to do.

4656

break;

4657

}

4658

4659

if (OpInfo.CallOperandVal) {

4660

llvm::Type *OpTy = OpInfo.CallOperandVal->getType();

4661

if (OpInfo.isIndirect) {

4662

llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);

4663

if (!PtrTy)

4664

report_fatal_error("Indirect operand for inline asm not a pointer!");

4665

OpTy = PtrTy->getElementType();

4666

}

4667

4668

// Look for vector wrapped in a struct. e.g. { <16 x i8> }.

4669

if (StructType *STy = dyn_cast<StructType>(OpTy))

4670

if (STy->getNumElements() == 1)

4671

OpTy = STy->getElementType(0);

4672

4673

// If OpTy is not a single value, it may be a struct/union that we

4674

// can tile with integers.

4675

if (!OpTy->isSingleValueType() && OpTy->isSized()) {

4676

unsigned BitSize = DL.getTypeSizeInBits(OpTy);

4677

switch (BitSize) {

4678

default: break;

4679

case 1:

4680

case 8:

4681

case 16:

4682

case 32:

4683

case 64:

4684

case 128:

4685

OpInfo.ConstraintVT =

4686

MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);

4687

break;

4688

}

4689

} else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {

4690

unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());

4691

OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);

4692

} else {

4693

OpInfo.ConstraintVT = MVT::getVT(OpTy, true);

4694

}

4695

}

4696

}

4697

4698

// If we have multiple alternative constraints, select the best alternative.

4699

if (!ConstraintOperands.empty()) {

4700

if (maCount) {

4701

unsigned bestMAIndex = 0;

4702

int bestWeight = -1;

4703

// weight: -1 = invalid match, and 0 = so-so match to 5 = good match.

4704

int weight = -1;

4705

unsigned maIndex;

4706

// Compute the sums of the weights for each alternative, keeping track

4707

// of the best (highest weight) one so far.

4708

for (maIndex = 0; maIndex < maCount; ++maIndex) {

4709

int weightSum = 0;

4710

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4711

cIndex != eIndex; ++cIndex) {

4712

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4713

if (OpInfo.Type == InlineAsm::isClobber)

4714

continue;

4715

4716

// If this is an output operand with a matching input operand,

4717

// look up the matching input. If their types mismatch, e.g. one

4718

// is an integer, the other is floating point, or their sizes are

4719

// different, flag it as an maCantMatch.

4720

if (OpInfo.hasMatchingInput()) {

4721

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4722

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4723

if ((OpInfo.ConstraintVT.isInteger() !=

4724

Input.ConstraintVT.isInteger()) ||

4725

(OpInfo.ConstraintVT.getSizeInBits() !=

4726

Input.ConstraintVT.getSizeInBits())) {

4727

weightSum = -1; // Can't match.

4728

break;

4729

}

4730

}

4731

}

4732

weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);

4733

if (weight == -1) {

4734

weightSum = -1;

4735

break;

4736

}

4737

weightSum += weight;

4738

}

4739

// Update best.

4740

if (weightSum > bestWeight) {

4741

bestWeight = weightSum;

4742

bestMAIndex = maIndex;

4743

}

4744

}

4745

4746

// Now select chosen alternative in each constraint.

4747

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4748

cIndex != eIndex; ++cIndex) {

4749

AsmOperandInfo &cInfo = ConstraintOperands[cIndex];

4750

if (cInfo.Type == InlineAsm::isClobber)

4751

continue;

4752

cInfo.selectAlternative(bestMAIndex);

4753

}

4754

}

4755

}

4756

4757

// Check and hook up tied operands, choose constraint code to use.

4758

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4759

cIndex != eIndex; ++cIndex) {

4760

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4761

4762

// If this is an output operand with a matching input operand, look up the

4763

// matching input. If their types mismatch, e.g. one is an integer, the

4764

// other is floating point, or their sizes are different, flag it as an

4765

// error.

4766

if (OpInfo.hasMatchingInput()) {

4767

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4768

4769

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4770

std::pair<unsigned, const TargetRegisterClass *> MatchRC =

4771

getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,

4772

OpInfo.ConstraintVT);

4773

std::pair<unsigned, const TargetRegisterClass *> InputRC =

4774

getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,

4775

Input.ConstraintVT);

4776

if ((OpInfo.ConstraintVT.isInteger() !=

4777

Input.ConstraintVT.isInteger()) ||

4778

(MatchRC.second != InputRC.second)) {

4779

report_fatal_error("Unsupported asm: input constraint"

4780

" with a matching output constraint of"

4781

" incompatible type!");

4782

}

4783

}

4784

}

4785

}

4786

4787

return ConstraintOperands;

4788

}

4789

4790

/// Return an integer indicating how general CT is.

4791

static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {

4792

switch (CT) {

4793

case TargetLowering::C_Immediate:

4794

case TargetLowering::C_Other:

4795

case TargetLowering::C_Unknown:

4796

return 0;

4797

case TargetLowering::C_Register:

4798

return 1;

4799

case TargetLowering::C_RegisterClass:

4800

return 2;

4801

case TargetLowering::C_Memory:

4802

return 3;

4803

}

4804

llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4804);

4805

}

4806

4807

/// Examine constraint type and operand type and determine a weight value.

4808

/// This object must already have been set up with the operand type

4809

/// and the current alternative constraint selected.

4810

TargetLowering::ConstraintWeight

4811

TargetLowering::getMultipleConstraintMatchWeight(

4812

AsmOperandInfo &info, int maIndex) const {

4813

InlineAsm::ConstraintCodeVector *rCodes;

4814

if (maIndex >= (int)info.multipleAlternatives.size())

4815

rCodes = &info.Codes;

4816

else

4817

rCodes = &info.multipleAlternatives[maIndex].Codes;

4818

ConstraintWeight BestWeight = CW_Invalid;

4819

4820

// Loop over the options, keeping track of the most general one.

4821

for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {

4822

ConstraintWeight weight =

4823

getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());

4824

if (weight > BestWeight)

4825

BestWeight = weight;

4826

}

4827

4828

return BestWeight;

4829

}

4830

4831

/// Examine constraint type and operand type and determine a weight value.

4832

/// This object must already have been set up with the operand type

4833

/// and the current alternative constraint selected.

4834

TargetLowering::ConstraintWeight

4835

TargetLowering::getSingleConstraintMatchWeight(

4836

AsmOperandInfo &info, const char *constraint) const {

4837

ConstraintWeight weight = CW_Invalid;

4838

Value *CallOperandVal = info.CallOperandVal;

4839

// If we don't have a value, we can't do a match,

4840

// but allow it at the lowest weight.

4841

if (!CallOperandVal)

4842

return CW_Default;

4843

// Look at the constraint type.

4844

switch (*constraint) {

4845

case 'i': // immediate integer.

4846

case 'n': // immediate integer with a known value.

4847

if (isa<ConstantInt>(CallOperandVal))

4848

weight = CW_Constant;

4849

break;

4850

case 's': // non-explicit intregal immediate.

4851

if (isa<GlobalValue>(CallOperandVal))

4852

weight = CW_Constant;

4853

break;

4854

case 'E': // immediate float if host format.

4855

case 'F': // immediate float.

4856

if (isa<ConstantFP>(CallOperandVal))

4857

weight = CW_Constant;

4858

break;

4859

case '<': // memory operand with autodecrement.

4860

case '>': // memory operand with autoincrement.

4861

case 'm': // memory operand.

4862

case 'o': // offsettable memory operand

4863

case 'V': // non-offsettable memory operand

4864

weight = CW_Memory;

4865

break;

4866

case 'r': // general register.

4867

case 'g': // general register, memory operand or immediate integer.

4868

// note: Clang converts "g" to "imr".

4869

if (CallOperandVal->getType()->isIntegerTy())

4870

weight = CW_Register;

4871

break;

4872

case 'X': // any operand.

4873

default:

4874

weight = CW_Default;

4875

break;

4876

}

4877

return weight;

4878

}

4879

4880

/// If there are multiple different constraints that we could pick for this

4881

/// operand (e.g. "imr") try to pick the 'best' one.

4882

/// This is somewhat tricky: constraints fall into four classes:

4883

/// Other -> immediates and magic values

4884

/// Register -> one specific register

4885

/// RegisterClass -> a group of regs

4886

/// Memory -> memory

4887

/// Ideally, we would pick the most specific constraint possible: if we have

4888

/// something that fits into a register, we would pick it. The problem here

4889

/// is that if we have something that could either be in a register or in

4890

/// memory that use of the register could cause selection of *other*

4891

/// operands to fail: they might only succeed if we pick memory. Because of

4892

/// this the heuristic we use is:

4893

///

4894

/// 1) If there is an 'other' constraint, and if the operand is valid for

4895

/// that constraint, use it. This makes us take advantage of 'i'

4896

/// constraints when available.

4897

/// 2) Otherwise, pick the most general constraint present. This prefers

4898

/// 'm' over 'r', for example.

4899

///

4900

static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,

4901

const TargetLowering &TLI,

4902

SDValue Op, SelectionDAG *DAG) {

4903

assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4903, __PRETTY_FUNCTION__));

4904

unsigned BestIdx = 0;

4905

TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;

4906

int BestGenerality = -1;

4907

4908

// Loop over the options, keeping track of the most general one.

4909

for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {

4910

TargetLowering::ConstraintType CType =

4911

TLI.getConstraintType(OpInfo.Codes[i]);

4912

4913

// Indirect 'other' or 'immediate' constraints are not allowed.

4914

if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||

4915

CType == TargetLowering::C_Register ||

4916

CType == TargetLowering::C_RegisterClass))

4917

continue;

4918

4919

// If this is an 'other' or 'immediate' constraint, see if the operand is

4920

// valid for it. For example, on X86 we might have an 'rI' constraint. If

4921

// the operand is an integer in the range [0..31] we want to use I (saving a

4922

// load of a register), otherwise we must use 'r'.

4923

if ((CType == TargetLowering::C_Other ||

4924

CType == TargetLowering::C_Immediate) && Op.getNode()) {

4925

assert(OpInfo.Codes[i].size() == 1 &&((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4926, __PRETTY_FUNCTION__))

4926

"Unhandled multi-letter 'other' constraint")((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4926, __PRETTY_FUNCTION__));

4927

std::vector<SDValue> ResultOps;

4928

TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],

4929

ResultOps, *DAG);

4930

if (!ResultOps.empty()) {

4931

BestType = CType;

4932

BestIdx = i;

4933

break;

4934

}

4935

}

4936

4937

// Things with matching constraints can only be registers, per gcc

4938

// documentation. This mainly affects "g" constraints.

4939

if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())

4940

continue;

4941

4942

// This constraint letter is more general than the previous one, use it.

4943

int Generality = getConstraintGenerality(CType);

4944

if (Generality > BestGenerality) {

4945

BestType = CType;

4946

BestIdx = i;

4947

BestGenerality = Generality;

4948

}

4949

}

4950

4951

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];

4952

OpInfo.ConstraintType = BestType;

4953

}

4954

4955

/// Determines the constraint code and constraint type to use for the specific

4956

/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.

4957

void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

4958

SDValue Op,

4959

SelectionDAG *DAG) const {

4960

assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? static_cast<void> (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4960, __PRETTY_FUNCTION__));

4961

4962

// Single-letter constraints ('r') are very common.

4963

if (OpInfo.Codes.size() == 1) {

4964

OpInfo.ConstraintCode = OpInfo.Codes[0];

4965

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

4966

} else {

4967

ChooseConstraint(OpInfo, *this, Op, DAG);

4968

}

4969

4970

// 'X' matches anything.

4971

if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {

4972

// Labels and constants are handled elsewhere ('X' is the only thing

4973

// that matches labels). For Functions, the type here is the type of

4974

// the result, which is not what we want to look at; leave them alone.

4975

Value *v = OpInfo.CallOperandVal;

4976

if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {

4977

OpInfo.CallOperandVal = v;

4978

return;

4979

}

4980

4981

if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)

4982

return;

4983

4984

// Otherwise, try to resolve it to something we know about by looking at

4985

// the actual operand type.

4986

if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {

4987

OpInfo.ConstraintCode = Repl;

4988

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

4989

}

4990

}

4991

}

4992

4993

/// Given an exact SDIV by a constant, create a multiplication

4994

/// with the multiplicative inverse of the constant.

4995

static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,

4996

const SDLoc &dl, SelectionDAG &DAG,

4997

SmallVectorImpl<SDNode *> &Created) {

4998

SDValue Op0 = N->getOperand(0);

4999

SDValue Op1 = N->getOperand(1);

5000

EVT VT = N->getValueType(0);

5001

EVT SVT = VT.getScalarType();

5002

EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());

5003

EVT ShSVT = ShVT.getScalarType();

5004

5005

bool UseSRA = false;

5006

SmallVector<SDValue, 16> Shifts, Factors;

5007

5008

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5009

if (C->isNullValue())

5010

return false;

5011

APInt Divisor = C->getAPIntValue();

5012

unsigned Shift = Divisor.countTrailingZeros();

5013

if (Shift) {

5014

Divisor.ashrInPlace(Shift);

5015

UseSRA = true;

5016

}

5017

// Calculate the multiplicative inverse, using Newton's method.

5018

APInt t;

5019

APInt Factor = Divisor;

5020

while ((t = Divisor * Factor) != 1)

5021

Factor *= APInt(Divisor.getBitWidth(), 2) - t;

5022

Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));

5023

Factors.push_back(DAG.getConstant(Factor, dl, SVT));

5024

return true;

5025

};

5026

5027

// Collect all magic values from the build vector.

5028

if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))

5029

return SDValue();

5030

5031

SDValue Shift, Factor;

5032

if (Op1.getOpcode() == ISD::BUILD_VECTOR) {

5033

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5034

Factor = DAG.getBuildVector(VT, dl, Factors);

5035

} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {

5036

assert(Shifts.size() == 1 && Factors.size() == 1 &&((Shifts.size() == 1 && Factors.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5038, __PRETTY_FUNCTION__))

5037

"Expected matchUnaryPredicate to return one element for scalable "((Shifts.size() == 1 && Factors.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5038, __PRETTY_FUNCTION__))

5038

"vectors")((Shifts.size() == 1 && Factors.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5038, __PRETTY_FUNCTION__));

5039

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5040

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5041

} else {

5042

assert(isa<ConstantSDNode>(Op1) && "Expected a constant")((isa<ConstantSDNode>(Op1) && "Expected a constant"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5042, __PRETTY_FUNCTION__));

5043

Shift = Shifts[0];

5044

Factor = Factors[0];

5045

}

5046

5047

SDValue Res = Op0;

5048

5049

// Shift the value upfront if it is even, so the LSB is one.

5050

if (UseSRA) {

5051

// TODO: For UDIV use SRL instead of SRA.

5052

SDNodeFlags Flags;

5053

Flags.setExact(true);

5054

Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);

5055

Created.push_back(Res.getNode());

5056

}

5057

5058

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);

5059

}

5060

5061

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,

5062

SelectionDAG &DAG,

5063

SmallVectorImpl<SDNode *> &Created) const {

5064

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

5065

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5066

if (TLI.isIntDivCheap(N->getValueType(0), Attr))

5067

return SDValue(N, 0); // Lower SDIV as SDIV

5068

return SDValue();

5069

}

5070

5071

/// Given an ISD::SDIV node expressing a divide by constant,

5072

/// return a DAG expression to select that will generate the same value by

5073

/// multiplying by a magic number.

5074

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5075

SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,

5076

bool IsAfterLegalization,

5077

SmallVectorImpl<SDNode *> &Created) const {

5078

SDLoc dl(N);

5079

EVT VT = N->getValueType(0);

5080

EVT SVT = VT.getScalarType();

5081

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5082

EVT ShSVT = ShVT.getScalarType();

5083

unsigned EltBits = VT.getScalarSizeInBits();

5084

EVT MulVT;

5085

5086

// Check to see if we can do this.

5087

// FIXME: We should be more aggressive here.

5088

if (!isTypeLegal(VT)) {

5089

// Limit this to simple scalars for now.

5090

if (VT.isVector() || !VT.isSimple())

5091

return SDValue();

5092

5093

// If this type will be promoted to a large enough type with a legal

5094

// multiply operation, we can go ahead and do this transform.

5095

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5096

return SDValue();

5097

5098

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5099

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5100

!isOperationLegal(ISD::MUL, MulVT))

5101

return SDValue();

5102

}

5103

5104

// If the sdiv has an 'exact' bit we can use a simpler lowering.

5105

if (N->getFlags().hasExact())

5106

return BuildExactSDIV(*this, N, dl, DAG, Created);

5107

5108

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

5109

5110

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5111

if (C->isNullValue())

5112

return false;

5113

5114

const APInt &Divisor = C->getAPIntValue();

5115

APInt::ms magics = Divisor.magic();

5116

int NumeratorFactor = 0;

5117

int ShiftMask = -1;

5118

5119

if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {

5120

// If d is +1/-1, we just multiply the numerator by +1/-1.

5121

NumeratorFactor = Divisor.getSExtValue();

5122

magics.m = 0;

5123

magics.s = 0;

5124

ShiftMask = 0;

5125

} else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {

5126

// If d > 0 and m < 0, add the numerator.

5127

NumeratorFactor = 1;

5128

} else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {

5129

// If d < 0 and m > 0, subtract the numerator.

5130

NumeratorFactor = -1;

5131

}

5132

5133

MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));

5134

Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));

5135

Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));

5136

ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));

5137

return true;

5138

};

5139

5140

SDValue N0 = N->getOperand(0);

5141

SDValue N1 = N->getOperand(1);

5142

5143

// Collect the shifts / magic values from each element.

5144

if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))

5145

return SDValue();

5146

5147

SDValue MagicFactor, Factor, Shift, ShiftMask;

5148

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5149

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5150

Factor = DAG.getBuildVector(VT, dl, Factors);

5151

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5152

ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);

5153

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5154

assert(MagicFactors.size() == 1 && Factors.size() == 1 &&((MagicFactors.size() == 1 && Factors.size() == 1 &&
Shifts.size() == 1 && ShiftMasks.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5157, __PRETTY_FUNCTION__))

5155

Shifts.size() == 1 && ShiftMasks.size() == 1 &&((MagicFactors.size() == 1 && Factors.size() == 1 &&
Shifts.size() == 1 && ShiftMasks.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5157, __PRETTY_FUNCTION__))

5156

"Expected matchUnaryPredicate to return one element for scalable "((MagicFactors.size() == 1 && Factors.size() == 1 &&
Shifts.size() == 1 && ShiftMasks.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5157, __PRETTY_FUNCTION__))

5157

"vectors")((MagicFactors.size() == 1 && Factors.size() == 1 &&
Shifts.size() == 1 && ShiftMasks.size() == 1 &&
"Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? static_cast<void> (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5157, __PRETTY_FUNCTION__));

5158

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5159

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5160

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5161

ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);

5162

} else {

5163

assert(isa<ConstantSDNode>(N1) && "Expected a constant")((isa<ConstantSDNode>(N1) && "Expected a constant"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5163, __PRETTY_FUNCTION__));

5164

MagicFactor = MagicFactors[0];

5165

Factor = Factors[0];

5166

Shift = Shifts[0];

5167

ShiftMask = ShiftMasks[0];

5168

}

5169

5170

// Multiply the numerator (operand 0) by the magic value.

5171

// FIXME: We should support doing a MUL in a wider type.

5172

auto GetMULHS = [&](SDValue X, SDValue Y) {

5173

// If the type isn't legal, use a wider mul of the the type calculated

5174

// earlier.

5175

if (!isTypeLegal(VT)) {

5176

X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);

5177

Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);

5178

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5179

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5180

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5181

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5182

}

5183

5184

if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))

5185

return DAG.getNode(ISD::MULHS, dl, VT, X, Y);

5186

if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {

5187

SDValue LoHi =

5188

DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5189

return SDValue(LoHi.getNode(), 1);

5190

}

5191

return SDValue();

5192

};

5193

5194

SDValue Q = GetMULHS(N0, MagicFactor);

5195

if (!Q)

5196

return SDValue();

5197

5198

Created.push_back(Q.getNode());

5199

5200

// (Optionally) Add/subtract the numerator using Factor.

5201

Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);

5202

Created.push_back(Factor.getNode());

5203

Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);

5204

Created.push_back(Q.getNode());

5205

5206

// Shift right algebraic by shift value.

5207

Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);

5208

Created.push_back(Q.getNode());

5209

5210

// Extract the sign bit, mask it and add it to the quotient.

5211

SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);

5212

SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);

5213

Created.push_back(T.getNode());

5214

T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);

5215

Created.push_back(T.getNode());

5216

return DAG.getNode(ISD::ADD, dl, VT, Q, T);

5217

}

5218

5219

/// Given an ISD::UDIV node expressing a divide by constant,

5220

/// return a DAG expression to select that will generate the same value by

5221

/// multiplying by a magic number.

5222

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5223

SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,

5224

bool IsAfterLegalization,

5225

SmallVectorImpl<SDNode *> &Created) const {

5226

SDLoc dl(N);

5227

EVT VT = N->getValueType(0);

5228

EVT SVT = VT.getScalarType();

5229

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5230

EVT ShSVT = ShVT.getScalarType();

5231

unsigned EltBits = VT.getScalarSizeInBits();

5232

EVT MulVT;

5233

5234

// Check to see if we can do this.

5235

// FIXME: We should be more aggressive here.

5236

if (!isTypeLegal(VT)) {

5237

// Limit this to simple scalars for now.

5238

if (VT.isVector() || !VT.isSimple())

5239

return SDValue();

5240

5241

// If this type will be promoted to a large enough type with a legal

5242

// multiply operation, we can go ahead and do this transform.

5243

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5244

return SDValue();

5245

5246

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5247

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5248

!isOperationLegal(ISD::MUL, MulVT))

5249

return SDValue();

5250

}

5251

5252

bool UseNPQ = false;

5253

SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

5254

5255

auto BuildUDIVPattern = [&](ConstantSDNode *C) {

5256

if (C->isNullValue())

5257

return false;

5258

// FIXME: We should use a narrower constant when the upper

5259

// bits are known to be zero.

5260

const APInt& Divisor = C->getAPIntValue();

5261

APInt::mu magics = Divisor.magicu();

5262

unsigned PreShift = 0, PostShift = 0;

5263

5264

// If the divisor is even, we can avoid using the expensive fixup by

5265

// shifting the divided value upfront.

5266

if (magics.a != 0 && !Divisor[0]) {

5267

PreShift = Divisor.countTrailingZeros();

5268

// Get magic number for the shifted divisor.

5269

magics = Divisor.lshr(PreShift).magicu(PreShift);

5270

assert(magics.a == 0 && "Should use cheap fixup now")((magics.a == 0 && "Should use cheap fixup now") ? static_cast
<void> (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5270, __PRETTY_FUNCTION__));

5271

}

5272

5273

APInt Magic = magics.m;

5274

5275

unsigned SelNPQ;

5276

if (magics.a == 0 || Divisor.isOneValue()) {

5277

assert(magics.s < Divisor.getBitWidth() &&((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5278, __PRETTY_FUNCTION__))

5278

"We shouldn't generate an undefined shift!")((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5278, __PRETTY_FUNCTION__));

5279

PostShift = magics.s;

5280

SelNPQ = false;

5281

} else {

5282

PostShift = magics.s - 1;

5283

SelNPQ = true;

5284

}

5285

5286

PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));

5287

MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));

5288

NPQFactors.push_back(

5289

DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)

5290

: APInt::getNullValue(EltBits),

5291

dl, SVT));

5292

PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));

5293

UseNPQ |= SelNPQ;

5294

return true;

5295

};

5296

5297

SDValue N0 = N->getOperand(0);

5298

SDValue N1 = N->getOperand(1);

5299

5300

// Collect the shifts/magic values from each element.

5301

if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))

5302

return SDValue();

5303

5304

SDValue PreShift, PostShift, MagicFactor, NPQFactor;

5305

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5306

PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);

5307

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5308

NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);

5309

PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);

5310

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5311

assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&((PreShifts.size() == 1 && MagicFactors.size() == 1 &&
NPQFactors.size() == 1 && PostShifts.size() == 1 &&
"Expected matchUnaryPredicate to return one for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5313, __PRETTY_FUNCTION__))

5312

NPQFactors.size() == 1 && PostShifts.size() == 1 &&((PreShifts.size() == 1 && MagicFactors.size() == 1 &&
NPQFactors.size() == 1 && PostShifts.size() == 1 &&
"Expected matchUnaryPredicate to return one for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5313, __PRETTY_FUNCTION__))

5313

"Expected matchUnaryPredicate to return one for scalable vectors")((PreShifts.size() == 1 && MagicFactors.size() == 1 &&
NPQFactors.size() == 1 && PostShifts.size() == 1 &&
"Expected matchUnaryPredicate to return one for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5313, __PRETTY_FUNCTION__));

5314

PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);

5315

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5316

NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);

5317

PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);

5318

} else {

5319

assert(isa<ConstantSDNode>(N1) && "Expected a constant")((isa<ConstantSDNode>(N1) && "Expected a constant"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5319, __PRETTY_FUNCTION__));

5320

PreShift = PreShifts[0];

5321

MagicFactor = MagicFactors[0];

5322

PostShift = PostShifts[0];

5323

}

5324

5325

SDValue Q = N0;

5326

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);

5327

Created.push_back(Q.getNode());

5328

5329

// FIXME: We should support doing a MUL in a wider type.

5330

auto GetMULHU = [&](SDValue X, SDValue Y) {

5331

// If the type isn't legal, use a wider mul of the the type calculated

5332

// earlier.

5333

if (!isTypeLegal(VT)) {

5334

X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);

5335

Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);

5336

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5337

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5338

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5339

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5340

}

5341

5342

if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))

5343

return DAG.getNode(ISD::MULHU, dl, VT, X, Y);

5344

if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {

5345

SDValue LoHi =

5346

DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5347

return SDValue(LoHi.getNode(), 1);

5348

}

5349

return SDValue(); // No mulhu or equivalent

5350

};

5351

5352

// Multiply the numerator (operand 0) by the magic value.

5353

Q = GetMULHU(Q, MagicFactor);

5354

if (!Q)

5355

return SDValue();

5356

5357

Created.push_back(Q.getNode());

5358

5359

if (UseNPQ) {

5360

SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);

5361

Created.push_back(NPQ.getNode());

5362

5363

// For vectors we might have a mix of non-NPQ/NPQ paths, so use

5364

// MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.

5365

if (VT.isVector())

5366

NPQ = GetMULHU(NPQ, NPQFactor);

5367

else

5368

NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

5369

5370

Created.push_back(NPQ.getNode());

5371

5372

Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);

5373

Created.push_back(Q.getNode());

5374

}

5375

5376

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);

5377

Created.push_back(Q.getNode());

5378

5379

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5380

5381

SDValue One = DAG.getConstant(1, dl, VT);

5382

SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);

5383

return DAG.getSelect(dl, VT, IsOne, N0, Q);

5384

}

5385

5386

/// If all values in Values that *don't* match the predicate are same 'splat'

5387

/// value, then replace all values with that splat value.

5388

/// Else, if AlternativeReplacement was provided, then replace all values that

5389

/// do match predicate with AlternativeReplacement value.

5390

static void

5391

turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,

5392

std::function<bool(SDValue)> Predicate,

5393

SDValue AlternativeReplacement = SDValue()) {

5394

SDValue Replacement;

5395

// Is there a value for which the Predicate does *NOT* match? What is it?

5396

auto SplatValue = llvm::find_if_not(Values, Predicate);

5397

if (SplatValue != Values.end()) {

5398

// Does Values consist only of SplatValue's and values matching Predicate?

5399

if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {

5400

return Value == *SplatValue || Predicate(Value);

5401

})) // Then we shall replace values matching predicate with SplatValue.

5402

Replacement = *SplatValue;

5403

}

5404

if (!Replacement) {

5405

// Oops, we did not find the "baseline" splat value.

5406

if (!AlternativeReplacement)

5407

return; // Nothing to do.

5408

// Let's replace with provided value then.

5409

Replacement = AlternativeReplacement;

5410

}

5411

std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);

5412

}

5413

5414

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE

5415

/// where the divisor is constant and the comparison target is zero,

5416

/// return a DAG expression that will generate the same comparison result

5417

/// using only multiplications, additions and shifts/rotations.

5418

/// Ref: "Hacker's Delight" 10-17.

5419

SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,

5420

SDValue CompTargetNode,

5421

ISD::CondCode Cond,

5422

DAGCombinerInfo &DCI,

5423

const SDLoc &DL) const {

5424

SmallVector<SDNode *, 5> Built;

5425

if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5426

DCI, DL, Built)) {

5427

for (SDNode *N : Built)

5428

DCI.AddToWorklist(N);

5429

return Folded;

5430

}

5431

5432

return SDValue();

5433

}

5434

5435

SDValue

5436

TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,

5437

SDValue CompTargetNode, ISD::CondCode Cond,

5438

DAGCombinerInfo &DCI, const SDLoc &DL,

5439

SmallVectorImpl<SDNode *> &Created) const {

5440

// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)

5441

// - D must be constant, with D = D0 * 2^K where D0 is odd

5442

// - P is the multiplicative inverse of D0 modulo 2^W

5443

// - Q = floor(((2^W) - 1) / D)

5444

// where W is the width of the common type of N and D.

5445

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5446, __PRETTY_FUNCTION__))

5446

"Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5446, __PRETTY_FUNCTION__));

5447

5448

SelectionDAG &DAG = DCI.DAG;

5449

5450

EVT VT = REMNode.getValueType();

5451

EVT SVT = VT.getScalarType();

5452

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5453

EVT ShSVT = ShVT.getScalarType();

5454

5455

// If MUL is unavailable, we cannot proceed in any case.

5456

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5457

return SDValue();

5458

5459

bool ComparingWithAllZeros = true;

5460

bool AllComparisonsWithNonZerosAreTautological = true;

5461

bool HadTautologicalLanes = false;

5462

bool AllLanesAreTautological = true;

5463

bool HadEvenDivisor = false;

5464

bool AllDivisorsArePowerOfTwo = true;

5465

bool HadTautologicalInvertedLanes = false;

5466

SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

5467

5468

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {

5469

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5470

if (CDiv->isNullValue())

5471

return false;

5472

5473

const APInt &D = CDiv->getAPIntValue();

5474

const APInt &Cmp = CCmp->getAPIntValue();

5475

5476

ComparingWithAllZeros &= Cmp.isNullValue();

5477

5478

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5479

// if C2 is not less than C1, the comparison is always false.

5480

// But we will only be able to produce the comparison that will give the

5481

// opposive tautological answer. So this lane would need to be fixed up.

5482

bool TautologicalInvertedLane = D.ule(Cmp);

5483

HadTautologicalInvertedLanes |= TautologicalInvertedLane;

5484

5485

// If all lanes are tautological (either all divisors are ones, or divisor

5486

// is not greater than the constant we are comparing with),

5487

// we will prefer to avoid the fold.

5488

bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;

5489

HadTautologicalLanes |= TautologicalLane;

5490

AllLanesAreTautological &= TautologicalLane;

5491

5492

// If we are comparing with non-zero, we need'll need to subtract said

5493

// comparison value from the LHS. But there is no point in doing that if

5494

// every lane where we are comparing with non-zero is tautological..

5495

if (!Cmp.isNullValue())

5496

AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

5497

5498

// Decompose D into D0 * 2^K

5499

unsigned K = D.countTrailingZeros();

5500

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __PRETTY_FUNCTION__));

5501

APInt D0 = D.lshr(K);

5502

5503

// D is even if it has trailing zeros.

5504

HadEvenDivisor |= (K != 0);

5505

// D is a power-of-two if D0 is one.

5506

// If all divisors are power-of-two, we will prefer to avoid the fold.

5507

AllDivisorsArePowerOfTwo &= D0.isOneValue();

5508

5509

// P = inv(D0, 2^W)

5510

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5511

unsigned W = D.getBitWidth();

5512

APInt P = D0.zext(W + 1)

5513

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5514

.trunc(W);

5515

assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5515, __PRETTY_FUNCTION__)); // unreachable

5516

assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5516, __PRETTY_FUNCTION__));

5517

5518

// Q = floor((2^W - 1) u/ D)

5519

// R = ((2^W - 1) u% D)

5520

APInt Q, R;

5521

APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);

5522

5523

// If we are comparing with zero, then that comparison constant is okay,

5524

// else it may need to be one less than that.

5525

if (Cmp.ugt(R))

5526

Q -= 1;

5527

5528

assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5529, __PRETTY_FUNCTION__))

5529

"We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5529, __PRETTY_FUNCTION__));

5530

5531

// If the lane is tautological the result can be constant-folded.

5532

if (TautologicalLane) {

5533

// Set P and K amount to a bogus values so we can try to splat them.

5534

P = 0;

5535

K = -1;

5536

// And ensure that comparison constant is tautological,

5537

// it will always compare true/false.

5538

Q = -1;

5539

}

5540

5541

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5542

KAmts.push_back(

5543

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5544

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5545

return true;

5546

};

5547

5548

SDValue N = REMNode.getOperand(0);

5549

SDValue D = REMNode.getOperand(1);

5550

5551

// Collect the values from each element.

5552

if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))

5553

return SDValue();

5554

5555

// If all lanes are tautological, the result can be constant-folded.

5556

if (AllLanesAreTautological)

5557

return SDValue();

5558

5559

// If this is a urem by a powers-of-two, avoid the fold since it can be

5560

// best implemented as a bit test.

5561

if (AllDivisorsArePowerOfTwo)

5562

return SDValue();

5563

5564

SDValue PVal, KVal, QVal;

5565

if (VT.isVector()) {

5566

if (HadTautologicalLanes) {

5567

// Try to turn PAmts into a splat, since we don't care about the values

5568

// that are currently '0'. If we can't, just keep '0'`s.

5569

turnVectorIntoSplatVector(PAmts, isNullConstant);

5570

// Try to turn KAmts into a splat, since we don't care about the values

5571

// that are currently '-1'. If we can't, change them to '0'`s.

5572

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5573

DAG.getConstant(0, DL, ShSVT));

5574

}

5575

5576

PVal = DAG.getBuildVector(VT, DL, PAmts);

5577

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5578

QVal = DAG.getBuildVector(VT, DL, QAmts);

5579

} else {

5580

PVal = PAmts[0];

5581

KVal = KAmts[0];

5582

QVal = QAmts[0];

5583

}

5584

5585

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {

5586

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))

5587

return SDValue(); // FIXME: Could/should use `ISD::ADD`?

5588

assert(CompTargetNode.getValueType() == N.getValueType() &&((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5589, __PRETTY_FUNCTION__))

5589

"Expecting that the types on LHS and RHS of comparisons match.")((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5589, __PRETTY_FUNCTION__));

5590

N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);

5591

}

5592

5593

// (mul N, P)

5594

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5595

Created.push_back(Op0.getNode());

5596

5597

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5598

// divisors as a performance improvement, since rotating by 0 is a no-op.

5599

if (HadEvenDivisor) {

5600

// We need ROTR to do this.

5601

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5602

return SDValue();

5603

SDNodeFlags Flags;

5604

Flags.setExact(true);

5605

// UREM: (rotr (mul N, P), K)

5606

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);

5607

Created.push_back(Op0.getNode());

5608

}

5609

5610

// UREM: (setule/setugt (rotr (mul N, P), K), Q)

5611

SDValue NewCC =

5612

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5613

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5614

if (!HadTautologicalInvertedLanes)

5615

return NewCC;

5616

5617

// If any lanes previously compared always-false, the NewCC will give

5618

// always-true result for them, so we need to fixup those lanes.

5619

// Or the other way around for inequality predicate.

5620

assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5620, __PRETTY_FUNCTION__));

5621

Created.push_back(NewCC.getNode());

5622

5623

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5624

// if C2 is not less than C1, the comparison is always false.

5625

// But we have produced the comparison that will give the

5626

// opposive tautological answer. So these lanes would need to be fixed up.

5627

SDValue TautologicalInvertedChannels =

5628

DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);

5629

Created.push_back(TautologicalInvertedChannels.getNode());

5630

5631

// NOTE: we avoid letting illegal types through even if we're before legalize

5632

// ops – legalization has a hard time producing good code for this.

5633

if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {

5634

// If we have a vector select, let's replace the comparison results in the

5635

// affected lanes with the correct tautological result.

5636

SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,

5637

DL, SETCCVT, SETCCVT);

5638

return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,

5639

Replacement, NewCC);

5640

}

5641

5642

// Else, we can just invert the comparison result in the appropriate lanes.

5643

//

5644

// NOTE: see the note above VSELECT above.

5645

if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))

5646

return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,

5647

TautologicalInvertedChannels);

5648

5649

return SDValue(); // Don't know how to lower.

5650

}

5651

5652

/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE

5653

/// where the divisor is constant and the comparison target is zero,

5654

/// return a DAG expression that will generate the same comparison result

5655

/// using only multiplications, additions and shifts/rotations.

5656

/// Ref: "Hacker's Delight" 10-17.

5657

SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,

5658

SDValue CompTargetNode,

5659

ISD::CondCode Cond,

5660

DAGCombinerInfo &DCI,

5661

const SDLoc &DL) const {

5662

SmallVector<SDNode *, 7> Built;

5663

if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5664

DCI, DL, Built)) {

5665

assert(Built.size() <= 7 && "Max size prediction failed.")((Built.size() <= 7 && "Max size prediction failed."
) ? static_cast<void> (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5665, __PRETTY_FUNCTION__));

5666

for (SDNode *N : Built)

5667

DCI.AddToWorklist(N);

5668

return Folded;

5669

}

5670

5671

return SDValue();

5672

}

5673

5674

SDValue

5675

TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,

5676

SDValue CompTargetNode, ISD::CondCode Cond,

5677

DAGCombinerInfo &DCI, const SDLoc &DL,

5678

SmallVectorImpl<SDNode *> &Created) const {

5679

// Fold:

5680

// (seteq/ne (srem N, D), 0)

5681

// To:

5682

// (setule/ugt (rotr (add (mul N, P), A), K), Q)

5683

//

5684

// - D must be constant, with D = D0 * 2^K where D0 is odd

5685

// - P is the multiplicative inverse of D0 modulo 2^W

5686

// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))

5687

// - Q = floor((2 * A) / (2^K))

5688

// where W is the width of the common type of N and D.

5689

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5690, __PRETTY_FUNCTION__))

5690

"Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5690, __PRETTY_FUNCTION__));

5691

5692

SelectionDAG &DAG = DCI.DAG;

5693

5694

EVT VT = REMNode.getValueType();

5695

EVT SVT = VT.getScalarType();

5696

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5697

EVT ShSVT = ShVT.getScalarType();

5698

5699

// If we are after ops legalization, and MUL is unavailable, we can not

5700

// proceed.

5701

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5702

return SDValue();

5703

5704

// TODO: Could support comparing with non-zero too.

5705

ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);

5706

if (!CompTarget || !CompTarget->isNullValue())

5707

return SDValue();

5708

5709

bool HadIntMinDivisor = false;

5710

bool HadOneDivisor = false;

5711

bool AllDivisorsAreOnes = true;

5712

bool HadEvenDivisor = false;

5713

bool NeedToApplyOffset = false;

5714

bool AllDivisorsArePowerOfTwo = true;

5715

SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

5716

5717

auto BuildSREMPattern = [&](ConstantSDNode *C) {

5718

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5719

if (C->isNullValue())

5720

return false;

5721

5722

// FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

5723

5724

// WARNING: this fold is only valid for positive divisors!

5725

APInt D = C->getAPIntValue();

5726

if (D.isNegative())

5727

D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`

5728

5729

HadIntMinDivisor |= D.isMinSignedValue();

5730

5731

// If all divisors are ones, we will prefer to avoid the fold.

5732

HadOneDivisor |= D.isOneValue();

5733

AllDivisorsAreOnes &= D.isOneValue();

5734

5735

// Decompose D into D0 * 2^K

5736

unsigned K = D.countTrailingZeros();

5737

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5737, __PRETTY_FUNCTION__));

5738

APInt D0 = D.lshr(K);

5739

5740

if (!D.isMinSignedValue()) {

5741

// D is even if it has trailing zeros; unless it's INT_MIN, in which case

5742

// we don't care about this lane in this fold, we'll special-handle it.

5743

HadEvenDivisor |= (K != 0);

5744

}

5745

5746

// D is a power-of-two if D0 is one. This includes INT_MIN.

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 7836, column 5 Value stored to 'IsScaledIndex' is never read

Bug Summary

Annotated Source Code