/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1

//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements the TargetLowering class.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "llvm/CodeGen/TargetLowering.h"

14

#include "llvm/ADT/STLExtras.h"

15

#include "llvm/CodeGen/CallingConvLower.h"

16

#include "llvm/CodeGen/MachineFrameInfo.h"

17

#include "llvm/CodeGen/MachineFunction.h"

18

#include "llvm/CodeGen/MachineJumpTableInfo.h"

19

#include "llvm/CodeGen/MachineRegisterInfo.h"

20

#include "llvm/CodeGen/SelectionDAG.h"

21

#include "llvm/CodeGen/TargetRegisterInfo.h"

22

#include "llvm/CodeGen/TargetSubtargetInfo.h"

23

#include "llvm/IR/DataLayout.h"

24

#include "llvm/IR/DerivedTypes.h"

25

#include "llvm/IR/GlobalVariable.h"

26

#include "llvm/IR/LLVMContext.h"

27

#include "llvm/MC/MCAsmInfo.h"

28

#include "llvm/MC/MCExpr.h"

29

#include "llvm/Support/ErrorHandling.h"

30

#include "llvm/Support/KnownBits.h"

31

#include "llvm/Support/MathExtras.h"

32

#include "llvm/Target/TargetLoweringObjectFile.h"

33

#include "llvm/Target/TargetMachine.h"

34

#include <cctype>

35

using namespace llvm;

36

37

/// NOTE: The TargetMachine owns TLOF.

38

TargetLowering::TargetLowering(const TargetMachine &tm)

39

: TargetLoweringBase(tm) {}

40

41

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {

42

return nullptr;

43

}

44

45

bool TargetLowering::isPositionIndependent() const {

46

return getTargetMachine().isPositionIndependent();

47

}

48

49

/// Check whether a given call node is in tail position within its function. If

50

/// so, it sets Chain to the input chain of the tail call.

51

bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,

52

SDValue &Chain) const {

53

const Function &F = DAG.getMachineFunction().getFunction();

54

55

// First, check if tail calls have been disabled in this function.

56

if (F.getFnAttribute("disable-tail-calls").getValueAsBool())

57

return false;

58

59

// Conservatively require the attributes of the call to match those of

60

// the return. Ignore following attributes because they don't affect the

61

// call sequence.

62

AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);

63

for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,

64

Attribute::DereferenceableOrNull, Attribute::NoAlias,

65

Attribute::NonNull})

66

CallerAttrs.removeAttribute(Attr);

67

68

if (CallerAttrs.hasAttributes())

69

return false;

70

71

// It's not safe to eliminate the sign / zero extension of the return value.

72

if (CallerAttrs.contains(Attribute::ZExt) ||

73

CallerAttrs.contains(Attribute::SExt))

74

return false;

75

76

// Check if the only use is a function return node.

77

return isUsedByReturnOnly(Node, Chain);

78

}

79

80

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,

81

const uint32_t *CallerPreservedMask,

82

const SmallVectorImpl<CCValAssign> &ArgLocs,

83

const SmallVectorImpl<SDValue> &OutVals) const {

84

for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {

85

const CCValAssign &ArgLoc = ArgLocs[I];

86

if (!ArgLoc.isRegLoc())

87

continue;

88

MCRegister Reg = ArgLoc.getLocReg();

89

// Only look at callee saved registers.

90

if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))

91

continue;

92

// Check that we pass the value used for the caller.

93

// (We look for a CopyFromReg reading a virtual register that is used

94

// for the function live-in value of register Reg)

95

SDValue Value = OutVals[I];

96

if (Value->getOpcode() != ISD::CopyFromReg)

97

return false;

98

Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();

99

if (MRI.getLiveInPhysReg(ArgReg) != Reg)

100

return false;

101

}

102

return true;

103

}

104

105

/// Set CallLoweringInfo attribute flags based on a call instruction

106

/// and called function attributes.

107

void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,

108

unsigned ArgIdx) {

109

IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);

110

IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);

111

IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);

112

IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);

113

IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);

114

IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);

115

IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);

116

IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);

117

IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);

118

IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);

119

IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);

120

IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);

121

Alignment = Call->getParamStackAlign(ArgIdx);

122

ByValType = nullptr;

123

if (IsByVal) {

124

ByValType = Call->getParamByValType(ArgIdx);

125

if (!Alignment)

126

Alignment = Call->getParamAlign(ArgIdx);

127

}

128

PreallocatedType = nullptr;

129

if (IsPreallocated)

130

PreallocatedType = Call->getParamPreallocatedType(ArgIdx);

131

}

132

133

/// Generate a libcall taking the given operands as arguments and returning a

134

/// result of type RetVT.

135

std::pair<SDValue, SDValue>

136

TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,

137

ArrayRef<SDValue> Ops,

138

MakeLibCallOptions CallOptions,

139

const SDLoc &dl,

140

SDValue InChain) const {

141

if (!InChain)

142

InChain = DAG.getEntryNode();

143

144

TargetLowering::ArgListTy Args;

145

Args.reserve(Ops.size());

146

147

TargetLowering::ArgListEntry Entry;

148

for (unsigned i = 0; i < Ops.size(); ++i) {

149

SDValue NewOp = Ops[i];

150

Entry.Node = NewOp;

151

Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());

152

Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),

153

CallOptions.IsSExt);

154

Entry.IsZExt = !Entry.IsSExt;

155

156

if (CallOptions.IsSoften &&

157

!shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {

158

Entry.IsSExt = Entry.IsZExt = false;

159

}

160

Args.push_back(Entry);

161

}

162

163

if (LC == RTLIB::UNKNOWN_LIBCALL)

164

report_fatal_error("Unsupported library call operation!");

165

SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),

166

getPointerTy(DAG.getDataLayout()));

167

168

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());

169

TargetLowering::CallLoweringInfo CLI(DAG);

170

bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);

171

bool zeroExtend = !signExtend;

172

173

if (CallOptions.IsSoften &&

174

!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {

175

signExtend = zeroExtend = false;

176

}

177

178

CLI.setDebugLoc(dl)

179

.setChain(InChain)

180

.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))

181

.setNoReturn(CallOptions.DoesNotReturn)

182

.setDiscardResult(!CallOptions.IsReturnValueUsed)

183

.setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)

184

.setSExtResult(signExtend)

185

.setZExtResult(zeroExtend);

186

return LowerCallTo(CLI);

187

}

188

189

bool TargetLowering::findOptimalMemOpLowering(

190

std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,

191

unsigned SrcAS, const AttributeList &FuncAttributes) const {

192

if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())

193

return false;

194

195

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

196

197

if (VT == MVT::Other) {

198

// Use the largest integer type whose alignment constraints are satisfied.

199

// We only need to check DstAlign here as SrcAlign is always greater or

200

// equal to DstAlign (or zero).

201

VT = MVT::i64;

202

if (Op.isFixedDstAlign())

203

while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

204

!allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))

205

VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);

206

assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 206, __extension__ __PRETTY_FUNCTION__));

207

208

// Find the largest legal integer type.

209

MVT LVT = MVT::i64;

210

while (!isTypeLegal(LVT))

211

LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);

212

assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 212, __extension__ __PRETTY_FUNCTION__));

213

214

// If the type we've chosen is larger than the largest legal integer type

215

// then use that instead.

216

if (VT.bitsGT(LVT))

217

VT = LVT;

218

}

219

220

unsigned NumMemOps = 0;

221

uint64_t Size = Op.size();

222

while (Size) {

223

unsigned VTSize = VT.getSizeInBits() / 8;

224

while (VTSize > Size) {

225

// For now, only use non-vector load / store's for the left-over pieces.

226

EVT NewVT = VT;

227

unsigned NewVTSize;

228

229

bool Found = false;

230

if (VT.isVector() || VT.isFloatingPoint()) {

231

NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;

232

if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&

233

isSafeMemOpType(NewVT.getSimpleVT()))

234

Found = true;

235

else if (NewVT == MVT::i64 &&

236

isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&

237

isSafeMemOpType(MVT::f64)) {

238

// i64 is usually not legal on 32-bit targets, but f64 may be.

239

NewVT = MVT::f64;

240

Found = true;

241

}

242

}

243

244

if (!Found) {

245

do {

246

NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);

247

if (NewVT == MVT::i8)

248

break;

249

} while (!isSafeMemOpType(NewVT.getSimpleVT()));

250

}

251

NewVTSize = NewVT.getSizeInBits() / 8;

252

253

// If the new VT cannot cover all of the remaining bits, then consider

254

// issuing a (or a pair of) unaligned and overlapping load / store.

255

bool Fast;

256

if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&

257

allowsMisalignedMemoryAccesses(

258

VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),

259

MachineMemOperand::MONone, &Fast) &&

260

Fast)

261

VTSize = Size;

262

else {

263

VT = NewVT;

264

VTSize = NewVTSize;

265

}

266

}

267

268

if (++NumMemOps > Limit)

269

return false;

270

271

MemOps.push_back(VT);

272

Size -= VTSize;

273

}

274

275

return true;

276

}

277

278

/// Soften the operands of a comparison. This code is shared among BR_CC,

279

/// SELECT_CC, and SETCC handlers.

280

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

281

SDValue &NewLHS, SDValue &NewRHS,

282

ISD::CondCode &CCCode,

283

const SDLoc &dl, const SDValue OldLHS,

284

const SDValue OldRHS) const {

285

SDValue Chain;

286

return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,

287

OldRHS, Chain);

288

}

289

290

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

291

SDValue &NewLHS, SDValue &NewRHS,

292

ISD::CondCode &CCCode,

293

const SDLoc &dl, const SDValue OldLHS,

294

const SDValue OldRHS,

295

SDValue &Chain,

296

bool IsSignaling) const {

297

// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc

298

// not supporting it. We can update this code when libgcc provides such

299

// functions.

300

301

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 302, __extension__ __PRETTY_FUNCTION__))

302

&& "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 302, __extension__ __PRETTY_FUNCTION__));

303

304

// Expand into one or more soft-fp libcall(s).

305

RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;

306

bool ShouldInvertCC = false;

307

switch (CCCode) {

308

case ISD::SETEQ:

309

case ISD::SETOEQ:

310

LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

311

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

312

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

313

break;

314

case ISD::SETNE:

315

case ISD::SETUNE:

316

LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :

317

(VT == MVT::f64) ? RTLIB::UNE_F64 :

318

(VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;

319

break;

320

case ISD::SETGE:

321

case ISD::SETOGE:

322

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

323

(VT == MVT::f64) ? RTLIB::OGE_F64 :

324

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

325

break;

326

case ISD::SETLT:

327

case ISD::SETOLT:

328

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

329

(VT == MVT::f64) ? RTLIB::OLT_F64 :

330

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

331

break;

332

case ISD::SETLE:

333

case ISD::SETOLE:

334

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

335

(VT == MVT::f64) ? RTLIB::OLE_F64 :

336

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

337

break;

338

case ISD::SETGT:

339

case ISD::SETOGT:

340

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

341

(VT == MVT::f64) ? RTLIB::OGT_F64 :

342

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

343

break;

344

case ISD::SETO:

345

ShouldInvertCC = true;

346

LLVM_FALLTHROUGH[[gnu::fallthrough]];

347

case ISD::SETUO:

348

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

349

(VT == MVT::f64) ? RTLIB::UO_F64 :

350

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

351

break;

352

case ISD::SETONE:

353

// SETONE = O && UNE

354

ShouldInvertCC = true;

355

LLVM_FALLTHROUGH[[gnu::fallthrough]];

356

case ISD::SETUEQ:

357

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

358

(VT == MVT::f64) ? RTLIB::UO_F64 :

359

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

360

LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

361

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

362

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

363

break;

364

default:

365

// Invert CC for unordered comparisons

366

ShouldInvertCC = true;

367

switch (CCCode) {

368

case ISD::SETULT:

369

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

370

(VT == MVT::f64) ? RTLIB::OGE_F64 :

371

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

372

break;

373

case ISD::SETULE:

374

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

375

(VT == MVT::f64) ? RTLIB::OGT_F64 :

376

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

377

break;

378

case ISD::SETUGT:

379

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

380

(VT == MVT::f64) ? RTLIB::OLE_F64 :

381

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

382

break;

383

case ISD::SETUGE:

384

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

385

(VT == MVT::f64) ? RTLIB::OLT_F64 :

386

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

387

break;

388

default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 388);

389

}

390

}

391

392

// Use the target specific return value for comparions lib calls.

393

EVT RetVT = getCmpLibcallReturnType();

394

SDValue Ops[2] = {NewLHS, NewRHS};

395

TargetLowering::MakeLibCallOptions CallOptions;

396

EVT OpsVT[2] = { OldLHS.getValueType(),

397

OldRHS.getValueType() };

398

CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);

399

auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);

400

NewLHS = Call.first;

401

NewRHS = DAG.getConstant(0, dl, RetVT);

402

403

CCCode = getCmpLibcallCC(LC1);

404

if (ShouldInvertCC) {

405

assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 405, __extension__ __PRETTY_FUNCTION__));

406

CCCode = getSetCCInverse(CCCode, RetVT);

407

}

408

409

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {

410

// Update Chain.

411

Chain = Call.second;

412

} else {

413

EVT SetCCVT =

414

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);

415

SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);

416

auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);

417

CCCode = getCmpLibcallCC(LC2);

418

if (ShouldInvertCC)

419

CCCode = getSetCCInverse(CCCode, RetVT);

420

NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);

421

if (Chain)

422

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,

423

Call2.second);

424

NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,

425

Tmp.getValueType(), Tmp, NewLHS);

426

NewRHS = SDValue();

427

}

428

}

429

430

/// Return the entry encoding for a jump table in the current function. The

431

/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.

432

unsigned TargetLowering::getJumpTableEncoding() const {

433

// In non-pic modes, just use the address of a block.

434

if (!isPositionIndependent())

435

return MachineJumpTableInfo::EK_BlockAddress;

436

437

// In PIC mode, if the target supports a GPRel32 directive, use it.

438

if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)

439

return MachineJumpTableInfo::EK_GPRel32BlockAddress;

440

441

// Otherwise, use a label difference.

442

return MachineJumpTableInfo::EK_LabelDifference32;

443

}

444

445

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,

446

SelectionDAG &DAG) const {

447

// If our PIC model is GP relative, use the global offset table as the base.

448

unsigned JTEncoding = getJumpTableEncoding();

449

450

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||

451

(JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))

452

return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

453

454

return Table;

455

}

456

457

/// This returns the relocation base for the given PIC jumptable, the same as

458

/// getPICJumpTableRelocBase, but as an MCExpr.

459

const MCExpr *

460

TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,

461

unsigned JTI,MCContext &Ctx) const{

462

// The normal PIC reloc base is the label at the start of the jump table.

463

return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);

464

}

465

466

bool

467

TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

468

const TargetMachine &TM = getTargetMachine();

469

const GlobalValue *GV = GA->getGlobal();

470

471

// If the address is not even local to this DSO we will have to load it from

472

// a got and then add the offset.

473

if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))

474

return false;

475

476

// If the code is position independent we will have to add a base register.

477

if (isPositionIndependent())

478

return false;

479

480

// Otherwise we can do it.

481

return true;

482

}

483

484

//===----------------------------------------------------------------------===//

485

// Optimization Methods

486

//===----------------------------------------------------------------------===//

487

488

/// If the specified instruction has a constant integer operand and there are

489

/// bits set in that constant that are not demanded, then clear those bits and

490

/// return true.

491

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

492

const APInt &DemandedBits,

493

const APInt &DemandedElts,

494

TargetLoweringOpt &TLO) const {

495

SDLoc DL(Op);

496

unsigned Opcode = Op.getOpcode();

497

498

// Do target-specific constant optimization.

499

if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

500

return TLO.New.getNode();

501

502

// FIXME: ISD::SELECT, ISD::SELECT_CC

503

switch (Opcode) {

504

default:

505

break;

506

case ISD::XOR:

507

case ISD::AND:

508

case ISD::OR: {

509

auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));

510

if (!Op1C)

511

return false;

512

513

// If this is a 'not' op, don't touch it because that's a canonical form.

514

const APInt &C = Op1C->getAPIntValue();

515

if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))

516

return false;

517

518

if (!C.isSubsetOf(DemandedBits)) {

519

EVT VT = Op.getValueType();

520

SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);

521

SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);

522

return TLO.CombineTo(Op, NewOp);

523

}

524

525

break;

526

}

527

}

528

529

return false;

530

}

531

532

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

533

const APInt &DemandedBits,

534

TargetLoweringOpt &TLO) const {

535

EVT VT = Op.getValueType();

536

APInt DemandedElts = VT.isVector()

537

? APInt::getAllOnesValue(VT.getVectorNumElements())

538

: APInt(1, 1);

539

return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);

540

}

541

542

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.

543

/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be

544

/// generalized for targets with other types of implicit widening casts.

545

bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,

546

const APInt &Demanded,

547

TargetLoweringOpt &TLO) const {

548

assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 549, __extension__ __PRETTY_FUNCTION__))

549

"ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 549, __extension__ __PRETTY_FUNCTION__));

550

assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 551, __extension__ __PRETTY_FUNCTION__))

551

"ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 551, __extension__ __PRETTY_FUNCTION__));

552

553

SelectionDAG &DAG = TLO.DAG;

554

SDLoc dl(Op);

555

556

// Early return, as this function cannot handle vector types.

557

if (Op.getValueType().isVector())

558

return false;

559

560

// Don't do this if the node has another user, which may require the

561

// full value.

562

if (!Op.getNode()->hasOneUse())

563

return false;

564

565

// Search for the smallest integer type with free casts to and from

566

// Op's type. For expedience, just check power-of-2 integer types.

567

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

568

unsigned DemandedSize = Demanded.getActiveBits();

569

unsigned SmallVTBits = DemandedSize;

570

if (!isPowerOf2_32(SmallVTBits))

571

SmallVTBits = NextPowerOf2(SmallVTBits);

572

for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {

573

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);

574

if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&

575

TLI.isZExtFree(SmallVT, Op.getValueType())) {

576

// We found a type with free casts.

577

SDValue X = DAG.getNode(

578

Op.getOpcode(), dl, SmallVT,

579

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),

580

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));

581

assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 581, __extension__ __PRETTY_FUNCTION__));

582

SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);

583

return TLO.CombineTo(Op, Z);

584

}

585

}

586

return false;

587

}

588

589

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

590

DAGCombinerInfo &DCI) const {

591

SelectionDAG &DAG = DCI.DAG;

592

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

593

!DCI.isBeforeLegalizeOps());

594

KnownBits Known;

595

596

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);

597

if (Simplified) {

598

DCI.AddToWorklist(Op.getNode());

599

DCI.CommitTargetLoweringOpt(TLO);

600

}

601

return Simplified;

602

}

603

604

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

605

KnownBits &Known,

606

TargetLoweringOpt &TLO,

607

unsigned Depth,

608

bool AssumeSingleUse) const {

609

EVT VT = Op.getValueType();

610

611

// TODO: We can probably do more work on calculating the known bits and

612

// simplifying the operations for scalable vectors, but for now we just

613

// bail out.

614

if (VT.isScalableVector()) {

615

// Pretend we don't know anything for now.

616

Known = KnownBits(DemandedBits.getBitWidth());

617

return false;

618

}

619

620

APInt DemandedElts = VT.isVector()

621

? APInt::getAllOnesValue(VT.getVectorNumElements())

622

: APInt(1, 1);

623

return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,

624

AssumeSingleUse);

625

}

626

627

// TODO: Can we merge SelectionDAG::GetDemandedBits into this?

628

// TODO: Under what circumstances can we create nodes? Constant folding?

629

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

630

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

631

SelectionDAG &DAG, unsigned Depth) const {

632

// Limit search depth.

633

if (Depth >= SelectionDAG::MaxRecursionDepth)

634

return SDValue();

635

636

// Ignore UNDEFs.

637

if (Op.isUndef())

638

return SDValue();

639

640

// Not demanding any bits/elts from Op.

641

if (DemandedBits == 0 || DemandedElts == 0)

642

return DAG.getUNDEF(Op.getValueType());

643

644

unsigned NumElts = DemandedElts.getBitWidth();

645

unsigned BitWidth = DemandedBits.getBitWidth();

646

KnownBits LHSKnown, RHSKnown;

647

switch (Op.getOpcode()) {

648

case ISD::BITCAST: {

649

SDValue Src = peekThroughBitcasts(Op.getOperand(0));

650

EVT SrcVT = Src.getValueType();

651

EVT DstVT = Op.getValueType();

652

if (SrcVT == DstVT)

653

return Src;

654

655

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

656

unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

657

if (NumSrcEltBits == NumDstEltBits)

658

if (SDValue V = SimplifyMultipleUseDemandedBits(

659

Src, DemandedBits, DemandedElts, DAG, Depth + 1))

660

return DAG.getBitcast(DstVT, V);

661

662

// TODO - bigendian once we have test coverage.

663

if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&

664

DAG.getDataLayout().isLittleEndian()) {

665

unsigned Scale = NumDstEltBits / NumSrcEltBits;

666

unsigned NumSrcElts = SrcVT.getVectorNumElements();

667

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

668

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

669

for (unsigned i = 0; i != Scale; ++i) {

670

unsigned Offset = i * NumSrcEltBits;

671

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

672

if (!Sub.isNullValue()) {

673

DemandedSrcBits |= Sub;

674

for (unsigned j = 0; j != NumElts; ++j)

675

if (DemandedElts[j])

676

DemandedSrcElts.setBit((j * Scale) + i);

677

}

678

}

679

680

if (SDValue V = SimplifyMultipleUseDemandedBits(

681

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

682

return DAG.getBitcast(DstVT, V);

683

}

684

685

// TODO - bigendian once we have test coverage.

686

if ((NumSrcEltBits % NumDstEltBits) == 0 &&

687

DAG.getDataLayout().isLittleEndian()) {

688

unsigned Scale = NumSrcEltBits / NumDstEltBits;

689

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

690

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

691

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

692

for (unsigned i = 0; i != NumElts; ++i)

693

if (DemandedElts[i]) {

694

unsigned Offset = (i % Scale) * NumDstEltBits;

695

DemandedSrcBits.insertBits(DemandedBits, Offset);

696

DemandedSrcElts.setBit(i / Scale);

697

}

698

699

if (SDValue V = SimplifyMultipleUseDemandedBits(

700

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

701

return DAG.getBitcast(DstVT, V);

702

}

703

704

break;

705

}

706

case ISD::AND: {

707

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

708

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

709

710

// If all of the demanded bits are known 1 on one side, return the other.

711

// These bits cannot contribute to the result of the 'and' in this

712

// context.

713

if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))

714

return Op.getOperand(0);

715

if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))

716

return Op.getOperand(1);

717

break;

718

}

719

case ISD::OR: {

720

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

721

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

722

723

// If all of the demanded bits are known zero on one side, return the

724

// other. These bits cannot contribute to the result of the 'or' in this

725

// context.

726

if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))

727

return Op.getOperand(0);

728

if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))

729

return Op.getOperand(1);

730

break;

731

}

732

case ISD::XOR: {

733

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

734

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

735

736

// If all of the demanded bits are known zero on one side, return the

737

// other.

738

if (DemandedBits.isSubsetOf(RHSKnown.Zero))

739

return Op.getOperand(0);

740

if (DemandedBits.isSubsetOf(LHSKnown.Zero))

741

return Op.getOperand(1);

742

break;

743

}

744

case ISD::SHL: {

745

// If we are only demanding sign bits then we can use the shift source

746

// directly.

747

if (const APInt *MaxSA =

748

DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

749

SDValue Op0 = Op.getOperand(0);

750

unsigned ShAmt = MaxSA->getZExtValue();

751

unsigned NumSignBits =

752

DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

753

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

754

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

755

return Op0;

756

}

757

break;

758

}

759

case ISD::SETCC: {

760

SDValue Op0 = Op.getOperand(0);

761

SDValue Op1 = Op.getOperand(1);

762

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

763

// If (1) we only need the sign-bit, (2) the setcc operands are the same

764

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

765

// -1, we may be able to bypass the setcc.

766

if (DemandedBits.isSignMask() &&

767

Op0.getScalarValueSizeInBits() == BitWidth &&

768

getBooleanContents(Op0.getValueType()) ==

769

BooleanContent::ZeroOrNegativeOneBooleanContent) {

770

// If we're testing X < 0, then this compare isn't needed - just use X!

771

// FIXME: We're limiting to integer types here, but this should also work

772

// if we don't care about FP signed-zero. The use of SETLT with FP means

773

// that we don't care about NaNs.

774

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

775

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

776

return Op0;

777

}

778

break;

779

}

780

case ISD::SIGN_EXTEND_INREG: {

781

// If none of the extended bits are demanded, eliminate the sextinreg.

782

SDValue Op0 = Op.getOperand(0);

783

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

784

unsigned ExBits = ExVT.getScalarSizeInBits();

785

if (DemandedBits.getActiveBits() <= ExBits)

786

return Op0;

787

// If the input is already sign extended, just drop the extension.

788

unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

789

if (NumSignBits >= (BitWidth - ExBits + 1))

790

return Op0;

791

break;

792

}

793

case ISD::ANY_EXTEND_VECTOR_INREG:

794

case ISD::SIGN_EXTEND_VECTOR_INREG:

795

case ISD::ZERO_EXTEND_VECTOR_INREG: {

796

// If we only want the lowest element and none of extended bits, then we can

797

// return the bitcasted source vector.

798

SDValue Src = Op.getOperand(0);

799

EVT SrcVT = Src.getValueType();

800

EVT DstVT = Op.getValueType();

801

if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&

802

DAG.getDataLayout().isLittleEndian() &&

803

DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {

804

return DAG.getBitcast(DstVT, Src);

805

}

806

break;

807

}

808

case ISD::INSERT_VECTOR_ELT: {

809

// If we don't demand the inserted element, return the base vector.

810

SDValue Vec = Op.getOperand(0);

811

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

812

EVT VecVT = Vec.getValueType();

813

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&

814

!DemandedElts[CIdx->getZExtValue()])

815

return Vec;

816

break;

817

}

818

case ISD::INSERT_SUBVECTOR: {

819

// If we don't demand the inserted subvector, return the base vector.

820

SDValue Vec = Op.getOperand(0);

821

SDValue Sub = Op.getOperand(1);

822

uint64_t Idx = Op.getConstantOperandVal(2);

823

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

824

if (DemandedElts.extractBits(NumSubElts, Idx) == 0)

825

return Vec;

826

break;

827

}

828

case ISD::VECTOR_SHUFFLE: {

829

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

830

831

// If all the demanded elts are from one operand and are inline,

832

// then we can use the operand directly.

833

bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;

834

for (unsigned i = 0; i != NumElts; ++i) {

835

int M = ShuffleMask[i];

836

if (M < 0 || !DemandedElts[i])

837

continue;

838

AllUndef = false;

839

IdentityLHS &= (M == (int)i);

840

IdentityRHS &= ((M - NumElts) == i);

841

}

842

843

if (AllUndef)

844

return DAG.getUNDEF(Op.getValueType());

845

if (IdentityLHS)

846

return Op.getOperand(0);

847

if (IdentityRHS)

848

return Op.getOperand(1);

849

break;

850

}

851

default:

852

if (Op.getOpcode() >= ISD::BUILTIN_OP_END)

853

if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(

854

Op, DemandedBits, DemandedElts, DAG, Depth))

855

return V;

856

break;

857

}

858

return SDValue();

859

}

860

861

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

862

SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,

863

unsigned Depth) const {

864

EVT VT = Op.getValueType();

865

APInt DemandedElts = VT.isVector()

866

? APInt::getAllOnesValue(VT.getVectorNumElements())

867

: APInt(1, 1);

868

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

869

Depth);

870

}

871

872

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(

873

SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,

874

unsigned Depth) const {

875

APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());

876

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

877

Depth);

878

}

879

880

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the

881

/// result of Op are ever used downstream. If we can use this information to

882

/// simplify Op, create a new simplified DAG node and return true, returning the

883

/// original and new nodes in Old and New. Otherwise, analyze the expression and

884

/// return a mask of Known bits for the expression (used to simplify the

885

/// caller). The Known bits may only be accurate for those bits in the

886

/// OriginalDemandedBits and OriginalDemandedElts.

887

bool TargetLowering::SimplifyDemandedBits(

888

SDValue Op, const APInt &OriginalDemandedBits,

889

const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,

890

unsigned Depth, bool AssumeSingleUse) const {

891

unsigned BitWidth = OriginalDemandedBits.getBitWidth();

892

assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 893, __extension__ __PRETTY_FUNCTION__))

893

"Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 893, __extension__ __PRETTY_FUNCTION__));

894

895

// Don't know anything.

896

Known = KnownBits(BitWidth);

897

898

// TODO: We can probably do more work on calculating the known bits and

899

// simplifying the operations for scalable vectors, but for now we just

900

// bail out.

901

if (Op.getValueType().isScalableVector())

902

return false;

903

904

unsigned NumElts = OriginalDemandedElts.getBitWidth();

905

assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))

906

NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))

907

"Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__));

908

909

APInt DemandedBits = OriginalDemandedBits;

910

APInt DemandedElts = OriginalDemandedElts;

911

SDLoc dl(Op);

912

auto &DL = TLO.DAG.getDataLayout();

913

914

// Undef operand.

915

if (Op.isUndef())

916

return false;

917

918

if (Op.getOpcode() == ISD::Constant) {

919

// We know all of the bits for a constant!

920

Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());

921

return false;

922

}

923

924

if (Op.getOpcode() == ISD::ConstantFP) {

925

// We know all of the bits for a floating point constant!

926

Known = KnownBits::makeConstant(

927

cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());

928

return false;

929

}

930

931

// Other users may use these bits.

932

EVT VT = Op.getValueType();

933

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {

934

if (Depth != 0) {

935

// If not at the root, Just compute the Known bits to

936

// simplify things downstream.

937

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

938

return false;

939

}

940

// If this is the root being simplified, allow it to have multiple uses,

941

// just set the DemandedBits/Elts to all bits.

942

DemandedBits = APInt::getAllOnesValue(BitWidth);

943

DemandedElts = APInt::getAllOnesValue(NumElts);

944

} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {

945

// Not demanding any bits/elts from Op.

946

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

947

} else if (Depth >= SelectionDAG::MaxRecursionDepth) {

948

// Limit search depth.

949

return false;

950

}

951

952

KnownBits Known2;

953

switch (Op.getOpcode()) {

954

case ISD::TargetConstant:

955

llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 955);

956

case ISD::SCALAR_TO_VECTOR: {

957

if (!DemandedElts[0])

958

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

959

960

KnownBits SrcKnown;

961

SDValue Src = Op.getOperand(0);

962

unsigned SrcBitWidth = Src.getScalarValueSizeInBits();

963

APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);

964

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))

965

return true;

966

967

// Upper elements are undef, so only get the knownbits if we just demand

968

// the bottom element.

969

if (DemandedElts == 1)

970

Known = SrcKnown.anyextOrTrunc(BitWidth);

971

break;

972

}

973

case ISD::BUILD_VECTOR:

974

// Collect the known bits that are shared by every demanded element.

975

// TODO: Call SimplifyDemandedBits for non-constant demanded elements.

976

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

977

return false; // Don't fall through, will infinitely loop.

978

case ISD::LOAD: {

979

auto *LD = cast<LoadSDNode>(Op);

980

if (getTargetConstantFromLoad(LD)) {

981

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

982

return false; // Don't fall through, will infinitely loop.

983

}

984

if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {

985

// If this is a ZEXTLoad and we are looking at the loaded value.

986

EVT MemVT = LD->getMemoryVT();

987

unsigned MemBits = MemVT.getScalarSizeInBits();

988

Known.Zero.setBitsFrom(MemBits);

989

return false; // Don't fall through, will infinitely loop.

990

}

991

break;

992

}

993

case ISD::INSERT_VECTOR_ELT: {

994

SDValue Vec = Op.getOperand(0);

995

SDValue Scl = Op.getOperand(1);

996

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

997

EVT VecVT = Vec.getValueType();

998

999

// If index isn't constant, assume we need all vector elements AND the

1000

// inserted element.

1001

APInt DemandedVecElts(DemandedElts);

1002

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {

1003

unsigned Idx = CIdx->getZExtValue();

1004

DemandedVecElts.clearBit(Idx);

1005

1006

// Inserted element is not required.

1007

if (!DemandedElts[Idx])

1008

return TLO.CombineTo(Op, Vec);

1009

}

1010

1011

KnownBits KnownScl;

1012

unsigned NumSclBits = Scl.getScalarValueSizeInBits();

1013

APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);

1014

if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))

1015

return true;

1016

1017

Known = KnownScl.anyextOrTrunc(BitWidth);

1018

1019

KnownBits KnownVec;

1020

if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,

1021

Depth + 1))

1022

return true;

1023

1024

if (!!DemandedVecElts)

1025

Known = KnownBits::commonBits(Known, KnownVec);

1026

1027

return false;

1028

}

1029

case ISD::INSERT_SUBVECTOR: {

1030

// Demand any elements from the subvector and the remainder from the src its

1031

// inserted into.

1032

SDValue Src = Op.getOperand(0);

1033

SDValue Sub = Op.getOperand(1);

1034

uint64_t Idx = Op.getConstantOperandVal(2);

1035

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

1036

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

1037

APInt DemandedSrcElts = DemandedElts;

1038

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

1039

1040

KnownBits KnownSub, KnownSrc;

1041

if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,

1042

Depth + 1))

1043

return true;

1044

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,

1045

Depth + 1))

1046

return true;

1047

1048

Known.Zero.setAllBits();

1049

Known.One.setAllBits();

1050

if (!!DemandedSubElts)

1051

Known = KnownBits::commonBits(Known, KnownSub);

1052

if (!!DemandedSrcElts)

1053

Known = KnownBits::commonBits(Known, KnownSrc);

1054

1055

// Attempt to avoid multi-use src if we don't need anything from it.

1056

if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||

1057

!DemandedSrcElts.isAllOnesValue()) {

1058

SDValue NewSub = SimplifyMultipleUseDemandedBits(

1059

Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);

1060

SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1061

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1062

if (NewSub || NewSrc) {

1063

NewSub = NewSub ? NewSub : Sub;

1064

NewSrc = NewSrc ? NewSrc : Src;

1065

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,

1066

Op.getOperand(2));

1067

return TLO.CombineTo(Op, NewOp);

1068

}

1069

}

1070

break;

1071

}

1072

case ISD::EXTRACT_SUBVECTOR: {

1073

// Offset the demanded elts by the subvector index.

1074

SDValue Src = Op.getOperand(0);

1075

if (Src.getValueType().isScalableVector())

1076

break;

1077

uint64_t Idx = Op.getConstantOperandVal(1);

1078

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

1079

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

1080

1081

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,

1082

Depth + 1))

1083

return true;

1084

1085

// Attempt to avoid multi-use src if we don't need anything from it.

1086

if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {

1087

SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

1088

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1089

if (DemandedSrc) {

1090

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,

1091

Op.getOperand(1));

1092

return TLO.CombineTo(Op, NewOp);

1093

}

1094

}

1095

break;

1096

}

1097

case ISD::CONCAT_VECTORS: {

1098

Known.Zero.setAllBits();

1099

Known.One.setAllBits();

1100

EVT SubVT = Op.getOperand(0).getValueType();

1101

unsigned NumSubVecs = Op.getNumOperands();

1102

unsigned NumSubElts = SubVT.getVectorNumElements();

1103

for (unsigned i = 0; i != NumSubVecs; ++i) {

1104

APInt DemandedSubElts =

1105

DemandedElts.extractBits(NumSubElts, i * NumSubElts);

1106

if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,

1107

Known2, TLO, Depth + 1))

1108

return true;

1109

// Known bits are shared by every demanded subvector element.

1110

if (!!DemandedSubElts)

1111

Known = KnownBits::commonBits(Known, Known2);

1112

}

1113

break;

1114

}

1115

case ISD::VECTOR_SHUFFLE: {

1116

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

1117

1118

// Collect demanded elements from shuffle operands..

1119

APInt DemandedLHS(NumElts, 0);

1120

APInt DemandedRHS(NumElts, 0);

1121

for (unsigned i = 0; i != NumElts; ++i) {

1122

if (!DemandedElts[i])

1123

continue;

1124

int M = ShuffleMask[i];

1125

if (M < 0) {

1126

// For UNDEF elements, we don't know anything about the common state of

1127

// the shuffle result.

1128

DemandedLHS.clearAllBits();

1129

DemandedRHS.clearAllBits();

1130

break;

1131

}

1132

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1132, __extension__ __PRETTY_FUNCTION__));

1133

if (M < (int)NumElts)

1134

DemandedLHS.setBit(M);

1135

else

1136

DemandedRHS.setBit(M - NumElts);

1137

}

1138

1139

if (!!DemandedLHS || !!DemandedRHS) {

1140

SDValue Op0 = Op.getOperand(0);

1141

SDValue Op1 = Op.getOperand(1);

1142

1143

Known.Zero.setAllBits();

1144

Known.One.setAllBits();

1145

if (!!DemandedLHS) {

1146

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,

1147

Depth + 1))

1148

return true;

1149

Known = KnownBits::commonBits(Known, Known2);

1150

}

1151

if (!!DemandedRHS) {

1152

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,

1153

Depth + 1))

1154

return true;

1155

Known = KnownBits::commonBits(Known, Known2);

1156

}

1157

1158

// Attempt to avoid multi-use ops if we don't need anything from them.

1159

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1160

Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);

1161

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1162

Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);

1163

if (DemandedOp0 || DemandedOp1) {

1164

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1165

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1166

SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);

1167

return TLO.CombineTo(Op, NewOp);

1168

}

1169

}

1170

break;

1171

}

1172

case ISD::AND: {

1173

SDValue Op0 = Op.getOperand(0);

1174

SDValue Op1 = Op.getOperand(1);

1175

1176

// If the RHS is a constant, check to see if the LHS would be zero without

1177

// using the bits from the RHS. Below, we use knowledge about the RHS to

1178

// simplify the LHS, here we're using information from the LHS to simplify

1179

// the RHS.

1180

if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {

1181

// Do not increment Depth here; that can cause an infinite loop.

1182

KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);

1183

// If the LHS already has zeros where RHSC does, this 'and' is dead.

1184

if ((LHSKnown.Zero & DemandedBits) ==

1185

(~RHSC->getAPIntValue() & DemandedBits))

1186

return TLO.CombineTo(Op, Op0);

1187

1188

// If any of the set bits in the RHS are known zero on the LHS, shrink

1189

// the constant.

1190

if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,

1191

DemandedElts, TLO))

1192

return true;

1193

1194

// Bitwise-not (xor X, -1) is a special case: we don't usually shrink its

1195

// constant, but if this 'and' is only clearing bits that were just set by

1196

// the xor, then this 'and' can be eliminated by shrinking the mask of

1197

// the xor. For example, for a 32-bit X:

1198

// and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1

1199

if (isBitwiseNot(Op0) && Op0.hasOneUse() &&

1200

LHSKnown.One == ~RHSC->getAPIntValue()) {

1201

SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);

1202

return TLO.CombineTo(Op, Xor);

1203

}

1204

}

1205

1206

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1207

Depth + 1))

1208

return true;

1209

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1209, __extension__ __PRETTY_FUNCTION__));

1210

if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,

1211

Known2, TLO, Depth + 1))

1212

return true;

1213

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1213, __extension__ __PRETTY_FUNCTION__));

1214

1215

// Attempt to avoid multi-use ops if we don't need anything from them.

1216

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1217

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1218

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1219

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1220

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1221

if (DemandedOp0 || DemandedOp1) {

1222

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1223

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1224

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1225

return TLO.CombineTo(Op, NewOp);

1226

}

1227

}

1228

1229

// If all of the demanded bits are known one on one side, return the other.

1230

// These bits cannot contribute to the result of the 'and'.

1231

if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))

1232

return TLO.CombineTo(Op, Op0);

1233

if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))

1234

return TLO.CombineTo(Op, Op1);

1235

// If all of the demanded bits in the inputs are known zeros, return zero.

1236

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1237

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));

1238

// If the RHS is a constant, see if we can simplify it.

1239

if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,

1240

TLO))

1241

return true;

1242

// If the operation can be done in a smaller type, do so.

1243

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1244

return true;

1245

1246

Known &= Known2;

1247

break;

1248

}

1249

case ISD::OR: {

1250

SDValue Op0 = Op.getOperand(0);

1251

SDValue Op1 = Op.getOperand(1);

1252

1253

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1254

Depth + 1))

1255

return true;

1256

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1256, __extension__ __PRETTY_FUNCTION__));

1257

if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,

1258

Known2, TLO, Depth + 1))

1259

return true;

1260

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1260, __extension__ __PRETTY_FUNCTION__));

1261

1262

// Attempt to avoid multi-use ops if we don't need anything from them.

1263

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1264

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1265

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1266

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1267

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1268

if (DemandedOp0 || DemandedOp1) {

1269

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1270

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1271

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1272

return TLO.CombineTo(Op, NewOp);

1273

}

1274

}

1275

1276

// If all of the demanded bits are known zero on one side, return the other.

1277

// These bits cannot contribute to the result of the 'or'.

1278

if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))

1279

return TLO.CombineTo(Op, Op0);

1280

if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))

1281

return TLO.CombineTo(Op, Op1);

1282

// If the RHS is a constant, see if we can simplify it.

1283

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1284

return true;

1285

// If the operation can be done in a smaller type, do so.

1286

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1287

return true;

1288

1289

Known |= Known2;

1290

break;

1291

}

1292

case ISD::XOR: {

1293

SDValue Op0 = Op.getOperand(0);

1294

SDValue Op1 = Op.getOperand(1);

1295

1296

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1297

Depth + 1))

1298

return true;

1299

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__));

1300

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,

1301

Depth + 1))

1302

return true;

1303

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1303, __extension__ __PRETTY_FUNCTION__));

1304

1305

// Attempt to avoid multi-use ops if we don't need anything from them.

1306

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1307

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1308

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1309

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1310

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1311

if (DemandedOp0 || DemandedOp1) {

1312

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1313

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1314

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1315

return TLO.CombineTo(Op, NewOp);

1316

}

1317

}

1318

1319

// If all of the demanded bits are known zero on one side, return the other.

1320

// These bits cannot contribute to the result of the 'xor'.

1321

if (DemandedBits.isSubsetOf(Known.Zero))

1322

return TLO.CombineTo(Op, Op0);

1323

if (DemandedBits.isSubsetOf(Known2.Zero))

1324

return TLO.CombineTo(Op, Op1);

1325

// If the operation can be done in a smaller type, do so.

1326

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1327

return true;

1328

1329

// If all of the unknown bits are known to be zero on one side or the other

1330

// turn this into an *inclusive* or.

1331

// e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0

1332

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1333

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

1334

1335

ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);

1336

if (C) {

1337

// If one side is a constant, and all of the set bits in the constant are

1338

// also known set on the other side, turn this into an AND, as we know

1339

// the bits will be cleared.

1340

// e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2

1341

// NB: it is okay if more bits are known than are requested

1342

if (C->getAPIntValue() == Known2.One) {

1343

SDValue ANDC =

1344

TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);

1345

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));

1346

}

1347

1348

// If the RHS is a constant, see if we can change it. Don't alter a -1

1349

// constant because that's a 'not' op, and that is better for combining

1350

// and codegen.

1351

if (!C->isAllOnesValue() &&

1352

DemandedBits.isSubsetOf(C->getAPIntValue())) {

1353

// We're flipping all demanded bits. Flip the undemanded bits too.

1354

SDValue New = TLO.DAG.getNOT(dl, Op0, VT);

1355

return TLO.CombineTo(Op, New);

1356

}

1357

}

1358

1359

// If we can't turn this into a 'not', try to shrink the constant.

1360

if (!C || !C->isAllOnesValue())

1361

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1362

return true;

1363

1364

Known ^= Known2;

1365

break;

1366

}

1367

case ISD::SELECT:

1368

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,

1369

Depth + 1))

1370

return true;

1371

if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,

1372

Depth + 1))

1373

return true;

1374

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1374, __extension__ __PRETTY_FUNCTION__));

1375

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1375, __extension__ __PRETTY_FUNCTION__));

1376

1377

// If the operands are constants, see if we can simplify them.

1378

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1379

return true;

1380

1381

// Only known if known in both the LHS and RHS.

1382

Known = KnownBits::commonBits(Known, Known2);

1383

break;

1384

case ISD::SELECT_CC:

1385

if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,

1386

Depth + 1))

1387

return true;

1388

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,

1389

Depth + 1))

1390

return true;

1391

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1391, __extension__ __PRETTY_FUNCTION__));

1392

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1392, __extension__ __PRETTY_FUNCTION__));

1393

1394

// If the operands are constants, see if we can simplify them.

1395

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1396

return true;

1397

1398

// Only known if known in both the LHS and RHS.

1399

Known = KnownBits::commonBits(Known, Known2);

1400

break;

1401

case ISD::SETCC: {

1402

SDValue Op0 = Op.getOperand(0);

1403

SDValue Op1 = Op.getOperand(1);

1404

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

1405

// If (1) we only need the sign-bit, (2) the setcc operands are the same

1406

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

1407

// -1, we may be able to bypass the setcc.

1408

if (DemandedBits.isSignMask() &&

1409

Op0.getScalarValueSizeInBits() == BitWidth &&

1410

getBooleanContents(Op0.getValueType()) ==

1411

BooleanContent::ZeroOrNegativeOneBooleanContent) {

1412

// If we're testing X < 0, then this compare isn't needed - just use X!

1413

// FIXME: We're limiting to integer types here, but this should also work

1414

// if we don't care about FP signed-zero. The use of SETLT with FP means

1415

// that we don't care about NaNs.

1416

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

1417

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

1418

return TLO.CombineTo(Op, Op0);

1419

1420

// TODO: Should we check for other forms of sign-bit comparisons?

1421

// Examples: X <= -1, X >= 0

1422

}

1423

if (getBooleanContents(Op0.getValueType()) ==

1424

TargetLowering::ZeroOrOneBooleanContent &&

1425

BitWidth > 1)

1426

Known.Zero.setBitsFrom(1);

1427

break;

1428

}

1429

case ISD::SHL: {

1430

SDValue Op0 = Op.getOperand(0);

1431

SDValue Op1 = Op.getOperand(1);

1432

EVT ShiftVT = Op1.getValueType();

1433

1434

if (const APInt *SA =

1435

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1436

unsigned ShAmt = SA->getZExtValue();

1437

if (ShAmt == 0)

1438

return TLO.CombineTo(Op, Op0);

1439

1440

// If this is ((X >>u C1) << ShAmt), see if we can simplify this into a

1441

// single shift. We can do this if the bottom bits (which are shifted

1442

// out) are never demanded.

1443

// TODO - support non-uniform vector amounts.

1444

if (Op0.getOpcode() == ISD::SRL) {

1445

if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {

1446

if (const APInt *SA2 =

1447

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1448

unsigned C1 = SA2->getZExtValue();

1449

unsigned Opc = ISD::SHL;

1450

int Diff = ShAmt - C1;

1451

if (Diff < 0) {

1452

Diff = -Diff;

1453

Opc = ISD::SRL;

1454

}

1455

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1456

return TLO.CombineTo(

1457

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1458

}

1459

}

1460

}

1461

1462

// Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits

1463

// are not demanded. This will likely allow the anyext to be folded away.

1464

// TODO - support non-uniform vector amounts.

1465

if (Op0.getOpcode() == ISD::ANY_EXTEND) {

1466

SDValue InnerOp = Op0.getOperand(0);

1467

EVT InnerVT = InnerOp.getValueType();

1468

unsigned InnerBits = InnerVT.getScalarSizeInBits();

1469

if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&

1470

isTypeDesirableForOp(ISD::SHL, InnerVT)) {

1471

EVT ShTy = getShiftAmountTy(InnerVT, DL);

1472

if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))

1473

ShTy = InnerVT;

1474

SDValue NarrowShl =

1475

TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,

1476

TLO.DAG.getConstant(ShAmt, dl, ShTy));

1477

return TLO.CombineTo(

1478

Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));

1479

}

1480

1481

// Repeat the SHL optimization above in cases where an extension

1482

// intervenes: (shl (anyext (shr x, c1)), c2) to

1483

// (shl (anyext x), c2-c1). This requires that the bottom c1 bits

1484

// aren't demanded (as above) and that the shifted upper c1 bits of

1485

// x aren't demanded.

1486

// TODO - support non-uniform vector amounts.

1487

if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&

1488

InnerOp.hasOneUse()) {

1489

if (const APInt *SA2 =

1490

TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {

1491

unsigned InnerShAmt = SA2->getZExtValue();

1492

if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&

1493

DemandedBits.getActiveBits() <=

1494

(InnerBits - InnerShAmt + ShAmt) &&

1495

DemandedBits.countTrailingZeros() >= ShAmt) {

1496

SDValue NewSA =

1497

TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);

1498

SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,

1499

InnerOp.getOperand(0));

1500

return TLO.CombineTo(

1501

Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));

1502

}

1503

}

1504

}

1505

}

1506

1507

APInt InDemandedMask = DemandedBits.lshr(ShAmt);

1508

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1509

Depth + 1))

1510

return true;

1511

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1511, __extension__ __PRETTY_FUNCTION__));

1512

Known.Zero <<= ShAmt;

1513

Known.One <<= ShAmt;

1514

// low bits known zero.

1515

Known.Zero.setLowBits(ShAmt);

1516

1517

// Try shrinking the operation as long as the shift amount will still be

1518

// in range.

1519

if ((ShAmt < DemandedBits.getActiveBits()) &&

1520

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1521

return true;

1522

}

1523

1524

// If we are only demanding sign bits then we can use the shift source

1525

// directly.

1526

if (const APInt *MaxSA =

1527

TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

1528

unsigned ShAmt = MaxSA->getZExtValue();

1529

unsigned NumSignBits =

1530

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1531

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1532

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

1533

return TLO.CombineTo(Op, Op0);

1534

}

1535

break;

1536

}

1537

case ISD::SRL: {

1538

SDValue Op0 = Op.getOperand(0);

1539

SDValue Op1 = Op.getOperand(1);

1540

EVT ShiftVT = Op1.getValueType();

1541

1542

if (const APInt *SA =

1543

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1544

unsigned ShAmt = SA->getZExtValue();

1545

if (ShAmt == 0)

1546

return TLO.CombineTo(Op, Op0);

1547

1548

// If this is ((X << C1) >>u ShAmt), see if we can simplify this into a

1549

// single shift. We can do this if the top bits (which are shifted out)

1550

// are never demanded.

1551

// TODO - support non-uniform vector amounts.

1552

if (Op0.getOpcode() == ISD::SHL) {

1553

if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {

1554

if (const APInt *SA2 =

1555

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1556

unsigned C1 = SA2->getZExtValue();

1557

unsigned Opc = ISD::SRL;

1558

int Diff = ShAmt - C1;

1559

if (Diff < 0) {

1560

Diff = -Diff;

1561

Opc = ISD::SHL;

1562

}

1563

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1564

return TLO.CombineTo(

1565

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1566

}

1567

}

1568

}

1569

1570

APInt InDemandedMask = (DemandedBits << ShAmt);

1571

1572

// If the shift is exact, then it does demand the low bits (and knows that

1573

// they are zero).

1574

if (Op->getFlags().hasExact())

1575

InDemandedMask.setLowBits(ShAmt);

1576

1577

// Compute the new bits that are at the top now.

1578

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1579

Depth + 1))

1580

return true;

1581

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1581, __extension__ __PRETTY_FUNCTION__));

1582

Known.Zero.lshrInPlace(ShAmt);

1583

Known.One.lshrInPlace(ShAmt);

1584

// High bits known zero.

1585

Known.Zero.setHighBits(ShAmt);

1586

}

1587

break;

1588

}

1589

case ISD::SRA: {

1590

SDValue Op0 = Op.getOperand(0);

1591

SDValue Op1 = Op.getOperand(1);

1592

EVT ShiftVT = Op1.getValueType();

1593

1594

// If we only want bits that already match the signbit then we don't need

1595

// to shift.

1596

unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1597

if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=

1598

NumHiDemandedBits)

1599

return TLO.CombineTo(Op, Op0);

1600

1601

// If this is an arithmetic shift right and only the low-bit is set, we can

1602

// always convert this into a logical shr, even if the shift amount is

1603

// variable. The low bit of the shift cannot be an input sign bit unless

1604

// the shift amount is >= the size of the datatype, which is undefined.

1605

if (DemandedBits.isOneValue())

1606

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

1607

1608

if (const APInt *SA =

1609

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1610

unsigned ShAmt = SA->getZExtValue();

1611

if (ShAmt == 0)

1612

return TLO.CombineTo(Op, Op0);

1613

1614

APInt InDemandedMask = (DemandedBits << ShAmt);

1615

1616

// If the shift is exact, then it does demand the low bits (and knows that

1617

// they are zero).

1618

if (Op->getFlags().hasExact())

1619

InDemandedMask.setLowBits(ShAmt);

1620

1621

// If any of the demanded bits are produced by the sign extension, we also

1622

// demand the input sign bit.

1623

if (DemandedBits.countLeadingZeros() < ShAmt)

1624

InDemandedMask.setSignBit();

1625

1626

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1627

Depth + 1))

1628

return true;

1629

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1629, __extension__ __PRETTY_FUNCTION__));

1630

Known.Zero.lshrInPlace(ShAmt);

1631

Known.One.lshrInPlace(ShAmt);

1632

1633

// If the input sign bit is known to be zero, or if none of the top bits

1634

// are demanded, turn this into an unsigned shift right.

1635

if (Known.Zero[BitWidth - ShAmt - 1] ||

1636

DemandedBits.countLeadingZeros() >= ShAmt) {

1637

SDNodeFlags Flags;

1638

Flags.setExact(Op->getFlags().hasExact());

1639

return TLO.CombineTo(

1640

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));

1641

}

1642

1643

int Log2 = DemandedBits.exactLogBase2();

1644

if (Log2 >= 0) {

1645

// The bit must come from the sign.

1646

SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);

1647

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));

1648

}

1649

1650

if (Known.One[BitWidth - ShAmt - 1])

1651

// New bits are known one.

1652

Known.One.setHighBits(ShAmt);

1653

1654

// Attempt to avoid multi-use ops if we don't need anything from them.

1655

if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1656

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1657

Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);

1658

if (DemandedOp0) {

1659

SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);

1660

return TLO.CombineTo(Op, NewOp);

1661

}

1662

}

1663

}

1664

break;

1665

}

1666

case ISD::FSHL:

1667

case ISD::FSHR: {

1668

SDValue Op0 = Op.getOperand(0);

1669

SDValue Op1 = Op.getOperand(1);

1670

SDValue Op2 = Op.getOperand(2);

1671

bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

1672

1673

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {

1674

unsigned Amt = SA->getAPIntValue().urem(BitWidth);

1675

1676

// For fshl, 0-shift returns the 1st arg.

1677

// For fshr, 0-shift returns the 2nd arg.

1678

if (Amt == 0) {

1679

if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,

1680

Known, TLO, Depth + 1))

1681

return true;

1682

break;

1683

}

1684

1685

// fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))

1686

// fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)

1687

APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));

1688

APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);

1689

if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,

1690

Depth + 1))

1691

return true;

1692

if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,

1693

Depth + 1))

1694

return true;

1695

1696

Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));

1697

Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));

1698

Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1699

Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1700

Known.One |= Known2.One;

1701

Known.Zero |= Known2.Zero;

1702

}

1703

1704

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1705

if (isPowerOf2_32(BitWidth)) {

1706

APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);

1707

if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,

1708

Known2, TLO, Depth + 1))

1709

return true;

1710

}

1711

break;

1712

}

1713

case ISD::ROTL:

1714

case ISD::ROTR: {

1715

SDValue Op0 = Op.getOperand(0);

1716

SDValue Op1 = Op.getOperand(1);

1717

1718

// If we're rotating an 0/-1 value, then it stays an 0/-1 value.

1719

if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))

1720

return TLO.CombineTo(Op, Op0);

1721

1722

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1723

if (isPowerOf2_32(BitWidth)) {

1724

APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);

1725

if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,

1726

Depth + 1))

1727

return true;

1728

}

1729

break;

1730

}

1731

case ISD::UMIN: {

1732

// Check if one arg is always less than (or equal) to the other arg.

1733

SDValue Op0 = Op.getOperand(0);

1734

SDValue Op1 = Op.getOperand(1);

1735

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1736

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1737

Known = KnownBits::umin(Known0, Known1);

1738

if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))

1739

return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);

1740

if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))

1741

return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);

1742

break;

1743

}

1744

case ISD::UMAX: {

1745

// Check if one arg is always greater than (or equal) to the other arg.

1746

SDValue Op0 = Op.getOperand(0);

1747

SDValue Op1 = Op.getOperand(1);

1748

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1749

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1750

Known = KnownBits::umax(Known0, Known1);

1751

if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))

1752

return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);

1753

if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))

1754

return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);

1755

break;

1756

}

1757

case ISD::BITREVERSE: {

1758

SDValue Src = Op.getOperand(0);

1759

APInt DemandedSrcBits = DemandedBits.reverseBits();

1760

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1761

Depth + 1))

1762

return true;

1763

Known.One = Known2.One.reverseBits();

1764

Known.Zero = Known2.Zero.reverseBits();

1765

break;

1766

}

1767

case ISD::BSWAP: {

1768

SDValue Src = Op.getOperand(0);

1769

APInt DemandedSrcBits = DemandedBits.byteSwap();

1770

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1771

Depth + 1))

1772

return true;

1773

Known.One = Known2.One.byteSwap();

1774

Known.Zero = Known2.Zero.byteSwap();

1775

break;

1776

}

1777

case ISD::CTPOP: {

1778

// If only 1 bit is demanded, replace with PARITY as long as we're before

1779

// op legalization.

1780

// FIXME: Limit to scalars for now.

1781

if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())

1782

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,

1783

Op.getOperand(0)));

1784

1785

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

1786

break;

1787

}

1788

case ISD::SIGN_EXTEND_INREG: {

1789

SDValue Op0 = Op.getOperand(0);

1790

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

1791

unsigned ExVTBits = ExVT.getScalarSizeInBits();

1792

1793

// If we only care about the highest bit, don't bother shifting right.

1794

if (DemandedBits.isSignMask()) {

1795

unsigned NumSignBits =

1796

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1797

bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;

1798

// However if the input is already sign extended we expect the sign

1799

// extension to be dropped altogether later and do not simplify.

1800

if (!AlreadySignExtended) {

1801

// Compute the correct shift amount type, which must be getShiftAmountTy

1802

// for scalar types after legalization.

1803

EVT ShiftAmtTy = VT;

1804

if (TLO.LegalTypes() && !ShiftAmtTy.isVector())

1805

ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

1806

1807

SDValue ShiftAmt =

1808

TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);

1809

return TLO.CombineTo(Op,

1810

TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));

1811

}

1812

}

1813

1814

// If none of the extended bits are demanded, eliminate the sextinreg.

1815

if (DemandedBits.getActiveBits() <= ExVTBits)

1816

return TLO.CombineTo(Op, Op0);

1817

1818

APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

1819

1820

// Since the sign extended bits are demanded, we know that the sign

1821

// bit is demanded.

1822

InputDemandedBits.setBit(ExVTBits - 1);

1823

1824

if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))

1825

return true;

1826

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1826, __extension__ __PRETTY_FUNCTION__));

1827

1828

// If the sign bit of the input is known set or clear, then we know the

1829

// top bits of the result.

1830

1831

// If the input sign bit is known zero, convert this into a zero extension.

1832

if (Known.Zero[ExVTBits - 1])

1833

return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

1834

1835

APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);

1836

if (Known.One[ExVTBits - 1]) { // Input sign bit known set

1837

Known.One.setBitsFrom(ExVTBits);

1838

Known.Zero &= Mask;

1839

} else { // Input sign bit unknown

1840

Known.Zero &= Mask;

1841

Known.One &= Mask;

1842

}

1843

break;

1844

}

1845

case ISD::BUILD_PAIR: {

1846

EVT HalfVT = Op.getOperand(0).getValueType();

1847

unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

1848

1849

APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);

1850

APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

1851

1852

KnownBits KnownLo, KnownHi;

1853

1854

if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))

1855

return true;

1856

1857

if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))

1858

return true;

1859

1860

Known.Zero = KnownLo.Zero.zext(BitWidth) |

1861

KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

1862

1863

Known.One = KnownLo.One.zext(BitWidth) |

1864

KnownHi.One.zext(BitWidth).shl(HalfBitWidth);

1865

break;

1866

}

1867

case ISD::ZERO_EXTEND:

1868

case ISD::ZERO_EXTEND_VECTOR_INREG: {

1869

SDValue Src = Op.getOperand(0);

1870

EVT SrcVT = Src.getValueType();

1871

unsigned InBits = SrcVT.getScalarSizeInBits();

1872

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1873

bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

1874

1875

// If none of the top bits are demanded, convert this into an any_extend.

1876

if (DemandedBits.getActiveBits() <= InBits) {

1877

// If we only need the non-extended bits of the bottom element

1878

// then we can just bitcast to the result.

1879

if (IsVecInReg && DemandedElts == 1 &&

1880

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1881

TLO.DAG.getDataLayout().isLittleEndian())

1882

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1883

1884

unsigned Opc =

1885

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1886

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1887

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1888

}

1889

1890

APInt InDemandedBits = DemandedBits.trunc(InBits);

1891

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1892

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1893

Depth + 1))

1894

return true;

1895

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__));

1896

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1896, __extension__ __PRETTY_FUNCTION__));

1897

Known = Known.zext(BitWidth);

1898

1899

// Attempt to avoid multi-use ops if we don't need anything from them.

1900

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1901

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1902

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1903

break;

1904

}

1905

case ISD::SIGN_EXTEND:

1906

case ISD::SIGN_EXTEND_VECTOR_INREG: {

1907

SDValue Src = Op.getOperand(0);

1908

EVT SrcVT = Src.getValueType();

1909

unsigned InBits = SrcVT.getScalarSizeInBits();

1910

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1911

bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

1912

1913

// If none of the top bits are demanded, convert this into an any_extend.

1914

if (DemandedBits.getActiveBits() <= InBits) {

1915

// If we only need the non-extended bits of the bottom element

1916

// then we can just bitcast to the result.

1917

if (IsVecInReg && DemandedElts == 1 &&

1918

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1919

TLO.DAG.getDataLayout().isLittleEndian())

1920

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1921

1922

unsigned Opc =

1923

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1924

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1925

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1926

}

1927

1928

APInt InDemandedBits = DemandedBits.trunc(InBits);

1929

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1930

1931

// Since some of the sign extended bits are demanded, we know that the sign

1932

// bit is demanded.

1933

InDemandedBits.setBit(InBits - 1);

1934

1935

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1936

Depth + 1))

1937

return true;

1938

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1938, __extension__ __PRETTY_FUNCTION__));

1939

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1939, __extension__ __PRETTY_FUNCTION__));

1940

1941

// If the sign bit is known one, the top bits match.

1942

Known = Known.sext(BitWidth);

1943

1944

// If the sign bit is known zero, convert this to a zero extend.

1945

if (Known.isNonNegative()) {

1946

unsigned Opc =

1947

IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;

1948

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1949

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1950

}

1951

1952

// Attempt to avoid multi-use ops if we don't need anything from them.

1953

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1954

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1955

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1956

break;

1957

}

1958

case ISD::ANY_EXTEND:

1959

case ISD::ANY_EXTEND_VECTOR_INREG: {

1960

SDValue Src = Op.getOperand(0);

1961

EVT SrcVT = Src.getValueType();

1962

unsigned InBits = SrcVT.getScalarSizeInBits();

1963

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1964

bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

1965

1966

// If we only need the bottom element then we can just bitcast.

1967

// TODO: Handle ANY_EXTEND?

1968

if (IsVecInReg && DemandedElts == 1 &&

1969

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1970

TLO.DAG.getDataLayout().isLittleEndian())

1971

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1972

1973

APInt InDemandedBits = DemandedBits.trunc(InBits);

1974

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1975

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1976

Depth + 1))

1977

return true;

1978

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1978, __extension__ __PRETTY_FUNCTION__));

1979

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1979, __extension__ __PRETTY_FUNCTION__));

1980

Known = Known.anyext(BitWidth);

1981

1982

// Attempt to avoid multi-use ops if we don't need anything from them.

1983

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1984

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1985

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1986

break;

1987

}

1988

case ISD::TRUNCATE: {

1989

SDValue Src = Op.getOperand(0);

1990

1991

// Simplify the input, using demanded bit information, and compute the known

1992

// zero/one bits live out.

1993

unsigned OperandBitWidth = Src.getScalarValueSizeInBits();

1994

APInt TruncMask = DemandedBits.zext(OperandBitWidth);

1995

if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,

1996

Depth + 1))

1997

return true;

1998

Known = Known.trunc(BitWidth);

1999

2000

// Attempt to avoid multi-use ops if we don't need anything from them.

2001

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

2002

Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))

2003

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

2004

2005

// If the input is only used by this truncate, see if we can shrink it based

2006

// on the known demanded bits.

2007

if (Src.getNode()->hasOneUse()) {

2008

switch (Src.getOpcode()) {

2009

default:

2010

break;

2011

case ISD::SRL:

2012

// Shrink SRL by a constant if none of the high bits shifted in are

2013

// demanded.

2014

if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))

2015

// Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is

2016

// undesirable.

2017

break;

2018

2019

const APInt *ShAmtC =

2020

TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);

2021

if (!ShAmtC || ShAmtC->uge(BitWidth))

2022

break;

2023

uint64_t ShVal = ShAmtC->getZExtValue();

2024

2025

APInt HighBits =

2026

APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);

2027

HighBits.lshrInPlace(ShVal);

2028

HighBits = HighBits.trunc(BitWidth);

2029

2030

if (!(HighBits & DemandedBits)) {

2031

// None of the shifted in bits are needed. Add a truncate of the

2032

// shift input, then shift it.

2033

SDValue NewShAmt = TLO.DAG.getConstant(

2034

ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));

2035

SDValue NewTrunc =

2036

TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));

2037

return TLO.CombineTo(

2038

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));

2039

}

2040

break;

2041

}

2042

}

2043

2044

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2044, __extension__ __PRETTY_FUNCTION__));

2045

break;

2046

}

2047

case ISD::AssertZext: {

2048

// AssertZext demands all of the high bits, plus any of the low bits

2049

// demanded by its users.

2050

EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

2051

APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());

2052

if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,

2053

TLO, Depth + 1))

2054

return true;

2055

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__));

2056

2057

Known.Zero |= ~InMask;

2058

break;

2059

}

2060

case ISD::EXTRACT_VECTOR_ELT: {

2061

SDValue Src = Op.getOperand(0);

2062

SDValue Idx = Op.getOperand(1);

2063

ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();

2064

unsigned EltBitWidth = Src.getScalarValueSizeInBits();

2065

2066

if (SrcEltCnt.isScalable())

2067

return false;

2068

2069

// Demand the bits from every vector element without a constant index.

2070

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2071

APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);

2072

if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))

2073

if (CIdx->getAPIntValue().ult(NumSrcElts))

2074

DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

2075

2076

// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know

2077

// anything about the extended bits.

2078

APInt DemandedSrcBits = DemandedBits;

2079

if (BitWidth > EltBitWidth)

2080

DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

2081

2082

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,

2083

Depth + 1))

2084

return true;

2085

2086

// Attempt to avoid multi-use ops if we don't need anything from them.

2087

if (!DemandedSrcBits.isAllOnesValue() ||

2088

!DemandedSrcElts.isAllOnesValue()) {

2089

if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

2090

Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {

2091

SDValue NewOp =

2092

TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);

2093

return TLO.CombineTo(Op, NewOp);

2094

}

2095

}

2096

2097

Known = Known2;

2098

if (BitWidth > EltBitWidth)

2099

Known = Known.anyext(BitWidth);

2100

break;

2101

}

2102

case ISD::BITCAST: {

2103

SDValue Src = Op.getOperand(0);

2104

EVT SrcVT = Src.getValueType();

2105

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

2106

2107

// If this is an FP->Int bitcast and if the sign bit is the only

2108

// thing demanded, turn this into a FGETSIGN.

2109

if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&

2110

DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&

2111

SrcVT.isFloatingPoint()) {

2112

bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);

2113

bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);

2114

if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&

2115

SrcVT != MVT::f128) {

2116

// Cannot eliminate/lower SHL for f128 yet.

2117

EVT Ty = OpVTLegal ? VT : MVT::i32;

2118

// Make a FGETSIGN + SHL to move the sign bit into the appropriate

2119

// place. We expect the SHL to be eliminated by other optimizations.

2120

SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);

2121

unsigned OpVTSizeInBits = Op.getValueSizeInBits();

2122

if (!OpVTLegal && OpVTSizeInBits > 32)

2123

Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);

2124

unsigned ShVal = Op.getValueSizeInBits() - 1;

2125

SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);

2126

return TLO.CombineTo(Op,

2127

TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));

2128

}

2129

}

2130

2131

// Bitcast from a vector using SimplifyDemanded Bits/VectorElts.

2132

// Demand the elt/bit if any of the original elts/bits are demanded.

2133

// TODO - bigendian once we have test coverage.

2134

if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&

2135

TLO.DAG.getDataLayout().isLittleEndian()) {

2136

unsigned Scale = BitWidth / NumSrcEltBits;

2137

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2138

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2139

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2140

for (unsigned i = 0; i != Scale; ++i) {

2141

unsigned Offset = i * NumSrcEltBits;

2142

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

2143

if (!Sub.isNullValue()) {

2144

DemandedSrcBits |= Sub;

2145

for (unsigned j = 0; j != NumElts; ++j)

2146

if (DemandedElts[j])

2147

DemandedSrcElts.setBit((j * Scale) + i);

2148

}

2149

}

2150

2151

APInt KnownSrcUndef, KnownSrcZero;

2152

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2153

KnownSrcZero, TLO, Depth + 1))

2154

return true;

2155

2156

KnownBits KnownSrcBits;

2157

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2158

KnownSrcBits, TLO, Depth + 1))

2159

return true;

2160

} else if ((NumSrcEltBits % BitWidth) == 0 &&

2161

TLO.DAG.getDataLayout().isLittleEndian()) {

2162

unsigned Scale = NumSrcEltBits / BitWidth;

2163

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

2164

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2165

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2166

for (unsigned i = 0; i != NumElts; ++i)

2167

if (DemandedElts[i]) {

2168

unsigned Offset = (i % Scale) * BitWidth;

2169

DemandedSrcBits.insertBits(DemandedBits, Offset);

2170

DemandedSrcElts.setBit(i / Scale);

2171

}

2172

2173

if (SrcVT.isVector()) {

2174

APInt KnownSrcUndef, KnownSrcZero;

2175

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2176

KnownSrcZero, TLO, Depth + 1))

2177

return true;

2178

}

2179

2180

KnownBits KnownSrcBits;

2181

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2182

KnownSrcBits, TLO, Depth + 1))

2183

return true;

2184

}

2185

2186

// If this is a bitcast, let computeKnownBits handle it. Only do this on a

2187

// recursive call where Known may be useful to the caller.

2188

if (Depth > 0) {

2189

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2190

return false;

2191

}

2192

break;

2193

}

2194

case ISD::ADD:

2195

case ISD::MUL:

2196

case ISD::SUB: {

2197

// Add, Sub, and Mul don't demand any bits in positions beyond that

2198

// of the highest bit demanded of them.

2199

SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);

2200

SDNodeFlags Flags = Op.getNode()->getFlags();

2201

unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();

2202

APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

2203

if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,

2204

Depth + 1) ||

2205

SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,

2206

Depth + 1) ||

2207

// See if the operation should be performed at a smaller bit width.

2208

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {

2209

if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {

2210

// Disable the nsw and nuw flags. We can no longer guarantee that we

2211

// won't wrap after simplification.

2212

Flags.setNoSignedWrap(false);

2213

Flags.setNoUnsignedWrap(false);

2214

SDValue NewOp =

2215

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2216

return TLO.CombineTo(Op, NewOp);

2217

}

2218

return true;

2219

}

2220

2221

// Attempt to avoid multi-use ops if we don't need anything from them.

2222

if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

2223

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

2224

Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2225

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

2226

Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2227

if (DemandedOp0 || DemandedOp1) {

2228

Flags.setNoSignedWrap(false);

2229

Flags.setNoUnsignedWrap(false);

2230

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

2231

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

2232

SDValue NewOp =

2233

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2234

return TLO.CombineTo(Op, NewOp);

2235

}

2236

}

2237

2238

// If we have a constant operand, we may be able to turn it into -1 if we

2239

// do not demand the high bits. This can make the constant smaller to

2240

// encode, allow more general folding, or match specialized instruction

2241

// patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that

2242

// is probably not useful (and could be detrimental).

2243

ConstantSDNode *C = isConstOrConstSplat(Op1);

2244

APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);

2245

if (C && !C->isAllOnesValue() && !C->isOne() &&

2246

(C->getAPIntValue() | HighMask).isAllOnesValue()) {

2247

SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);

2248

// Disable the nsw and nuw flags. We can no longer guarantee that we

2249

// won't wrap after simplification.

2250

Flags.setNoSignedWrap(false);

2251

Flags.setNoUnsignedWrap(false);

2252

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);

2253

return TLO.CombineTo(Op, NewOp);

2254

}

2255

2256

LLVM_FALLTHROUGH[[gnu::fallthrough]];

2257

}

2258

default:

2259

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2260

if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,

2261

Known, TLO, Depth))

2262

return true;

2263

break;

2264

}

2265

2266

// Just use computeKnownBits to compute output bits.

2267

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2268

break;

2269

}

2270

2271

// If we know the value of all of the demanded bits, return this as a

2272

// constant.

2273

if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {

2274

// Avoid folding to a constant if any OpaqueConstant is involved.

2275

const SDNode *N = Op.getNode();

2276

for (SDNode *Op :

2277

llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {

2278

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))

2279

if (C->isOpaque())

2280

return false;

2281

}

2282

if (VT.isInteger())

2283

return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));

2284

if (VT.isFloatingPoint())

2285

return TLO.CombineTo(

2286

Op,

2287

TLO.DAG.getConstantFP(

2288

APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));

2289

}

2290

2291

return false;

2292

}

2293

2294

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,

2295

const APInt &DemandedElts,

2296

APInt &KnownUndef,

2297

APInt &KnownZero,

2298

DAGCombinerInfo &DCI) const {

2299

SelectionDAG &DAG = DCI.DAG;

2300

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

2301

!DCI.isBeforeLegalizeOps());

2302

2303

bool Simplified =

2304

SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);

2305

if (Simplified) {

2306

DCI.AddToWorklist(Op.getNode());

2307

DCI.CommitTargetLoweringOpt(TLO);

2308

}

2309

2310

return Simplified;

2311

}

2312

2313

/// Given a vector binary operation and known undefined elements for each input

2314

/// operand, compute whether each element of the output is undefined.

2315

static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,

2316

const APInt &UndefOp0,

2317

const APInt &UndefOp1) {

2318

EVT VT = BO.getValueType();

2319

assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2320, __extension__ __PRETTY_FUNCTION__))

2320

"Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2320, __extension__ __PRETTY_FUNCTION__));

2321

2322

EVT EltVT = VT.getVectorElementType();

2323

unsigned NumElts = VT.getVectorNumElements();

2324

assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2325, __extension__ __PRETTY_FUNCTION__))

2325

UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2325, __extension__ __PRETTY_FUNCTION__));

2326

2327

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,

2328

const APInt &UndefVals) {

2329

if (UndefVals[Index])

2330

return DAG.getUNDEF(EltVT);

2331

2332

if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {

2333

// Try hard to make sure that the getNode() call is not creating temporary

2334

// nodes. Ignore opaque integers because they do not constant fold.

2335

SDValue Elt = BV->getOperand(Index);

2336

auto *C = dyn_cast<ConstantSDNode>(Elt);

2337

if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))

2338

return Elt;

2339

}

2340

2341

return SDValue();

2342

};

2343

2344

APInt KnownUndef = APInt::getNullValue(NumElts);

2345

for (unsigned i = 0; i != NumElts; ++i) {

2346

// If both inputs for this element are either constant or undef and match

2347

// the element type, compute the constant/undef result for this element of

2348

// the vector.

2349

// TODO: Ideally we would use FoldConstantArithmetic() here, but that does

2350

// not handle FP constants. The code within getNode() should be refactored

2351

// to avoid the danger of creating a bogus temporary node here.

2352

SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);

2353

SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);

2354

if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)

2355

if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())

2356

KnownUndef.setBit(i);

2357

}

2358

return KnownUndef;

2359

}

2360

2361

bool TargetLowering::SimplifyDemandedVectorElts(

2362

SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,

2363

APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,

2364

bool AssumeSingleUse) const {

2365

EVT VT = Op.getValueType();

2366

unsigned Opcode = Op.getOpcode();

2367

APInt DemandedElts = OriginalDemandedElts;

2368

unsigned NumElts = DemandedElts.getBitWidth();

2369

assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2369, __extension__ __PRETTY_FUNCTION__));

2370

2371

KnownUndef = KnownZero = APInt::getNullValue(NumElts);

2372

2373

// TODO: For now we assume we know nothing about scalable vectors.

2374

if (VT.isScalableVector())

2375

return false;

2376

2377

assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2378, __extension__ __PRETTY_FUNCTION__))

2378

"Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2378, __extension__ __PRETTY_FUNCTION__));

2379

2380

// Undef operand.

2381

if (Op.isUndef()) {

2382

KnownUndef.setAllBits();

2383

return false;

2384

}

2385

2386

// If Op has other users, assume that all elements are needed.

2387

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)

2388

DemandedElts.setAllBits();

2389

2390

// Not demanding any elements from Op.

2391

if (DemandedElts == 0) {

2392

KnownUndef.setAllBits();

2393

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2394

}

2395

2396

// Limit search depth.

2397

if (Depth >= SelectionDAG::MaxRecursionDepth)

2398

return false;

2399

2400

SDLoc DL(Op);

2401

unsigned EltSizeInBits = VT.getScalarSizeInBits();

2402

2403

// Helper for demanding the specified elements and all the bits of both binary

2404

// operands.

2405

auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {

2406

SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,

2407

TLO.DAG, Depth + 1);

2408

SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,

2409

TLO.DAG, Depth + 1);

2410

if (NewOp0 || NewOp1) {

2411

SDValue NewOp = TLO.DAG.getNode(

2412

Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);

2413

return TLO.CombineTo(Op, NewOp);

2414

}

2415

return false;

2416

};

2417

2418

switch (Opcode) {

2419

case ISD::SCALAR_TO_VECTOR: {

2420

if (!DemandedElts[0]) {

2421

KnownUndef.setAllBits();

2422

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2423

}

2424

SDValue ScalarSrc = Op.getOperand(0);

2425

if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {

2426

SDValue Src = ScalarSrc.getOperand(0);

2427

SDValue Idx = ScalarSrc.getOperand(1);

2428

EVT SrcVT = Src.getValueType();

2429

2430

ElementCount SrcEltCnt = SrcVT.getVectorElementCount();

2431

2432

if (SrcEltCnt.isScalable())

2433

return false;

2434

2435

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2436

if (isNullConstant(Idx)) {

2437

APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);

2438

APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);

2439

APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);

2440

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2441

TLO, Depth + 1))

2442

return true;

2443

}

2444

}

2445

KnownUndef.setHighBits(NumElts - 1);

2446

break;

2447

}

2448

case ISD::BITCAST: {

2449

SDValue Src = Op.getOperand(0);

2450

EVT SrcVT = Src.getValueType();

2451

2452

// We only handle vectors here.

2453

// TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?

2454

if (!SrcVT.isVector())

2455

break;

2456

2457

// Fast handling of 'identity' bitcasts.

2458

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2459

if (NumSrcElts == NumElts)

2460

return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,

2461

KnownZero, TLO, Depth + 1);

2462

2463

APInt SrcZero, SrcUndef;

2464

APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);

2465

2466

// Bitcast from 'large element' src vector to 'small element' vector, we

2467

// must demand a source element if any DemandedElt maps to it.

2468

if ((NumElts % NumSrcElts) == 0) {

2469

unsigned Scale = NumElts / NumSrcElts;

2470

for (unsigned i = 0; i != NumElts; ++i)

2471

if (DemandedElts[i])

2472

SrcDemandedElts.setBit(i / Scale);

2473

2474

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2475

TLO, Depth + 1))

2476

return true;

2477

2478

// Try calling SimplifyDemandedBits, converting demanded elts to the bits

2479

// of the large element.

2480

// TODO - bigendian once we have test coverage.

2481

if (TLO.DAG.getDataLayout().isLittleEndian()) {

2482

unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();

2483

APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);

2484

for (unsigned i = 0; i != NumElts; ++i)

2485

if (DemandedElts[i]) {

2486

unsigned Ofs = (i % Scale) * EltSizeInBits;

2487

SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);

2488

}

2489

2490

KnownBits Known;

2491

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,

2492

TLO, Depth + 1))

2493

return true;

2494

}

2495

2496

// If the src element is zero/undef then all the output elements will be -

2497

// only demanded elements are guaranteed to be correct.

2498

for (unsigned i = 0; i != NumSrcElts; ++i) {

2499

if (SrcDemandedElts[i]) {

2500

if (SrcZero[i])

2501

KnownZero.setBits(i * Scale, (i + 1) * Scale);

2502

if (SrcUndef[i])

2503

KnownUndef.setBits(i * Scale, (i + 1) * Scale);

2504

}

2505

}

2506

}

2507

2508

// Bitcast from 'small element' src vector to 'large element' vector, we

2509

// demand all smaller source elements covered by the larger demanded element

2510

// of this vector.

2511

if ((NumSrcElts % NumElts) == 0) {

2512

unsigned Scale = NumSrcElts / NumElts;

2513

for (unsigned i = 0; i != NumElts; ++i)

2514

if (DemandedElts[i])

2515

SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);

2516

2517

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2518

TLO, Depth + 1))

2519

return true;

2520

2521

// If all the src elements covering an output element are zero/undef, then

2522

// the output element will be as well, assuming it was demanded.

2523

for (unsigned i = 0; i != NumElts; ++i) {

2524

if (DemandedElts[i]) {

2525

if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())

2526

KnownZero.setBit(i);

2527

if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())

2528

KnownUndef.setBit(i);

2529

}

2530

}

2531

}

2532

break;

2533

}

2534

case ISD::BUILD_VECTOR: {

2535

// Check all elements and simplify any unused elements with UNDEF.

2536

if (!DemandedElts.isAllOnesValue()) {

2537

// Don't simplify BROADCASTS.

2538

if (llvm::any_of(Op->op_values(),

2539

[&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {

2540

SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());

2541

bool Updated = false;

2542

for (unsigned i = 0; i != NumElts; ++i) {

2543

if (!DemandedElts[i] && !Ops[i].isUndef()) {

2544

Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());

2545

KnownUndef.setBit(i);

2546

Updated = true;

2547

}

2548

}

2549

if (Updated)

2550

return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));

2551

}

2552

}

2553

for (unsigned i = 0; i != NumElts; ++i) {

2554

SDValue SrcOp = Op.getOperand(i);

2555

if (SrcOp.isUndef()) {

2556

KnownUndef.setBit(i);

2557

} else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&

2558

(isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {

2559

KnownZero.setBit(i);

2560

}

2561

}

2562

break;

2563

}

2564

case ISD::CONCAT_VECTORS: {

2565

EVT SubVT = Op.getOperand(0).getValueType();

2566

unsigned NumSubVecs = Op.getNumOperands();

2567

unsigned NumSubElts = SubVT.getVectorNumElements();

2568

for (unsigned i = 0; i != NumSubVecs; ++i) {

2569

SDValue SubOp = Op.getOperand(i);

2570

APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);

2571

APInt SubUndef, SubZero;

2572

if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,

2573

Depth + 1))

2574

return true;

2575

KnownUndef.insertBits(SubUndef, i * NumSubElts);

2576

KnownZero.insertBits(SubZero, i * NumSubElts);

2577

}

2578

break;

2579

}

2580

case ISD::INSERT_SUBVECTOR: {

2581

// Demand any elements from the subvector and the remainder from the src its

2582

// inserted into.

2583

SDValue Src = Op.getOperand(0);

2584

SDValue Sub = Op.getOperand(1);

2585

uint64_t Idx = Op.getConstantOperandVal(2);

2586

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

2587

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

2588

APInt DemandedSrcElts = DemandedElts;

2589

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

2590

2591

APInt SubUndef, SubZero;

2592

if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,

2593

Depth + 1))

2594

return true;

2595

2596

// If none of the src operand elements are demanded, replace it with undef.

2597

if (!DemandedSrcElts && !Src.isUndef())

2598

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,

2599

TLO.DAG.getUNDEF(VT), Sub,

2600

Op.getOperand(2)));

2601

2602

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,

2603

TLO, Depth + 1))

2604

return true;

2605

KnownUndef.insertBits(SubUndef, Idx);

2606

KnownZero.insertBits(SubZero, Idx);

2607

2608

// Attempt to avoid multi-use ops if we don't need anything from them.

2609

if (!DemandedSrcElts.isAllOnesValue() ||

2610

!DemandedSubElts.isAllOnesValue()) {

2611

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2612

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2613

SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(

2614

Sub, DemandedSubElts, TLO.DAG, Depth + 1);

2615

if (NewSrc || NewSub) {

2616

NewSrc = NewSrc ? NewSrc : Src;

2617

NewSub = NewSub ? NewSub : Sub;

2618

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2619

NewSub, Op.getOperand(2));

2620

return TLO.CombineTo(Op, NewOp);

2621

}

2622

}

2623

break;

2624

}

2625

case ISD::EXTRACT_SUBVECTOR: {

2626

// Offset the demanded elts by the subvector index.

2627

SDValue Src = Op.getOperand(0);

2628

if (Src.getValueType().isScalableVector())

2629

break;

2630

uint64_t Idx = Op.getConstantOperandVal(1);

2631

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2632

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

2633

2634

APInt SrcUndef, SrcZero;

2635

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2636

Depth + 1))

2637

return true;

2638

KnownUndef = SrcUndef.extractBits(NumElts, Idx);

2639

KnownZero = SrcZero.extractBits(NumElts, Idx);

2640

2641

// Attempt to avoid multi-use ops if we don't need anything from them.

2642

if (!DemandedElts.isAllOnesValue()) {

2643

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2644

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2645

if (NewSrc) {

2646

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2647

Op.getOperand(1));

2648

return TLO.CombineTo(Op, NewOp);

2649

}

2650

}

2651

break;

2652

}

2653

case ISD::INSERT_VECTOR_ELT: {

2654

SDValue Vec = Op.getOperand(0);

2655

SDValue Scl = Op.getOperand(1);

2656

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

2657

2658

// For a legal, constant insertion index, if we don't need this insertion

2659

// then strip it, else remove it from the demanded elts.

2660

if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {

2661

unsigned Idx = CIdx->getZExtValue();

2662

if (!DemandedElts[Idx])

2663

return TLO.CombineTo(Op, Vec);

2664

2665

APInt DemandedVecElts(DemandedElts);

2666

DemandedVecElts.clearBit(Idx);

2667

if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,

2668

KnownZero, TLO, Depth + 1))

2669

return true;

2670

2671

KnownUndef.setBitVal(Idx, Scl.isUndef());

2672

2673

KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));

2674

break;

2675

}

2676

2677

APInt VecUndef, VecZero;

2678

if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,

2679

Depth + 1))

2680

return true;

2681

// Without knowing the insertion index we can't set KnownUndef/KnownZero.

2682

break;

2683

}

2684

case ISD::VSELECT: {

2685

// Try to transform the select condition based on the current demanded

2686

// elements.

2687

// TODO: If a condition element is undef, we can choose from one arm of the

2688

// select (and if one arm is undef, then we can propagate that to the

2689

// result).

2690

// TODO - add support for constant vselect masks (see IR version of this).

2691

APInt UnusedUndef, UnusedZero;

2692

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,

2693

UnusedZero, TLO, Depth + 1))

2694

return true;

2695

2696

// See if we can simplify either vselect operand.

2697

APInt DemandedLHS(DemandedElts);

2698

APInt DemandedRHS(DemandedElts);

2699

APInt UndefLHS, ZeroLHS;

2700

APInt UndefRHS, ZeroRHS;

2701

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,

2702

ZeroLHS, TLO, Depth + 1))

2703

return true;

2704

if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,

2705

ZeroRHS, TLO, Depth + 1))

2706

return true;

2707

2708

KnownUndef = UndefLHS & UndefRHS;

2709

KnownZero = ZeroLHS & ZeroRHS;

2710

break;

2711

}

2712

case ISD::VECTOR_SHUFFLE: {

2713

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

2714

2715

// Collect demanded elements from shuffle operands..

2716

APInt DemandedLHS(NumElts, 0);

2717

APInt DemandedRHS(NumElts, 0);

2718

for (unsigned i = 0; i != NumElts; ++i) {

2719

int M = ShuffleMask[i];

2720

if (M < 0 || !DemandedElts[i])

2721

continue;

2722

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2722, __extension__ __PRETTY_FUNCTION__));

2723

if (M < (int)NumElts)

2724

DemandedLHS.setBit(M);

2725

else

2726

DemandedRHS.setBit(M - NumElts);

2727

}

2728

2729

// See if we can simplify either shuffle operand.

2730

APInt UndefLHS, ZeroLHS;

2731

APInt UndefRHS, ZeroRHS;

2732

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,

2733

ZeroLHS, TLO, Depth + 1))

2734

return true;

2735

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,

2736

ZeroRHS, TLO, Depth + 1))

2737

return true;

2738

2739

// Simplify mask using undef elements from LHS/RHS.

2740

bool Updated = false;

2741

bool IdentityLHS = true, IdentityRHS = true;

2742

SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());

2743

for (unsigned i = 0; i != NumElts; ++i) {

2744

int &M = NewMask[i];

2745

if (M < 0)

2746

continue;

2747

if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||

2748

(M >= (int)NumElts && UndefRHS[M - NumElts])) {

2749

Updated = true;

2750

M = -1;

2751

}

2752

IdentityLHS &= (M < 0) || (M == (int)i);

2753

IdentityRHS &= (M < 0) || ((M - NumElts) == i);

2754

}

2755

2756

// Update legal shuffle masks based on demanded elements if it won't reduce

2757

// to Identity which can cause premature removal of the shuffle mask.

2758

if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {

2759

SDValue LegalShuffle =

2760

buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),

2761

NewMask, TLO.DAG);

2762

if (LegalShuffle)

2763

return TLO.CombineTo(Op, LegalShuffle);

2764

}

2765

2766

// Propagate undef/zero elements from LHS/RHS.

2767

for (unsigned i = 0; i != NumElts; ++i) {

2768

int M = ShuffleMask[i];

2769

if (M < 0) {

2770

KnownUndef.setBit(i);

2771

} else if (M < (int)NumElts) {

2772

if (UndefLHS[M])

2773

KnownUndef.setBit(i);

2774

if (ZeroLHS[M])

2775

KnownZero.setBit(i);

2776

} else {

2777

if (UndefRHS[M - NumElts])

2778

KnownUndef.setBit(i);

2779

if (ZeroRHS[M - NumElts])

2780

KnownZero.setBit(i);

2781

}

2782

}

2783

break;

2784

}

2785

case ISD::ANY_EXTEND_VECTOR_INREG:

2786

case ISD::SIGN_EXTEND_VECTOR_INREG:

2787

case ISD::ZERO_EXTEND_VECTOR_INREG: {

2788

APInt SrcUndef, SrcZero;

2789

SDValue Src = Op.getOperand(0);

2790

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2791

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);

2792

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2793

Depth + 1))

2794

return true;

2795

KnownZero = SrcZero.zextOrTrunc(NumElts);

2796

KnownUndef = SrcUndef.zextOrTrunc(NumElts);

2797

2798

if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&

2799

Op.getValueSizeInBits() == Src.getValueSizeInBits() &&

2800

DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {

2801

// aext - if we just need the bottom element then we can bitcast.

2802

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

2803

}

2804

2805

if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {

2806

// zext(undef) upper bits are guaranteed to be zero.

2807

if (DemandedElts.isSubsetOf(KnownUndef))

2808

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2809

KnownUndef.clearAllBits();

2810

}

2811

break;

2812

}

2813

2814

// TODO: There are more binop opcodes that could be handled here - MIN,

2815

// MAX, saturated math, etc.

2816

case ISD::OR:

2817

case ISD::XOR:

2818

case ISD::ADD:

2819

case ISD::SUB:

2820

case ISD::FADD:

2821

case ISD::FSUB:

2822

case ISD::FMUL:

2823

case ISD::FDIV:

2824

case ISD::FREM: {

2825

SDValue Op0 = Op.getOperand(0);

2826

SDValue Op1 = Op.getOperand(1);

2827

2828

APInt UndefRHS, ZeroRHS;

2829

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2830

Depth + 1))

2831

return true;

2832

APInt UndefLHS, ZeroLHS;

2833

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2834

Depth + 1))

2835

return true;

2836

2837

KnownZero = ZeroLHS & ZeroRHS;

2838

KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

2839

2840

// Attempt to avoid multi-use ops if we don't need anything from them.

2841

// TODO - use KnownUndef to relax the demandedelts?

2842

if (!DemandedElts.isAllOnesValue())

2843

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2844

return true;

2845

break;

2846

}

2847

case ISD::SHL:

2848

case ISD::SRL:

2849

case ISD::SRA:

2850

case ISD::ROTL:

2851

case ISD::ROTR: {

2852

SDValue Op0 = Op.getOperand(0);

2853

SDValue Op1 = Op.getOperand(1);

2854

2855

APInt UndefRHS, ZeroRHS;

2856

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2857

Depth + 1))

2858

return true;

2859

APInt UndefLHS, ZeroLHS;

2860

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2861

Depth + 1))

2862

return true;

2863

2864

KnownZero = ZeroLHS;

2865

KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

2866

2867

// Attempt to avoid multi-use ops if we don't need anything from them.

2868

// TODO - use KnownUndef to relax the demandedelts?

2869

if (!DemandedElts.isAllOnesValue())

2870

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2871

return true;

2872

break;

2873

}

2874

case ISD::MUL:

2875

case ISD::AND: {

2876

SDValue Op0 = Op.getOperand(0);

2877

SDValue Op1 = Op.getOperand(1);

2878

2879

APInt SrcUndef, SrcZero;

2880

if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,

2881

Depth + 1))

2882

return true;

2883

if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,

2884

TLO, Depth + 1))

2885

return true;

2886

2887

// If either side has a zero element, then the result element is zero, even

2888

// if the other is an UNDEF.

2889

// TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros

2890

// and then handle 'and' nodes with the rest of the binop opcodes.

2891

KnownZero |= SrcZero;

2892

KnownUndef &= SrcUndef;

2893

KnownUndef &= ~KnownZero;

2894

2895

// Attempt to avoid multi-use ops if we don't need anything from them.

2896

// TODO - use KnownUndef to relax the demandedelts?

2897

if (!DemandedElts.isAllOnesValue())

2898

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2899

return true;

2900

break;

2901

}

2902

case ISD::TRUNCATE:

2903

case ISD::SIGN_EXTEND:

2904

case ISD::ZERO_EXTEND:

2905

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,

2906

KnownZero, TLO, Depth + 1))

2907

return true;

2908

2909

if (Op.getOpcode() == ISD::ZERO_EXTEND) {

2910

// zext(undef) upper bits are guaranteed to be zero.

2911

if (DemandedElts.isSubsetOf(KnownUndef))

2912

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2913

KnownUndef.clearAllBits();

2914

}

2915

break;

2916

default: {

2917

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2918

if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,

2919

KnownZero, TLO, Depth))

2920

return true;

2921

} else {

2922

KnownBits Known;

2923

APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);

2924

if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,

2925

TLO, Depth, AssumeSingleUse))

2926

return true;

2927

}

2928

break;

2929

}

2930

}

2931

assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2931, __extension__ __PRETTY_FUNCTION__));

2932

2933

// Constant fold all undef cases.

2934

// TODO: Handle zero cases as well.

2935

if (DemandedElts.isSubsetOf(KnownUndef))

2936

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2937

2938

return false;

2939

}

2940

2941

/// Determine which of the bits specified in Mask are known to be either zero or

2942

/// one and return them in the Known.

2943

void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,

2944

KnownBits &Known,

2945

const APInt &DemandedElts,

2946

const SelectionDAG &DAG,

2947

unsigned Depth) const {

2948

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))

2949

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))

2950

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))

2951

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))

2952

"Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))

2953

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__));

2954

Known.resetAll();

2955

}

2956

2957

void TargetLowering::computeKnownBitsForTargetInstr(

2958

GISelKnownBits &Analysis, Register R, KnownBits &Known,

2959

const APInt &DemandedElts, const MachineRegisterInfo &MRI,

2960

unsigned Depth) const {

2961

Known.resetAll();

2962

}

2963

2964

void TargetLowering::computeKnownBitsForFrameIndex(

2965

const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {

2966

// The low bits are known zero if the pointer is aligned.

2967

Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));

2968

}

2969

2970

Align TargetLowering::computeKnownAlignForTargetInstr(

2971

GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,

2972

unsigned Depth) const {

2973

return Align(1);

2974

}

2975

2976

/// This method can be implemented by targets that want to expose additional

2977

/// information about sign bits to the DAG Combiner.

2978

unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,

2979

const APInt &,

2980

const SelectionDAG &,

2981

unsigned Depth) const {

2982

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))

2983

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))

2984

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))

2985

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))

2986

"Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))

2987

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__));

2988

return 1;

2989

}

2990

2991

unsigned TargetLowering::computeNumSignBitsForTargetInstr(

2992

GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,

2993

const MachineRegisterInfo &MRI, unsigned Depth) const {

2994

return 1;

2995

}

2996

2997

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(

2998

SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,

2999

TargetLoweringOpt &TLO, unsigned Depth) const {

3000

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))

3001

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))

3002

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))

3003

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))

3004

"Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))

3005

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__));

3006

return false;

3007

}

3008

3009

bool TargetLowering::SimplifyDemandedBitsForTargetNode(

3010

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3011

KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {

3012

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))

3013

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))

3014

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))

3015

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))

3016

"Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))

3017

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__));

3018

computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);

3019

return false;

3020

}

3021

3022

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(

3023

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3024

SelectionDAG &DAG, unsigned Depth) const {

3025

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3026

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3027

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3028

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3029

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3030

"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))

3031

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__));

3032

return SDValue();

3033

}

3034

3035

SDValue

3036

TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,

3037

SDValue N1, MutableArrayRef<int> Mask,

3038

SelectionDAG &DAG) const {

3039

bool LegalMask = isShuffleMaskLegal(Mask, VT);

3040

if (!LegalMask) {

3041

std::swap(N0, N1);

3042

ShuffleVectorSDNode::commuteMask(Mask);

3043

LegalMask = isShuffleMaskLegal(Mask, VT);

3044

}

3045

3046

if (!LegalMask)

3047

return SDValue();

3048

3049

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);

3050

}

3051

3052

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {

3053

return nullptr;

3054

}

3055

3056

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,

3057

const SelectionDAG &DAG,

3058

bool SNaN,

3059

unsigned Depth) const {

3060

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))

3061

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))

3062

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))

3063

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))

3064

"Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))

3065

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__));

3066

return false;

3067

}

3068

3069

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must

3070

// work with truncating build vectors and vectors with elements of less than

3071

// 8 bits.

3072

bool TargetLowering::isConstTrueVal(const SDNode *N) const {

3073

if (!N)

3074

return false;

3075

3076

APInt CVal;

3077

if (auto *CN = dyn_cast<ConstantSDNode>(N)) {

3078

CVal = CN->getAPIntValue();

3079

} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {

3080

auto *CN = BV->getConstantSplatNode();

3081

if (!CN)

3082

return false;

3083

3084

// If this is a truncating build vector, truncate the splat value.

3085

// Otherwise, we may fail to match the expected values below.

3086

unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();

3087

CVal = CN->getAPIntValue();

3088

if (BVEltWidth < CVal.getBitWidth())

3089

CVal = CVal.trunc(BVEltWidth);

3090

} else {

3091

return false;

3092

}

3093

3094

switch (getBooleanContents(N->getValueType(0))) {

3095

case UndefinedBooleanContent:

3096

return CVal[0];

3097

case ZeroOrOneBooleanContent:

3098

return CVal.isOneValue();

3099

case ZeroOrNegativeOneBooleanContent:

3100

return CVal.isAllOnesValue();

3101

}

3102

3103

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3103);

3104

}

3105

3106

bool TargetLowering::isConstFalseVal(const SDNode *N) const {

3107

if (!N)

3108

return false;

3109

3110

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);

3111

if (!CN) {

3112

const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);

3113

if (!BV)

3114

return false;

3115

3116

// Only interested in constant splats, we don't care about undef

3117

// elements in identifying boolean constants and getConstantSplatNode

3118

// returns NULL if all ops are undef;

3119

CN = BV->getConstantSplatNode();

3120

if (!CN)

3121

return false;

3122

}

3123

3124

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)

3125

return !CN->getAPIntValue()[0];

3126

3127

return CN->isNullValue();

3128

}

3129

3130

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,

3131

bool SExt) const {

3132

if (VT == MVT::i1)

3133

return N->isOne();

3134

3135

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);

3136

switch (Cnt) {

3137

case TargetLowering::ZeroOrOneBooleanContent:

3138

// An extended value of 1 is always true, unless its original type is i1,

3139

// in which case it will be sign extended to -1.

3140

return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));

3141

case TargetLowering::UndefinedBooleanContent:

3142

case TargetLowering::ZeroOrNegativeOneBooleanContent:

3143

return N->isAllOnesValue() && SExt;

3144

}

3145

llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3145);

3146

}

3147

3148

/// This helper function of SimplifySetCC tries to optimize the comparison when

3149

/// either operand of the SetCC node is a bitwise-and instruction.

3150

SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,

3151

ISD::CondCode Cond, const SDLoc &DL,

3152

DAGCombinerInfo &DCI) const {

3153

// Match these patterns in any of their permutations:

3154

// (X & Y) == Y

3155

// (X & Y) != Y

3156

if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)

3157

std::swap(N0, N1);

3158

3159

EVT OpVT = N0.getValueType();

3160

if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||

3161

(Cond != ISD::SETEQ && Cond != ISD::SETNE))

3162

return SDValue();

3163

3164

SDValue X, Y;

3165

if (N0.getOperand(0) == N1) {

3166

X = N0.getOperand(1);

3167

Y = N0.getOperand(0);

3168

} else if (N0.getOperand(1) == N1) {

3169

X = N0.getOperand(0);

3170

Y = N0.getOperand(1);

3171

} else {

3172

return SDValue();

3173

}

3174

3175

SelectionDAG &DAG = DCI.DAG;

3176

SDValue Zero = DAG.getConstant(0, DL, OpVT);

3177

if (DAG.isKnownToBeAPowerOfTwo(Y)) {

3178

// Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.

3179

// Note that where Y is variable and is known to have at most one bit set

3180

// (for example, if it is Z & 1) we cannot do this; the expressions are not

3181

// equivalent when Y == 0.

3182

assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3182, __extension__ __PRETTY_FUNCTION__));

3183

Cond = ISD::getSetCCInverse(Cond, OpVT);

3184

if (DCI.isBeforeLegalizeOps() ||

3185

isCondCodeLegal(Cond, N0.getSimpleValueType()))

3186

return DAG.getSetCC(DL, VT, N0, Zero, Cond);

3187

} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {

3188

// If the target supports an 'and-not' or 'and-complement' logic operation,

3189

// try to use that to make a comparison operation more efficient.

3190

// But don't do this transform if the mask is a single bit because there are

3191

// more efficient ways to deal with that case (for example, 'bt' on x86 or

3192

// 'rlwinm' on PPC).

3193

3194

// Bail out if the compare operand that we want to turn into a zero is

3195

// already a zero (otherwise, infinite loop).

3196

auto *YConst = dyn_cast<ConstantSDNode>(Y);

3197

if (YConst && YConst->isNullValue())

3198

return SDValue();

3199

3200

// Transform this into: ~X & Y == 0.

3201

SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);

3202

SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);

3203

return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);

3204

}

3205

3206

return SDValue();

3207

}

3208

3209

/// There are multiple IR patterns that could be checking whether certain

3210

/// truncation of a signed number would be lossy or not. The pattern which is

3211

/// best at IR level, may not lower optimally. Thus, we want to unfold it.

3212

/// We are looking for the following pattern: (KeptBits is a constant)

3213

/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)

3214

/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.

3215

/// KeptBits also can't be 1, that would have been folded to %x dstcond 0

3216

/// We will unfold it into the natural trunc+sext pattern:

3217

/// ((%x << C) a>> C) dstcond %x

3218

/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)

3219

SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(

3220

EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,

3221

const SDLoc &DL) const {

3222

// We must be comparing with a constant.

3223

ConstantSDNode *C1;

3224

if (!(C1 = dyn_cast<ConstantSDNode>(N1)))

3225

return SDValue();

3226

3227

// N0 should be: add %x, (1 << (KeptBits-1))

3228

if (N0->getOpcode() != ISD::ADD)

3229

return SDValue();

3230

3231

// And we must be 'add'ing a constant.

3232

ConstantSDNode *C01;

3233

if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))

3234

return SDValue();

3235

3236

SDValue X = N0->getOperand(0);

3237

EVT XVT = X.getValueType();

3238

3239

// Validate constants ...

3240

3241

APInt I1 = C1->getAPIntValue();

3242

3243

ISD::CondCode NewCond;

3244

if (Cond == ISD::CondCode::SETULT) {

3245

NewCond = ISD::CondCode::SETEQ;

3246

} else if (Cond == ISD::CondCode::SETULE) {

3247

NewCond = ISD::CondCode::SETEQ;

3248

// But need to 'canonicalize' the constant.

3249

I1 += 1;

3250

} else if (Cond == ISD::CondCode::SETUGT) {

3251

NewCond = ISD::CondCode::SETNE;

3252

// But need to 'canonicalize' the constant.

3253

I1 += 1;

3254

} else if (Cond == ISD::CondCode::SETUGE) {

3255

NewCond = ISD::CondCode::SETNE;

3256

} else

3257

return SDValue();

3258

3259

APInt I01 = C01->getAPIntValue();

3260

3261

auto checkConstants = [&I1, &I01]() -> bool {

3262

// Both of them must be power-of-two, and the constant from setcc is bigger.

3263

return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();

3264

};

3265

3266

if (checkConstants()) {

3267

// Great, e.g. got icmp ult i16 (add i16 %x, 128), 256

3268

} else {

3269

// What if we invert constants? (and the target predicate)

3270

I1.negate();

3271

I01.negate();

3272

assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3272, __extension__ __PRETTY_FUNCTION__));

3273

NewCond = getSetCCInverse(NewCond, XVT);

3274

if (!checkConstants())

3275

return SDValue();

3276

// Great, e.g. got icmp uge i16 (add i16 %x, -128), -256

3277

}

3278

3279

// They are power-of-two, so which bit is set?

3280

const unsigned KeptBits = I1.logBase2();

3281

const unsigned KeptBitsMinusOne = I01.logBase2();

3282

3283

// Magic!

3284

if (KeptBits != (KeptBitsMinusOne + 1))

3285

return SDValue();

3286

assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3286, __extension__ __PRETTY_FUNCTION__));

3287

3288

// We don't want to do this in every single case.

3289

SelectionDAG &DAG = DCI.DAG;

3290

if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(

3291

XVT, KeptBits))

3292

return SDValue();

3293

3294

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;

3295

assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3295, __extension__ __PRETTY_FUNCTION__));

3296

3297

// Unfold into: ((%x << C) a>> C) cond %x

3298

// Where 'cond' will be either 'eq' or 'ne'.

3299

SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);

3300

SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);

3301

SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);

3302

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

3303

3304

return T2;

3305

}

3306

3307

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3308

SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(

3309

EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,

3310

DAGCombinerInfo &DCI, const SDLoc &DL) const {

3311

assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__))

3312

isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__))

3313

"Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__));

3314

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3315, __extension__ __PRETTY_FUNCTION__))

3315

"Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3315, __extension__ __PRETTY_FUNCTION__));

3316

3317

unsigned NewShiftOpcode;

3318

SDValue X, C, Y;

3319

3320

SelectionDAG &DAG = DCI.DAG;

3321

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3322

3323

// Look for '(C l>>/<< Y)'.

3324

auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {

3325

// The shift should be one-use.

3326

if (!V.hasOneUse())

3327

return false;

3328

unsigned OldShiftOpcode = V.getOpcode();

3329

switch (OldShiftOpcode) {

3330

case ISD::SHL:

3331

NewShiftOpcode = ISD::SRL;

3332

break;

3333

case ISD::SRL:

3334

NewShiftOpcode = ISD::SHL;

3335

break;

3336

default:

3337

return false; // must be a logical shift.

3338

}

3339

// We should be shifting a constant.

3340

// FIXME: best to use isConstantOrConstantVector().

3341

C = V.getOperand(0);

3342

ConstantSDNode *CC =

3343

isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3344

if (!CC)

3345

return false;

3346

Y = V.getOperand(1);

3347

3348

ConstantSDNode *XC =

3349

isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3350

return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(

3351

X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);

3352

};

3353

3354

// LHS of comparison should be an one-use 'and'.

3355

if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())

3356

return SDValue();

3357

3358

X = N0.getOperand(0);

3359

SDValue Mask = N0.getOperand(1);

3360

3361

// 'and' is commutative!

3362

if (!Match(Mask)) {

3363

std::swap(X, Mask);

3364

if (!Match(Mask))

3365

return SDValue();

3366

}

3367

3368

EVT VT = X.getValueType();

3369

3370

// Produce:

3371

// ((X 'OppositeShiftOpcode' Y) & C) Cond 0

3372

SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);

3373

SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);

3374

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);

3375

return T2;

3376

}

3377

3378

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as

3379

/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to

3380

/// handle the commuted versions of these patterns.

3381

SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,

3382

ISD::CondCode Cond, const SDLoc &DL,

3383

DAGCombinerInfo &DCI) const {

3384

unsigned BOpcode = N0.getOpcode();

3385

assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3386, __extension__ __PRETTY_FUNCTION__))

3386

"Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3386, __extension__ __PRETTY_FUNCTION__));

3387

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3387, __extension__ __PRETTY_FUNCTION__));

3388

3389

// (X + Y) == X --> Y == 0

3390

// (X - Y) == X --> Y == 0

3391

// (X ^ Y) == X --> Y == 0

3392

SelectionDAG &DAG = DCI.DAG;

3393

EVT OpVT = N0.getValueType();

3394

SDValue X = N0.getOperand(0);

3395

SDValue Y = N0.getOperand(1);

3396

if (X == N1)

3397

return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

3398

3399

if (Y != N1)

3400

return SDValue();

3401

3402

// (X + Y) == Y --> X == 0

3403

// (X ^ Y) == Y --> X == 0

3404

if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)

3405

return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

3406

3407

// The shift would not be valid if the operands are boolean (i1).

3408

if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)

3409

return SDValue();

3410

3411

// (X - Y) == Y --> X == Y << 1

3412

EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),

3413

!DCI.isBeforeLegalize());

3414

SDValue One = DAG.getConstant(1, DL, ShiftVT);

3415

SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);

3416

if (!DCI.isCalledByLegalizer())

3417

DCI.AddToWorklist(YShl1.getNode());

3418

return DAG.getSetCC(DL, VT, X, YShl1, Cond);

3419

}

3420

3421

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,

3422

SDValue N0, const APInt &C1,

3423

ISD::CondCode Cond, const SDLoc &dl,

3424

SelectionDAG &DAG) {

3425

// Look through truncs that don't change the value of a ctpop.

3426

// FIXME: Add vector support? Need to be careful with setcc result type below.

3427

SDValue CTPOP = N0;

3428

if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&

3429

N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))

3430

CTPOP = N0.getOperand(0);

3431

3432

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())

3433

return SDValue();

3434

3435

EVT CTVT = CTPOP.getValueType();

3436

SDValue CTOp = CTPOP.getOperand(0);

3437

3438

// If this is a vector CTPOP, keep the CTPOP if it is legal.

3439

// TODO: Should we check if CTPOP is legal(or custom) for scalars?

3440

if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))

3441

return SDValue();

3442

3443

// (ctpop x) u< 2 -> (x & x-1) == 0

3444

// (ctpop x) u> 1 -> (x & x-1) != 0

3445

if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {

3446

unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);

3447

if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))

3448

return SDValue();

3449

if (C1 == 0 && (Cond == ISD::SETULT))

3450

return SDValue(); // This is handled elsewhere.

3451

3452

unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

3453

3454

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3455

SDValue Result = CTOp;

3456

for (unsigned i = 0; i < Passes; i++) {

3457

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);

3458

Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);

3459

}

3460

ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;

3461

return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);

3462

}

3463

3464

// If ctpop is not supported, expand a power-of-2 comparison based on it.

3465

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {

3466

// For scalars, keep CTPOP if it is legal or custom.

3467

if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))

3468

return SDValue();

3469

// This is based on X86's custom lowering for CTPOP which produces more

3470

// instructions than the expansion here.

3471

3472

// (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)

3473

// (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)

3474

SDValue Zero = DAG.getConstant(0, dl, CTVT);

3475

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3476

assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3476, __extension__ __PRETTY_FUNCTION__));

3477

ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);

3478

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

3479

SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

3480

SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);

3481

SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);

3482

unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;

3483

return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);

3484

}

3485

3486

return SDValue();

3487

}

3488

3489

/// Try to simplify a setcc built with the specified operands and cc. If it is

3490

/// unable to simplify it, return a null SDValue.

3491

SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,

3492

ISD::CondCode Cond, bool foldBooleans,

3493

DAGCombinerInfo &DCI,

3494

const SDLoc &dl) const {

3495

SelectionDAG &DAG = DCI.DAG;

3496

const DataLayout &Layout = DAG.getDataLayout();

3497

EVT OpVT = N0.getValueType();

3498

3499

// Constant fold or commute setcc.

3500

if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))

3501

return Fold;

3502

3503

// Ensure that the constant occurs on the RHS and fold constant comparisons.

3504

// TODO: Handle non-splat vector constants. All undef causes trouble.

3505

// FIXME: We can't yet fold constant scalable vector splats, so avoid an

3506

// infinite loop here when we encounter one.

3507

ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);

3508

if (isConstOrConstSplat(N0) &&

3509

(!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&

3510

(DCI.isBeforeLegalizeOps() ||

3511

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))

3512

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3513

3514

// If we have a subtract with the same 2 non-constant operands as this setcc

3515

// -- but in reverse order -- then try to commute the operands of this setcc

3516

// to match. A matching pair of setcc (cmp) and sub may be combined into 1

3517

// instruction on some targets.

3518

if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&

3519

(DCI.isBeforeLegalizeOps() ||

3520

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&

3521

DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&

3522

!DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))

3523

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3524

3525

if (auto *N1C = isConstOrConstSplat(N1)) {

3526

const APInt &C1 = N1C->getAPIntValue();

3527

3528

// Optimize some CTPOP cases.

3529

if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))

3530

return V;

3531

3532

// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an

3533

// equality comparison, then we're just comparing whether X itself is

3534

// zero.

3535

if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&

3536

N0.getOperand(0).getOpcode() == ISD::CTLZ &&

3537

isPowerOf2_32(N0.getScalarValueSizeInBits())) {

3538

if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {

3539

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3540

ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {

3541

if ((C1 == 0) == (Cond == ISD::SETEQ)) {

3542

// (srl (ctlz x), 5) == 0 -> X != 0

3543

// (srl (ctlz x), 5) != 1 -> X != 0

3544

Cond = ISD::SETNE;

3545

} else {

3546

// (srl (ctlz x), 5) != 0 -> X == 0

3547

// (srl (ctlz x), 5) == 1 -> X == 0

3548

Cond = ISD::SETEQ;

3549

}

3550

SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());

3551

return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,

3552

Cond);

3553

}

3554

}

3555

}

3556

}

3557

3558

// FIXME: Support vectors.

3559

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

3560

const APInt &C1 = N1C->getAPIntValue();

3561

3562

// (zext x) == C --> x == (trunc C)

3563

// (sext x) == C --> x == (trunc C)

3564

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3565

DCI.isBeforeLegalize() && N0->hasOneUse()) {

3566

unsigned MinBits = N0.getValueSizeInBits();

3567

SDValue PreExt;

3568

bool Signed = false;

3569

if (N0->getOpcode() == ISD::ZERO_EXTEND) {

3570

// ZExt

3571

MinBits = N0->getOperand(0).getValueSizeInBits();

3572

PreExt = N0->getOperand(0);

3573

} else if (N0->getOpcode() == ISD::AND) {

3574

// DAGCombine turns costly ZExts into ANDs

3575

if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))

3576

if ((C->getAPIntValue()+1).isPowerOf2()) {

3577

MinBits = C->getAPIntValue().countTrailingOnes();

3578

PreExt = N0->getOperand(0);

3579

}

3580

} else if (N0->getOpcode() == ISD::SIGN_EXTEND) {

3581

// SExt

3582

MinBits = N0->getOperand(0).getValueSizeInBits();

3583

PreExt = N0->getOperand(0);

3584

Signed = true;

3585

} else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {

3586

// ZEXTLOAD / SEXTLOAD

3587

if (LN0->getExtensionType() == ISD::ZEXTLOAD) {

3588

MinBits = LN0->getMemoryVT().getSizeInBits();

3589

PreExt = N0;

3590

} else if (LN0->getExtensionType() == ISD::SEXTLOAD) {

3591

Signed = true;

3592

MinBits = LN0->getMemoryVT().getSizeInBits();

3593

PreExt = N0;

3594

}

3595

}

3596

3597

// Figure out how many bits we need to preserve this constant.

3598

unsigned ReqdBits = Signed ?

3599

C1.getBitWidth() - C1.getNumSignBits() + 1 :

3600

C1.getActiveBits();

3601

3602

// Make sure we're not losing bits from the constant.

3603

if (MinBits > 0 &&

3604

MinBits < C1.getBitWidth() &&

3605

MinBits >= ReqdBits) {

3606

EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);

3607

if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {

3608

// Will get folded away.

3609

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);

3610

if (MinBits == 1 && C1 == 1)

3611

// Invert the condition.

3612

return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),

3613

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3614

SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);

3615

return DAG.getSetCC(dl, VT, Trunc, C, Cond);

3616

}

3617

3618

// If truncating the setcc operands is not desirable, we can still

3619

// simplify the expression in some cases:

3620

// setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)

3621

// setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))

3622

// setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))

3623

// setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)

3624

// setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))

3625

// setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)

3626

SDValue TopSetCC = N0->getOperand(0);

3627

unsigned N0Opc = N0->getOpcode();

3628

bool SExt = (N0Opc == ISD::SIGN_EXTEND);

3629

if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&

3630

TopSetCC.getOpcode() == ISD::SETCC &&

3631

(N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&

3632

(isConstFalseVal(N1C) ||

3633

isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

3634

3635

bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||

3636

(!N1C->isNullValue() && Cond == ISD::SETNE);

3637

3638

if (!Inverse)

3639

return TopSetCC;

3640

3641

ISD::CondCode InvCond = ISD::getSetCCInverse(

3642

cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),

3643

TopSetCC.getOperand(0).getValueType());

3644

return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),

3645

TopSetCC.getOperand(1),

3646

InvCond);

3647

}

3648

}

3649

}

3650

3651

// If the LHS is '(and load, const)', the RHS is 0, the test is for

3652

// equality or unsigned, and all 1 bits of the const are in the same

3653

// partial word, see if we can shorten the load.

3654

if (DCI.isBeforeLegalize() &&

3655

!ISD::isSignedIntSetCC(Cond) &&

3656

N0.getOpcode() == ISD::AND && C1 == 0 &&

3657

N0.getNode()->hasOneUse() &&

3658

isa<LoadSDNode>(N0.getOperand(0)) &&

3659

N0.getOperand(0).getNode()->hasOneUse() &&

3660

isa<ConstantSDNode>(N0.getOperand(1))) {

3661

LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));

3662

APInt bestMask;

3663

unsigned bestWidth = 0, bestOffset = 0;

3664

if (Lod->isSimple() && Lod->isUnindexed()) {

3665

unsigned origWidth = N0.getValueSizeInBits();

3666

unsigned maskWidth = origWidth;

3667

// We can narrow (e.g.) 16-bit extending loads on 32-bit target to

3668

// 8 bits, but have to be careful...

3669

if (Lod->getExtensionType() != ISD::NON_EXTLOAD)

3670

origWidth = Lod->getMemoryVT().getSizeInBits();

3671

const APInt &Mask = N0.getConstantOperandAPInt(1);

3672

for (unsigned width = origWidth / 2; width>=8; width /= 2) {

3673

APInt newMask = APInt::getLowBitsSet(maskWidth, width);

3674

for (unsigned offset=0; offset<origWidth/width; offset++) {

3675

if (Mask.isSubsetOf(newMask)) {

3676

if (Layout.isLittleEndian())

3677

bestOffset = (uint64_t)offset * (width/8);

3678

else

3679

bestOffset = (origWidth/width - offset - 1) * (width/8);

3680

bestMask = Mask.lshr(offset * (width/8) * 8);

3681

bestWidth = width;

3682

break;

3683

}

3684

newMask <<= width;

3685

}

3686

}

3687

}

3688

if (bestWidth) {

3689

EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);

3690

if (newVT.isRound() &&

3691

shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {

3692

SDValue Ptr = Lod->getBasePtr();

3693

if (bestOffset != 0)

3694

Ptr =

3695

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);

3696

SDValue NewLoad =

3697

DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,

3698

Lod->getPointerInfo().getWithOffset(bestOffset),

3699

Lod->getOriginalAlign());

3700

return DAG.getSetCC(dl, VT,

3701

DAG.getNode(ISD::AND, dl, newVT, NewLoad,

3702

DAG.getConstant(bestMask.trunc(bestWidth),

3703

dl, newVT)),

3704

DAG.getConstant(0LL, dl, newVT), Cond);

3705

}

3706

}

3707

}

3708

3709

// If the LHS is a ZERO_EXTEND, perform the comparison on the input.

3710

if (N0.getOpcode() == ISD::ZERO_EXTEND) {

3711

unsigned InSize = N0.getOperand(0).getValueSizeInBits();

3712

3713

// If the comparison constant has bits in the upper part, the

3714

// zero-extended value could never match.

3715

if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),

3716

C1.getBitWidth() - InSize))) {

3717

switch (Cond) {

3718

case ISD::SETUGT:

3719

case ISD::SETUGE:

3720

case ISD::SETEQ:

3721

return DAG.getConstant(0, dl, VT);

3722

case ISD::SETULT:

3723

case ISD::SETULE:

3724

case ISD::SETNE:

3725

return DAG.getConstant(1, dl, VT);

3726

case ISD::SETGT:

3727

case ISD::SETGE:

3728

// True if the sign bit of C1 is set.

3729

return DAG.getConstant(C1.isNegative(), dl, VT);

3730

case ISD::SETLT:

3731

case ISD::SETLE:

3732

// True if the sign bit of C1 isn't set.

3733

return DAG.getConstant(C1.isNonNegative(), dl, VT);

3734

default:

3735

break;

3736

}

3737

}

3738

3739

// Otherwise, we can perform the comparison with the low bits.

3740

switch (Cond) {

3741

case ISD::SETEQ:

3742

case ISD::SETNE:

3743

case ISD::SETUGT:

3744

case ISD::SETUGE:

3745

case ISD::SETULT:

3746

case ISD::SETULE: {

3747

EVT newVT = N0.getOperand(0).getValueType();

3748

if (DCI.isBeforeLegalizeOps() ||

3749

(isOperationLegal(ISD::SETCC, newVT) &&

3750

isCondCodeLegal(Cond, newVT.getSimpleVT()))) {

3751

EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);

3752

SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

3753

3754

SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),

3755

NewConst, Cond);

3756

return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());

3757

}

3758

break;

3759

}

3760

default:

3761

break; // todo, be more careful with signed comparisons

3762

}

3763

} else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&

3764

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3765

!isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),

3766

OpVT)) {

3767

EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();

3768

unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();

3769

EVT ExtDstTy = N0.getValueType();

3770

unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

3771

3772

// If the constant doesn't fit into the number of bits for the source of

3773

// the sign extension, it is impossible for both sides to be equal.

3774

if (C1.getMinSignedBits() > ExtSrcTyBits)

3775

return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

3776

3777

assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3778, __extension__ __PRETTY_FUNCTION__))

3778

ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3778, __extension__ __PRETTY_FUNCTION__));

3779

APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);

3780

SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),

3781

DAG.getConstant(Imm, dl, ExtDstTy));

3782

if (!DCI.isCalledByLegalizer())

3783

DCI.AddToWorklist(ZextOp.getNode());

3784

// Otherwise, make this a use of a zext.

3785

return DAG.getSetCC(dl, VT, ZextOp,

3786

DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);

3787

} else if ((N1C->isNullValue() || N1C->isOne()) &&

3788

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3789

// SETCC (SETCC), [0|1], [EQ|NE] -> SETCC

3790

if (N0.getOpcode() == ISD::SETCC &&

3791

isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&

3792

(N0.getValueType() == MVT::i1 ||

3793

getBooleanContents(N0.getOperand(0).getValueType()) ==

3794

ZeroOrOneBooleanContent)) {

3795

bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());

3796

if (TrueWhenTrue)

3797

return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);

3798

// Invert the condition.

3799

ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();

3800

CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());

3801

if (DCI.isBeforeLegalizeOps() ||

3802

isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))

3803

return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);

3804

}

3805

3806

if ((N0.getOpcode() == ISD::XOR ||

3807

(N0.getOpcode() == ISD::AND &&

3808

N0.getOperand(0).getOpcode() == ISD::XOR &&

3809

N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&

3810

isOneConstant(N0.getOperand(1))) {

3811

// If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We

3812

// can only do this if the top bits are known zero.

3813

unsigned BitWidth = N0.getValueSizeInBits();

3814

if (DAG.MaskedValueIsZero(N0,

3815

APInt::getHighBitsSet(BitWidth,

3816

BitWidth-1))) {

3817

// Okay, get the un-inverted input value.

3818

SDValue Val;

3819

if (N0.getOpcode() == ISD::XOR) {

3820

Val = N0.getOperand(0);

3821

} else {

3822

assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3823, __extension__ __PRETTY_FUNCTION__))

3823

N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3823, __extension__ __PRETTY_FUNCTION__));

3824

// ((X^1)&1)^1 -> X & 1

3825

Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),

3826

N0.getOperand(0).getOperand(0),

3827

N0.getOperand(1));

3828

}

3829

3830

return DAG.getSetCC(dl, VT, Val, N1,

3831

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3832

}

3833

} else if (N1C->isOne()) {

3834

SDValue Op0 = N0;

3835

if (Op0.getOpcode() == ISD::TRUNCATE)

3836

Op0 = Op0.getOperand(0);

3837

3838

if ((Op0.getOpcode() == ISD::XOR) &&

3839

Op0.getOperand(0).getOpcode() == ISD::SETCC &&

3840

Op0.getOperand(1).getOpcode() == ISD::SETCC) {

3841

SDValue XorLHS = Op0.getOperand(0);

3842

SDValue XorRHS = Op0.getOperand(1);

3843

// Ensure that the input setccs return an i1 type or 0/1 value.

3844

if (Op0.getValueType() == MVT::i1 ||

3845

(getBooleanContents(XorLHS.getOperand(0).getValueType()) ==

3846

ZeroOrOneBooleanContent &&

3847

getBooleanContents(XorRHS.getOperand(0).getValueType()) ==

3848

ZeroOrOneBooleanContent)) {

3849

// (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)

3850

Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;

3851

return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);

3852

}

3853

}

3854

if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {

3855

// If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.

3856

if (Op0.getValueType().bitsGT(VT))

3857

Op0 = DAG.getNode(ISD::AND, dl, VT,

3858

DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),

3859

DAG.getConstant(1, dl, VT));

3860

else if (Op0.getValueType().bitsLT(VT))

3861

Op0 = DAG.getNode(ISD::AND, dl, VT,

3862

DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),

3863

DAG.getConstant(1, dl, VT));

3864

3865

return DAG.getSetCC(dl, VT, Op0,

3866

DAG.getConstant(0, dl, Op0.getValueType()),

3867

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3868

}

3869

if (Op0.getOpcode() == ISD::AssertZext &&

3870

cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)

3871

return DAG.getSetCC(dl, VT, Op0,

3872

DAG.getConstant(0, dl, Op0.getValueType()),

3873

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3874

}

3875

}

3876

3877

// Given:

3878

// icmp eq/ne (urem %x, %y), 0

3879

// Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':

3880

// icmp eq/ne %x, 0

3881

if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&

3882

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3883

KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));

3884

KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));

3885

if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)

3886

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);

3887

}

3888

3889

if (SDValue V =

3890

optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))

3891

return V;

3892

}

3893

3894

// These simplifications apply to splat vectors as well.

3895

// TODO: Handle more splat vector cases.

3896

if (auto *N1C = isConstOrConstSplat(N1)) {

3897

const APInt &C1 = N1C->getAPIntValue();

3898

3899

APInt MinVal, MaxVal;

3900

unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();

3901

if (ISD::isSignedIntSetCC(Cond)) {

3902

MinVal = APInt::getSignedMinValue(OperandBitSize);

3903

MaxVal = APInt::getSignedMaxValue(OperandBitSize);

3904

} else {

3905

MinVal = APInt::getMinValue(OperandBitSize);

3906

MaxVal = APInt::getMaxValue(OperandBitSize);

3907

}

3908

3909

// Canonicalize GE/LE comparisons to use GT/LT comparisons.

3910

if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {

3911

// X >= MIN --> true

3912

if (C1 == MinVal)

3913

return DAG.getBoolConstant(true, dl, VT, OpVT);

3914

3915

if (!VT.isVector()) { // TODO: Support this for vectors.

3916

// X >= C0 --> X > (C0 - 1)

3917

APInt C = C1 - 1;

3918

ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;

3919

if ((DCI.isBeforeLegalizeOps() ||

3920

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3921

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3922

isLegalICmpImmediate(C.getSExtValue())))) {

3923

return DAG.getSetCC(dl, VT, N0,

3924

DAG.getConstant(C, dl, N1.getValueType()),

3925

NewCC);

3926

}

3927

}

3928

}

3929

3930

if (Cond == ISD::SETLE || Cond == ISD::SETULE) {

3931

// X <= MAX --> true

3932

if (C1 == MaxVal)

3933

return DAG.getBoolConstant(true, dl, VT, OpVT);

3934

3935

// X <= C0 --> X < (C0 + 1)

3936

if (!VT.isVector()) { // TODO: Support this for vectors.

3937

APInt C = C1 + 1;

3938

ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;

3939

if ((DCI.isBeforeLegalizeOps() ||

3940

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3941

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3942

isLegalICmpImmediate(C.getSExtValue())))) {

3943

return DAG.getSetCC(dl, VT, N0,

3944

DAG.getConstant(C, dl, N1.getValueType()),

3945

NewCC);

3946

}

3947

}

3948

}

3949

3950

if (Cond == ISD::SETLT || Cond == ISD::SETULT) {

3951

if (C1 == MinVal)

3952

return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

3953

3954

// TODO: Support this for vectors after legalize ops.

3955

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3956

// Canonicalize setlt X, Max --> setne X, Max

3957

if (C1 == MaxVal)

3958

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3959

3960

// If we have setult X, 1, turn it into seteq X, 0

3961

if (C1 == MinVal+1)

3962

return DAG.getSetCC(dl, VT, N0,

3963

DAG.getConstant(MinVal, dl, N0.getValueType()),

3964

ISD::SETEQ);

3965

}

3966

}

3967

3968

if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {

3969

if (C1 == MaxVal)

3970

return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

3971

3972

// TODO: Support this for vectors after legalize ops.

3973

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3974

// Canonicalize setgt X, Min --> setne X, Min

3975

if (C1 == MinVal)

3976

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3977

3978

// If we have setugt X, Max-1, turn it into seteq X, Max

3979

if (C1 == MaxVal-1)

3980

return DAG.getSetCC(dl, VT, N0,

3981

DAG.getConstant(MaxVal, dl, N0.getValueType()),

3982

ISD::SETEQ);

3983

}

3984

}

3985

3986

if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {

3987

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3988

if (C1.isNullValue())

3989

if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(

3990

VT, N0, N1, Cond, DCI, dl))

3991

return CC;

3992

3993

// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).

3994

// For example, when high 32-bits of i64 X are known clear:

3995

// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0

3996

// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1

3997

bool CmpZero = N1C->getAPIntValue().isNullValue();

3998

bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();

3999

if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {

4000

// Match or(lo,shl(hi,bw/2)) pattern.

4001

auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {

4002

unsigned EltBits = V.getScalarValueSizeInBits();

4003

if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)

4004

return false;

4005

SDValue LHS = V.getOperand(0);

4006

SDValue RHS = V.getOperand(1);

4007

APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);

4008

// Unshifted element must have zero upperbits.

4009

if (RHS.getOpcode() == ISD::SHL &&

4010

isa<ConstantSDNode>(RHS.getOperand(1)) &&

4011

RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4012

DAG.MaskedValueIsZero(LHS, HiBits)) {

4013

Lo = LHS;

4014

Hi = RHS.getOperand(0);

4015

return true;

4016

}

4017

if (LHS.getOpcode() == ISD::SHL &&

4018

isa<ConstantSDNode>(LHS.getOperand(1)) &&

4019

LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4020

DAG.MaskedValueIsZero(RHS, HiBits)) {

4021

Lo = RHS;

4022

Hi = LHS.getOperand(0);

4023

return true;

4024

}

4025

return false;

4026

};

4027

4028

auto MergeConcat = [&](SDValue Lo, SDValue Hi) {

4029

unsigned EltBits = N0.getScalarValueSizeInBits();

4030

unsigned HalfBits = EltBits / 2;

4031

APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);

4032

SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);

4033

SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);

4034

SDValue NewN0 =

4035

DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);

4036

SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;

4037

return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);

4038

};

4039

4040

SDValue Lo, Hi;

4041

if (IsConcat(N0, Lo, Hi))

4042

return MergeConcat(Lo, Hi);

4043

4044

if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {

4045

SDValue Lo0, Lo1, Hi0, Hi1;

4046

if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&

4047

IsConcat(N0.getOperand(1), Lo1, Hi1)) {

4048

return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),

4049

DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));

4050

}

4051

}

4052

}

4053

}

4054

4055

// If we have "setcc X, C0", check to see if we can shrink the immediate

4056

// by changing cc.

4057

// TODO: Support this for vectors after legalize ops.

4058

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4059

// SETUGT X, SINTMAX -> SETLT X, 0

4060

// SETUGE X, SINTMIN -> SETLT X, 0

4061

if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||

4062

(Cond == ISD::SETUGE && C1.isMinSignedValue()))

4063

return DAG.getSetCC(dl, VT, N0,

4064

DAG.getConstant(0, dl, N1.getValueType()),

4065

ISD::SETLT);

4066

4067

// SETULT X, SINTMIN -> SETGT X, -1

4068

// SETULE X, SINTMAX -> SETGT X, -1

4069

if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||

4070

(Cond == ISD::SETULE && C1.isMaxSignedValue()))

4071

return DAG.getSetCC(dl, VT, N0,

4072

DAG.getAllOnesConstant(dl, N1.getValueType()),

4073

ISD::SETGT);

4074

}

4075

}

4076

4077

// Back to non-vector simplifications.

4078

// TODO: Can we do these for vector splats?

4079

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

4080

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4081

const APInt &C1 = N1C->getAPIntValue();

4082

EVT ShValTy = N0.getValueType();

4083

4084

// Fold bit comparisons when we can. This will result in an

4085

// incorrect value when boolean false is negative one, unless

4086

// the bitsize is 1 in which case the false value is the same

4087

// in practice regardless of the representation.

4088

if ((VT.getSizeInBits() == 1 ||

4089

getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&

4090

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4091

(VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&

4092

N0.getOpcode() == ISD::AND) {

4093

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4094

EVT ShiftTy =

4095

getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4096

if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3

4097

// Perform the xform if the AND RHS is a single bit.

4098

unsigned ShCt = AndRHS->getAPIntValue().logBase2();

4099

if (AndRHS->getAPIntValue().isPowerOf2() &&

4100

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4101

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4102

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4103

DAG.getConstant(ShCt, dl, ShiftTy)));

4104

}

4105

} else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {

4106

// (X & 8) == 8 --> (X & 8) >> 3

4107

// Perform the xform if C1 is a single bit.

4108

unsigned ShCt = C1.logBase2();

4109

if (C1.isPowerOf2() &&

4110

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4111

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4112

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4113

DAG.getConstant(ShCt, dl, ShiftTy)));

4114

}

4115

}

4116

}

4117

}

4118

4119

if (C1.getMinSignedBits() <= 64 &&

4120

!isLegalICmpImmediate(C1.getSExtValue())) {

4121

EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4122

// (X & -256) == 256 -> (X >> 8) == 1

4123

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4124

N0.getOpcode() == ISD::AND && N0.hasOneUse()) {

4125

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4126

const APInt &AndRHSC = AndRHS->getAPIntValue();

4127

if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {

4128

unsigned ShiftBits = AndRHSC.countTrailingZeros();

4129

if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4130

SDValue Shift =

4131

DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),

4132

DAG.getConstant(ShiftBits, dl, ShiftTy));

4133

SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);

4134

return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);

4135

}

4136

}

4137

}

4138

} else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||

4139

Cond == ISD::SETULE || Cond == ISD::SETUGT) {

4140

bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);

4141

// X < 0x100000000 -> (X >> 32) < 1

4142

// X >= 0x100000000 -> (X >> 32) >= 1

4143

// X <= 0x0ffffffff -> (X >> 32) < 1

4144

// X > 0x0ffffffff -> (X >> 32) >= 1

4145

unsigned ShiftBits;

4146

APInt NewC = C1;

4147

ISD::CondCode NewCond = Cond;

4148

if (AdjOne) {

4149

ShiftBits = C1.countTrailingOnes();

4150

NewC = NewC + 1;

4151

NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;

4152

} else {

4153

ShiftBits = C1.countTrailingZeros();

4154

}

4155

NewC.lshrInPlace(ShiftBits);

4156

if (ShiftBits && NewC.getMinSignedBits() <= 64 &&

4157

isLegalICmpImmediate(NewC.getSExtValue()) &&

4158

!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4159

SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4160

DAG.getConstant(ShiftBits, dl, ShiftTy));

4161

SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);

4162

return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);

4163

}

4164

}

4165

}

4166

}

4167

4168

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {

4169

auto *CFP = cast<ConstantFPSDNode>(N1);

4170

assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4170, __extension__ __PRETTY_FUNCTION__));

4171

4172

// Otherwise, we know the RHS is not a NaN. Simplify the node to drop the

4173

// constant if knowing that the operand is non-nan is enough. We prefer to

4174

// have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to

4175

// materialize 0.0.

4176

if (Cond == ISD::SETO || Cond == ISD::SETUO)

4177

return DAG.getSetCC(dl, VT, N0, N0, Cond);

4178

4179

// setcc (fneg x), C -> setcc swap(pred) x, -C

4180

if (N0.getOpcode() == ISD::FNEG) {

4181

ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);

4182

if (DCI.isBeforeLegalizeOps() ||

4183

isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {

4184

SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);

4185

return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);

4186

}

4187

}

4188

4189

// If the condition is not legal, see if we can find an equivalent one

4190

// which is legal.

4191

if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {

4192

// If the comparison was an awkward floating-point == or != and one of

4193

// the comparison operands is infinity or negative infinity, convert the

4194

// condition to a less-awkward <= or >=.

4195

if (CFP->getValueAPF().isInfinity()) {

4196

bool IsNegInf = CFP->getValueAPF().isNegative();

4197

ISD::CondCode NewCond = ISD::SETCC_INVALID;

4198

switch (Cond) {

4199

case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;

4200

case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;

4201

case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;

4202

case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;

4203

default: break;

4204

}

4205

if (NewCond != ISD::SETCC_INVALID &&

4206

isCondCodeLegal(NewCond, N0.getSimpleValueType()))

4207

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4208

}

4209

}

4210

}

4211

4212

if (N0 == N1) {

4213

// The sext(setcc()) => setcc() optimization relies on the appropriate

4214

// constant being emitted.

4215

assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4216, __extension__ __PRETTY_FUNCTION__))

4216

"Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4216, __extension__ __PRETTY_FUNCTION__));

4217

4218

bool EqTrue = ISD::isTrueWhenEqual(Cond);

4219

unsigned UOF = ISD::getUnorderedFlavor(Cond);

4220

if (UOF == 2) // FP operators that are undefined on NaNs.

4221

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4222

if (UOF == unsigned(EqTrue))

4223

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4224

// Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO

4225

// if it is not already.

4226

ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;

4227

if (NewCond != Cond &&

4228

(DCI.isBeforeLegalizeOps() ||

4229

isCondCodeLegal(NewCond, N0.getSimpleValueType())))

4230

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4231

}

4232

4233

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4234

N0.getValueType().isInteger()) {

4235

if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||

4236

N0.getOpcode() == ISD::XOR) {

4237

// Simplify (X+Y) == (X+Z) --> Y == Z

4238

if (N0.getOpcode() == N1.getOpcode()) {

4239

if (N0.getOperand(0) == N1.getOperand(0))

4240

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);

4241

if (N0.getOperand(1) == N1.getOperand(1))

4242

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);

4243

if (isCommutativeBinOp(N0.getOpcode())) {

4244

// If X op Y == Y op X, try other combinations.

4245

if (N0.getOperand(0) == N1.getOperand(1))

4246

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),

4247

Cond);

4248

if (N0.getOperand(1) == N1.getOperand(0))

4249

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),

4250

Cond);

4251

}

4252

}

4253

4254

// If RHS is a legal immediate value for a compare instruction, we need

4255

// to be careful about increasing register pressure needlessly.

4256

bool LegalRHSImm = false;

4257

4258

if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {

4259

if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4260

// Turn (X+C1) == C2 --> X == C2-C1

4261

if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {

4262

return DAG.getSetCC(dl, VT, N0.getOperand(0),

4263

DAG.getConstant(RHSC->getAPIntValue()-

4264

LHSR->getAPIntValue(),

4265

dl, N0.getValueType()), Cond);

4266

}

4267

4268

// Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.

4269

if (N0.getOpcode() == ISD::XOR)

4270

// If we know that all of the inverted bits are zero, don't bother

4271

// performing the inversion.

4272

if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))

4273

return

4274

DAG.getSetCC(dl, VT, N0.getOperand(0),

4275

DAG.getConstant(LHSR->getAPIntValue() ^

4276

RHSC->getAPIntValue(),

4277

dl, N0.getValueType()),

4278

Cond);

4279

}

4280

4281

// Turn (C1-X) == C2 --> X == C1-C2

4282

if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {

4283

if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {

4284

return

4285

DAG.getSetCC(dl, VT, N0.getOperand(1),

4286

DAG.getConstant(SUBC->getAPIntValue() -

4287

RHSC->getAPIntValue(),

4288

dl, N0.getValueType()),

4289

Cond);

4290

}

4291

}

4292

4293

// Could RHSC fold directly into a compare?

4294

if (RHSC->getValueType(0).getSizeInBits() <= 64)

4295

LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());

4296

}

4297

4298

// (X+Y) == X --> Y == 0 and similar folds.

4299

// Don't do this if X is an immediate that can fold into a cmp

4300

// instruction and X+Y has other uses. It could be an induction variable

4301

// chain, and the transform would increase register pressure.

4302

if (!LegalRHSImm || N0.hasOneUse())

4303

if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))

4304

return V;

4305

}

4306

4307

if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||

4308

N1.getOpcode() == ISD::XOR)

4309

if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))

4310

return V;

4311

4312

if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))

4313

return V;

4314

}

4315

4316

// Fold remainder of division by a constant.

4317

if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&

4318

N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

4319

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

4320

4321

// When division is cheap or optimizing for minimum size,

4322

// fall through to DIVREM creation by skipping this fold.

4323

if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {

4324

if (N0.getOpcode() == ISD::UREM) {

4325

if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))

4326

return Folded;

4327

} else if (N0.getOpcode() == ISD::SREM) {

4328

if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))

4329

return Folded;

4330

}

4331

}

4332

}

4333

4334

// Fold away ALL boolean setcc's.

4335

if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {

4336

SDValue Temp;

4337

switch (Cond) {

4338

default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4338);

4339

case ISD::SETEQ: // X == Y -> ~(X^Y)

4340

Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4341

N0 = DAG.getNOT(dl, Temp, OpVT);

4342

if (!DCI.isCalledByLegalizer())

4343

DCI.AddToWorklist(Temp.getNode());

4344

break;

4345

case ISD::SETNE: // X != Y --> (X^Y)

4346

N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4347

break;

4348

case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y

4349

case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y

4350

Temp = DAG.getNOT(dl, N0, OpVT);

4351

N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);

4352

if (!DCI.isCalledByLegalizer())

4353

DCI.AddToWorklist(Temp.getNode());

4354

break;

4355

case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X

4356

case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X

4357

Temp = DAG.getNOT(dl, N1, OpVT);

4358

N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);

4359

if (!DCI.isCalledByLegalizer())

4360

DCI.AddToWorklist(Temp.getNode());

4361

break;

4362

case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y

4363

case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y

4364

Temp = DAG.getNOT(dl, N0, OpVT);

4365

N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);

4366

if (!DCI.isCalledByLegalizer())

4367

DCI.AddToWorklist(Temp.getNode());

4368

break;

4369

case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X

4370

case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X

4371

Temp = DAG.getNOT(dl, N1, OpVT);

4372

N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);

4373

break;

4374

}

4375

if (VT.getScalarType() != MVT::i1) {

4376

if (!DCI.isCalledByLegalizer())

4377

DCI.AddToWorklist(N0.getNode());

4378

// FIXME: If running after legalize, we probably can't do this.

4379

ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));

4380

N0 = DAG.getNode(ExtendCode, dl, VT, N0);

4381

}

4382

return N0;

4383

}

4384

4385

// Could not fold it.

4386

return SDValue();

4387

}

4388

4389

/// Returns true (and the GlobalValue and the offset) if the node is a

4390

/// GlobalAddress + offset.

4391

bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,

4392

int64_t &Offset) const {

4393

4394

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

4395

4396

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {

4397

GA = GASD->getGlobal();

4398

Offset += GASD->getOffset();

4399

return true;

4400

}

4401

4402

if (N->getOpcode() == ISD::ADD) {

4403

SDValue N1 = N->getOperand(0);

4404

SDValue N2 = N->getOperand(1);

4405

if (isGAPlusOffset(N1.getNode(), GA, Offset)) {

4406

if (auto *V = dyn_cast<ConstantSDNode>(N2)) {

4407

Offset += V->getSExtValue();

4408

return true;

4409

}

4410

} else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {

4411

if (auto *V = dyn_cast<ConstantSDNode>(N1)) {

4412

Offset += V->getSExtValue();

4413

return true;

4414

}

4415

}

4416

}

4417

4418

return false;

4419

}

4420

4421

SDValue TargetLowering::PerformDAGCombine(SDNode *N,

4422

DAGCombinerInfo &DCI) const {

4423

// Default implementation: no optimization.

4424

return SDValue();

4425

}

4426

4427

//===----------------------------------------------------------------------===//

4428

// Inline Assembler Implementation Methods

4429

//===----------------------------------------------------------------------===//

4430

4431

TargetLowering::ConstraintType

4432

TargetLowering::getConstraintType(StringRef Constraint) const {

4433

unsigned S = Constraint.size();

4434

4435

if (S == 1) {

4436

switch (Constraint[0]) {

4437

default: break;

4438

case 'r':

4439

return C_RegisterClass;

4440

case 'm': // memory

4441

case 'o': // offsetable

4442

case 'V': // not offsetable

4443

return C_Memory;

4444

case 'n': // Simple Integer

4445

case 'E': // Floating Point Constant

4446

case 'F': // Floating Point Constant

4447

return C_Immediate;

4448

case 'i': // Simple Integer or Relocatable Constant

4449

case 's': // Relocatable Constant

4450

case 'p': // Address.

4451

case 'X': // Allow ANY value.

4452

case 'I': // Target registers.

4453

case 'J':

4454

case 'K':

4455

case 'L':

4456

case 'M':

4457

case 'N':

4458

case 'O':

4459

case 'P':

4460

case '<':

4461

case '>':

4462

return C_Other;

4463

}

4464

}

4465

4466

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {

4467

if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"

4468

return C_Memory;

4469

return C_Register;

4470

}

4471

return C_Unknown;

4472

}

4473

4474

/// Try to replace an X constraint, which matches anything, with another that

4475

/// has more specific requirements based on the type of the corresponding

4476

/// operand.

4477

const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {

4478

if (ConstraintVT.isInteger())

4479

return "r";

4480

if (ConstraintVT.isFloatingPoint())

4481

return "f"; // works for many targets

4482

return nullptr;

4483

}

4484

4485

SDValue TargetLowering::LowerAsmOutputForConstraint(

4486

SDValue &Chain, SDValue &Flag, const SDLoc &DL,

4487

const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {

4488

return SDValue();

4489

}

4490

4491

/// Lower the specified operand into the Ops vector.

4492

/// If it is invalid, don't add anything to Ops.

4493

void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,

4494

std::string &Constraint,

4495

std::vector<SDValue> &Ops,

4496

SelectionDAG &DAG) const {

4497

4498

if (Constraint.length() > 1) return;

4499

4500

char ConstraintLetter = Constraint[0];

4501

switch (ConstraintLetter) {

4502

default: break;

4503

case 'X': // Allows any operand; labels (basic block) use this.

4504

if (Op.getOpcode() == ISD::BasicBlock ||

4505

Op.getOpcode() == ISD::TargetBlockAddress) {

4506

Ops.push_back(Op);

4507

return;

4508

}

4509

LLVM_FALLTHROUGH[[gnu::fallthrough]];

4510

case 'i': // Simple Integer or Relocatable Constant

4511

case 'n': // Simple Integer

4512

case 's': { // Relocatable Constant

4513

4514

GlobalAddressSDNode *GA;

4515

ConstantSDNode *C;

4516

BlockAddressSDNode *BA;

4517

uint64_t Offset = 0;

4518

4519

// Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),

4520

// etc., since getelementpointer is variadic. We can't use

4521

// SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible

4522

// while in this case the GA may be furthest from the root node which is

4523

// likely an ISD::ADD.

4524

while (1) {

4525

if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4526

Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),

4527

GA->getValueType(0),

4528

Offset + GA->getOffset()));

4529

return;

4530

}

4531

if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {

4532

// gcc prints these as sign extended. Sign extend value to 64 bits

4533

// now; without this it would get ZExt'd later in

4534

// ScheduleDAGSDNodes::EmitNode, which is very generic.

4535

bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;

4536

BooleanContent BCont = getBooleanContents(MVT::i64);

4537

ISD::NodeType ExtOpc =

4538

IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;

4539

int64_t ExtVal =

4540

ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();

4541

Ops.push_back(

4542

DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));

4543

return;

4544

}

4545

if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4546

Ops.push_back(DAG.getTargetBlockAddress(

4547

BA->getBlockAddress(), BA->getValueType(0),

4548

Offset + BA->getOffset(), BA->getTargetFlags()));

4549

return;

4550

}

4551

const unsigned OpCode = Op.getOpcode();

4552

if (OpCode == ISD::ADD || OpCode == ISD::SUB) {

4553

if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))

4554

Op = Op.getOperand(1);

4555

// Subtraction is not commutative.

4556

else if (OpCode == ISD::ADD &&

4557

(C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))

4558

Op = Op.getOperand(0);

4559

else

4560

return;

4561

Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();

4562

continue;

4563

}

4564

return;

4565

}

4566

break;

4567

}

4568

}

4569

}

4570

4571

std::pair<unsigned, const TargetRegisterClass *>

4572

TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,

4573

StringRef Constraint,

4574

MVT VT) const {

4575

if (Constraint.empty() || Constraint[0] != '{')

4576

return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));

4577

assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4577, __extension__ __PRETTY_FUNCTION__));

4578

4579

// Remove the braces from around the name.

4580

StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

4581

4582

std::pair<unsigned, const TargetRegisterClass *> R =

4583

std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

4584

4585

// Figure out which register class contains this reg.

4586

for (const TargetRegisterClass *RC : RI->regclasses()) {

4587

// If none of the value types for this register class are valid, we

4588

// can't use it. For example, 64-bit reg classes on 32-bit targets.

4589

if (!isLegalRC(*RI, *RC))

4590

continue;

4591

4592

for (const MCPhysReg &PR : *RC) {

4593

if (RegName.equals_lower(RI->getRegAsmName(PR))) {

4594

std::pair<unsigned, const TargetRegisterClass *> S =

4595

std::make_pair(PR, RC);

4596

4597

// If this register class has the requested value type, return it,

4598

// otherwise keep searching and return the first class found

4599

// if no other is found which explicitly has the requested type.

4600

if (RI->isTypeLegalForClass(*RC, VT))

4601

return S;

4602

if (!R.second)

4603

R = S;

4604

}

4605

}

4606

}

4607

4608

return R;

4609

}

4610

4611

//===----------------------------------------------------------------------===//

4612

// Constraint Selection.

4613

4614

/// Return true of this is an input operand that is a matching constraint like

4615

/// "4".

4616

bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {

4617

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4617, __extension__ __PRETTY_FUNCTION__));

4618

return isdigit(static_cast<unsigned char>(ConstraintCode[0]));

4619

}

4620

4621

/// If this is an input matching constraint, this method returns the output

4622

/// operand it matches.

4623

unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {

4624

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4624, __extension__ __PRETTY_FUNCTION__));

4625

return atoi(ConstraintCode.c_str());

4626

}

4627

4628

/// Split up the constraint string from the inline assembly value into the

4629

/// specific constraints and their prefixes, and also tie in the associated

4630

/// operand values.

4631

/// If this returns an empty vector, and if the constraint string itself

4632

/// isn't empty, there was an error parsing.

4633

TargetLowering::AsmOperandInfoVector

4634

TargetLowering::ParseConstraints(const DataLayout &DL,

4635

const TargetRegisterInfo *TRI,

4636

const CallBase &Call) const {

4637

/// Information about all of the constraints.

4638

AsmOperandInfoVector ConstraintOperands;

4639

const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());

4640

unsigned maCount = 0; // Largest number of multiple alternative constraints.

4641

4642

// Do a prepass over the constraints, canonicalizing them, and building up the

4643

// ConstraintOperands list.

4644

unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.

4645

unsigned ResNo = 0; // ResNo - The result number of the next output.

4646

4647

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {

4648

ConstraintOperands.emplace_back(std::move(CI));

4649

AsmOperandInfo &OpInfo = ConstraintOperands.back();

4650

4651

// Update multiple alternative constraint count.

4652

if (OpInfo.multipleAlternatives.size() > maCount)

4653

maCount = OpInfo.multipleAlternatives.size();

4654

4655

OpInfo.ConstraintVT = MVT::Other;

4656

4657

// Compute the value type for each operand.

4658

switch (OpInfo.Type) {

4659

case InlineAsm::isOutput:

4660

// Indirect outputs just consume an argument.

4661

if (OpInfo.isIndirect) {

4662

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4663

break;

4664

}

4665

4666

// The return value of the call is this value. As such, there is no

4667

// corresponding argument.

4668

assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4668, __extension__ __PRETTY_FUNCTION__));

4669

if (StructType *STy = dyn_cast<StructType>(Call.getType())) {

4670

OpInfo.ConstraintVT =

4671

getSimpleValueType(DL, STy->getElementType(ResNo));

4672

} else {

4673

assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4673, __extension__ __PRETTY_FUNCTION__));

4674

OpInfo.ConstraintVT = getSimpleValueType(DL, Call.getType());

4675

}

4676

++ResNo;

4677

break;

4678

case InlineAsm::isInput:

4679

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4680

break;

4681

case InlineAsm::isClobber:

4682

// Nothing to do.

4683

break;

4684

}

4685

4686

if (OpInfo.CallOperandVal) {

4687

llvm::Type *OpTy = OpInfo.CallOperandVal->getType();

4688

if (OpInfo.isIndirect) {

4689

llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);

4690

if (!PtrTy)

4691

report_fatal_error("Indirect operand for inline asm not a pointer!");

4692

OpTy = PtrTy->getElementType();

4693

}

4694

4695

// Look for vector wrapped in a struct. e.g. { <16 x i8> }.

4696

if (StructType *STy = dyn_cast<StructType>(OpTy))

4697

if (STy->getNumElements() == 1)

4698

OpTy = STy->getElementType(0);

4699

4700

// If OpTy is not a single value, it may be a struct/union that we

4701

// can tile with integers.

4702

if (!OpTy->isSingleValueType() && OpTy->isSized()) {

4703

unsigned BitSize = DL.getTypeSizeInBits(OpTy);

4704

switch (BitSize) {

4705

default: break;

4706

case 1:

4707

case 8:

4708

case 16:

4709

case 32:

4710

case 64:

4711

case 128:

4712

OpInfo.ConstraintVT =

4713

MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);

4714

break;

4715

}

4716

} else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {

4717

unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());

4718

OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);

4719

} else {

4720

OpInfo.ConstraintVT = MVT::getVT(OpTy, true);

4721

}

4722

}

4723

}

4724

4725

// If we have multiple alternative constraints, select the best alternative.

4726

if (!ConstraintOperands.empty()) {

4727

if (maCount) {

4728

unsigned bestMAIndex = 0;

4729

int bestWeight = -1;

4730

// weight: -1 = invalid match, and 0 = so-so match to 5 = good match.

4731

int weight = -1;

4732

unsigned maIndex;

4733

// Compute the sums of the weights for each alternative, keeping track

4734

// of the best (highest weight) one so far.

4735

for (maIndex = 0; maIndex < maCount; ++maIndex) {

4736

int weightSum = 0;

4737

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4738

cIndex != eIndex; ++cIndex) {

4739

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4740

if (OpInfo.Type == InlineAsm::isClobber)

4741

continue;

4742

4743

// If this is an output operand with a matching input operand,

4744

// look up the matching input. If their types mismatch, e.g. one

4745

// is an integer, the other is floating point, or their sizes are

4746

// different, flag it as an maCantMatch.

4747

if (OpInfo.hasMatchingInput()) {

4748

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4749

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4750

if ((OpInfo.ConstraintVT.isInteger() !=

4751

Input.ConstraintVT.isInteger()) ||

4752

(OpInfo.ConstraintVT.getSizeInBits() !=

4753

Input.ConstraintVT.getSizeInBits())) {

4754

weightSum = -1; // Can't match.

4755

break;

4756

}

4757

}

4758

}

4759

weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);

4760

if (weight == -1) {

4761

weightSum = -1;

4762

break;

4763

}

4764

weightSum += weight;

4765

}

4766

// Update best.

4767

if (weightSum > bestWeight) {

4768

bestWeight = weightSum;

4769

bestMAIndex = maIndex;

4770

}

4771

}

4772

4773

// Now select chosen alternative in each constraint.

4774

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4775

cIndex != eIndex; ++cIndex) {

4776

AsmOperandInfo &cInfo = ConstraintOperands[cIndex];

4777

if (cInfo.Type == InlineAsm::isClobber)

4778

continue;

4779

cInfo.selectAlternative(bestMAIndex);

4780

}

4781

}

4782

}

4783

4784

// Check and hook up tied operands, choose constraint code to use.

4785

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4786

cIndex != eIndex; ++cIndex) {

4787

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4788

4789

// If this is an output operand with a matching input operand, look up the

4790

// matching input. If their types mismatch, e.g. one is an integer, the

4791

// other is floating point, or their sizes are different, flag it as an

4792

// error.

4793

if (OpInfo.hasMatchingInput()) {

4794

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4795

4796

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4797

std::pair<unsigned, const TargetRegisterClass *> MatchRC =

4798

getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,

4799

OpInfo.ConstraintVT);

4800

std::pair<unsigned, const TargetRegisterClass *> InputRC =

4801

getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,

4802

Input.ConstraintVT);

4803

if ((OpInfo.ConstraintVT.isInteger() !=

4804

Input.ConstraintVT.isInteger()) ||

4805

(MatchRC.second != InputRC.second)) {

4806

report_fatal_error("Unsupported asm: input constraint"

4807

" with a matching output constraint of"

4808

" incompatible type!");

4809

}

4810

}

4811

}

4812

}

4813

4814

return ConstraintOperands;

4815

}

4816

4817

/// Return an integer indicating how general CT is.

4818

static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {

4819

switch (CT) {

4820

case TargetLowering::C_Immediate:

4821

case TargetLowering::C_Other:

4822

case TargetLowering::C_Unknown:

4823

return 0;

4824

case TargetLowering::C_Register:

4825

return 1;

4826

case TargetLowering::C_RegisterClass:

4827

return 2;

4828

case TargetLowering::C_Memory:

4829

return 3;

4830

}

4831

llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4831);

4832

}

4833

4834

/// Examine constraint type and operand type and determine a weight value.

4835

/// This object must already have been set up with the operand type

4836

/// and the current alternative constraint selected.

4837

TargetLowering::ConstraintWeight

4838

TargetLowering::getMultipleConstraintMatchWeight(

4839

AsmOperandInfo &info, int maIndex) const {

4840

InlineAsm::ConstraintCodeVector *rCodes;

4841

if (maIndex >= (int)info.multipleAlternatives.size())

4842

rCodes = &info.Codes;

4843

else

4844

rCodes = &info.multipleAlternatives[maIndex].Codes;

4845

ConstraintWeight BestWeight = CW_Invalid;

4846

4847

// Loop over the options, keeping track of the most general one.

4848

for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {

4849

ConstraintWeight weight =

4850

getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());

4851

if (weight > BestWeight)

4852

BestWeight = weight;

4853

}

4854

4855

return BestWeight;

4856

}

4857

4858

/// Examine constraint type and operand type and determine a weight value.

4859

/// This object must already have been set up with the operand type

4860

/// and the current alternative constraint selected.

4861

TargetLowering::ConstraintWeight

4862

TargetLowering::getSingleConstraintMatchWeight(

4863

AsmOperandInfo &info, const char *constraint) const {

4864

ConstraintWeight weight = CW_Invalid;

4865

Value *CallOperandVal = info.CallOperandVal;

4866

// If we don't have a value, we can't do a match,

4867

// but allow it at the lowest weight.

4868

if (!CallOperandVal)

4869

return CW_Default;

4870

// Look at the constraint type.

4871

switch (*constraint) {

4872

case 'i': // immediate integer.

4873

case 'n': // immediate integer with a known value.

4874

if (isa<ConstantInt>(CallOperandVal))

4875

weight = CW_Constant;

4876

break;

4877

case 's': // non-explicit intregal immediate.

4878

if (isa<GlobalValue>(CallOperandVal))

4879

weight = CW_Constant;

4880

break;

4881

case 'E': // immediate float if host format.

4882

case 'F': // immediate float.

4883

if (isa<ConstantFP>(CallOperandVal))

4884

weight = CW_Constant;

4885

break;

4886

case '<': // memory operand with autodecrement.

4887

case '>': // memory operand with autoincrement.

4888

case 'm': // memory operand.

4889

case 'o': // offsettable memory operand

4890

case 'V': // non-offsettable memory operand

4891

weight = CW_Memory;

4892

break;

4893

case 'r': // general register.

4894

case 'g': // general register, memory operand or immediate integer.

4895

// note: Clang converts "g" to "imr".

4896

if (CallOperandVal->getType()->isIntegerTy())

4897

weight = CW_Register;

4898

break;

4899

case 'X': // any operand.

4900

default:

4901

weight = CW_Default;

4902

break;

4903

}

4904

return weight;

4905

}

4906

4907

/// If there are multiple different constraints that we could pick for this

4908

/// operand (e.g. "imr") try to pick the 'best' one.

4909

/// This is somewhat tricky: constraints fall into four classes:

4910

/// Other -> immediates and magic values

4911

/// Register -> one specific register

4912

/// RegisterClass -> a group of regs

4913

/// Memory -> memory

4914

/// Ideally, we would pick the most specific constraint possible: if we have

4915

/// something that fits into a register, we would pick it. The problem here

4916

/// is that if we have something that could either be in a register or in

4917

/// memory that use of the register could cause selection of *other*

4918

/// operands to fail: they might only succeed if we pick memory. Because of

4919

/// this the heuristic we use is:

4920

///

4921

/// 1) If there is an 'other' constraint, and if the operand is valid for

4922

/// that constraint, use it. This makes us take advantage of 'i'

4923

/// constraints when available.

4924

/// 2) Otherwise, pick the most general constraint present. This prefers

4925

/// 'm' over 'r', for example.

4926

///

4927

static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,

4928

const TargetLowering &TLI,

4929

SDValue Op, SelectionDAG *DAG) {

4930

assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__));

4931

unsigned BestIdx = 0;

4932

TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;

4933

int BestGenerality = -1;

4934

4935

// Loop over the options, keeping track of the most general one.

4936

for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {

4937

TargetLowering::ConstraintType CType =

4938

TLI.getConstraintType(OpInfo.Codes[i]);

4939

4940

// Indirect 'other' or 'immediate' constraints are not allowed.

4941

if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||

4942

CType == TargetLowering::C_Register ||

4943

CType == TargetLowering::C_RegisterClass))

4944

continue;

4945

4946

// If this is an 'other' or 'immediate' constraint, see if the operand is

4947

// valid for it. For example, on X86 we might have an 'rI' constraint. If

4948

// the operand is an integer in the range [0..31] we want to use I (saving a

4949

// load of a register), otherwise we must use 'r'.

4950

if ((CType == TargetLowering::C_Other ||

4951

CType == TargetLowering::C_Immediate) && Op.getNode()) {

4952

assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4953, __extension__ __PRETTY_FUNCTION__))

4953

"Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4953, __extension__ __PRETTY_FUNCTION__));

4954

std::vector<SDValue> ResultOps;

4955

TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],

4956

ResultOps, *DAG);

4957

if (!ResultOps.empty()) {

4958

BestType = CType;

4959

BestIdx = i;

4960

break;

4961

}

4962

}

4963

4964

// Things with matching constraints can only be registers, per gcc

4965

// documentation. This mainly affects "g" constraints.

4966

if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())

4967

continue;

4968

4969

// This constraint letter is more general than the previous one, use it.

4970

int Generality = getConstraintGenerality(CType);

4971

if (Generality > BestGenerality) {

4972

BestType = CType;

4973

BestIdx = i;

4974

BestGenerality = Generality;

4975

}

4976

}

4977

4978

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];

4979

OpInfo.ConstraintType = BestType;

4980

}

4981

4982

/// Determines the constraint code and constraint type to use for the specific

4983

/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.

4984

void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

4985

SDValue Op,

4986

SelectionDAG *DAG) const {

4987

assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4987, __extension__ __PRETTY_FUNCTION__));

4988

4989

// Single-letter constraints ('r') are very common.

4990

if (OpInfo.Codes.size() == 1) {

4991

OpInfo.ConstraintCode = OpInfo.Codes[0];

4992

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

4993

} else {

4994

ChooseConstraint(OpInfo, *this, Op, DAG);

4995

}

4996

4997

// 'X' matches anything.

4998

if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {

4999

// Labels and constants are handled elsewhere ('X' is the only thing

5000

// that matches labels). For Functions, the type here is the type of

5001

// the result, which is not what we want to look at; leave them alone.

5002

Value *v = OpInfo.CallOperandVal;

5003

if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {

5004

OpInfo.CallOperandVal = v;

5005

return;

5006

}

5007

5008

if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)

5009

return;

5010

5011

// Otherwise, try to resolve it to something we know about by looking at

5012

// the actual operand type.

5013

if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {

5014

OpInfo.ConstraintCode = Repl;

5015

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5016

}

5017

}

5018

}

5019

5020

/// Given an exact SDIV by a constant, create a multiplication

5021

/// with the multiplicative inverse of the constant.

5022

static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,

5023

const SDLoc &dl, SelectionDAG &DAG,

5024

SmallVectorImpl<SDNode *> &Created) {

5025

SDValue Op0 = N->getOperand(0);

5026

SDValue Op1 = N->getOperand(1);

5027

EVT VT = N->getValueType(0);

5028

EVT SVT = VT.getScalarType();

5029

EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());

5030

EVT ShSVT = ShVT.getScalarType();

5031

5032

bool UseSRA = false;

5033

SmallVector<SDValue, 16> Shifts, Factors;

5034

5035

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5036

if (C->isNullValue())

5037

return false;

5038

APInt Divisor = C->getAPIntValue();

5039

unsigned Shift = Divisor.countTrailingZeros();

5040

if (Shift) {

5041

Divisor.ashrInPlace(Shift);

5042

UseSRA = true;

5043

}

5044

// Calculate the multiplicative inverse, using Newton's method.

5045

APInt t;

5046

APInt Factor = Divisor;

5047

while ((t = Divisor * Factor) != 1)

5048

Factor *= APInt(Divisor.getBitWidth(), 2) - t;

5049

Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));

5050

Factors.push_back(DAG.getConstant(Factor, dl, SVT));

5051

return true;

5052

};

5053

5054

// Collect all magic values from the build vector.

5055

if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))

5056

return SDValue();

5057

5058

SDValue Shift, Factor;

5059

if (Op1.getOpcode() == ISD::BUILD_VECTOR) {

5060

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5061

Factor = DAG.getBuildVector(VT, dl, Factors);

5062

} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {

5063

assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))

5064

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))

5065

"vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__));

5066

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5067

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5068

} else {

5069

assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5069, __extension__ __PRETTY_FUNCTION__));

5070

Shift = Shifts[0];

5071

Factor = Factors[0];

5072

}

5073

5074

SDValue Res = Op0;

5075

5076

// Shift the value upfront if it is even, so the LSB is one.

5077

if (UseSRA) {

5078

// TODO: For UDIV use SRL instead of SRA.

5079

SDNodeFlags Flags;

5080

Flags.setExact(true);

5081

Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);

5082

Created.push_back(Res.getNode());

5083

}

5084

5085

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);

5086

}

5087

5088

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,

5089

SelectionDAG &DAG,

5090

SmallVectorImpl<SDNode *> &Created) const {

5091

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

5092

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5093

if (TLI.isIntDivCheap(N->getValueType(0), Attr))

5094

return SDValue(N, 0); // Lower SDIV as SDIV

5095

return SDValue();

5096

}

5097

5098

/// Given an ISD::SDIV node expressing a divide by constant,

5099

/// return a DAG expression to select that will generate the same value by

5100

/// multiplying by a magic number.

5101

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5102

SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,

5103

bool IsAfterLegalization,

5104

SmallVectorImpl<SDNode *> &Created) const {

5105

SDLoc dl(N);

5106

EVT VT = N->getValueType(0);

5107

EVT SVT = VT.getScalarType();

5108

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5109

EVT ShSVT = ShVT.getScalarType();

5110

unsigned EltBits = VT.getScalarSizeInBits();

5111

EVT MulVT;

5112

5113

// Check to see if we can do this.

5114

// FIXME: We should be more aggressive here.

5115

if (!isTypeLegal(VT)) {

5116

// Limit this to simple scalars for now.

5117

if (VT.isVector() || !VT.isSimple())

5118

return SDValue();

5119

5120

// If this type will be promoted to a large enough type with a legal

5121

// multiply operation, we can go ahead and do this transform.

5122

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5123

return SDValue();

5124

5125

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5126

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5127

!isOperationLegal(ISD::MUL, MulVT))

5128

return SDValue();

5129

}

5130

5131

// If the sdiv has an 'exact' bit we can use a simpler lowering.

5132

if (N->getFlags().hasExact())

5133

return BuildExactSDIV(*this, N, dl, DAG, Created);

5134

5135

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

5136

5137

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5138

if (C->isNullValue())

5139

return false;

5140

5141

const APInt &Divisor = C->getAPIntValue();

5142

APInt::ms magics = Divisor.magic();

5143

int NumeratorFactor = 0;

5144

int ShiftMask = -1;

5145

5146

if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {

5147

// If d is +1/-1, we just multiply the numerator by +1/-1.

5148

NumeratorFactor = Divisor.getSExtValue();

5149

magics.m = 0;

5150

magics.s = 0;

5151

ShiftMask = 0;

5152

} else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {

5153

// If d > 0 and m < 0, add the numerator.

5154

NumeratorFactor = 1;

5155

} else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {

5156

// If d < 0 and m > 0, subtract the numerator.

5157

NumeratorFactor = -1;

5158

}

5159

5160

MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));

5161

Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));

5162

Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));

5163

ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));

5164

return true;

5165

};

5166

5167

SDValue N0 = N->getOperand(0);

5168

SDValue N1 = N->getOperand(1);

5169

5170

// Collect the shifts / magic values from each element.

5171

if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))

5172

return SDValue();

5173

5174

SDValue MagicFactor, Factor, Shift, ShiftMask;

5175

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5176

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5177

Factor = DAG.getBuildVector(VT, dl, Factors);

5178

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5179

ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);

5180

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5181

assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))

5182

Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))

5183

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))

5184

"vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__));

5185

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5186

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5187

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5188

ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);

5189

} else {

5190

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5190, __extension__ __PRETTY_FUNCTION__));

5191

MagicFactor = MagicFactors[0];

5192

Factor = Factors[0];

5193

Shift = Shifts[0];

5194

ShiftMask = ShiftMasks[0];

5195

}

5196

5197

// Multiply the numerator (operand 0) by the magic value.

5198

// FIXME: We should support doing a MUL in a wider type.

5199

auto GetMULHS = [&](SDValue X, SDValue Y) {

5200

// If the type isn't legal, use a wider mul of the the type calculated

5201

// earlier.

5202

if (!isTypeLegal(VT)) {

5203

X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);

5204

Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);

5205

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5206

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5207

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5208

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5209

}

5210

5211

if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))

5212

return DAG.getNode(ISD::MULHS, dl, VT, X, Y);

5213

if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {

5214

SDValue LoHi =

5215

DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5216

return SDValue(LoHi.getNode(), 1);

5217

}

5218

return SDValue();

5219

};

5220

5221

SDValue Q = GetMULHS(N0, MagicFactor);

5222

if (!Q)

5223

return SDValue();

5224

5225

Created.push_back(Q.getNode());

5226

5227

// (Optionally) Add/subtract the numerator using Factor.

5228

Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);

5229

Created.push_back(Factor.getNode());

5230

Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);

5231

Created.push_back(Q.getNode());

5232

5233

// Shift right algebraic by shift value.

5234

Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);

5235

Created.push_back(Q.getNode());

5236

5237

// Extract the sign bit, mask it and add it to the quotient.

5238

SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);

5239

SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);

5240

Created.push_back(T.getNode());

5241

T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);

5242

Created.push_back(T.getNode());

5243

return DAG.getNode(ISD::ADD, dl, VT, Q, T);

5244

}

5245

5246

/// Given an ISD::UDIV node expressing a divide by constant,

5247

/// return a DAG expression to select that will generate the same value by

5248

/// multiplying by a magic number.

5249

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5250

SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,

5251

bool IsAfterLegalization,

5252

SmallVectorImpl<SDNode *> &Created) const {

5253

SDLoc dl(N);

5254

EVT VT = N->getValueType(0);

5255

EVT SVT = VT.getScalarType();

5256

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5257

EVT ShSVT = ShVT.getScalarType();

5258

unsigned EltBits = VT.getScalarSizeInBits();

5259

EVT MulVT;

5260

5261

// Check to see if we can do this.

5262

// FIXME: We should be more aggressive here.

5263

if (!isTypeLegal(VT)) {

5264

// Limit this to simple scalars for now.

5265

if (VT.isVector() || !VT.isSimple())

5266

return SDValue();

5267

5268

// If this type will be promoted to a large enough type with a legal

5269

// multiply operation, we can go ahead and do this transform.

5270

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5271

return SDValue();

5272

5273

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5274

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5275

!isOperationLegal(ISD::MUL, MulVT))

5276

return SDValue();

5277

}

5278

5279

bool UseNPQ = false;

5280

SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

5281

5282

auto BuildUDIVPattern = [&](ConstantSDNode *C) {

5283

if (C->isNullValue())

5284

return false;

5285

// FIXME: We should use a narrower constant when the upper

5286

// bits are known to be zero.

5287

const APInt& Divisor = C->getAPIntValue();

5288

APInt::mu magics = Divisor.magicu();

5289

unsigned PreShift = 0, PostShift = 0;

5290

5291

// If the divisor is even, we can avoid using the expensive fixup by

5292

// shifting the divided value upfront.

5293

if (magics.a != 0 && !Divisor[0]) {

5294

PreShift = Divisor.countTrailingZeros();

5295

// Get magic number for the shifted divisor.

5296

magics = Divisor.lshr(PreShift).magicu(PreShift);

5297

assert(magics.a == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.a == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5297, __extension__ __PRETTY_FUNCTION__));

5298

}

5299

5300

APInt Magic = magics.m;

5301

5302

unsigned SelNPQ;

5303

if (magics.a == 0 || Divisor.isOneValue()) {

5304

assert(magics.s < Divisor.getBitWidth() &&(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__))

5305

"We shouldn't generate an undefined shift!")(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__));

5306

PostShift = magics.s;

5307

SelNPQ = false;

5308

} else {

5309

PostShift = magics.s - 1;

5310

SelNPQ = true;

5311

}

5312

5313

PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));

5314

MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));

5315

NPQFactors.push_back(

5316

DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)

5317

: APInt::getNullValue(EltBits),

5318

dl, SVT));

5319

PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));

5320

UseNPQ |= SelNPQ;

5321

return true;

5322

};

5323

5324

SDValue N0 = N->getOperand(0);

5325

SDValue N1 = N->getOperand(1);

5326

5327

// Collect the shifts/magic values from each element.

5328

if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))

5329

return SDValue();

5330

5331

SDValue PreShift, PostShift, MagicFactor, NPQFactor;

5332

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5333

PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);

5334

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5335

NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);

5336

PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);

5337

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5338

assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))

5339

NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))

5340

"Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__));

5341

PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);

5342

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5343

NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);

5344

PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);

5345

} else {

5346

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5346, __extension__ __PRETTY_FUNCTION__));

5347

PreShift = PreShifts[0];

5348

MagicFactor = MagicFactors[0];

5349

PostShift = PostShifts[0];

5350

}

5351

5352

SDValue Q = N0;

5353

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);

5354

Created.push_back(Q.getNode());

5355

5356

// FIXME: We should support doing a MUL in a wider type.

5357

auto GetMULHU = [&](SDValue X, SDValue Y) {

5358

// If the type isn't legal, use a wider mul of the the type calculated

5359

// earlier.

5360

if (!isTypeLegal(VT)) {

5361

X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);

5362

Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);

5363

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5364

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5365

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5366

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5367

}

5368

5369

if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))

5370

return DAG.getNode(ISD::MULHU, dl, VT, X, Y);

5371

if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {

5372

SDValue LoHi =

5373

DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5374

return SDValue(LoHi.getNode(), 1);

5375

}

5376

return SDValue(); // No mulhu or equivalent

5377

};

5378

5379

// Multiply the numerator (operand 0) by the magic value.

5380

Q = GetMULHU(Q, MagicFactor);

5381

if (!Q)

5382

return SDValue();

5383

5384

Created.push_back(Q.getNode());

5385

5386

if (UseNPQ) {

5387

SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);

5388

Created.push_back(NPQ.getNode());

5389

5390

// For vectors we might have a mix of non-NPQ/NPQ paths, so use

5391

// MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.

5392

if (VT.isVector())

5393

NPQ = GetMULHU(NPQ, NPQFactor);

5394

else

5395

NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

5396

5397

Created.push_back(NPQ.getNode());

5398

5399

Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);

5400

Created.push_back(Q.getNode());

5401

}

5402

5403

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);

5404

Created.push_back(Q.getNode());

5405

5406

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5407

5408

SDValue One = DAG.getConstant(1, dl, VT);

5409

SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);

5410

return DAG.getSelect(dl, VT, IsOne, N0, Q);

5411

}

5412

5413

/// If all values in Values that *don't* match the predicate are same 'splat'

5414

/// value, then replace all values with that splat value.

5415

/// Else, if AlternativeReplacement was provided, then replace all values that

5416

/// do match predicate with AlternativeReplacement value.

5417

static void

5418

turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,

5419

std::function<bool(SDValue)> Predicate,

5420

SDValue AlternativeReplacement = SDValue()) {

5421

SDValue Replacement;

5422

// Is there a value for which the Predicate does *NOT* match? What is it?

5423

auto SplatValue = llvm::find_if_not(Values, Predicate);

5424

if (SplatValue != Values.end()) {

5425

// Does Values consist only of SplatValue's and values matching Predicate?

5426

if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {

5427

return Value == *SplatValue || Predicate(Value);

5428

})) // Then we shall replace values matching predicate with SplatValue.

5429

Replacement = *SplatValue;

5430

}

5431

if (!Replacement) {

5432

// Oops, we did not find the "baseline" splat value.

5433

if (!AlternativeReplacement)

5434

return; // Nothing to do.

5435

// Let's replace with provided value then.

5436

Replacement = AlternativeReplacement;

5437

}

5438

std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);

5439

}

5440

5441

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE

5442

/// where the divisor is constant and the comparison target is zero,

5443

/// return a DAG expression that will generate the same comparison result

5444

/// using only multiplications, additions and shifts/rotations.

5445

/// Ref: "Hacker's Delight" 10-17.

5446

SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,

5447

SDValue CompTargetNode,

5448

ISD::CondCode Cond,

5449

DAGCombinerInfo &DCI,

5450

const SDLoc &DL) const {

5451

SmallVector<SDNode *, 5> Built;

5452

if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5453

DCI, DL, Built)) {

5454

for (SDNode *N : Built)

5455

DCI.AddToWorklist(N);

5456

return Folded;

5457

}

5458

5459

return SDValue();

5460

}

5461

5462

SDValue

5463

TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,

5464

SDValue CompTargetNode, ISD::CondCode Cond,

5465

DAGCombinerInfo &DCI, const SDLoc &DL,

5466

SmallVectorImpl<SDNode *> &Created) const {

5467

// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)

5468

// - D must be constant, with D = D0 * 2^K where D0 is odd

5469

// - P is the multiplicative inverse of D0 modulo 2^W

5470

// - Q = floor(((2^W) - 1) / D)

5471

// where W is the width of the common type of N and D.

5472

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5473, __extension__ __PRETTY_FUNCTION__))

5473

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5473, __extension__ __PRETTY_FUNCTION__));

5474

5475

SelectionDAG &DAG = DCI.DAG;

5476

5477

EVT VT = REMNode.getValueType();

5478

EVT SVT = VT.getScalarType();

5479

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5480

EVT ShSVT = ShVT.getScalarType();

5481

5482

// If MUL is unavailable, we cannot proceed in any case.

5483

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5484

return SDValue();

5485

5486

bool ComparingWithAllZeros = true;

5487

bool AllComparisonsWithNonZerosAreTautological = true;

5488

bool HadTautologicalLanes = false;

5489

bool AllLanesAreTautological = true;

5490

bool HadEvenDivisor = false;

5491

bool AllDivisorsArePowerOfTwo = true;

5492

bool HadTautologicalInvertedLanes = false;

5493

SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

5494

5495

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {

5496

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5497

if (CDiv->isNullValue())

5498

return false;

5499

5500

const APInt &D = CDiv->getAPIntValue();

5501

const APInt &Cmp = CCmp->getAPIntValue();

5502

5503

ComparingWithAllZeros &= Cmp.isNullValue();

5504

5505

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5506

// if C2 is not less than C1, the comparison is always false.

5507

// But we will only be able to produce the comparison that will give the

5508

// opposive tautological answer. So this lane would need to be fixed up.

5509

bool TautologicalInvertedLane = D.ule(Cmp);

5510

HadTautologicalInvertedLanes |= TautologicalInvertedLane;

5511

5512

// If all lanes are tautological (either all divisors are ones, or divisor

5513

// is not greater than the constant we are comparing with),

5514

// we will prefer to avoid the fold.

5515

bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;

5516

HadTautologicalLanes |= TautologicalLane;

5517

AllLanesAreTautological &= TautologicalLane;

5518

5519

// If we are comparing with non-zero, we need'll need to subtract said

5520

// comparison value from the LHS. But there is no point in doing that if

5521

// every lane where we are comparing with non-zero is tautological..

5522

if (!Cmp.isNullValue())

5523

AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

5524

5525

// Decompose D into D0 * 2^K

5526

unsigned K = D.countTrailingZeros();

5527

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5527, __extension__ __PRETTY_FUNCTION__));

5528

APInt D0 = D.lshr(K);

5529

5530

// D is even if it has trailing zeros.

5531

HadEvenDivisor |= (K != 0);

5532

// D is a power-of-two if D0 is one.

5533

// If all divisors are power-of-two, we will prefer to avoid the fold.

5534

AllDivisorsArePowerOfTwo &= D0.isOneValue();

5535

5536

// P = inv(D0, 2^W)

5537

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5538

unsigned W = D.getBitWidth();

5539

APInt P = D0.zext(W + 1)

5540

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5541

.trunc(W);

5542

assert(!P.isNullValue() && "No multiplicative inverse!")(static_cast <bool> (!P.isNullValue() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5542, __extension__ __PRETTY_FUNCTION__)); // unreachable

5543

assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5543, __extension__ __PRETTY_FUNCTION__));

5544

5545

// Q = floor((2^W - 1) u/ D)

5546

// R = ((2^W - 1) u% D)

5547

APInt Q, R;

5548

APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);

5549

5550

// If we are comparing with zero, then that comparison constant is okay,

5551

// else it may need to be one less than that.

5552

if (Cmp.ugt(R))

5553

Q -= 1;

5554

5555

assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5556, __extension__ __PRETTY_FUNCTION__))

5556

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5556, __extension__ __PRETTY_FUNCTION__));

5557

5558

// If the lane is tautological the result can be constant-folded.

5559

if (TautologicalLane) {

5560

// Set P and K amount to a bogus values so we can try to splat them.

5561

P = 0;

5562

K = -1;

5563

// And ensure that comparison constant is tautological,

5564

// it will always compare true/false.

5565

Q = -1;

5566

}

5567

5568

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5569

KAmts.push_back(

5570

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5571

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5572

return true;

5573

};

5574

5575

SDValue N = REMNode.getOperand(0);

5576

SDValue D = REMNode.getOperand(1);

5577

5578

// Collect the values from each element.

5579

if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))

5580

return SDValue();

5581

5582

// If all lanes are tautological, the result can be constant-folded.

5583

if (AllLanesAreTautological)

5584

return SDValue();

5585

5586

// If this is a urem by a powers-of-two, avoid the fold since it can be

5587

// best implemented as a bit test.

5588

if (AllDivisorsArePowerOfTwo)

5589

return SDValue();

5590

5591

SDValue PVal, KVal, QVal;

5592

if (VT.isVector()) {

5593

if (HadTautologicalLanes) {

5594

// Try to turn PAmts into a splat, since we don't care about the values

5595

// that are currently '0'. If we can't, just keep '0'`s.

5596

turnVectorIntoSplatVector(PAmts, isNullConstant);

5597

// Try to turn KAmts into a splat, since we don't care about the values

5598

// that are currently '-1'. If we can't, change them to '0'`s.

5599

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5600

DAG.getConstant(0, DL, ShSVT));

5601

}

5602

5603

PVal = DAG.getBuildVector(VT, DL, PAmts);

5604

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5605

QVal = DAG.getBuildVector(VT, DL, QAmts);

5606

} else {

5607

PVal = PAmts[0];

5608

KVal = KAmts[0];

5609

QVal = QAmts[0];

5610

}

5611

5612

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {

5613

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))

5614

return SDValue(); // FIXME: Could/should use `ISD::ADD`?

5615

assert(CompTargetNode.getValueType() == N.getValueType() &&(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5616, __extension__ __PRETTY_FUNCTION__))

5616

"Expecting that the types on LHS and RHS of comparisons match.")(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5616, __extension__ __PRETTY_FUNCTION__));

5617

N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);

5618

}

5619

5620

// (mul N, P)

5621

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5622

Created.push_back(Op0.getNode());

5623

5624

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5625

// divisors as a performance improvement, since rotating by 0 is a no-op.

5626

if (HadEvenDivisor) {

5627

// We need ROTR to do this.

5628

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5629

return SDValue();

5630

SDNodeFlags Flags;

5631

Flags.setExact(true);

5632

// UREM: (rotr (mul N, P), K)

5633

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);

5634

Created.push_back(Op0.getNode());

5635

}

5636

5637

// UREM: (setule/setugt (rotr (mul N, P), K), Q)

5638

SDValue NewCC =

5639

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5640

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5641

if (!HadTautologicalInvertedLanes)

5642

return NewCC;

5643

5644

// If any lanes previously compared always-false, the NewCC will give

5645

// always-true result for them, so we need to fixup those lanes.

5646

// Or the other way around for inequality predicate.

5647

assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5647, __extension__ __PRETTY_FUNCTION__));

5648

Created.push_back(NewCC.getNode());

5649

5650

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5651

// if C2 is not less than C1, the comparison is always false.

5652

// But we have produced the comparison that will give the

5653

// opposive tautological answer. So these lanes would need to be fixed up.

5654

SDValue TautologicalInvertedChannels =

5655

DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);

5656

Created.push_back(TautologicalInvertedChannels.getNode());

5657

5658

// NOTE: we avoid letting illegal types through even if we're before legalize

5659

// ops – legalization has a hard time producing good code for this.

5660

if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {

5661

// If we have a vector select, let's replace the comparison results in the

5662

// affected lanes with the correct tautological result.

5663

SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,

5664

DL, SETCCVT, SETCCVT);

5665

return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,

5666

Replacement, NewCC);

5667

}

5668

5669

// Else, we can just invert the comparison result in the appropriate lanes.

5670

//

5671

// NOTE: see the note above VSELECT above.

5672

if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))

5673

return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,

5674

TautologicalInvertedChannels);

5675

5676

return SDValue(); // Don't know how to lower.

5677

}

5678

5679

/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE

5680

/// where the divisor is constant and the comparison target is zero,

5681

/// return a DAG expression that will generate the same comparison result

5682

/// using only multiplications, additions and shifts/rotations.

5683

/// Ref: "Hacker's Delight" 10-17.

5684

SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,

5685

SDValue CompTargetNode,

5686

ISD::CondCode Cond,

5687

DAGCombinerInfo &DCI,

5688

const SDLoc &DL) const {

5689

SmallVector<SDNode *, 7> Built;

5690

if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5691

DCI, DL, Built)) {

5692

assert(Built.size() <= 7 && "Max size prediction failed.")(static_cast <bool> (Built.size() <= 7 && "Max size prediction failed."
) ? void (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5692, __extension__ __PRETTY_FUNCTION__));

5693

for (SDNode *N : Built)

5694

DCI.AddToWorklist(N);

5695

return Folded;

5696

}

5697

5698

return SDValue();

5699

}

5700

5701

SDValue

5702

TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,

5703

SDValue CompTargetNode, ISD::CondCode Cond,

5704

DAGCombinerInfo &DCI, const SDLoc &DL,

5705

SmallVectorImpl<SDNode *> &Created) const {

5706

// Fold:

5707

// (seteq/ne (srem N, D), 0)

5708

// To:

5709

// (setule/ugt (rotr (add (mul N, P), A), K), Q)

5710

//

5711

// - D must be constant, with D = D0 * 2^K where D0 is odd

5712

// - P is the multiplicative inverse of D0 modulo 2^W

5713

// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))

5714

// - Q = floor((2 * A) / (2^K))

5715

// where W is the width of the common type of N and D.

5716

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5717, __extension__ __PRETTY_FUNCTION__))

5717

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5717, __extension__ __PRETTY_FUNCTION__));

5718

5719

SelectionDAG &DAG = DCI.DAG;

5720

5721

EVT VT = REMNode.getValueType();

5722

EVT SVT = VT.getScalarType();

5723

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5724

EVT ShSVT = ShVT.getScalarType();

5725

5726

// If we are after ops legalization, and MUL is unavailable, we can not

5727

// proceed.

5728

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5729

return SDValue();

5730

5731

// TODO: Could support comparing with non-zero too.

5732

ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);

5733

if (!CompTarget || !CompTarget->isNullValue())

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 7915, column 5 Value stored to 'IsScaledIndex' is never read

Bug Summary

Annotated Source Code