/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1

//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements the TargetLowering class.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "llvm/CodeGen/TargetLowering.h"

14

#include "llvm/ADT/STLExtras.h"

15

#include "llvm/CodeGen/CallingConvLower.h"

16

#include "llvm/CodeGen/MachineFrameInfo.h"

17

#include "llvm/CodeGen/MachineFunction.h"

18

#include "llvm/CodeGen/MachineJumpTableInfo.h"

19

#include "llvm/CodeGen/MachineRegisterInfo.h"

20

#include "llvm/CodeGen/SelectionDAG.h"

21

#include "llvm/CodeGen/TargetRegisterInfo.h"

22

#include "llvm/CodeGen/TargetSubtargetInfo.h"

23

#include "llvm/IR/DataLayout.h"

24

#include "llvm/IR/DerivedTypes.h"

25

#include "llvm/IR/GlobalVariable.h"

26

#include "llvm/IR/LLVMContext.h"

27

#include "llvm/MC/MCAsmInfo.h"

28

#include "llvm/MC/MCExpr.h"

29

#include "llvm/Support/ErrorHandling.h"

30

#include "llvm/Support/KnownBits.h"

31

#include "llvm/Support/MathExtras.h"

32

#include "llvm/Target/TargetLoweringObjectFile.h"

33

#include "llvm/Target/TargetMachine.h"

34

#include <cctype>

35

using namespace llvm;

36

37

/// NOTE: The TargetMachine owns TLOF.

38

TargetLowering::TargetLowering(const TargetMachine &tm)

39

: TargetLoweringBase(tm) {}

40

41

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {

42

return nullptr;

43

}

44

45

bool TargetLowering::isPositionIndependent() const {

46

return getTargetMachine().isPositionIndependent();

47

}

48

49

/// Check whether a given call node is in tail position within its function. If

50

/// so, it sets Chain to the input chain of the tail call.

51

bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,

52

SDValue &Chain) const {

53

const Function &F = DAG.getMachineFunction().getFunction();

54

55

// First, check if tail calls have been disabled in this function.

56

if (F.getFnAttribute("disable-tail-calls").getValueAsBool())

57

return false;

58

59

// Conservatively require the attributes of the call to match those of

60

// the return. Ignore following attributes because they don't affect the

61

// call sequence.

62

AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);

63

for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,

64

Attribute::DereferenceableOrNull, Attribute::NoAlias,

65

Attribute::NonNull})

66

CallerAttrs.removeAttribute(Attr);

67

68

if (CallerAttrs.hasAttributes())

69

return false;

70

71

// It's not safe to eliminate the sign / zero extension of the return value.

72

if (CallerAttrs.contains(Attribute::ZExt) ||

73

CallerAttrs.contains(Attribute::SExt))

74

return false;

75

76

// Check if the only use is a function return node.

77

return isUsedByReturnOnly(Node, Chain);

78

}

79

80

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,

81

const uint32_t *CallerPreservedMask,

82

const SmallVectorImpl<CCValAssign> &ArgLocs,

83

const SmallVectorImpl<SDValue> &OutVals) const {

84

for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {

85

const CCValAssign &ArgLoc = ArgLocs[I];

86

if (!ArgLoc.isRegLoc())

87

continue;

88

MCRegister Reg = ArgLoc.getLocReg();

89

// Only look at callee saved registers.

90

if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))

91

continue;

92

// Check that we pass the value used for the caller.

93

// (We look for a CopyFromReg reading a virtual register that is used

94

// for the function live-in value of register Reg)

95

SDValue Value = OutVals[I];

96

if (Value->getOpcode() != ISD::CopyFromReg)

97

return false;

98

Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();

99

if (MRI.getLiveInPhysReg(ArgReg) != Reg)

100

return false;

101

}

102

return true;

103

}

104

105

/// Set CallLoweringInfo attribute flags based on a call instruction

106

/// and called function attributes.

107

void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,

108

unsigned ArgIdx) {

109

IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);

110

IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);

111

IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);

112

IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);

113

IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);

114

IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);

115

IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);

116

IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);

117

IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);

118

IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);

119

IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);

120

IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);

121

Alignment = Call->getParamStackAlign(ArgIdx);

122

IndirectType = nullptr;

123

assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 124, __extension__ __PRETTY_FUNCTION__))

124

"multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 124, __extension__ __PRETTY_FUNCTION__));

125

if (IsByVal) {

126

IndirectType = Call->getParamByValType(ArgIdx);

127

if (!Alignment)

128

Alignment = Call->getParamAlign(ArgIdx);

129

}

130

if (IsPreallocated)

131

IndirectType = Call->getParamPreallocatedType(ArgIdx);

132

if (IsInAlloca)

133

IndirectType = Call->getParamInAllocaType(ArgIdx);

134

}

135

136

/// Generate a libcall taking the given operands as arguments and returning a

137

/// result of type RetVT.

138

std::pair<SDValue, SDValue>

139

TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,

140

ArrayRef<SDValue> Ops,

141

MakeLibCallOptions CallOptions,

142

const SDLoc &dl,

143

SDValue InChain) const {

144

if (!InChain)

145

InChain = DAG.getEntryNode();

146

147

TargetLowering::ArgListTy Args;

148

Args.reserve(Ops.size());

149

150

TargetLowering::ArgListEntry Entry;

151

for (unsigned i = 0; i < Ops.size(); ++i) {

152

SDValue NewOp = Ops[i];

153

Entry.Node = NewOp;

154

Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());

155

Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),

156

CallOptions.IsSExt);

157

Entry.IsZExt = !Entry.IsSExt;

158

159

if (CallOptions.IsSoften &&

160

!shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {

161

Entry.IsSExt = Entry.IsZExt = false;

162

}

163

Args.push_back(Entry);

164

}

165

166

if (LC == RTLIB::UNKNOWN_LIBCALL)

167

report_fatal_error("Unsupported library call operation!");

168

SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),

169

getPointerTy(DAG.getDataLayout()));

170

171

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());

172

TargetLowering::CallLoweringInfo CLI(DAG);

173

bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);

174

bool zeroExtend = !signExtend;

175

176

if (CallOptions.IsSoften &&

177

!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {

178

signExtend = zeroExtend = false;

179

}

180

181

CLI.setDebugLoc(dl)

182

.setChain(InChain)

183

.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))

184

.setNoReturn(CallOptions.DoesNotReturn)

185

.setDiscardResult(!CallOptions.IsReturnValueUsed)

186

.setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)

187

.setSExtResult(signExtend)

188

.setZExtResult(zeroExtend);

189

return LowerCallTo(CLI);

190

}

191

192

bool TargetLowering::findOptimalMemOpLowering(

193

std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,

194

unsigned SrcAS, const AttributeList &FuncAttributes) const {

195

if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())

196

return false;

197

198

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

199

200

if (VT == MVT::Other) {

201

// Use the largest integer type whose alignment constraints are satisfied.

202

// We only need to check DstAlign here as SrcAlign is always greater or

203

// equal to DstAlign (or zero).

204

VT = MVT::i64;

205

if (Op.isFixedDstAlign())

206

while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

207

!allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))

208

VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);

209

assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 209, __extension__ __PRETTY_FUNCTION__));

210

211

// Find the largest legal integer type.

212

MVT LVT = MVT::i64;

213

while (!isTypeLegal(LVT))

214

LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);

215

assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 215, __extension__ __PRETTY_FUNCTION__));

216

217

// If the type we've chosen is larger than the largest legal integer type

218

// then use that instead.

219

if (VT.bitsGT(LVT))

220

VT = LVT;

221

}

222

223

unsigned NumMemOps = 0;

224

uint64_t Size = Op.size();

225

while (Size) {

226

unsigned VTSize = VT.getSizeInBits() / 8;

227

while (VTSize > Size) {

228

// For now, only use non-vector load / store's for the left-over pieces.

229

EVT NewVT = VT;

230

unsigned NewVTSize;

231

232

bool Found = false;

233

if (VT.isVector() || VT.isFloatingPoint()) {

234

NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;

235

if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&

236

isSafeMemOpType(NewVT.getSimpleVT()))

237

Found = true;

238

else if (NewVT == MVT::i64 &&

239

isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&

240

isSafeMemOpType(MVT::f64)) {

241

// i64 is usually not legal on 32-bit targets, but f64 may be.

242

NewVT = MVT::f64;

243

Found = true;

244

}

245

}

246

247

if (!Found) {

248

do {

249

NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);

250

if (NewVT == MVT::i8)

251

break;

252

} while (!isSafeMemOpType(NewVT.getSimpleVT()));

253

}

254

NewVTSize = NewVT.getSizeInBits() / 8;

255

256

// If the new VT cannot cover all of the remaining bits, then consider

257

// issuing a (or a pair of) unaligned and overlapping load / store.

258

bool Fast;

259

if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&

260

allowsMisalignedMemoryAccesses(

261

VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),

262

MachineMemOperand::MONone, &Fast) &&

263

Fast)

264

VTSize = Size;

265

else {

266

VT = NewVT;

267

VTSize = NewVTSize;

268

}

269

}

270

271

if (++NumMemOps > Limit)

272

return false;

273

274

MemOps.push_back(VT);

275

Size -= VTSize;

276

}

277

278

return true;

279

}

280

281

/// Soften the operands of a comparison. This code is shared among BR_CC,

282

/// SELECT_CC, and SETCC handlers.

283

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

284

SDValue &NewLHS, SDValue &NewRHS,

285

ISD::CondCode &CCCode,

286

const SDLoc &dl, const SDValue OldLHS,

287

const SDValue OldRHS) const {

288

SDValue Chain;

289

return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,

290

OldRHS, Chain);

291

}

292

293

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

294

SDValue &NewLHS, SDValue &NewRHS,

295

ISD::CondCode &CCCode,

296

const SDLoc &dl, const SDValue OldLHS,

297

const SDValue OldRHS,

298

SDValue &Chain,

299

bool IsSignaling) const {

300

// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc

301

// not supporting it. We can update this code when libgcc provides such

302

// functions.

303

304

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __extension__ __PRETTY_FUNCTION__))

305

&& "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __extension__ __PRETTY_FUNCTION__));

306

307

// Expand into one or more soft-fp libcall(s).

308

RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;

309

bool ShouldInvertCC = false;

310

switch (CCCode) {

311

case ISD::SETEQ:

312

case ISD::SETOEQ:

313

LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

314

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

315

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

316

break;

317

case ISD::SETNE:

318

case ISD::SETUNE:

319

LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :

320

(VT == MVT::f64) ? RTLIB::UNE_F64 :

321

(VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;

322

break;

323

case ISD::SETGE:

324

case ISD::SETOGE:

325

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

326

(VT == MVT::f64) ? RTLIB::OGE_F64 :

327

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

328

break;

329

case ISD::SETLT:

330

case ISD::SETOLT:

331

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

332

(VT == MVT::f64) ? RTLIB::OLT_F64 :

333

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

334

break;

335

case ISD::SETLE:

336

case ISD::SETOLE:

337

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

338

(VT == MVT::f64) ? RTLIB::OLE_F64 :

339

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

340

break;

341

case ISD::SETGT:

342

case ISD::SETOGT:

343

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

344

(VT == MVT::f64) ? RTLIB::OGT_F64 :

345

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

346

break;

347

case ISD::SETO:

348

ShouldInvertCC = true;

349

LLVM_FALLTHROUGH[[gnu::fallthrough]];

350

case ISD::SETUO:

351

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

352

(VT == MVT::f64) ? RTLIB::UO_F64 :

353

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

354

break;

355

case ISD::SETONE:

356

// SETONE = O && UNE

357

ShouldInvertCC = true;

358

LLVM_FALLTHROUGH[[gnu::fallthrough]];

359

case ISD::SETUEQ:

360

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

361

(VT == MVT::f64) ? RTLIB::UO_F64 :

362

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

363

LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

364

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

365

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

366

break;

367

default:

368

// Invert CC for unordered comparisons

369

ShouldInvertCC = true;

370

switch (CCCode) {

371

case ISD::SETULT:

372

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

373

(VT == MVT::f64) ? RTLIB::OGE_F64 :

374

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

375

break;

376

case ISD::SETULE:

377

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

378

(VT == MVT::f64) ? RTLIB::OGT_F64 :

379

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

380

break;

381

case ISD::SETUGT:

382

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

383

(VT == MVT::f64) ? RTLIB::OLE_F64 :

384

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

385

break;

386

case ISD::SETUGE:

387

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

388

(VT == MVT::f64) ? RTLIB::OLT_F64 :

389

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

390

break;

391

default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 391);

392

}

393

}

394

395

// Use the target specific return value for comparions lib calls.

396

EVT RetVT = getCmpLibcallReturnType();

397

SDValue Ops[2] = {NewLHS, NewRHS};

398

TargetLowering::MakeLibCallOptions CallOptions;

399

EVT OpsVT[2] = { OldLHS.getValueType(),

400

OldRHS.getValueType() };

401

CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);

402

auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);

403

NewLHS = Call.first;

404

NewRHS = DAG.getConstant(0, dl, RetVT);

405

406

CCCode = getCmpLibcallCC(LC1);

407

if (ShouldInvertCC) {

408

assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 408, __extension__ __PRETTY_FUNCTION__));

409

CCCode = getSetCCInverse(CCCode, RetVT);

410

}

411

412

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {

413

// Update Chain.

414

Chain = Call.second;

415

} else {

416

EVT SetCCVT =

417

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);

418

SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);

419

auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);

420

CCCode = getCmpLibcallCC(LC2);

421

if (ShouldInvertCC)

422

CCCode = getSetCCInverse(CCCode, RetVT);

423

NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);

424

if (Chain)

425

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,

426

Call2.second);

427

NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,

428

Tmp.getValueType(), Tmp, NewLHS);

429

NewRHS = SDValue();

430

}

431

}

432

433

/// Return the entry encoding for a jump table in the current function. The

434

/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.

435

unsigned TargetLowering::getJumpTableEncoding() const {

436

// In non-pic modes, just use the address of a block.

437

if (!isPositionIndependent())

438

return MachineJumpTableInfo::EK_BlockAddress;

439

440

// In PIC mode, if the target supports a GPRel32 directive, use it.

441

if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)

442

return MachineJumpTableInfo::EK_GPRel32BlockAddress;

443

444

// Otherwise, use a label difference.

445

return MachineJumpTableInfo::EK_LabelDifference32;

446

}

447

448

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,

449

SelectionDAG &DAG) const {

450

// If our PIC model is GP relative, use the global offset table as the base.

451

unsigned JTEncoding = getJumpTableEncoding();

452

453

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||

454

(JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))

455

return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

456

457

return Table;

458

}

459

460

/// This returns the relocation base for the given PIC jumptable, the same as

461

/// getPICJumpTableRelocBase, but as an MCExpr.

462

const MCExpr *

463

TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,

464

unsigned JTI,MCContext &Ctx) const{

465

// The normal PIC reloc base is the label at the start of the jump table.

466

return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);

467

}

468

469

bool

470

TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

471

const TargetMachine &TM = getTargetMachine();

472

const GlobalValue *GV = GA->getGlobal();

473

474

// If the address is not even local to this DSO we will have to load it from

475

// a got and then add the offset.

476

if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))

477

return false;

478

479

// If the code is position independent we will have to add a base register.

480

if (isPositionIndependent())

481

return false;

482

483

// Otherwise we can do it.

484

return true;

485

}

486

487

//===----------------------------------------------------------------------===//

488

// Optimization Methods

489

//===----------------------------------------------------------------------===//

490

491

/// If the specified instruction has a constant integer operand and there are

492

/// bits set in that constant that are not demanded, then clear those bits and

493

/// return true.

494

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

495

const APInt &DemandedBits,

496

const APInt &DemandedElts,

497

TargetLoweringOpt &TLO) const {

498

SDLoc DL(Op);

499

unsigned Opcode = Op.getOpcode();

500

501

// Do target-specific constant optimization.

502

if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

503

return TLO.New.getNode();

504

505

// FIXME: ISD::SELECT, ISD::SELECT_CC

506

switch (Opcode) {

507

default:

508

break;

509

case ISD::XOR:

510

case ISD::AND:

511

case ISD::OR: {

512

auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));

513

if (!Op1C || Op1C->isOpaque())

514

return false;

515

516

// If this is a 'not' op, don't touch it because that's a canonical form.

517

const APInt &C = Op1C->getAPIntValue();

518

if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))

519

return false;

520

521

if (!C.isSubsetOf(DemandedBits)) {

522

EVT VT = Op.getValueType();

523

SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);

524

SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);

525

return TLO.CombineTo(Op, NewOp);

526

}

527

528

break;

529

}

530

}

531

532

return false;

533

}

534

535

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

536

const APInt &DemandedBits,

537

TargetLoweringOpt &TLO) const {

538

EVT VT = Op.getValueType();

539

APInt DemandedElts = VT.isVector()

540

? APInt::getAllOnesValue(VT.getVectorNumElements())

541

: APInt(1, 1);

542

return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);

543

}

544

545

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.

546

/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be

547

/// generalized for targets with other types of implicit widening casts.

548

bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,

549

const APInt &Demanded,

550

TargetLoweringOpt &TLO) const {

551

assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))

552

"ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 552, __extension__ __PRETTY_FUNCTION__));

553

assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 554, __extension__ __PRETTY_FUNCTION__))

554

"ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 554, __extension__ __PRETTY_FUNCTION__));

555

556

SelectionDAG &DAG = TLO.DAG;

557

SDLoc dl(Op);

558

559

// Early return, as this function cannot handle vector types.

560

if (Op.getValueType().isVector())

561

return false;

562

563

// Don't do this if the node has another user, which may require the

564

// full value.

565

if (!Op.getNode()->hasOneUse())

566

return false;

567

568

// Search for the smallest integer type with free casts to and from

569

// Op's type. For expedience, just check power-of-2 integer types.

570

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

571

unsigned DemandedSize = Demanded.getActiveBits();

572

unsigned SmallVTBits = DemandedSize;

573

if (!isPowerOf2_32(SmallVTBits))

574

SmallVTBits = NextPowerOf2(SmallVTBits);

575

for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {

576

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);

577

if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&

578

TLI.isZExtFree(SmallVT, Op.getValueType())) {

579

// We found a type with free casts.

580

SDValue X = DAG.getNode(

581

Op.getOpcode(), dl, SmallVT,

582

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),

583

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));

584

assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 584, __extension__ __PRETTY_FUNCTION__));

585

SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);

586

return TLO.CombineTo(Op, Z);

587

}

588

}

589

return false;

590

}

591

592

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

593

DAGCombinerInfo &DCI) const {

594

SelectionDAG &DAG = DCI.DAG;

595

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

596

!DCI.isBeforeLegalizeOps());

597

KnownBits Known;

598

599

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);

600

if (Simplified) {

601

DCI.AddToWorklist(Op.getNode());

602

DCI.CommitTargetLoweringOpt(TLO);

603

}

604

return Simplified;

605

}

606

607

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

608

KnownBits &Known,

609

TargetLoweringOpt &TLO,

610

unsigned Depth,

611

bool AssumeSingleUse) const {

612

EVT VT = Op.getValueType();

613

614

// TODO: We can probably do more work on calculating the known bits and

615

// simplifying the operations for scalable vectors, but for now we just

616

// bail out.

617

if (VT.isScalableVector()) {

618

// Pretend we don't know anything for now.

619

Known = KnownBits(DemandedBits.getBitWidth());

620

return false;

621

}

622

623

APInt DemandedElts = VT.isVector()

624

? APInt::getAllOnesValue(VT.getVectorNumElements())

625

: APInt(1, 1);

626

return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,

627

AssumeSingleUse);

628

}

629

630

// TODO: Can we merge SelectionDAG::GetDemandedBits into this?

631

// TODO: Under what circumstances can we create nodes? Constant folding?

632

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

633

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

634

SelectionDAG &DAG, unsigned Depth) const {

635

// Limit search depth.

636

if (Depth >= SelectionDAG::MaxRecursionDepth)

637

return SDValue();

638

639

// Ignore UNDEFs.

640

if (Op.isUndef())

641

return SDValue();

642

643

// Not demanding any bits/elts from Op.

644

if (DemandedBits == 0 || DemandedElts == 0)

645

return DAG.getUNDEF(Op.getValueType());

646

647

unsigned NumElts = DemandedElts.getBitWidth();

648

unsigned BitWidth = DemandedBits.getBitWidth();

649

KnownBits LHSKnown, RHSKnown;

650

switch (Op.getOpcode()) {

651

case ISD::BITCAST: {

652

SDValue Src = peekThroughBitcasts(Op.getOperand(0));

653

EVT SrcVT = Src.getValueType();

654

EVT DstVT = Op.getValueType();

655

if (SrcVT == DstVT)

656

return Src;

657

658

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

659

unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

660

if (NumSrcEltBits == NumDstEltBits)

661

if (SDValue V = SimplifyMultipleUseDemandedBits(

662

Src, DemandedBits, DemandedElts, DAG, Depth + 1))

663

return DAG.getBitcast(DstVT, V);

664

665

// TODO - bigendian once we have test coverage.

666

if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&

667

DAG.getDataLayout().isLittleEndian()) {

668

unsigned Scale = NumDstEltBits / NumSrcEltBits;

669

unsigned NumSrcElts = SrcVT.getVectorNumElements();

670

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

671

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

672

for (unsigned i = 0; i != Scale; ++i) {

673

unsigned Offset = i * NumSrcEltBits;

674

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

675

if (!Sub.isNullValue()) {

676

DemandedSrcBits |= Sub;

677

for (unsigned j = 0; j != NumElts; ++j)

678

if (DemandedElts[j])

679

DemandedSrcElts.setBit((j * Scale) + i);

680

}

681

}

682

683

if (SDValue V = SimplifyMultipleUseDemandedBits(

684

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

685

return DAG.getBitcast(DstVT, V);

686

}

687

688

// TODO - bigendian once we have test coverage.

689

if ((NumSrcEltBits % NumDstEltBits) == 0 &&

690

DAG.getDataLayout().isLittleEndian()) {

691

unsigned Scale = NumSrcEltBits / NumDstEltBits;

692

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

693

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

694

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

695

for (unsigned i = 0; i != NumElts; ++i)

696

if (DemandedElts[i]) {

697

unsigned Offset = (i % Scale) * NumDstEltBits;

698

DemandedSrcBits.insertBits(DemandedBits, Offset);

699

DemandedSrcElts.setBit(i / Scale);

700

}

701

702

if (SDValue V = SimplifyMultipleUseDemandedBits(

703

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

704

return DAG.getBitcast(DstVT, V);

705

}

706

707

break;

708

}

709

case ISD::AND: {

710

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

711

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

712

713

// If all of the demanded bits are known 1 on one side, return the other.

714

// These bits cannot contribute to the result of the 'and' in this

715

// context.

716

if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))

717

return Op.getOperand(0);

718

if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))

719

return Op.getOperand(1);

720

break;

721

}

722

case ISD::OR: {

723

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

724

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

725

726

// If all of the demanded bits are known zero on one side, return the

727

// other. These bits cannot contribute to the result of the 'or' in this

728

// context.

729

if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))

730

return Op.getOperand(0);

731

if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))

732

return Op.getOperand(1);

733

break;

734

}

735

case ISD::XOR: {

736

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

737

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

738

739

// If all of the demanded bits are known zero on one side, return the

740

// other.

741

if (DemandedBits.isSubsetOf(RHSKnown.Zero))

742

return Op.getOperand(0);

743

if (DemandedBits.isSubsetOf(LHSKnown.Zero))

744

return Op.getOperand(1);

745

break;

746

}

747

case ISD::SHL: {

748

// If we are only demanding sign bits then we can use the shift source

749

// directly.

750

if (const APInt *MaxSA =

751

DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

752

SDValue Op0 = Op.getOperand(0);

753

unsigned ShAmt = MaxSA->getZExtValue();

754

unsigned NumSignBits =

755

DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

756

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

757

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

758

return Op0;

759

}

760

break;

761

}

762

case ISD::SETCC: {

763

SDValue Op0 = Op.getOperand(0);

764

SDValue Op1 = Op.getOperand(1);

765

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

766

// If (1) we only need the sign-bit, (2) the setcc operands are the same

767

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

768

// -1, we may be able to bypass the setcc.

769

if (DemandedBits.isSignMask() &&

770

Op0.getScalarValueSizeInBits() == BitWidth &&

771

getBooleanContents(Op0.getValueType()) ==

772

BooleanContent::ZeroOrNegativeOneBooleanContent) {

773

// If we're testing X < 0, then this compare isn't needed - just use X!

774

// FIXME: We're limiting to integer types here, but this should also work

775

// if we don't care about FP signed-zero. The use of SETLT with FP means

776

// that we don't care about NaNs.

777

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

778

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

779

return Op0;

780

}

781

break;

782

}

783

case ISD::SIGN_EXTEND_INREG: {

784

// If none of the extended bits are demanded, eliminate the sextinreg.

785

SDValue Op0 = Op.getOperand(0);

786

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

787

unsigned ExBits = ExVT.getScalarSizeInBits();

788

if (DemandedBits.getActiveBits() <= ExBits)

789

return Op0;

790

// If the input is already sign extended, just drop the extension.

791

unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

792

if (NumSignBits >= (BitWidth - ExBits + 1))

793

return Op0;

794

break;

795

}

796

case ISD::ANY_EXTEND_VECTOR_INREG:

797

case ISD::SIGN_EXTEND_VECTOR_INREG:

798

case ISD::ZERO_EXTEND_VECTOR_INREG: {

799

// If we only want the lowest element and none of extended bits, then we can

800

// return the bitcasted source vector.

801

SDValue Src = Op.getOperand(0);

802

EVT SrcVT = Src.getValueType();

803

EVT DstVT = Op.getValueType();

804

if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&

805

DAG.getDataLayout().isLittleEndian() &&

806

DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {

807

return DAG.getBitcast(DstVT, Src);

808

}

809

break;

810

}

811

case ISD::INSERT_VECTOR_ELT: {

812

// If we don't demand the inserted element, return the base vector.

813

SDValue Vec = Op.getOperand(0);

814

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

815

EVT VecVT = Vec.getValueType();

816

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&

817

!DemandedElts[CIdx->getZExtValue()])

818

return Vec;

819

break;

820

}

821

case ISD::INSERT_SUBVECTOR: {

822

// If we don't demand the inserted subvector, return the base vector.

823

SDValue Vec = Op.getOperand(0);

824

SDValue Sub = Op.getOperand(1);

825

uint64_t Idx = Op.getConstantOperandVal(2);

826

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

827

if (DemandedElts.extractBits(NumSubElts, Idx) == 0)

828

return Vec;

829

break;

830

}

831

case ISD::VECTOR_SHUFFLE: {

832

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

833

834

// If all the demanded elts are from one operand and are inline,

835

// then we can use the operand directly.

836

bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;

837

for (unsigned i = 0; i != NumElts; ++i) {

838

int M = ShuffleMask[i];

839

if (M < 0 || !DemandedElts[i])

840

continue;

841

AllUndef = false;

842

IdentityLHS &= (M == (int)i);

843

IdentityRHS &= ((M - NumElts) == i);

844

}

845

846

if (AllUndef)

847

return DAG.getUNDEF(Op.getValueType());

848

if (IdentityLHS)

849

return Op.getOperand(0);

850

if (IdentityRHS)

851

return Op.getOperand(1);

852

break;

853

}

854

default:

855

if (Op.getOpcode() >= ISD::BUILTIN_OP_END)

856

if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(

857

Op, DemandedBits, DemandedElts, DAG, Depth))

858

return V;

859

break;

860

}

861

return SDValue();

862

}

863

864

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

865

SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,

866

unsigned Depth) const {

867

EVT VT = Op.getValueType();

868

APInt DemandedElts = VT.isVector()

869

? APInt::getAllOnesValue(VT.getVectorNumElements())

870

: APInt(1, 1);

871

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

872

Depth);

873

}

874

875

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(

876

SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,

877

unsigned Depth) const {

878

APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());

879

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

880

Depth);

881

}

882

883

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the

884

/// result of Op are ever used downstream. If we can use this information to

885

/// simplify Op, create a new simplified DAG node and return true, returning the

886

/// original and new nodes in Old and New. Otherwise, analyze the expression and

887

/// return a mask of Known bits for the expression (used to simplify the

888

/// caller). The Known bits may only be accurate for those bits in the

889

/// OriginalDemandedBits and OriginalDemandedElts.

890

bool TargetLowering::SimplifyDemandedBits(

891

SDValue Op, const APInt &OriginalDemandedBits,

892

const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,

893

unsigned Depth, bool AssumeSingleUse) const {

894

unsigned BitWidth = OriginalDemandedBits.getBitWidth();

895

assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 896, __extension__ __PRETTY_FUNCTION__))

896

"Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 896, __extension__ __PRETTY_FUNCTION__));

897

898

// Don't know anything.

899

Known = KnownBits(BitWidth);

900

901

// TODO: We can probably do more work on calculating the known bits and

902

// simplifying the operations for scalable vectors, but for now we just

903

// bail out.

904

if (Op.getValueType().isScalableVector())

905

return false;

906

907

unsigned NumElts = OriginalDemandedElts.getBitWidth();

908

assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))

909

NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))

910

"Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__));

911

912

APInt DemandedBits = OriginalDemandedBits;

913

APInt DemandedElts = OriginalDemandedElts;

914

SDLoc dl(Op);

915

auto &DL = TLO.DAG.getDataLayout();

916

917

// Undef operand.

918

if (Op.isUndef())

919

return false;

920

921

if (Op.getOpcode() == ISD::Constant) {

922

// We know all of the bits for a constant!

923

Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());

924

return false;

925

}

926

927

if (Op.getOpcode() == ISD::ConstantFP) {

928

// We know all of the bits for a floating point constant!

929

Known = KnownBits::makeConstant(

930

cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());

931

return false;

932

}

933

934

// Other users may use these bits.

935

EVT VT = Op.getValueType();

936

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {

937

if (Depth != 0) {

938

// If not at the root, Just compute the Known bits to

939

// simplify things downstream.

940

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

941

return false;

942

}

943

// If this is the root being simplified, allow it to have multiple uses,

944

// just set the DemandedBits/Elts to all bits.

945

DemandedBits = APInt::getAllOnesValue(BitWidth);

946

DemandedElts = APInt::getAllOnesValue(NumElts);

947

} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {

948

// Not demanding any bits/elts from Op.

949

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

950

} else if (Depth >= SelectionDAG::MaxRecursionDepth) {

951

// Limit search depth.

952

return false;

953

}

954

955

KnownBits Known2;

956

switch (Op.getOpcode()) {

957

case ISD::TargetConstant:

958

llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 958);

959

case ISD::SCALAR_TO_VECTOR: {

960

if (!DemandedElts[0])

961

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

962

963

KnownBits SrcKnown;

964

SDValue Src = Op.getOperand(0);

965

unsigned SrcBitWidth = Src.getScalarValueSizeInBits();

966

APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);

967

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))

968

return true;

969

970

// Upper elements are undef, so only get the knownbits if we just demand

971

// the bottom element.

972

if (DemandedElts == 1)

973

Known = SrcKnown.anyextOrTrunc(BitWidth);

974

break;

975

}

976

case ISD::BUILD_VECTOR:

977

// Collect the known bits that are shared by every demanded element.

978

// TODO: Call SimplifyDemandedBits for non-constant demanded elements.

979

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

980

return false; // Don't fall through, will infinitely loop.

981

case ISD::LOAD: {

982

auto *LD = cast<LoadSDNode>(Op);

983

if (getTargetConstantFromLoad(LD)) {

984

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

985

return false; // Don't fall through, will infinitely loop.

986

}

987

if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {

988

// If this is a ZEXTLoad and we are looking at the loaded value.

989

EVT MemVT = LD->getMemoryVT();

990

unsigned MemBits = MemVT.getScalarSizeInBits();

991

Known.Zero.setBitsFrom(MemBits);

992

return false; // Don't fall through, will infinitely loop.

993

}

994

break;

995

}

996

case ISD::INSERT_VECTOR_ELT: {

997

SDValue Vec = Op.getOperand(0);

998

SDValue Scl = Op.getOperand(1);

999

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

1000

EVT VecVT = Vec.getValueType();

1001

1002

// If index isn't constant, assume we need all vector elements AND the

1003

// inserted element.

1004

APInt DemandedVecElts(DemandedElts);

1005

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {

1006

unsigned Idx = CIdx->getZExtValue();

1007

DemandedVecElts.clearBit(Idx);

1008

1009

// Inserted element is not required.

1010

if (!DemandedElts[Idx])

1011

return TLO.CombineTo(Op, Vec);

1012

}

1013

1014

KnownBits KnownScl;

1015

unsigned NumSclBits = Scl.getScalarValueSizeInBits();

1016

APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);

1017

if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))

1018

return true;

1019

1020

Known = KnownScl.anyextOrTrunc(BitWidth);

1021

1022

KnownBits KnownVec;

1023

if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,

1024

Depth + 1))

1025

return true;

1026

1027

if (!!DemandedVecElts)

1028

Known = KnownBits::commonBits(Known, KnownVec);

1029

1030

return false;

1031

}

1032

case ISD::INSERT_SUBVECTOR: {

1033

// Demand any elements from the subvector and the remainder from the src its

1034

// inserted into.

1035

SDValue Src = Op.getOperand(0);

1036

SDValue Sub = Op.getOperand(1);

1037

uint64_t Idx = Op.getConstantOperandVal(2);

1038

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

1039

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

1040

APInt DemandedSrcElts = DemandedElts;

1041

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

1042

1043

KnownBits KnownSub, KnownSrc;

1044

if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,

1045

Depth + 1))

1046

return true;

1047

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,

1048

Depth + 1))

1049

return true;

1050

1051

Known.Zero.setAllBits();

1052

Known.One.setAllBits();

1053

if (!!DemandedSubElts)

1054

Known = KnownBits::commonBits(Known, KnownSub);

1055

if (!!DemandedSrcElts)

1056

Known = KnownBits::commonBits(Known, KnownSrc);

1057

1058

// Attempt to avoid multi-use src if we don't need anything from it.

1059

if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||

1060

!DemandedSrcElts.isAllOnesValue()) {

1061

SDValue NewSub = SimplifyMultipleUseDemandedBits(

1062

Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);

1063

SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1064

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1065

if (NewSub || NewSrc) {

1066

NewSub = NewSub ? NewSub : Sub;

1067

NewSrc = NewSrc ? NewSrc : Src;

1068

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,

1069

Op.getOperand(2));

1070

return TLO.CombineTo(Op, NewOp);

1071

}

1072

}

1073

break;

1074

}

1075

case ISD::EXTRACT_SUBVECTOR: {

1076

// Offset the demanded elts by the subvector index.

1077

SDValue Src = Op.getOperand(0);

1078

if (Src.getValueType().isScalableVector())

1079

break;

1080

uint64_t Idx = Op.getConstantOperandVal(1);

1081

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

1082

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

1083

1084

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,

1085

Depth + 1))

1086

return true;

1087

1088

// Attempt to avoid multi-use src if we don't need anything from it.

1089

if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {

1090

SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

1091

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1092

if (DemandedSrc) {

1093

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,

1094

Op.getOperand(1));

1095

return TLO.CombineTo(Op, NewOp);

1096

}

1097

}

1098

break;

1099

}

1100

case ISD::CONCAT_VECTORS: {

1101

Known.Zero.setAllBits();

1102

Known.One.setAllBits();

1103

EVT SubVT = Op.getOperand(0).getValueType();

1104

unsigned NumSubVecs = Op.getNumOperands();

1105

unsigned NumSubElts = SubVT.getVectorNumElements();

1106

for (unsigned i = 0; i != NumSubVecs; ++i) {

1107

APInt DemandedSubElts =

1108

DemandedElts.extractBits(NumSubElts, i * NumSubElts);

1109

if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,

1110

Known2, TLO, Depth + 1))

1111

return true;

1112

// Known bits are shared by every demanded subvector element.

1113

if (!!DemandedSubElts)

1114

Known = KnownBits::commonBits(Known, Known2);

1115

}

1116

break;

1117

}

1118

case ISD::VECTOR_SHUFFLE: {

1119

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

1120

1121

// Collect demanded elements from shuffle operands..

1122

APInt DemandedLHS(NumElts, 0);

1123

APInt DemandedRHS(NumElts, 0);

1124

for (unsigned i = 0; i != NumElts; ++i) {

1125

if (!DemandedElts[i])

1126

continue;

1127

int M = ShuffleMask[i];

1128

if (M < 0) {

1129

// For UNDEF elements, we don't know anything about the common state of

1130

// the shuffle result.

1131

DemandedLHS.clearAllBits();

1132

DemandedRHS.clearAllBits();

1133

break;

1134

}

1135

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1135, __extension__ __PRETTY_FUNCTION__));

1136

if (M < (int)NumElts)

1137

DemandedLHS.setBit(M);

1138

else

1139

DemandedRHS.setBit(M - NumElts);

1140

}

1141

1142

if (!!DemandedLHS || !!DemandedRHS) {

1143

SDValue Op0 = Op.getOperand(0);

1144

SDValue Op1 = Op.getOperand(1);

1145

1146

Known.Zero.setAllBits();

1147

Known.One.setAllBits();

1148

if (!!DemandedLHS) {

1149

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,

1150

Depth + 1))

1151

return true;

1152

Known = KnownBits::commonBits(Known, Known2);

1153

}

1154

if (!!DemandedRHS) {

1155

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,

1156

Depth + 1))

1157

return true;

1158

Known = KnownBits::commonBits(Known, Known2);

1159

}

1160

1161

// Attempt to avoid multi-use ops if we don't need anything from them.

1162

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1163

Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);

1164

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1165

Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);

1166

if (DemandedOp0 || DemandedOp1) {

1167

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1168

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1169

SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);

1170

return TLO.CombineTo(Op, NewOp);

1171

}

1172

}

1173

break;

1174

}

1175

case ISD::AND: {

1176

SDValue Op0 = Op.getOperand(0);

1177

SDValue Op1 = Op.getOperand(1);

1178

1179

// If the RHS is a constant, check to see if the LHS would be zero without

1180

// using the bits from the RHS. Below, we use knowledge about the RHS to

1181

// simplify the LHS, here we're using information from the LHS to simplify

1182

// the RHS.

1183

if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {

1184

// Do not increment Depth here; that can cause an infinite loop.

1185

KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);

1186

// If the LHS already has zeros where RHSC does, this 'and' is dead.

1187

if ((LHSKnown.Zero & DemandedBits) ==

1188

(~RHSC->getAPIntValue() & DemandedBits))

1189

return TLO.CombineTo(Op, Op0);

1190

1191

// If any of the set bits in the RHS are known zero on the LHS, shrink

1192

// the constant.

1193

if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,

1194

DemandedElts, TLO))

1195

return true;

1196

1197

// Bitwise-not (xor X, -1) is a special case: we don't usually shrink its

1198

// constant, but if this 'and' is only clearing bits that were just set by

1199

// the xor, then this 'and' can be eliminated by shrinking the mask of

1200

// the xor. For example, for a 32-bit X:

1201

// and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1

1202

if (isBitwiseNot(Op0) && Op0.hasOneUse() &&

1203

LHSKnown.One == ~RHSC->getAPIntValue()) {

1204

SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);

1205

return TLO.CombineTo(Op, Xor);

1206

}

1207

}

1208

1209

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1210

Depth + 1))

1211

return true;

1212

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1212, __extension__ __PRETTY_FUNCTION__));

1213

if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,

1214

Known2, TLO, Depth + 1))

1215

return true;

1216

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1216, __extension__ __PRETTY_FUNCTION__));

1217

1218

// Attempt to avoid multi-use ops if we don't need anything from them.

1219

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1220

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1221

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1222

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1223

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1224

if (DemandedOp0 || DemandedOp1) {

1225

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1226

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1227

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1228

return TLO.CombineTo(Op, NewOp);

1229

}

1230

}

1231

1232

// If all of the demanded bits are known one on one side, return the other.

1233

// These bits cannot contribute to the result of the 'and'.

1234

if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))

1235

return TLO.CombineTo(Op, Op0);

1236

if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))

1237

return TLO.CombineTo(Op, Op1);

1238

// If all of the demanded bits in the inputs are known zeros, return zero.

1239

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1240

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));

1241

// If the RHS is a constant, see if we can simplify it.

1242

if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,

1243

TLO))

1244

return true;

1245

// If the operation can be done in a smaller type, do so.

1246

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1247

return true;

1248

1249

Known &= Known2;

1250

break;

1251

}

1252

case ISD::OR: {

1253

SDValue Op0 = Op.getOperand(0);

1254

SDValue Op1 = Op.getOperand(1);

1255

1256

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1257

Depth + 1))

1258

return true;

1259

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1259, __extension__ __PRETTY_FUNCTION__));

1260

if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,

1261

Known2, TLO, Depth + 1))

1262

return true;

1263

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1263, __extension__ __PRETTY_FUNCTION__));

1264

1265

// Attempt to avoid multi-use ops if we don't need anything from them.

1266

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1267

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1268

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1269

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1270

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1271

if (DemandedOp0 || DemandedOp1) {

1272

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1273

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1274

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1275

return TLO.CombineTo(Op, NewOp);

1276

}

1277

}

1278

1279

// If all of the demanded bits are known zero on one side, return the other.

1280

// These bits cannot contribute to the result of the 'or'.

1281

if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))

1282

return TLO.CombineTo(Op, Op0);

1283

if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))

1284

return TLO.CombineTo(Op, Op1);

1285

// If the RHS is a constant, see if we can simplify it.

1286

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1287

return true;

1288

// If the operation can be done in a smaller type, do so.

1289

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1290

return true;

1291

1292

Known |= Known2;

1293

break;

1294

}

1295

case ISD::XOR: {

1296

SDValue Op0 = Op.getOperand(0);

1297

SDValue Op1 = Op.getOperand(1);

1298

1299

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1300

Depth + 1))

1301

return true;

1302

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1302, __extension__ __PRETTY_FUNCTION__));

1303

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,

1304

Depth + 1))

1305

return true;

1306

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1306, __extension__ __PRETTY_FUNCTION__));

1307

1308

// Attempt to avoid multi-use ops if we don't need anything from them.

1309

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1310

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1311

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1312

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1313

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1314

if (DemandedOp0 || DemandedOp1) {

1315

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1316

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1317

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1318

return TLO.CombineTo(Op, NewOp);

1319

}

1320

}

1321

1322

// If all of the demanded bits are known zero on one side, return the other.

1323

// These bits cannot contribute to the result of the 'xor'.

1324

if (DemandedBits.isSubsetOf(Known.Zero))

1325

return TLO.CombineTo(Op, Op0);

1326

if (DemandedBits.isSubsetOf(Known2.Zero))

1327

return TLO.CombineTo(Op, Op1);

1328

// If the operation can be done in a smaller type, do so.

1329

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1330

return true;

1331

1332

// If all of the unknown bits are known to be zero on one side or the other

1333

// turn this into an *inclusive* or.

1334

// e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0

1335

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1336

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

1337

1338

ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);

1339

if (C) {

1340

// If one side is a constant, and all of the set bits in the constant are

1341

// also known set on the other side, turn this into an AND, as we know

1342

// the bits will be cleared.

1343

// e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2

1344

// NB: it is okay if more bits are known than are requested

1345

if (C->getAPIntValue() == Known2.One) {

1346

SDValue ANDC =

1347

TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);

1348

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));

1349

}

1350

1351

// If the RHS is a constant, see if we can change it. Don't alter a -1

1352

// constant because that's a 'not' op, and that is better for combining

1353

// and codegen.

1354

if (!C->isAllOnesValue() &&

1355

DemandedBits.isSubsetOf(C->getAPIntValue())) {

1356

// We're flipping all demanded bits. Flip the undemanded bits too.

1357

SDValue New = TLO.DAG.getNOT(dl, Op0, VT);

1358

return TLO.CombineTo(Op, New);

1359

}

1360

}

1361

1362

// If we can't turn this into a 'not', try to shrink the constant.

1363

if (!C || !C->isAllOnesValue())

1364

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1365

return true;

1366

1367

Known ^= Known2;

1368

break;

1369

}

1370

case ISD::SELECT:

1371

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,

1372

Depth + 1))

1373

return true;

1374

if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,

1375

Depth + 1))

1376

return true;

1377

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1377, __extension__ __PRETTY_FUNCTION__));

1378

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1378, __extension__ __PRETTY_FUNCTION__));

1379

1380

// If the operands are constants, see if we can simplify them.

1381

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1382

return true;

1383

1384

// Only known if known in both the LHS and RHS.

1385

Known = KnownBits::commonBits(Known, Known2);

1386

break;

1387

case ISD::SELECT_CC:

1388

if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,

1389

Depth + 1))

1390

return true;

1391

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,

1392

Depth + 1))

1393

return true;

1394

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1394, __extension__ __PRETTY_FUNCTION__));

1395

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1395, __extension__ __PRETTY_FUNCTION__));

1396

1397

// If the operands are constants, see if we can simplify them.

1398

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1399

return true;

1400

1401

// Only known if known in both the LHS and RHS.

1402

Known = KnownBits::commonBits(Known, Known2);

1403

break;

1404

case ISD::SETCC: {

1405

SDValue Op0 = Op.getOperand(0);

1406

SDValue Op1 = Op.getOperand(1);

1407

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

1408

// If (1) we only need the sign-bit, (2) the setcc operands are the same

1409

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

1410

// -1, we may be able to bypass the setcc.

1411

if (DemandedBits.isSignMask() &&

1412

Op0.getScalarValueSizeInBits() == BitWidth &&

1413

getBooleanContents(Op0.getValueType()) ==

1414

BooleanContent::ZeroOrNegativeOneBooleanContent) {

1415

// If we're testing X < 0, then this compare isn't needed - just use X!

1416

// FIXME: We're limiting to integer types here, but this should also work

1417

// if we don't care about FP signed-zero. The use of SETLT with FP means

1418

// that we don't care about NaNs.

1419

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

1420

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

1421

return TLO.CombineTo(Op, Op0);

1422

1423

// TODO: Should we check for other forms of sign-bit comparisons?

1424

// Examples: X <= -1, X >= 0

1425

}

1426

if (getBooleanContents(Op0.getValueType()) ==

1427

TargetLowering::ZeroOrOneBooleanContent &&

1428

BitWidth > 1)

1429

Known.Zero.setBitsFrom(1);

1430

break;

1431

}

1432

case ISD::SHL: {

1433

SDValue Op0 = Op.getOperand(0);

1434

SDValue Op1 = Op.getOperand(1);

1435

EVT ShiftVT = Op1.getValueType();

1436

1437

if (const APInt *SA =

1438

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1439

unsigned ShAmt = SA->getZExtValue();

1440

if (ShAmt == 0)

1441

return TLO.CombineTo(Op, Op0);

1442

1443

// If this is ((X >>u C1) << ShAmt), see if we can simplify this into a

1444

// single shift. We can do this if the bottom bits (which are shifted

1445

// out) are never demanded.

1446

// TODO - support non-uniform vector amounts.

1447

if (Op0.getOpcode() == ISD::SRL) {

1448

if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {

1449

if (const APInt *SA2 =

1450

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1451

unsigned C1 = SA2->getZExtValue();

1452

unsigned Opc = ISD::SHL;

1453

int Diff = ShAmt - C1;

1454

if (Diff < 0) {

1455

Diff = -Diff;

1456

Opc = ISD::SRL;

1457

}

1458

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1459

return TLO.CombineTo(

1460

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1461

}

1462

}

1463

}

1464

1465

// Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits

1466

// are not demanded. This will likely allow the anyext to be folded away.

1467

// TODO - support non-uniform vector amounts.

1468

if (Op0.getOpcode() == ISD::ANY_EXTEND) {

1469

SDValue InnerOp = Op0.getOperand(0);

1470

EVT InnerVT = InnerOp.getValueType();

1471

unsigned InnerBits = InnerVT.getScalarSizeInBits();

1472

if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&

1473

isTypeDesirableForOp(ISD::SHL, InnerVT)) {

1474

EVT ShTy = getShiftAmountTy(InnerVT, DL);

1475

if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))

1476

ShTy = InnerVT;

1477

SDValue NarrowShl =

1478

TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,

1479

TLO.DAG.getConstant(ShAmt, dl, ShTy));

1480

return TLO.CombineTo(

1481

Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));

1482

}

1483

1484

// Repeat the SHL optimization above in cases where an extension

1485

// intervenes: (shl (anyext (shr x, c1)), c2) to

1486

// (shl (anyext x), c2-c1). This requires that the bottom c1 bits

1487

// aren't demanded (as above) and that the shifted upper c1 bits of

1488

// x aren't demanded.

1489

// TODO - support non-uniform vector amounts.

1490

if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&

1491

InnerOp.hasOneUse()) {

1492

if (const APInt *SA2 =

1493

TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {

1494

unsigned InnerShAmt = SA2->getZExtValue();

1495

if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&

1496

DemandedBits.getActiveBits() <=

1497

(InnerBits - InnerShAmt + ShAmt) &&

1498

DemandedBits.countTrailingZeros() >= ShAmt) {

1499

SDValue NewSA =

1500

TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);

1501

SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,

1502

InnerOp.getOperand(0));

1503

return TLO.CombineTo(

1504

Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));

1505

}

1506

}

1507

}

1508

}

1509

1510

APInt InDemandedMask = DemandedBits.lshr(ShAmt);

1511

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1512

Depth + 1))

1513

return true;

1514

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1514, __extension__ __PRETTY_FUNCTION__));

1515

Known.Zero <<= ShAmt;

1516

Known.One <<= ShAmt;

1517

// low bits known zero.

1518

Known.Zero.setLowBits(ShAmt);

1519

1520

// Try shrinking the operation as long as the shift amount will still be

1521

// in range.

1522

if ((ShAmt < DemandedBits.getActiveBits()) &&

1523

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1524

return true;

1525

}

1526

1527

// If we are only demanding sign bits then we can use the shift source

1528

// directly.

1529

if (const APInt *MaxSA =

1530

TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

1531

unsigned ShAmt = MaxSA->getZExtValue();

1532

unsigned NumSignBits =

1533

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1534

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1535

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

1536

return TLO.CombineTo(Op, Op0);

1537

}

1538

break;

1539

}

1540

case ISD::SRL: {

1541

SDValue Op0 = Op.getOperand(0);

1542

SDValue Op1 = Op.getOperand(1);

1543

EVT ShiftVT = Op1.getValueType();

1544

1545

if (const APInt *SA =

1546

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1547

unsigned ShAmt = SA->getZExtValue();

1548

if (ShAmt == 0)

1549

return TLO.CombineTo(Op, Op0);

1550

1551

// If this is ((X << C1) >>u ShAmt), see if we can simplify this into a

1552

// single shift. We can do this if the top bits (which are shifted out)

1553

// are never demanded.

1554

// TODO - support non-uniform vector amounts.

1555

if (Op0.getOpcode() == ISD::SHL) {

1556

if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {

1557

if (const APInt *SA2 =

1558

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1559

unsigned C1 = SA2->getZExtValue();

1560

unsigned Opc = ISD::SRL;

1561

int Diff = ShAmt - C1;

1562

if (Diff < 0) {

1563

Diff = -Diff;

1564

Opc = ISD::SHL;

1565

}

1566

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1567

return TLO.CombineTo(

1568

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1569

}

1570

}

1571

}

1572

1573

APInt InDemandedMask = (DemandedBits << ShAmt);

1574

1575

// If the shift is exact, then it does demand the low bits (and knows that

1576

// they are zero).

1577

if (Op->getFlags().hasExact())

1578

InDemandedMask.setLowBits(ShAmt);

1579

1580

// Compute the new bits that are at the top now.

1581

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1582

Depth + 1))

1583

return true;

1584

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1584, __extension__ __PRETTY_FUNCTION__));

1585

Known.Zero.lshrInPlace(ShAmt);

1586

Known.One.lshrInPlace(ShAmt);

1587

// High bits known zero.

1588

Known.Zero.setHighBits(ShAmt);

1589

}

1590

break;

1591

}

1592

case ISD::SRA: {

1593

SDValue Op0 = Op.getOperand(0);

1594

SDValue Op1 = Op.getOperand(1);

1595

EVT ShiftVT = Op1.getValueType();

1596

1597

// If we only want bits that already match the signbit then we don't need

1598

// to shift.

1599

unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1600

if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=

1601

NumHiDemandedBits)

1602

return TLO.CombineTo(Op, Op0);

1603

1604

// If this is an arithmetic shift right and only the low-bit is set, we can

1605

// always convert this into a logical shr, even if the shift amount is

1606

// variable. The low bit of the shift cannot be an input sign bit unless

1607

// the shift amount is >= the size of the datatype, which is undefined.

1608

if (DemandedBits.isOneValue())

1609

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

1610

1611

if (const APInt *SA =

1612

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1613

unsigned ShAmt = SA->getZExtValue();

1614

if (ShAmt == 0)

1615

return TLO.CombineTo(Op, Op0);

1616

1617

APInt InDemandedMask = (DemandedBits << ShAmt);

1618

1619

// If the shift is exact, then it does demand the low bits (and knows that

1620

// they are zero).

1621

if (Op->getFlags().hasExact())

1622

InDemandedMask.setLowBits(ShAmt);

1623

1624

// If any of the demanded bits are produced by the sign extension, we also

1625

// demand the input sign bit.

1626

if (DemandedBits.countLeadingZeros() < ShAmt)

1627

InDemandedMask.setSignBit();

1628

1629

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1630

Depth + 1))

1631

return true;

1632

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__));

1633

Known.Zero.lshrInPlace(ShAmt);

1634

Known.One.lshrInPlace(ShAmt);

1635

1636

// If the input sign bit is known to be zero, or if none of the top bits

1637

// are demanded, turn this into an unsigned shift right.

1638

if (Known.Zero[BitWidth - ShAmt - 1] ||

1639

DemandedBits.countLeadingZeros() >= ShAmt) {

1640

SDNodeFlags Flags;

1641

Flags.setExact(Op->getFlags().hasExact());

1642

return TLO.CombineTo(

1643

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));

1644

}

1645

1646

int Log2 = DemandedBits.exactLogBase2();

1647

if (Log2 >= 0) {

1648

// The bit must come from the sign.

1649

SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);

1650

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));

1651

}

1652

1653

if (Known.One[BitWidth - ShAmt - 1])

1654

// New bits are known one.

1655

Known.One.setHighBits(ShAmt);

1656

1657

// Attempt to avoid multi-use ops if we don't need anything from them.

1658

if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1659

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1660

Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);

1661

if (DemandedOp0) {

1662

SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);

1663

return TLO.CombineTo(Op, NewOp);

1664

}

1665

}

1666

}

1667

break;

1668

}

1669

case ISD::FSHL:

1670

case ISD::FSHR: {

1671

SDValue Op0 = Op.getOperand(0);

1672

SDValue Op1 = Op.getOperand(1);

1673

SDValue Op2 = Op.getOperand(2);

1674

bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

1675

1676

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {

1677

unsigned Amt = SA->getAPIntValue().urem(BitWidth);

1678

1679

// For fshl, 0-shift returns the 1st arg.

1680

// For fshr, 0-shift returns the 2nd arg.

1681

if (Amt == 0) {

1682

if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,

1683

Known, TLO, Depth + 1))

1684

return true;

1685

break;

1686

}

1687

1688

// fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))

1689

// fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)

1690

APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));

1691

APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);

1692

if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,

1693

Depth + 1))

1694

return true;

1695

if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,

1696

Depth + 1))

1697

return true;

1698

1699

Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));

1700

Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));

1701

Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1702

Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1703

Known.One |= Known2.One;

1704

Known.Zero |= Known2.Zero;

1705

}

1706

1707

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1708

if (isPowerOf2_32(BitWidth)) {

1709

APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);

1710

if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,

1711

Known2, TLO, Depth + 1))

1712

return true;

1713

}

1714

break;

1715

}

1716

case ISD::ROTL:

1717

case ISD::ROTR: {

1718

SDValue Op0 = Op.getOperand(0);

1719

SDValue Op1 = Op.getOperand(1);

1720

1721

// If we're rotating an 0/-1 value, then it stays an 0/-1 value.

1722

if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))

1723

return TLO.CombineTo(Op, Op0);

1724

1725

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1726

if (isPowerOf2_32(BitWidth)) {

1727

APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);

1728

if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,

1729

Depth + 1))

1730

return true;

1731

}

1732

break;

1733

}

1734

case ISD::UMIN: {

1735

// Check if one arg is always less than (or equal) to the other arg.

1736

SDValue Op0 = Op.getOperand(0);

1737

SDValue Op1 = Op.getOperand(1);

1738

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1739

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1740

Known = KnownBits::umin(Known0, Known1);

1741

if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))

1742

return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);

1743

if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))

1744

return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);

1745

break;

1746

}

1747

case ISD::UMAX: {

1748

// Check if one arg is always greater than (or equal) to the other arg.

1749

SDValue Op0 = Op.getOperand(0);

1750

SDValue Op1 = Op.getOperand(1);

1751

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1752

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1753

Known = KnownBits::umax(Known0, Known1);

1754

if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))

1755

return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);

1756

if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))

1757

return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);

1758

break;

1759

}

1760

case ISD::BITREVERSE: {

1761

SDValue Src = Op.getOperand(0);

1762

APInt DemandedSrcBits = DemandedBits.reverseBits();

1763

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1764

Depth + 1))

1765

return true;

1766

Known.One = Known2.One.reverseBits();

1767

Known.Zero = Known2.Zero.reverseBits();

1768

break;

1769

}

1770

case ISD::BSWAP: {

1771

SDValue Src = Op.getOperand(0);

1772

APInt DemandedSrcBits = DemandedBits.byteSwap();

1773

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1774

Depth + 1))

1775

return true;

1776

Known.One = Known2.One.byteSwap();

1777

Known.Zero = Known2.Zero.byteSwap();

1778

break;

1779

}

1780

case ISD::CTPOP: {

1781

// If only 1 bit is demanded, replace with PARITY as long as we're before

1782

// op legalization.

1783

// FIXME: Limit to scalars for now.

1784

if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())

1785

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,

1786

Op.getOperand(0)));

1787

1788

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

1789

break;

1790

}

1791

case ISD::SIGN_EXTEND_INREG: {

1792

SDValue Op0 = Op.getOperand(0);

1793

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

1794

unsigned ExVTBits = ExVT.getScalarSizeInBits();

1795

1796

// If we only care about the highest bit, don't bother shifting right.

1797

if (DemandedBits.isSignMask()) {

1798

unsigned NumSignBits =

1799

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1800

bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;

1801

// However if the input is already sign extended we expect the sign

1802

// extension to be dropped altogether later and do not simplify.

1803

if (!AlreadySignExtended) {

1804

// Compute the correct shift amount type, which must be getShiftAmountTy

1805

// for scalar types after legalization.

1806

EVT ShiftAmtTy = VT;

1807

if (TLO.LegalTypes() && !ShiftAmtTy.isVector())

1808

ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

1809

1810

SDValue ShiftAmt =

1811

TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);

1812

return TLO.CombineTo(Op,

1813

TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));

1814

}

1815

}

1816

1817

// If none of the extended bits are demanded, eliminate the sextinreg.

1818

if (DemandedBits.getActiveBits() <= ExVTBits)

1819

return TLO.CombineTo(Op, Op0);

1820

1821

APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

1822

1823

// Since the sign extended bits are demanded, we know that the sign

1824

// bit is demanded.

1825

InputDemandedBits.setBit(ExVTBits - 1);

1826

1827

if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))

1828

return true;

1829

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1829, __extension__ __PRETTY_FUNCTION__));

1830

1831

// If the sign bit of the input is known set or clear, then we know the

1832

// top bits of the result.

1833

1834

// If the input sign bit is known zero, convert this into a zero extension.

1835

if (Known.Zero[ExVTBits - 1])

1836

return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

1837

1838

APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);

1839

if (Known.One[ExVTBits - 1]) { // Input sign bit known set

1840

Known.One.setBitsFrom(ExVTBits);

1841

Known.Zero &= Mask;

1842

} else { // Input sign bit unknown

1843

Known.Zero &= Mask;

1844

Known.One &= Mask;

1845

}

1846

break;

1847

}

1848

case ISD::BUILD_PAIR: {

1849

EVT HalfVT = Op.getOperand(0).getValueType();

1850

unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

1851

1852

APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);

1853

APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

1854

1855

KnownBits KnownLo, KnownHi;

1856

1857

if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))

1858

return true;

1859

1860

if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))

1861

return true;

1862

1863

Known.Zero = KnownLo.Zero.zext(BitWidth) |

1864

KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

1865

1866

Known.One = KnownLo.One.zext(BitWidth) |

1867

KnownHi.One.zext(BitWidth).shl(HalfBitWidth);

1868

break;

1869

}

1870

case ISD::ZERO_EXTEND:

1871

case ISD::ZERO_EXTEND_VECTOR_INREG: {

1872

SDValue Src = Op.getOperand(0);

1873

EVT SrcVT = Src.getValueType();

1874

unsigned InBits = SrcVT.getScalarSizeInBits();

1875

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1876

bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

1877

1878

// If none of the top bits are demanded, convert this into an any_extend.

1879

if (DemandedBits.getActiveBits() <= InBits) {

1880

// If we only need the non-extended bits of the bottom element

1881

// then we can just bitcast to the result.

1882

if (IsVecInReg && DemandedElts == 1 &&

1883

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1884

TLO.DAG.getDataLayout().isLittleEndian())

1885

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1886

1887

unsigned Opc =

1888

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1889

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1890

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1891

}

1892

1893

APInt InDemandedBits = DemandedBits.trunc(InBits);

1894

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1895

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1896

Depth + 1))

1897

return true;

1898

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1898, __extension__ __PRETTY_FUNCTION__));

1899

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1899, __extension__ __PRETTY_FUNCTION__));

1900

Known = Known.zext(BitWidth);

1901

1902

// Attempt to avoid multi-use ops if we don't need anything from them.

1903

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1904

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1905

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1906

break;

1907

}

1908

case ISD::SIGN_EXTEND:

1909

case ISD::SIGN_EXTEND_VECTOR_INREG: {

1910

SDValue Src = Op.getOperand(0);

1911

EVT SrcVT = Src.getValueType();

1912

unsigned InBits = SrcVT.getScalarSizeInBits();

1913

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1914

bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

1915

1916

// If none of the top bits are demanded, convert this into an any_extend.

1917

if (DemandedBits.getActiveBits() <= InBits) {

1918

// If we only need the non-extended bits of the bottom element

1919

// then we can just bitcast to the result.

1920

if (IsVecInReg && DemandedElts == 1 &&

1921

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1922

TLO.DAG.getDataLayout().isLittleEndian())

1923

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1924

1925

unsigned Opc =

1926

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1927

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1928

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1929

}

1930

1931

APInt InDemandedBits = DemandedBits.trunc(InBits);

1932

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1933

1934

// Since some of the sign extended bits are demanded, we know that the sign

1935

// bit is demanded.

1936

InDemandedBits.setBit(InBits - 1);

1937

1938

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1939

Depth + 1))

1940

return true;

1941

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1941, __extension__ __PRETTY_FUNCTION__));

1942

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1942, __extension__ __PRETTY_FUNCTION__));

1943

1944

// If the sign bit is known one, the top bits match.

1945

Known = Known.sext(BitWidth);

1946

1947

// If the sign bit is known zero, convert this to a zero extend.

1948

if (Known.isNonNegative()) {

1949

unsigned Opc =

1950

IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;

1951

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1952

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1953

}

1954

1955

// Attempt to avoid multi-use ops if we don't need anything from them.

1956

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1957

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1958

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1959

break;

1960

}

1961

case ISD::ANY_EXTEND:

1962

case ISD::ANY_EXTEND_VECTOR_INREG: {

1963

SDValue Src = Op.getOperand(0);

1964

EVT SrcVT = Src.getValueType();

1965

unsigned InBits = SrcVT.getScalarSizeInBits();

1966

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1967

bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

1968

1969

// If we only need the bottom element then we can just bitcast.

1970

// TODO: Handle ANY_EXTEND?

1971

if (IsVecInReg && DemandedElts == 1 &&

1972

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1973

TLO.DAG.getDataLayout().isLittleEndian())

1974

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1975

1976

APInt InDemandedBits = DemandedBits.trunc(InBits);

1977

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1978

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1979

Depth + 1))

1980

return true;

1981

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1981, __extension__ __PRETTY_FUNCTION__));

1982

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1982, __extension__ __PRETTY_FUNCTION__));

1983

Known = Known.anyext(BitWidth);

1984

1985

// Attempt to avoid multi-use ops if we don't need anything from them.

1986

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1987

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1988

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1989

break;

1990

}

1991

case ISD::TRUNCATE: {

1992

SDValue Src = Op.getOperand(0);

1993

1994

// Simplify the input, using demanded bit information, and compute the known

1995

// zero/one bits live out.

1996

unsigned OperandBitWidth = Src.getScalarValueSizeInBits();

1997

APInt TruncMask = DemandedBits.zext(OperandBitWidth);

1998

if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,

1999

Depth + 1))

2000

return true;

2001

Known = Known.trunc(BitWidth);

2002

2003

// Attempt to avoid multi-use ops if we don't need anything from them.

2004

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

2005

Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))

2006

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

2007

2008

// If the input is only used by this truncate, see if we can shrink it based

2009

// on the known demanded bits.

2010

if (Src.getNode()->hasOneUse()) {

2011

switch (Src.getOpcode()) {

2012

default:

2013

break;

2014

case ISD::SRL:

2015

// Shrink SRL by a constant if none of the high bits shifted in are

2016

// demanded.

2017

if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))

2018

// Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is

2019

// undesirable.

2020

break;

2021

2022

const APInt *ShAmtC =

2023

TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);

2024

if (!ShAmtC || ShAmtC->uge(BitWidth))

2025

break;

2026

uint64_t ShVal = ShAmtC->getZExtValue();

2027

2028

APInt HighBits =

2029

APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);

2030

HighBits.lshrInPlace(ShVal);

2031

HighBits = HighBits.trunc(BitWidth);

2032

2033

if (!(HighBits & DemandedBits)) {

2034

// None of the shifted in bits are needed. Add a truncate of the

2035

// shift input, then shift it.

2036

SDValue NewShAmt = TLO.DAG.getConstant(

2037

ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));

2038

SDValue NewTrunc =

2039

TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));

2040

return TLO.CombineTo(

2041

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));

2042

}

2043

break;

2044

}

2045

}

2046

2047

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2047, __extension__ __PRETTY_FUNCTION__));

2048

break;

2049

}

2050

case ISD::AssertZext: {

2051

// AssertZext demands all of the high bits, plus any of the low bits

2052

// demanded by its users.

2053

EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

2054

APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());

2055

if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,

2056

TLO, Depth + 1))

2057

return true;

2058

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2058, __extension__ __PRETTY_FUNCTION__));

2059

2060

Known.Zero |= ~InMask;

2061

break;

2062

}

2063

case ISD::EXTRACT_VECTOR_ELT: {

2064

SDValue Src = Op.getOperand(0);

2065

SDValue Idx = Op.getOperand(1);

2066

ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();

2067

unsigned EltBitWidth = Src.getScalarValueSizeInBits();

2068

2069

if (SrcEltCnt.isScalable())

2070

return false;

2071

2072

// Demand the bits from every vector element without a constant index.

2073

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2074

APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);

2075

if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))

2076

if (CIdx->getAPIntValue().ult(NumSrcElts))

2077

DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

2078

2079

// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know

2080

// anything about the extended bits.

2081

APInt DemandedSrcBits = DemandedBits;

2082

if (BitWidth > EltBitWidth)

2083

DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

2084

2085

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,

2086

Depth + 1))

2087

return true;

2088

2089

// Attempt to avoid multi-use ops if we don't need anything from them.

2090

if (!DemandedSrcBits.isAllOnesValue() ||

2091

!DemandedSrcElts.isAllOnesValue()) {

2092

if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

2093

Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {

2094

SDValue NewOp =

2095

TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);

2096

return TLO.CombineTo(Op, NewOp);

2097

}

2098

}

2099

2100

Known = Known2;

2101

if (BitWidth > EltBitWidth)

2102

Known = Known.anyext(BitWidth);

2103

break;

2104

}

2105

case ISD::BITCAST: {

2106

SDValue Src = Op.getOperand(0);

2107

EVT SrcVT = Src.getValueType();

2108

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

2109

2110

// If this is an FP->Int bitcast and if the sign bit is the only

2111

// thing demanded, turn this into a FGETSIGN.

2112

if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&

2113

DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&

2114

SrcVT.isFloatingPoint()) {

2115

bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);

2116

bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);

2117

if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&

2118

SrcVT != MVT::f128) {

2119

// Cannot eliminate/lower SHL for f128 yet.

2120

EVT Ty = OpVTLegal ? VT : MVT::i32;

2121

// Make a FGETSIGN + SHL to move the sign bit into the appropriate

2122

// place. We expect the SHL to be eliminated by other optimizations.

2123

SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);

2124

unsigned OpVTSizeInBits = Op.getValueSizeInBits();

2125

if (!OpVTLegal && OpVTSizeInBits > 32)

2126

Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);

2127

unsigned ShVal = Op.getValueSizeInBits() - 1;

2128

SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);

2129

return TLO.CombineTo(Op,

2130

TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));

2131

}

2132

}

2133

2134

// Bitcast from a vector using SimplifyDemanded Bits/VectorElts.

2135

// Demand the elt/bit if any of the original elts/bits are demanded.

2136

// TODO - bigendian once we have test coverage.

2137

if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&

2138

TLO.DAG.getDataLayout().isLittleEndian()) {

2139

unsigned Scale = BitWidth / NumSrcEltBits;

2140

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2141

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2142

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2143

for (unsigned i = 0; i != Scale; ++i) {

2144

unsigned Offset = i * NumSrcEltBits;

2145

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

2146

if (!Sub.isNullValue()) {

2147

DemandedSrcBits |= Sub;

2148

for (unsigned j = 0; j != NumElts; ++j)

2149

if (DemandedElts[j])

2150

DemandedSrcElts.setBit((j * Scale) + i);

2151

}

2152

}

2153

2154

APInt KnownSrcUndef, KnownSrcZero;

2155

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2156

KnownSrcZero, TLO, Depth + 1))

2157

return true;

2158

2159

KnownBits KnownSrcBits;

2160

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2161

KnownSrcBits, TLO, Depth + 1))

2162

return true;

2163

} else if ((NumSrcEltBits % BitWidth) == 0 &&

2164

TLO.DAG.getDataLayout().isLittleEndian()) {

2165

unsigned Scale = NumSrcEltBits / BitWidth;

2166

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

2167

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2168

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2169

for (unsigned i = 0; i != NumElts; ++i)

2170

if (DemandedElts[i]) {

2171

unsigned Offset = (i % Scale) * BitWidth;

2172

DemandedSrcBits.insertBits(DemandedBits, Offset);

2173

DemandedSrcElts.setBit(i / Scale);

2174

}

2175

2176

if (SrcVT.isVector()) {

2177

APInt KnownSrcUndef, KnownSrcZero;

2178

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2179

KnownSrcZero, TLO, Depth + 1))

2180

return true;

2181

}

2182

2183

KnownBits KnownSrcBits;

2184

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2185

KnownSrcBits, TLO, Depth + 1))

2186

return true;

2187

}

2188

2189

// If this is a bitcast, let computeKnownBits handle it. Only do this on a

2190

// recursive call where Known may be useful to the caller.

2191

if (Depth > 0) {

2192

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2193

return false;

2194

}

2195

break;

2196

}

2197

case ISD::ADD:

2198

case ISD::MUL:

2199

case ISD::SUB: {

2200

// Add, Sub, and Mul don't demand any bits in positions beyond that

2201

// of the highest bit demanded of them.

2202

SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);

2203

SDNodeFlags Flags = Op.getNode()->getFlags();

2204

unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();

2205

APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

2206

if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,

2207

Depth + 1) ||

2208

SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,

2209

Depth + 1) ||

2210

// See if the operation should be performed at a smaller bit width.

2211

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {

2212

if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {

2213

// Disable the nsw and nuw flags. We can no longer guarantee that we

2214

// won't wrap after simplification.

2215

Flags.setNoSignedWrap(false);

2216

Flags.setNoUnsignedWrap(false);

2217

SDValue NewOp =

2218

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2219

return TLO.CombineTo(Op, NewOp);

2220

}

2221

return true;

2222

}

2223

2224

// Attempt to avoid multi-use ops if we don't need anything from them.

2225

if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

2226

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

2227

Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2228

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

2229

Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2230

if (DemandedOp0 || DemandedOp1) {

2231

Flags.setNoSignedWrap(false);

2232

Flags.setNoUnsignedWrap(false);

2233

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

2234

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

2235

SDValue NewOp =

2236

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2237

return TLO.CombineTo(Op, NewOp);

2238

}

2239

}

2240

2241

// If we have a constant operand, we may be able to turn it into -1 if we

2242

// do not demand the high bits. This can make the constant smaller to

2243

// encode, allow more general folding, or match specialized instruction

2244

// patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that

2245

// is probably not useful (and could be detrimental).

2246

ConstantSDNode *C = isConstOrConstSplat(Op1);

2247

APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);

2248

if (C && !C->isAllOnesValue() && !C->isOne() &&

2249

(C->getAPIntValue() | HighMask).isAllOnesValue()) {

2250

SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);

2251

// Disable the nsw and nuw flags. We can no longer guarantee that we

2252

// won't wrap after simplification.

2253

Flags.setNoSignedWrap(false);

2254

Flags.setNoUnsignedWrap(false);

2255

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);

2256

return TLO.CombineTo(Op, NewOp);

2257

}

2258

2259

LLVM_FALLTHROUGH[[gnu::fallthrough]];

2260

}

2261

default:

2262

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2263

if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,

2264

Known, TLO, Depth))

2265

return true;

2266

break;

2267

}

2268

2269

// Just use computeKnownBits to compute output bits.

2270

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2271

break;

2272

}

2273

2274

// If we know the value of all of the demanded bits, return this as a

2275

// constant.

2276

if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {

2277

// Avoid folding to a constant if any OpaqueConstant is involved.

2278

const SDNode *N = Op.getNode();

2279

for (SDNode *Op :

2280

llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {

2281

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))

2282

if (C->isOpaque())

2283

return false;

2284

}

2285

if (VT.isInteger())

2286

return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));

2287

if (VT.isFloatingPoint())

2288

return TLO.CombineTo(

2289

Op,

2290

TLO.DAG.getConstantFP(

2291

APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));

2292

}

2293

2294

return false;

2295

}

2296

2297

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,

2298

const APInt &DemandedElts,

2299

APInt &KnownUndef,

2300

APInt &KnownZero,

2301

DAGCombinerInfo &DCI) const {

2302

SelectionDAG &DAG = DCI.DAG;

2303

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

2304

!DCI.isBeforeLegalizeOps());

2305

2306

bool Simplified =

2307

SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);

2308

if (Simplified) {

2309

DCI.AddToWorklist(Op.getNode());

2310

DCI.CommitTargetLoweringOpt(TLO);

2311

}

2312

2313

return Simplified;

2314

}

2315

2316

/// Given a vector binary operation and known undefined elements for each input

2317

/// operand, compute whether each element of the output is undefined.

2318

static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,

2319

const APInt &UndefOp0,

2320

const APInt &UndefOp1) {

2321

EVT VT = BO.getValueType();

2322

assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2323, __extension__ __PRETTY_FUNCTION__))

2323

"Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2323, __extension__ __PRETTY_FUNCTION__));

2324

2325

EVT EltVT = VT.getVectorElementType();

2326

unsigned NumElts = VT.getVectorNumElements();

2327

assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__))

2328

UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__));

2329

2330

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,

2331

const APInt &UndefVals) {

2332

if (UndefVals[Index])

2333

return DAG.getUNDEF(EltVT);

2334

2335

if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {

2336

// Try hard to make sure that the getNode() call is not creating temporary

2337

// nodes. Ignore opaque integers because they do not constant fold.

2338

SDValue Elt = BV->getOperand(Index);

2339

auto *C = dyn_cast<ConstantSDNode>(Elt);

2340

if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))

2341

return Elt;

2342

}

2343

2344

return SDValue();

2345

};

2346

2347

APInt KnownUndef = APInt::getNullValue(NumElts);

2348

for (unsigned i = 0; i != NumElts; ++i) {

2349

// If both inputs for this element are either constant or undef and match

2350

// the element type, compute the constant/undef result for this element of

2351

// the vector.

2352

// TODO: Ideally we would use FoldConstantArithmetic() here, but that does

2353

// not handle FP constants. The code within getNode() should be refactored

2354

// to avoid the danger of creating a bogus temporary node here.

2355

SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);

2356

SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);

2357

if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)

2358

if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())

2359

KnownUndef.setBit(i);

2360

}

2361

return KnownUndef;

2362

}

2363

2364

bool TargetLowering::SimplifyDemandedVectorElts(

2365

SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,

2366

APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,

2367

bool AssumeSingleUse) const {

2368

EVT VT = Op.getValueType();

2369

unsigned Opcode = Op.getOpcode();

2370

APInt DemandedElts = OriginalDemandedElts;

2371

unsigned NumElts = DemandedElts.getBitWidth();

2372

assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2372, __extension__ __PRETTY_FUNCTION__));

2373

2374

KnownUndef = KnownZero = APInt::getNullValue(NumElts);

2375

2376

// TODO: For now we assume we know nothing about scalable vectors.

2377

if (VT.isScalableVector())

2378

return false;

2379

2380

assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2381, __extension__ __PRETTY_FUNCTION__))

2381

"Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2381, __extension__ __PRETTY_FUNCTION__));

2382

2383

// Undef operand.

2384

if (Op.isUndef()) {

2385

KnownUndef.setAllBits();

2386

return false;

2387

}

2388

2389

// If Op has other users, assume that all elements are needed.

2390

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)

2391

DemandedElts.setAllBits();

2392

2393

// Not demanding any elements from Op.

2394

if (DemandedElts == 0) {

2395

KnownUndef.setAllBits();

2396

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2397

}

2398

2399

// Limit search depth.

2400

if (Depth >= SelectionDAG::MaxRecursionDepth)

2401

return false;

2402

2403

SDLoc DL(Op);

2404

unsigned EltSizeInBits = VT.getScalarSizeInBits();

2405

2406

// Helper for demanding the specified elements and all the bits of both binary

2407

// operands.

2408

auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {

2409

SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,

2410

TLO.DAG, Depth + 1);

2411

SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,

2412

TLO.DAG, Depth + 1);

2413

if (NewOp0 || NewOp1) {

2414

SDValue NewOp = TLO.DAG.getNode(

2415

Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);

2416

return TLO.CombineTo(Op, NewOp);

2417

}

2418

return false;

2419

};

2420

2421

switch (Opcode) {

2422

case ISD::SCALAR_TO_VECTOR: {

2423

if (!DemandedElts[0]) {

2424

KnownUndef.setAllBits();

2425

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2426

}

2427

SDValue ScalarSrc = Op.getOperand(0);

2428

if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {

2429

SDValue Src = ScalarSrc.getOperand(0);

2430

SDValue Idx = ScalarSrc.getOperand(1);

2431

EVT SrcVT = Src.getValueType();

2432

2433

ElementCount SrcEltCnt = SrcVT.getVectorElementCount();

2434

2435

if (SrcEltCnt.isScalable())

2436

return false;

2437

2438

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2439

if (isNullConstant(Idx)) {

2440

APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);

2441

APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);

2442

APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);

2443

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2444

TLO, Depth + 1))

2445

return true;

2446

}

2447

}

2448

KnownUndef.setHighBits(NumElts - 1);

2449

break;

2450

}

2451

case ISD::BITCAST: {

2452

SDValue Src = Op.getOperand(0);

2453

EVT SrcVT = Src.getValueType();

2454

2455

// We only handle vectors here.

2456

// TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?

2457

if (!SrcVT.isVector())

2458

break;

2459

2460

// Fast handling of 'identity' bitcasts.

2461

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2462

if (NumSrcElts == NumElts)

2463

return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,

2464

KnownZero, TLO, Depth + 1);

2465

2466

APInt SrcZero, SrcUndef;

2467

APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);

2468

2469

// Bitcast from 'large element' src vector to 'small element' vector, we

2470

// must demand a source element if any DemandedElt maps to it.

2471

if ((NumElts % NumSrcElts) == 0) {

2472

unsigned Scale = NumElts / NumSrcElts;

2473

for (unsigned i = 0; i != NumElts; ++i)

2474

if (DemandedElts[i])

2475

SrcDemandedElts.setBit(i / Scale);

2476

2477

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2478

TLO, Depth + 1))

2479

return true;

2480

2481

// Try calling SimplifyDemandedBits, converting demanded elts to the bits

2482

// of the large element.

2483

// TODO - bigendian once we have test coverage.

2484

if (TLO.DAG.getDataLayout().isLittleEndian()) {

2485

unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();

2486

APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);

2487

for (unsigned i = 0; i != NumElts; ++i)

2488

if (DemandedElts[i]) {

2489

unsigned Ofs = (i % Scale) * EltSizeInBits;

2490

SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);

2491

}

2492

2493

KnownBits Known;

2494

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,

2495

TLO, Depth + 1))

2496

return true;

2497

}

2498

2499

// If the src element is zero/undef then all the output elements will be -

2500

// only demanded elements are guaranteed to be correct.

2501

for (unsigned i = 0; i != NumSrcElts; ++i) {

2502

if (SrcDemandedElts[i]) {

2503

if (SrcZero[i])

2504

KnownZero.setBits(i * Scale, (i + 1) * Scale);

2505

if (SrcUndef[i])

2506

KnownUndef.setBits(i * Scale, (i + 1) * Scale);

2507

}

2508

}

2509

}

2510

2511

// Bitcast from 'small element' src vector to 'large element' vector, we

2512

// demand all smaller source elements covered by the larger demanded element

2513

// of this vector.

2514

if ((NumSrcElts % NumElts) == 0) {

2515

unsigned Scale = NumSrcElts / NumElts;

2516

for (unsigned i = 0; i != NumElts; ++i)

2517

if (DemandedElts[i])

2518

SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);

2519

2520

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2521

TLO, Depth + 1))

2522

return true;

2523

2524

// If all the src elements covering an output element are zero/undef, then

2525

// the output element will be as well, assuming it was demanded.

2526

for (unsigned i = 0; i != NumElts; ++i) {

2527

if (DemandedElts[i]) {

2528

if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())

2529

KnownZero.setBit(i);

2530

if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())

2531

KnownUndef.setBit(i);

2532

}

2533

}

2534

}

2535

break;

2536

}

2537

case ISD::BUILD_VECTOR: {

2538

// Check all elements and simplify any unused elements with UNDEF.

2539

if (!DemandedElts.isAllOnesValue()) {

2540

// Don't simplify BROADCASTS.

2541

if (llvm::any_of(Op->op_values(),

2542

[&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {

2543

SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());

2544

bool Updated = false;

2545

for (unsigned i = 0; i != NumElts; ++i) {

2546

if (!DemandedElts[i] && !Ops[i].isUndef()) {

2547

Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());

2548

KnownUndef.setBit(i);

2549

Updated = true;

2550

}

2551

}

2552

if (Updated)

2553

return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));

2554

}

2555

}

2556

for (unsigned i = 0; i != NumElts; ++i) {

2557

SDValue SrcOp = Op.getOperand(i);

2558

if (SrcOp.isUndef()) {

2559

KnownUndef.setBit(i);

2560

} else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&

2561

(isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {

2562

KnownZero.setBit(i);

2563

}

2564

}

2565

break;

2566

}

2567

case ISD::CONCAT_VECTORS: {

2568

EVT SubVT = Op.getOperand(0).getValueType();

2569

unsigned NumSubVecs = Op.getNumOperands();

2570

unsigned NumSubElts = SubVT.getVectorNumElements();

2571

for (unsigned i = 0; i != NumSubVecs; ++i) {

2572

SDValue SubOp = Op.getOperand(i);

2573

APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);

2574

APInt SubUndef, SubZero;

2575

if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,

2576

Depth + 1))

2577

return true;

2578

KnownUndef.insertBits(SubUndef, i * NumSubElts);

2579

KnownZero.insertBits(SubZero, i * NumSubElts);

2580

}

2581

break;

2582

}

2583

case ISD::INSERT_SUBVECTOR: {

2584

// Demand any elements from the subvector and the remainder from the src its

2585

// inserted into.

2586

SDValue Src = Op.getOperand(0);

2587

SDValue Sub = Op.getOperand(1);

2588

uint64_t Idx = Op.getConstantOperandVal(2);

2589

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

2590

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

2591

APInt DemandedSrcElts = DemandedElts;

2592

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

2593

2594

APInt SubUndef, SubZero;

2595

if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,

2596

Depth + 1))

2597

return true;

2598

2599

// If none of the src operand elements are demanded, replace it with undef.

2600

if (!DemandedSrcElts && !Src.isUndef())

2601

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,

2602

TLO.DAG.getUNDEF(VT), Sub,

2603

Op.getOperand(2)));

2604

2605

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,

2606

TLO, Depth + 1))

2607

return true;

2608

KnownUndef.insertBits(SubUndef, Idx);

2609

KnownZero.insertBits(SubZero, Idx);

2610

2611

// Attempt to avoid multi-use ops if we don't need anything from them.

2612

if (!DemandedSrcElts.isAllOnesValue() ||

2613

!DemandedSubElts.isAllOnesValue()) {

2614

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2615

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2616

SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(

2617

Sub, DemandedSubElts, TLO.DAG, Depth + 1);

2618

if (NewSrc || NewSub) {

2619

NewSrc = NewSrc ? NewSrc : Src;

2620

NewSub = NewSub ? NewSub : Sub;

2621

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2622

NewSub, Op.getOperand(2));

2623

return TLO.CombineTo(Op, NewOp);

2624

}

2625

}

2626

break;

2627

}

2628

case ISD::EXTRACT_SUBVECTOR: {

2629

// Offset the demanded elts by the subvector index.

2630

SDValue Src = Op.getOperand(0);

2631

if (Src.getValueType().isScalableVector())

2632

break;

2633

uint64_t Idx = Op.getConstantOperandVal(1);

2634

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2635

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

2636

2637

APInt SrcUndef, SrcZero;

2638

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2639

Depth + 1))

2640

return true;

2641

KnownUndef = SrcUndef.extractBits(NumElts, Idx);

2642

KnownZero = SrcZero.extractBits(NumElts, Idx);

2643

2644

// Attempt to avoid multi-use ops if we don't need anything from them.

2645

if (!DemandedElts.isAllOnesValue()) {

2646

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2647

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2648

if (NewSrc) {

2649

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2650

Op.getOperand(1));

2651

return TLO.CombineTo(Op, NewOp);

2652

}

2653

}

2654

break;

2655

}

2656

case ISD::INSERT_VECTOR_ELT: {

2657

SDValue Vec = Op.getOperand(0);

2658

SDValue Scl = Op.getOperand(1);

2659

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

2660

2661

// For a legal, constant insertion index, if we don't need this insertion

2662

// then strip it, else remove it from the demanded elts.

2663

if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {

2664

unsigned Idx = CIdx->getZExtValue();

2665

if (!DemandedElts[Idx])

2666

return TLO.CombineTo(Op, Vec);

2667

2668

APInt DemandedVecElts(DemandedElts);

2669

DemandedVecElts.clearBit(Idx);

2670

if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,

2671

KnownZero, TLO, Depth + 1))

2672

return true;

2673

2674

KnownUndef.setBitVal(Idx, Scl.isUndef());

2675

2676

KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));

2677

break;

2678

}

2679

2680

APInt VecUndef, VecZero;

2681

if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,

2682

Depth + 1))

2683

return true;

2684

// Without knowing the insertion index we can't set KnownUndef/KnownZero.

2685

break;

2686

}

2687

case ISD::VSELECT: {

2688

// Try to transform the select condition based on the current demanded

2689

// elements.

2690

// TODO: If a condition element is undef, we can choose from one arm of the

2691

// select (and if one arm is undef, then we can propagate that to the

2692

// result).

2693

// TODO - add support for constant vselect masks (see IR version of this).

2694

APInt UnusedUndef, UnusedZero;

2695

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,

2696

UnusedZero, TLO, Depth + 1))

2697

return true;

2698

2699

// See if we can simplify either vselect operand.

2700

APInt DemandedLHS(DemandedElts);

2701

APInt DemandedRHS(DemandedElts);

2702

APInt UndefLHS, ZeroLHS;

2703

APInt UndefRHS, ZeroRHS;

2704

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,

2705

ZeroLHS, TLO, Depth + 1))

2706

return true;

2707

if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,

2708

ZeroRHS, TLO, Depth + 1))

2709

return true;

2710

2711

KnownUndef = UndefLHS & UndefRHS;

2712

KnownZero = ZeroLHS & ZeroRHS;

2713

break;

2714

}

2715

case ISD::VECTOR_SHUFFLE: {

2716

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

2717

2718

// Collect demanded elements from shuffle operands..

2719

APInt DemandedLHS(NumElts, 0);

2720

APInt DemandedRHS(NumElts, 0);

2721

for (unsigned i = 0; i != NumElts; ++i) {

2722

int M = ShuffleMask[i];

2723

if (M < 0 || !DemandedElts[i])

2724

continue;

2725

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2725, __extension__ __PRETTY_FUNCTION__));

2726

if (M < (int)NumElts)

2727

DemandedLHS.setBit(M);

2728

else

2729

DemandedRHS.setBit(M - NumElts);

2730

}

2731

2732

// See if we can simplify either shuffle operand.

2733

APInt UndefLHS, ZeroLHS;

2734

APInt UndefRHS, ZeroRHS;

2735

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,

2736

ZeroLHS, TLO, Depth + 1))

2737

return true;

2738

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,

2739

ZeroRHS, TLO, Depth + 1))

2740

return true;

2741

2742

// Simplify mask using undef elements from LHS/RHS.

2743

bool Updated = false;

2744

bool IdentityLHS = true, IdentityRHS = true;

2745

SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());

2746

for (unsigned i = 0; i != NumElts; ++i) {

2747

int &M = NewMask[i];

2748

if (M < 0)

2749

continue;

2750

if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||

2751

(M >= (int)NumElts && UndefRHS[M - NumElts])) {

2752

Updated = true;

2753

M = -1;

2754

}

2755

IdentityLHS &= (M < 0) || (M == (int)i);

2756

IdentityRHS &= (M < 0) || ((M - NumElts) == i);

2757

}

2758

2759

// Update legal shuffle masks based on demanded elements if it won't reduce

2760

// to Identity which can cause premature removal of the shuffle mask.

2761

if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {

2762

SDValue LegalShuffle =

2763

buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),

2764

NewMask, TLO.DAG);

2765

if (LegalShuffle)

2766

return TLO.CombineTo(Op, LegalShuffle);

2767

}

2768

2769

// Propagate undef/zero elements from LHS/RHS.

2770

for (unsigned i = 0; i != NumElts; ++i) {

2771

int M = ShuffleMask[i];

2772

if (M < 0) {

2773

KnownUndef.setBit(i);

2774

} else if (M < (int)NumElts) {

2775

if (UndefLHS[M])

2776

KnownUndef.setBit(i);

2777

if (ZeroLHS[M])

2778

KnownZero.setBit(i);

2779

} else {

2780

if (UndefRHS[M - NumElts])

2781

KnownUndef.setBit(i);

2782

if (ZeroRHS[M - NumElts])

2783

KnownZero.setBit(i);

2784

}

2785

}

2786

break;

2787

}

2788

case ISD::ANY_EXTEND_VECTOR_INREG:

2789

case ISD::SIGN_EXTEND_VECTOR_INREG:

2790

case ISD::ZERO_EXTEND_VECTOR_INREG: {

2791

APInt SrcUndef, SrcZero;

2792

SDValue Src = Op.getOperand(0);

2793

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2794

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);

2795

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2796

Depth + 1))

2797

return true;

2798

KnownZero = SrcZero.zextOrTrunc(NumElts);

2799

KnownUndef = SrcUndef.zextOrTrunc(NumElts);

2800

2801

if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&

2802

Op.getValueSizeInBits() == Src.getValueSizeInBits() &&

2803

DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {

2804

// aext - if we just need the bottom element then we can bitcast.

2805

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

2806

}

2807

2808

if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {

2809

// zext(undef) upper bits are guaranteed to be zero.

2810

if (DemandedElts.isSubsetOf(KnownUndef))

2811

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2812

KnownUndef.clearAllBits();

2813

}

2814

break;

2815

}

2816

2817

// TODO: There are more binop opcodes that could be handled here - MIN,

2818

// MAX, saturated math, etc.

2819

case ISD::OR:

2820

case ISD::XOR:

2821

case ISD::ADD:

2822

case ISD::SUB:

2823

case ISD::FADD:

2824

case ISD::FSUB:

2825

case ISD::FMUL:

2826

case ISD::FDIV:

2827

case ISD::FREM: {

2828

SDValue Op0 = Op.getOperand(0);

2829

SDValue Op1 = Op.getOperand(1);

2830

2831

APInt UndefRHS, ZeroRHS;

2832

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2833

Depth + 1))

2834

return true;

2835

APInt UndefLHS, ZeroLHS;

2836

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2837

Depth + 1))

2838

return true;

2839

2840

KnownZero = ZeroLHS & ZeroRHS;

2841

KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

2842

2843

// Attempt to avoid multi-use ops if we don't need anything from them.

2844

// TODO - use KnownUndef to relax the demandedelts?

2845

if (!DemandedElts.isAllOnesValue())

2846

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2847

return true;

2848

break;

2849

}

2850

case ISD::SHL:

2851

case ISD::SRL:

2852

case ISD::SRA:

2853

case ISD::ROTL:

2854

case ISD::ROTR: {

2855

SDValue Op0 = Op.getOperand(0);

2856

SDValue Op1 = Op.getOperand(1);

2857

2858

APInt UndefRHS, ZeroRHS;

2859

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2860

Depth + 1))

2861

return true;

2862

APInt UndefLHS, ZeroLHS;

2863

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2864

Depth + 1))

2865

return true;

2866

2867

KnownZero = ZeroLHS;

2868

KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

2869

2870

// Attempt to avoid multi-use ops if we don't need anything from them.

2871

// TODO - use KnownUndef to relax the demandedelts?

2872

if (!DemandedElts.isAllOnesValue())

2873

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2874

return true;

2875

break;

2876

}

2877

case ISD::MUL:

2878

case ISD::AND: {

2879

SDValue Op0 = Op.getOperand(0);

2880

SDValue Op1 = Op.getOperand(1);

2881

2882

APInt SrcUndef, SrcZero;

2883

if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,

2884

Depth + 1))

2885

return true;

2886

if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,

2887

TLO, Depth + 1))

2888

return true;

2889

2890

// If either side has a zero element, then the result element is zero, even

2891

// if the other is an UNDEF.

2892

// TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros

2893

// and then handle 'and' nodes with the rest of the binop opcodes.

2894

KnownZero |= SrcZero;

2895

KnownUndef &= SrcUndef;

2896

KnownUndef &= ~KnownZero;

2897

2898

// Attempt to avoid multi-use ops if we don't need anything from them.

2899

// TODO - use KnownUndef to relax the demandedelts?

2900

if (!DemandedElts.isAllOnesValue())

2901

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2902

return true;

2903

break;

2904

}

2905

case ISD::TRUNCATE:

2906

case ISD::SIGN_EXTEND:

2907

case ISD::ZERO_EXTEND:

2908

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,

2909

KnownZero, TLO, Depth + 1))

2910

return true;

2911

2912

if (Op.getOpcode() == ISD::ZERO_EXTEND) {

2913

// zext(undef) upper bits are guaranteed to be zero.

2914

if (DemandedElts.isSubsetOf(KnownUndef))

2915

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2916

KnownUndef.clearAllBits();

2917

}

2918

break;

2919

default: {

2920

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2921

if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,

2922

KnownZero, TLO, Depth))

2923

return true;

2924

} else {

2925

KnownBits Known;

2926

APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);

2927

if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,

2928

TLO, Depth, AssumeSingleUse))

2929

return true;

2930

}

2931

break;

2932

}

2933

}

2934

assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2934, __extension__ __PRETTY_FUNCTION__));

2935

2936

// Constant fold all undef cases.

2937

// TODO: Handle zero cases as well.

2938

if (DemandedElts.isSubsetOf(KnownUndef))

2939

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2940

2941

return false;

2942

}

2943

2944

/// Determine which of the bits specified in Mask are known to be either zero or

2945

/// one and return them in the Known.

2946

void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,

2947

KnownBits &Known,

2948

const APInt &DemandedElts,

2949

const SelectionDAG &DAG,

2950

unsigned Depth) const {

2951

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2952

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2953

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2954

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2955

"Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2956

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__));

2957

Known.resetAll();

2958

}

2959

2960

void TargetLowering::computeKnownBitsForTargetInstr(

2961

GISelKnownBits &Analysis, Register R, KnownBits &Known,

2962

const APInt &DemandedElts, const MachineRegisterInfo &MRI,

2963

unsigned Depth) const {

2964

Known.resetAll();

2965

}

2966

2967

void TargetLowering::computeKnownBitsForFrameIndex(

2968

const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {

2969

// The low bits are known zero if the pointer is aligned.

2970

Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));

2971

}

2972

2973

Align TargetLowering::computeKnownAlignForTargetInstr(

2974

GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,

2975

unsigned Depth) const {

2976

return Align(1);

2977

}

2978

2979

/// This method can be implemented by targets that want to expose additional

2980

/// information about sign bits to the DAG Combiner.

2981

unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,

2982

const APInt &,

2983

const SelectionDAG &,

2984

unsigned Depth) const {

2985

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2986

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2987

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2988

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2989

"Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2990

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__));

2991

return 1;

2992

}

2993

2994

unsigned TargetLowering::computeNumSignBitsForTargetInstr(

2995

GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,

2996

const MachineRegisterInfo &MRI, unsigned Depth) const {

2997

return 1;

2998

}

2999

3000

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(

3001

SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,

3002

TargetLoweringOpt &TLO, unsigned Depth) const {

3003

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3004

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3005

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3006

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3007

"Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3008

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__));

3009

return false;

3010

}

3011

3012

bool TargetLowering::SimplifyDemandedBitsForTargetNode(

3013

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3014

KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {

3015

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3016

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3017

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3018

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3019

"Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3020

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__));

3021

computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);

3022

return false;

3023

}

3024

3025

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(

3026

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3027

SelectionDAG &DAG, unsigned Depth) const {

3028

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3029

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3030

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3031

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3032

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3033

"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3034

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__));

3035

return SDValue();

3036

}

3037

3038

SDValue

3039

TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,

3040

SDValue N1, MutableArrayRef<int> Mask,

3041

SelectionDAG &DAG) const {

3042

bool LegalMask = isShuffleMaskLegal(Mask, VT);

3043

if (!LegalMask) {

3044

std::swap(N0, N1);

3045

ShuffleVectorSDNode::commuteMask(Mask);

3046

LegalMask = isShuffleMaskLegal(Mask, VT);

3047

}

3048

3049

if (!LegalMask)

3050

return SDValue();

3051

3052

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);

3053

}

3054

3055

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {

3056

return nullptr;

3057

}

3058

3059

bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(

3060

SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,

3061

bool PoisonOnly, unsigned Depth) const {

3062

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3063

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3064

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3065

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3066

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3067

"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3068

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__));

3069

return false;

3070

}

3071

3072

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,

3073

const SelectionDAG &DAG,

3074

bool SNaN,

3075

unsigned Depth) const {

3076

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3077

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3078

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3079

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3080

"Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3081

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__));

3082

return false;

3083

}

3084

3085

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must

3086

// work with truncating build vectors and vectors with elements of less than

3087

// 8 bits.

3088

bool TargetLowering::isConstTrueVal(const SDNode *N) const {

3089

if (!N)

3090

return false;

3091

3092

APInt CVal;

3093

if (auto *CN = dyn_cast<ConstantSDNode>(N)) {

3094

CVal = CN->getAPIntValue();

3095

} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {

3096

auto *CN = BV->getConstantSplatNode();

3097

if (!CN)

3098

return false;

3099

3100

// If this is a truncating build vector, truncate the splat value.

3101

// Otherwise, we may fail to match the expected values below.

3102

unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();

3103

CVal = CN->getAPIntValue();

3104

if (BVEltWidth < CVal.getBitWidth())

3105

CVal = CVal.trunc(BVEltWidth);

3106

} else {

3107

return false;

3108

}

3109

3110

switch (getBooleanContents(N->getValueType(0))) {

3111

case UndefinedBooleanContent:

3112

return CVal[0];

3113

case ZeroOrOneBooleanContent:

3114

return CVal.isOneValue();

3115

case ZeroOrNegativeOneBooleanContent:

3116

return CVal.isAllOnesValue();

3117

}

3118

3119

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3119);

3120

}

3121

3122

bool TargetLowering::isConstFalseVal(const SDNode *N) const {

3123

if (!N)

3124

return false;

3125

3126

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);

3127

if (!CN) {

3128

const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);

3129

if (!BV)

3130

return false;

3131

3132

// Only interested in constant splats, we don't care about undef

3133

// elements in identifying boolean constants and getConstantSplatNode

3134

// returns NULL if all ops are undef;

3135

CN = BV->getConstantSplatNode();

3136

if (!CN)

3137

return false;

3138

}

3139

3140

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)

3141

return !CN->getAPIntValue()[0];

3142

3143

return CN->isNullValue();

3144

}

3145

3146

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,

3147

bool SExt) const {

3148

if (VT == MVT::i1)

3149

return N->isOne();

3150

3151

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);

3152

switch (Cnt) {

3153

case TargetLowering::ZeroOrOneBooleanContent:

3154

// An extended value of 1 is always true, unless its original type is i1,

3155

// in which case it will be sign extended to -1.

3156

return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));

3157

case TargetLowering::UndefinedBooleanContent:

3158

case TargetLowering::ZeroOrNegativeOneBooleanContent:

3159

return N->isAllOnesValue() && SExt;

3160

}

3161

llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3161);

3162

}

3163

3164

/// This helper function of SimplifySetCC tries to optimize the comparison when

3165

/// either operand of the SetCC node is a bitwise-and instruction.

3166

SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,

3167

ISD::CondCode Cond, const SDLoc &DL,

3168

DAGCombinerInfo &DCI) const {

3169

// Match these patterns in any of their permutations:

3170

// (X & Y) == Y

3171

// (X & Y) != Y

3172

if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)

3173

std::swap(N0, N1);

3174

3175

EVT OpVT = N0.getValueType();

3176

if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||

3177

(Cond != ISD::SETEQ && Cond != ISD::SETNE))

3178

return SDValue();

3179

3180

SDValue X, Y;

3181

if (N0.getOperand(0) == N1) {

3182

X = N0.getOperand(1);

3183

Y = N0.getOperand(0);

3184

} else if (N0.getOperand(1) == N1) {

3185

X = N0.getOperand(0);

3186

Y = N0.getOperand(1);

3187

} else {

3188

return SDValue();

3189

}

3190

3191

SelectionDAG &DAG = DCI.DAG;

3192

SDValue Zero = DAG.getConstant(0, DL, OpVT);

3193

if (DAG.isKnownToBeAPowerOfTwo(Y)) {

3194

// Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.

3195

// Note that where Y is variable and is known to have at most one bit set

3196

// (for example, if it is Z & 1) we cannot do this; the expressions are not

3197

// equivalent when Y == 0.

3198

assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3198, __extension__ __PRETTY_FUNCTION__));

3199

Cond = ISD::getSetCCInverse(Cond, OpVT);

3200

if (DCI.isBeforeLegalizeOps() ||

3201

isCondCodeLegal(Cond, N0.getSimpleValueType()))

3202

return DAG.getSetCC(DL, VT, N0, Zero, Cond);

3203

} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {

3204

// If the target supports an 'and-not' or 'and-complement' logic operation,

3205

// try to use that to make a comparison operation more efficient.

3206

// But don't do this transform if the mask is a single bit because there are

3207

// more efficient ways to deal with that case (for example, 'bt' on x86 or

3208

// 'rlwinm' on PPC).

3209

3210

// Bail out if the compare operand that we want to turn into a zero is

3211

// already a zero (otherwise, infinite loop).

3212

auto *YConst = dyn_cast<ConstantSDNode>(Y);

3213

if (YConst && YConst->isNullValue())

3214

return SDValue();

3215

3216

// Transform this into: ~X & Y == 0.

3217

SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);

3218

SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);

3219

return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);

3220

}

3221

3222

return SDValue();

3223

}

3224

3225

/// There are multiple IR patterns that could be checking whether certain

3226

/// truncation of a signed number would be lossy or not. The pattern which is

3227

/// best at IR level, may not lower optimally. Thus, we want to unfold it.

3228

/// We are looking for the following pattern: (KeptBits is a constant)

3229

/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)

3230

/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.

3231

/// KeptBits also can't be 1, that would have been folded to %x dstcond 0

3232

/// We will unfold it into the natural trunc+sext pattern:

3233

/// ((%x << C) a>> C) dstcond %x

3234

/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)

3235

SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(

3236

EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,

3237

const SDLoc &DL) const {

3238

// We must be comparing with a constant.

3239

ConstantSDNode *C1;

3240

if (!(C1 = dyn_cast<ConstantSDNode>(N1)))

3241

return SDValue();

3242

3243

// N0 should be: add %x, (1 << (KeptBits-1))

3244

if (N0->getOpcode() != ISD::ADD)

3245

return SDValue();

3246

3247

// And we must be 'add'ing a constant.

3248

ConstantSDNode *C01;

3249

if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))

3250

return SDValue();

3251

3252

SDValue X = N0->getOperand(0);

3253

EVT XVT = X.getValueType();

3254

3255

// Validate constants ...

3256

3257

APInt I1 = C1->getAPIntValue();

3258

3259

ISD::CondCode NewCond;

3260

if (Cond == ISD::CondCode::SETULT) {

3261

NewCond = ISD::CondCode::SETEQ;

3262

} else if (Cond == ISD::CondCode::SETULE) {

3263

NewCond = ISD::CondCode::SETEQ;

3264

// But need to 'canonicalize' the constant.

3265

I1 += 1;

3266

} else if (Cond == ISD::CondCode::SETUGT) {

3267

NewCond = ISD::CondCode::SETNE;

3268

// But need to 'canonicalize' the constant.

3269

I1 += 1;

3270

} else if (Cond == ISD::CondCode::SETUGE) {

3271

NewCond = ISD::CondCode::SETNE;

3272

} else

3273

return SDValue();

3274

3275

APInt I01 = C01->getAPIntValue();

3276

3277

auto checkConstants = [&I1, &I01]() -> bool {

3278

// Both of them must be power-of-two, and the constant from setcc is bigger.

3279

return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();

3280

};

3281

3282

if (checkConstants()) {

3283

// Great, e.g. got icmp ult i16 (add i16 %x, 128), 256

3284

} else {

3285

// What if we invert constants? (and the target predicate)

3286

I1.negate();

3287

I01.negate();

3288

assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3288, __extension__ __PRETTY_FUNCTION__));

3289

NewCond = getSetCCInverse(NewCond, XVT);

3290

if (!checkConstants())

3291

return SDValue();

3292

// Great, e.g. got icmp uge i16 (add i16 %x, -128), -256

3293

}

3294

3295

// They are power-of-two, so which bit is set?

3296

const unsigned KeptBits = I1.logBase2();

3297

const unsigned KeptBitsMinusOne = I01.logBase2();

3298

3299

// Magic!

3300

if (KeptBits != (KeptBitsMinusOne + 1))

3301

return SDValue();

3302

assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3302, __extension__ __PRETTY_FUNCTION__));

3303

3304

// We don't want to do this in every single case.

3305

SelectionDAG &DAG = DCI.DAG;

3306

if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(

3307

XVT, KeptBits))

3308

return SDValue();

3309

3310

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;

3311

assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3311, __extension__ __PRETTY_FUNCTION__));

3312

3313

// Unfold into: ((%x << C) a>> C) cond %x

3314

// Where 'cond' will be either 'eq' or 'ne'.

3315

SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);

3316

SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);

3317

SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);

3318

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

3319

3320

return T2;

3321

}

3322

3323

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3324

SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(

3325

EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,

3326

DAGCombinerInfo &DCI, const SDLoc &DL) const {

3327

assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__))

3328

isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__))

3329

"Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__));

3330

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__))

3331

"Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__));

3332

3333

unsigned NewShiftOpcode;

3334

SDValue X, C, Y;

3335

3336

SelectionDAG &DAG = DCI.DAG;

3337

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3338

3339

// Look for '(C l>>/<< Y)'.

3340

auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {

3341

// The shift should be one-use.

3342

if (!V.hasOneUse())

3343

return false;

3344

unsigned OldShiftOpcode = V.getOpcode();

3345

switch (OldShiftOpcode) {

3346

case ISD::SHL:

3347

NewShiftOpcode = ISD::SRL;

3348

break;

3349

case ISD::SRL:

3350

NewShiftOpcode = ISD::SHL;

3351

break;

3352

default:

3353

return false; // must be a logical shift.

3354

}

3355

// We should be shifting a constant.

3356

// FIXME: best to use isConstantOrConstantVector().

3357

C = V.getOperand(0);

3358

ConstantSDNode *CC =

3359

isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3360

if (!CC)

3361

return false;

3362

Y = V.getOperand(1);

3363

3364

ConstantSDNode *XC =

3365

isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3366

return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(

3367

X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);

3368

};

3369

3370

// LHS of comparison should be an one-use 'and'.

3371

if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())

3372

return SDValue();

3373

3374

X = N0.getOperand(0);

3375

SDValue Mask = N0.getOperand(1);

3376

3377

// 'and' is commutative!

3378

if (!Match(Mask)) {

3379

std::swap(X, Mask);

3380

if (!Match(Mask))

3381

return SDValue();

3382

}

3383

3384

EVT VT = X.getValueType();

3385

3386

// Produce:

3387

// ((X 'OppositeShiftOpcode' Y) & C) Cond 0

3388

SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);

3389

SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);

3390

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);

3391

return T2;

3392

}

3393

3394

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as

3395

/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to

3396

/// handle the commuted versions of these patterns.

3397

SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,

3398

ISD::CondCode Cond, const SDLoc &DL,

3399

DAGCombinerInfo &DCI) const {

3400

unsigned BOpcode = N0.getOpcode();

3401

assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__))

3402

"Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__));

3403

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3403, __extension__ __PRETTY_FUNCTION__));

3404

3405

// (X + Y) == X --> Y == 0

3406

// (X - Y) == X --> Y == 0

3407

// (X ^ Y) == X --> Y == 0

3408

SelectionDAG &DAG = DCI.DAG;

3409

EVT OpVT = N0.getValueType();

3410

SDValue X = N0.getOperand(0);

3411

SDValue Y = N0.getOperand(1);

3412

if (X == N1)

3413

return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

3414

3415

if (Y != N1)

3416

return SDValue();

3417

3418

// (X + Y) == Y --> X == 0

3419

// (X ^ Y) == Y --> X == 0

3420

if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)

3421

return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

3422

3423

// The shift would not be valid if the operands are boolean (i1).

3424

if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)

3425

return SDValue();

3426

3427

// (X - Y) == Y --> X == Y << 1

3428

EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),

3429

!DCI.isBeforeLegalize());

3430

SDValue One = DAG.getConstant(1, DL, ShiftVT);

3431

SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);

3432

if (!DCI.isCalledByLegalizer())

3433

DCI.AddToWorklist(YShl1.getNode());

3434

return DAG.getSetCC(DL, VT, X, YShl1, Cond);

3435

}

3436

3437

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,

3438

SDValue N0, const APInt &C1,

3439

ISD::CondCode Cond, const SDLoc &dl,

3440

SelectionDAG &DAG) {

3441

// Look through truncs that don't change the value of a ctpop.

3442

// FIXME: Add vector support? Need to be careful with setcc result type below.

3443

SDValue CTPOP = N0;

3444

if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&

3445

N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))

3446

CTPOP = N0.getOperand(0);

3447

3448

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())

3449

return SDValue();

3450

3451

EVT CTVT = CTPOP.getValueType();

3452

SDValue CTOp = CTPOP.getOperand(0);

3453

3454

// If this is a vector CTPOP, keep the CTPOP if it is legal.

3455

// TODO: Should we check if CTPOP is legal(or custom) for scalars?

3456

if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))

3457

return SDValue();

3458

3459

// (ctpop x) u< 2 -> (x & x-1) == 0

3460

// (ctpop x) u> 1 -> (x & x-1) != 0

3461

if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {

3462

unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);

3463

if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))

3464

return SDValue();

3465

if (C1 == 0 && (Cond == ISD::SETULT))

3466

return SDValue(); // This is handled elsewhere.

3467

3468

unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

3469

3470

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3471

SDValue Result = CTOp;

3472

for (unsigned i = 0; i < Passes; i++) {

3473

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);

3474

Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);

3475

}

3476

ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;

3477

return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);

3478

}

3479

3480

// If ctpop is not supported, expand a power-of-2 comparison based on it.

3481

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {

3482

// For scalars, keep CTPOP if it is legal or custom.

3483

if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))

3484

return SDValue();

3485

// This is based on X86's custom lowering for CTPOP which produces more

3486

// instructions than the expansion here.

3487

3488

// (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)

3489

// (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)

3490

SDValue Zero = DAG.getConstant(0, dl, CTVT);

3491

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3492

assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3492, __extension__ __PRETTY_FUNCTION__));

3493

ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);

3494

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

3495

SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

3496

SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);

3497

SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);

3498

unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;

3499

return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);

3500

}

3501

3502

return SDValue();

3503

}

3504

3505

/// Try to simplify a setcc built with the specified operands and cc. If it is

3506

/// unable to simplify it, return a null SDValue.

3507

SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,

3508

ISD::CondCode Cond, bool foldBooleans,

3509

DAGCombinerInfo &DCI,

3510

const SDLoc &dl) const {

3511

SelectionDAG &DAG = DCI.DAG;

3512

const DataLayout &Layout = DAG.getDataLayout();

3513

EVT OpVT = N0.getValueType();

3514

3515

// Constant fold or commute setcc.

3516

if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))

3517

return Fold;

3518

3519

// Ensure that the constant occurs on the RHS and fold constant comparisons.

3520

// TODO: Handle non-splat vector constants. All undef causes trouble.

3521

// FIXME: We can't yet fold constant scalable vector splats, so avoid an

3522

// infinite loop here when we encounter one.

3523

ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);

3524

if (isConstOrConstSplat(N0) &&

3525

(!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&

3526

(DCI.isBeforeLegalizeOps() ||

3527

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))

3528

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3529

3530

// If we have a subtract with the same 2 non-constant operands as this setcc

3531

// -- but in reverse order -- then try to commute the operands of this setcc

3532

// to match. A matching pair of setcc (cmp) and sub may be combined into 1

3533

// instruction on some targets.

3534

if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&

3535

(DCI.isBeforeLegalizeOps() ||

3536

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&

3537

DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&

3538

!DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))

3539

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3540

3541

if (auto *N1C = isConstOrConstSplat(N1)) {

3542

const APInt &C1 = N1C->getAPIntValue();

3543

3544

// Optimize some CTPOP cases.

3545

if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))

3546

return V;

3547

3548

// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an

3549

// equality comparison, then we're just comparing whether X itself is

3550

// zero.

3551

if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&

3552

N0.getOperand(0).getOpcode() == ISD::CTLZ &&

3553

isPowerOf2_32(N0.getScalarValueSizeInBits())) {

3554

if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {

3555

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3556

ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {

3557

if ((C1 == 0) == (Cond == ISD::SETEQ)) {

3558

// (srl (ctlz x), 5) == 0 -> X != 0

3559

// (srl (ctlz x), 5) != 1 -> X != 0

3560

Cond = ISD::SETNE;

3561

} else {

3562

// (srl (ctlz x), 5) != 0 -> X == 0

3563

// (srl (ctlz x), 5) == 1 -> X == 0

3564

Cond = ISD::SETEQ;

3565

}

3566

SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());

3567

return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,

3568

Cond);

3569

}

3570

}

3571

}

3572

}

3573

3574

// FIXME: Support vectors.

3575

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

3576

const APInt &C1 = N1C->getAPIntValue();

3577

3578

// (zext x) == C --> x == (trunc C)

3579

// (sext x) == C --> x == (trunc C)

3580

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3581

DCI.isBeforeLegalize() && N0->hasOneUse()) {

3582

unsigned MinBits = N0.getValueSizeInBits();

3583

SDValue PreExt;

3584

bool Signed = false;

3585

if (N0->getOpcode() == ISD::ZERO_EXTEND) {

3586

// ZExt

3587

MinBits = N0->getOperand(0).getValueSizeInBits();

3588

PreExt = N0->getOperand(0);

3589

} else if (N0->getOpcode() == ISD::AND) {

3590

// DAGCombine turns costly ZExts into ANDs

3591

if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))

3592

if ((C->getAPIntValue()+1).isPowerOf2()) {

3593

MinBits = C->getAPIntValue().countTrailingOnes();

3594

PreExt = N0->getOperand(0);

3595

}

3596

} else if (N0->getOpcode() == ISD::SIGN_EXTEND) {

3597

// SExt

3598

MinBits = N0->getOperand(0).getValueSizeInBits();

3599

PreExt = N0->getOperand(0);

3600

Signed = true;

3601

} else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {

3602

// ZEXTLOAD / SEXTLOAD

3603

if (LN0->getExtensionType() == ISD::ZEXTLOAD) {

3604

MinBits = LN0->getMemoryVT().getSizeInBits();

3605

PreExt = N0;

3606

} else if (LN0->getExtensionType() == ISD::SEXTLOAD) {

3607

Signed = true;

3608

MinBits = LN0->getMemoryVT().getSizeInBits();

3609

PreExt = N0;

3610

}

3611

}

3612

3613

// Figure out how many bits we need to preserve this constant.

3614

unsigned ReqdBits = Signed ?

3615

C1.getBitWidth() - C1.getNumSignBits() + 1 :

3616

C1.getActiveBits();

3617

3618

// Make sure we're not losing bits from the constant.

3619

if (MinBits > 0 &&

3620

MinBits < C1.getBitWidth() &&

3621

MinBits >= ReqdBits) {

3622

EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);

3623

if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {

3624

// Will get folded away.

3625

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);

3626

if (MinBits == 1 && C1 == 1)

3627

// Invert the condition.

3628

return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),

3629

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3630

SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);

3631

return DAG.getSetCC(dl, VT, Trunc, C, Cond);

3632

}

3633

3634

// If truncating the setcc operands is not desirable, we can still

3635

// simplify the expression in some cases:

3636

// setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)

3637

// setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))

3638

// setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))

3639

// setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)

3640

// setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))

3641

// setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)

3642

SDValue TopSetCC = N0->getOperand(0);

3643

unsigned N0Opc = N0->getOpcode();

3644

bool SExt = (N0Opc == ISD::SIGN_EXTEND);

3645

if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&

3646

TopSetCC.getOpcode() == ISD::SETCC &&

3647

(N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&

3648

(isConstFalseVal(N1C) ||

3649

isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

3650

3651

bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||

3652

(!N1C->isNullValue() && Cond == ISD::SETNE);

3653

3654

if (!Inverse)

3655

return TopSetCC;

3656

3657

ISD::CondCode InvCond = ISD::getSetCCInverse(

3658

cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),

3659

TopSetCC.getOperand(0).getValueType());

3660

return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),

3661

TopSetCC.getOperand(1),

3662

InvCond);

3663

}

3664

}

3665

}

3666

3667

// If the LHS is '(and load, const)', the RHS is 0, the test is for

3668

// equality or unsigned, and all 1 bits of the const are in the same

3669

// partial word, see if we can shorten the load.

3670

if (DCI.isBeforeLegalize() &&

3671

!ISD::isSignedIntSetCC(Cond) &&

3672

N0.getOpcode() == ISD::AND && C1 == 0 &&

3673

N0.getNode()->hasOneUse() &&

3674

isa<LoadSDNode>(N0.getOperand(0)) &&

3675

N0.getOperand(0).getNode()->hasOneUse() &&

3676

isa<ConstantSDNode>(N0.getOperand(1))) {

3677

LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));

3678

APInt bestMask;

3679

unsigned bestWidth = 0, bestOffset = 0;

3680

if (Lod->isSimple() && Lod->isUnindexed()) {

3681

unsigned origWidth = N0.getValueSizeInBits();

3682

unsigned maskWidth = origWidth;

3683

// We can narrow (e.g.) 16-bit extending loads on 32-bit target to

3684

// 8 bits, but have to be careful...

3685

if (Lod->getExtensionType() != ISD::NON_EXTLOAD)

3686

origWidth = Lod->getMemoryVT().getSizeInBits();

3687

const APInt &Mask = N0.getConstantOperandAPInt(1);

3688

for (unsigned width = origWidth / 2; width>=8; width /= 2) {

3689

APInt newMask = APInt::getLowBitsSet(maskWidth, width);

3690

for (unsigned offset=0; offset<origWidth/width; offset++) {

3691

if (Mask.isSubsetOf(newMask)) {

3692

if (Layout.isLittleEndian())

3693

bestOffset = (uint64_t)offset * (width/8);

3694

else

3695

bestOffset = (origWidth/width - offset - 1) * (width/8);

3696

bestMask = Mask.lshr(offset * (width/8) * 8);

3697

bestWidth = width;

3698

break;

3699

}

3700

newMask <<= width;

3701

}

3702

}

3703

}

3704

if (bestWidth) {

3705

EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);

3706

if (newVT.isRound() &&

3707

shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {

3708

SDValue Ptr = Lod->getBasePtr();

3709

if (bestOffset != 0)

3710

Ptr =

3711

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);

3712

SDValue NewLoad =

3713

DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,

3714

Lod->getPointerInfo().getWithOffset(bestOffset),

3715

Lod->getOriginalAlign());

3716

return DAG.getSetCC(dl, VT,

3717

DAG.getNode(ISD::AND, dl, newVT, NewLoad,

3718

DAG.getConstant(bestMask.trunc(bestWidth),

3719

dl, newVT)),

3720

DAG.getConstant(0LL, dl, newVT), Cond);

3721

}

3722

}

3723

}

3724

3725

// If the LHS is a ZERO_EXTEND, perform the comparison on the input.

3726

if (N0.getOpcode() == ISD::ZERO_EXTEND) {

3727

unsigned InSize = N0.getOperand(0).getValueSizeInBits();

3728

3729

// If the comparison constant has bits in the upper part, the

3730

// zero-extended value could never match.

3731

if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),

3732

C1.getBitWidth() - InSize))) {

3733

switch (Cond) {

3734

case ISD::SETUGT:

3735

case ISD::SETUGE:

3736

case ISD::SETEQ:

3737

return DAG.getConstant(0, dl, VT);

3738

case ISD::SETULT:

3739

case ISD::SETULE:

3740

case ISD::SETNE:

3741

return DAG.getConstant(1, dl, VT);

3742

case ISD::SETGT:

3743

case ISD::SETGE:

3744

// True if the sign bit of C1 is set.

3745

return DAG.getConstant(C1.isNegative(), dl, VT);

3746

case ISD::SETLT:

3747

case ISD::SETLE:

3748

// True if the sign bit of C1 isn't set.

3749

return DAG.getConstant(C1.isNonNegative(), dl, VT);

3750

default:

3751

break;

3752

}

3753

}

3754

3755

// Otherwise, we can perform the comparison with the low bits.

3756

switch (Cond) {

3757

case ISD::SETEQ:

3758

case ISD::SETNE:

3759

case ISD::SETUGT:

3760

case ISD::SETUGE:

3761

case ISD::SETULT:

3762

case ISD::SETULE: {

3763

EVT newVT = N0.getOperand(0).getValueType();

3764

if (DCI.isBeforeLegalizeOps() ||

3765

(isOperationLegal(ISD::SETCC, newVT) &&

3766

isCondCodeLegal(Cond, newVT.getSimpleVT()))) {

3767

EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);

3768

SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

3769

3770

SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),

3771

NewConst, Cond);

3772

return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());

3773

}

3774

break;

3775

}

3776

default:

3777

break; // todo, be more careful with signed comparisons

3778

}

3779

} else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&

3780

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3781

!isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),

3782

OpVT)) {

3783

EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();

3784

unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();

3785

EVT ExtDstTy = N0.getValueType();

3786

unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

3787

3788

// If the constant doesn't fit into the number of bits for the source of

3789

// the sign extension, it is impossible for both sides to be equal.

3790

if (C1.getMinSignedBits() > ExtSrcTyBits)

3791

return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

3792

3793

assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__))

3794

ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__));

3795

APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);

3796

SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),

3797

DAG.getConstant(Imm, dl, ExtDstTy));

3798

if (!DCI.isCalledByLegalizer())

3799

DCI.AddToWorklist(ZextOp.getNode());

3800

// Otherwise, make this a use of a zext.

3801

return DAG.getSetCC(dl, VT, ZextOp,

3802

DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);

3803

} else if ((N1C->isNullValue() || N1C->isOne()) &&

3804

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3805

// SETCC (SETCC), [0|1], [EQ|NE] -> SETCC

3806

if (N0.getOpcode() == ISD::SETCC &&

3807

isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&

3808

(N0.getValueType() == MVT::i1 ||

3809

getBooleanContents(N0.getOperand(0).getValueType()) ==

3810

ZeroOrOneBooleanContent)) {

3811

bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());

3812

if (TrueWhenTrue)

3813

return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);

3814

// Invert the condition.

3815

ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();

3816

CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());

3817

if (DCI.isBeforeLegalizeOps() ||

3818

isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))

3819

return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);

3820

}

3821

3822

if ((N0.getOpcode() == ISD::XOR ||

3823

(N0.getOpcode() == ISD::AND &&

3824

N0.getOperand(0).getOpcode() == ISD::XOR &&

3825

N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&

3826

isOneConstant(N0.getOperand(1))) {

3827

// If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We

3828

// can only do this if the top bits are known zero.

3829

unsigned BitWidth = N0.getValueSizeInBits();

3830

if (DAG.MaskedValueIsZero(N0,

3831

APInt::getHighBitsSet(BitWidth,

3832

BitWidth-1))) {

3833

// Okay, get the un-inverted input value.

3834

SDValue Val;

3835

if (N0.getOpcode() == ISD::XOR) {

3836

Val = N0.getOperand(0);

3837

} else {

3838

assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3839, __extension__ __PRETTY_FUNCTION__))

3839

N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3839, __extension__ __PRETTY_FUNCTION__));

3840

// ((X^1)&1)^1 -> X & 1

3841

Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),

3842

N0.getOperand(0).getOperand(0),

3843

N0.getOperand(1));

3844

}

3845

3846

return DAG.getSetCC(dl, VT, Val, N1,

3847

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3848

}

3849

} else if (N1C->isOne()) {

3850

SDValue Op0 = N0;

3851

if (Op0.getOpcode() == ISD::TRUNCATE)

3852

Op0 = Op0.getOperand(0);

3853

3854

if ((Op0.getOpcode() == ISD::XOR) &&

3855

Op0.getOperand(0).getOpcode() == ISD::SETCC &&

3856

Op0.getOperand(1).getOpcode() == ISD::SETCC) {

3857

SDValue XorLHS = Op0.getOperand(0);

3858

SDValue XorRHS = Op0.getOperand(1);

3859

// Ensure that the input setccs return an i1 type or 0/1 value.

3860

if (Op0.getValueType() == MVT::i1 ||

3861

(getBooleanContents(XorLHS.getOperand(0).getValueType()) ==

3862

ZeroOrOneBooleanContent &&

3863

getBooleanContents(XorRHS.getOperand(0).getValueType()) ==

3864

ZeroOrOneBooleanContent)) {

3865

// (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)

3866

Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;

3867

return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);

3868

}

3869

}

3870

if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {

3871

// If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.

3872

if (Op0.getValueType().bitsGT(VT))

3873

Op0 = DAG.getNode(ISD::AND, dl, VT,

3874

DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),

3875

DAG.getConstant(1, dl, VT));

3876

else if (Op0.getValueType().bitsLT(VT))

3877

Op0 = DAG.getNode(ISD::AND, dl, VT,

3878

DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),

3879

DAG.getConstant(1, dl, VT));

3880

3881

return DAG.getSetCC(dl, VT, Op0,

3882

DAG.getConstant(0, dl, Op0.getValueType()),

3883

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3884

}

3885

if (Op0.getOpcode() == ISD::AssertZext &&

3886

cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)

3887

return DAG.getSetCC(dl, VT, Op0,

3888

DAG.getConstant(0, dl, Op0.getValueType()),

3889

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3890

}

3891

}

3892

3893

// Given:

3894

// icmp eq/ne (urem %x, %y), 0

3895

// Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':

3896

// icmp eq/ne %x, 0

3897

if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&

3898

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3899

KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));

3900

KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));

3901

if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)

3902

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);

3903

}

3904

3905

if (SDValue V =

3906

optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))

3907

return V;

3908

}

3909

3910

// These simplifications apply to splat vectors as well.

3911

// TODO: Handle more splat vector cases.

3912

if (auto *N1C = isConstOrConstSplat(N1)) {

3913

const APInt &C1 = N1C->getAPIntValue();

3914

3915

APInt MinVal, MaxVal;

3916

unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();

3917

if (ISD::isSignedIntSetCC(Cond)) {

3918

MinVal = APInt::getSignedMinValue(OperandBitSize);

3919

MaxVal = APInt::getSignedMaxValue(OperandBitSize);

3920

} else {

3921

MinVal = APInt::getMinValue(OperandBitSize);

3922

MaxVal = APInt::getMaxValue(OperandBitSize);

3923

}

3924

3925

// Canonicalize GE/LE comparisons to use GT/LT comparisons.

3926

if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {

3927

// X >= MIN --> true

3928

if (C1 == MinVal)

3929

return DAG.getBoolConstant(true, dl, VT, OpVT);

3930

3931

if (!VT.isVector()) { // TODO: Support this for vectors.

3932

// X >= C0 --> X > (C0 - 1)

3933

APInt C = C1 - 1;

3934

ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;

3935

if ((DCI.isBeforeLegalizeOps() ||

3936

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3937

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3938

isLegalICmpImmediate(C.getSExtValue())))) {

3939

return DAG.getSetCC(dl, VT, N0,

3940

DAG.getConstant(C, dl, N1.getValueType()),

3941

NewCC);

3942

}

3943

}

3944

}

3945

3946

if (Cond == ISD::SETLE || Cond == ISD::SETULE) {

3947

// X <= MAX --> true

3948

if (C1 == MaxVal)

3949

return DAG.getBoolConstant(true, dl, VT, OpVT);

3950

3951

// X <= C0 --> X < (C0 + 1)

3952

if (!VT.isVector()) { // TODO: Support this for vectors.

3953

APInt C = C1 + 1;

3954

ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;

3955

if ((DCI.isBeforeLegalizeOps() ||

3956

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3957

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3958

isLegalICmpImmediate(C.getSExtValue())))) {

3959

return DAG.getSetCC(dl, VT, N0,

3960

DAG.getConstant(C, dl, N1.getValueType()),

3961

NewCC);

3962

}

3963

}

3964

}

3965

3966

if (Cond == ISD::SETLT || Cond == ISD::SETULT) {

3967

if (C1 == MinVal)

3968

return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

3969

3970

// TODO: Support this for vectors after legalize ops.

3971

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3972

// Canonicalize setlt X, Max --> setne X, Max

3973

if (C1 == MaxVal)

3974

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3975

3976

// If we have setult X, 1, turn it into seteq X, 0

3977

if (C1 == MinVal+1)

3978

return DAG.getSetCC(dl, VT, N0,

3979

DAG.getConstant(MinVal, dl, N0.getValueType()),

3980

ISD::SETEQ);

3981

}

3982

}

3983

3984

if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {

3985

if (C1 == MaxVal)

3986

return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

3987

3988

// TODO: Support this for vectors after legalize ops.

3989

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3990

// Canonicalize setgt X, Min --> setne X, Min

3991

if (C1 == MinVal)

3992

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3993

3994

// If we have setugt X, Max-1, turn it into seteq X, Max

3995

if (C1 == MaxVal-1)

3996

return DAG.getSetCC(dl, VT, N0,

3997

DAG.getConstant(MaxVal, dl, N0.getValueType()),

3998

ISD::SETEQ);

3999

}

4000

}

4001

4002

if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {

4003

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

4004

if (C1.isNullValue())

4005

if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(

4006

VT, N0, N1, Cond, DCI, dl))

4007

return CC;

4008

4009

// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).

4010

// For example, when high 32-bits of i64 X are known clear:

4011

// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0

4012

// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1

4013

bool CmpZero = N1C->getAPIntValue().isNullValue();

4014

bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();

4015

if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {

4016

// Match or(lo,shl(hi,bw/2)) pattern.

4017

auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {

4018

unsigned EltBits = V.getScalarValueSizeInBits();

4019

if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)

4020

return false;

4021

SDValue LHS = V.getOperand(0);

4022

SDValue RHS = V.getOperand(1);

4023

APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);

4024

// Unshifted element must have zero upperbits.

4025

if (RHS.getOpcode() == ISD::SHL &&

4026

isa<ConstantSDNode>(RHS.getOperand(1)) &&

4027

RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4028

DAG.MaskedValueIsZero(LHS, HiBits)) {

4029

Lo = LHS;

4030

Hi = RHS.getOperand(0);

4031

return true;

4032

}

4033

if (LHS.getOpcode() == ISD::SHL &&

4034

isa<ConstantSDNode>(LHS.getOperand(1)) &&

4035

LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4036

DAG.MaskedValueIsZero(RHS, HiBits)) {

4037

Lo = RHS;

4038

Hi = LHS.getOperand(0);

4039

return true;

4040

}

4041

return false;

4042

};

4043

4044

auto MergeConcat = [&](SDValue Lo, SDValue Hi) {

4045

unsigned EltBits = N0.getScalarValueSizeInBits();

4046

unsigned HalfBits = EltBits / 2;

4047

APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);

4048

SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);

4049

SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);

4050

SDValue NewN0 =

4051

DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);

4052

SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;

4053

return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);

4054

};

4055

4056

SDValue Lo, Hi;

4057

if (IsConcat(N0, Lo, Hi))

4058

return MergeConcat(Lo, Hi);

4059

4060

if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {

4061

SDValue Lo0, Lo1, Hi0, Hi1;

4062

if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&

4063

IsConcat(N0.getOperand(1), Lo1, Hi1)) {

4064

return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),

4065

DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));

4066

}

4067

}

4068

}

4069

}

4070

4071

// If we have "setcc X, C0", check to see if we can shrink the immediate

4072

// by changing cc.

4073

// TODO: Support this for vectors after legalize ops.

4074

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4075

// SETUGT X, SINTMAX -> SETLT X, 0

4076

// SETUGE X, SINTMIN -> SETLT X, 0

4077

if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||

4078

(Cond == ISD::SETUGE && C1.isMinSignedValue()))

4079

return DAG.getSetCC(dl, VT, N0,

4080

DAG.getConstant(0, dl, N1.getValueType()),

4081

ISD::SETLT);

4082

4083

// SETULT X, SINTMIN -> SETGT X, -1

4084

// SETULE X, SINTMAX -> SETGT X, -1

4085

if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||

4086

(Cond == ISD::SETULE && C1.isMaxSignedValue()))

4087

return DAG.getSetCC(dl, VT, N0,

4088

DAG.getAllOnesConstant(dl, N1.getValueType()),

4089

ISD::SETGT);

4090

}

4091

}

4092

4093

// Back to non-vector simplifications.

4094

// TODO: Can we do these for vector splats?

4095

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

4096

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4097

const APInt &C1 = N1C->getAPIntValue();

4098

EVT ShValTy = N0.getValueType();

4099

4100

// Fold bit comparisons when we can. This will result in an

4101

// incorrect value when boolean false is negative one, unless

4102

// the bitsize is 1 in which case the false value is the same

4103

// in practice regardless of the representation.

4104

if ((VT.getSizeInBits() == 1 ||

4105

getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&

4106

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4107

(VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&

4108

N0.getOpcode() == ISD::AND) {

4109

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4110

EVT ShiftTy =

4111

getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4112

if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3

4113

// Perform the xform if the AND RHS is a single bit.

4114

unsigned ShCt = AndRHS->getAPIntValue().logBase2();

4115

if (AndRHS->getAPIntValue().isPowerOf2() &&

4116

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4117

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4118

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4119

DAG.getConstant(ShCt, dl, ShiftTy)));

4120

}

4121

} else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {

4122

// (X & 8) == 8 --> (X & 8) >> 3

4123

// Perform the xform if C1 is a single bit.

4124

unsigned ShCt = C1.logBase2();

4125

if (C1.isPowerOf2() &&

4126

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4127

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4128

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4129

DAG.getConstant(ShCt, dl, ShiftTy)));

4130

}

4131

}

4132

}

4133

}

4134

4135

if (C1.getMinSignedBits() <= 64 &&

4136

!isLegalICmpImmediate(C1.getSExtValue())) {

4137

EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4138

// (X & -256) == 256 -> (X >> 8) == 1

4139

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4140

N0.getOpcode() == ISD::AND && N0.hasOneUse()) {

4141

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4142

const APInt &AndRHSC = AndRHS->getAPIntValue();

4143

if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {

4144

unsigned ShiftBits = AndRHSC.countTrailingZeros();

4145

if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4146

SDValue Shift =

4147

DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),

4148

DAG.getConstant(ShiftBits, dl, ShiftTy));

4149

SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);

4150

return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);

4151

}

4152

}

4153

}

4154

} else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||

4155

Cond == ISD::SETULE || Cond == ISD::SETUGT) {

4156

bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);

4157

// X < 0x100000000 -> (X >> 32) < 1

4158

// X >= 0x100000000 -> (X >> 32) >= 1

4159

// X <= 0x0ffffffff -> (X >> 32) < 1

4160

// X > 0x0ffffffff -> (X >> 32) >= 1

4161

unsigned ShiftBits;

4162

APInt NewC = C1;

4163

ISD::CondCode NewCond = Cond;

4164

if (AdjOne) {

4165

ShiftBits = C1.countTrailingOnes();

4166

NewC = NewC + 1;

4167

NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;

4168

} else {

4169

ShiftBits = C1.countTrailingZeros();

4170

}

4171

NewC.lshrInPlace(ShiftBits);

4172

if (ShiftBits && NewC.getMinSignedBits() <= 64 &&

4173

isLegalICmpImmediate(NewC.getSExtValue()) &&

4174

!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4175

SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4176

DAG.getConstant(ShiftBits, dl, ShiftTy));

4177

SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);

4178

return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);

4179

}

4180

}

4181

}

4182

}

4183

4184

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {

4185

auto *CFP = cast<ConstantFPSDNode>(N1);

4186

assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4186, __extension__ __PRETTY_FUNCTION__));

4187

4188

// Otherwise, we know the RHS is not a NaN. Simplify the node to drop the

4189

// constant if knowing that the operand is non-nan is enough. We prefer to

4190

// have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to

4191

// materialize 0.0.

4192

if (Cond == ISD::SETO || Cond == ISD::SETUO)

4193

return DAG.getSetCC(dl, VT, N0, N0, Cond);

4194

4195

// setcc (fneg x), C -> setcc swap(pred) x, -C

4196

if (N0.getOpcode() == ISD::FNEG) {

4197

ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);

4198

if (DCI.isBeforeLegalizeOps() ||

4199

isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {

4200

SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);

4201

return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);

4202

}

4203

}

4204

4205

// If the condition is not legal, see if we can find an equivalent one

4206

// which is legal.

4207

if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {

4208

// If the comparison was an awkward floating-point == or != and one of

4209

// the comparison operands is infinity or negative infinity, convert the

4210

// condition to a less-awkward <= or >=.

4211

if (CFP->getValueAPF().isInfinity()) {

4212

bool IsNegInf = CFP->getValueAPF().isNegative();

4213

ISD::CondCode NewCond = ISD::SETCC_INVALID;

4214

switch (Cond) {

4215

case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;

4216

case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;

4217

case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;

4218

case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;

4219

default: break;

4220

}

4221

if (NewCond != ISD::SETCC_INVALID &&

4222

isCondCodeLegal(NewCond, N0.getSimpleValueType()))

4223

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4224

}

4225

}

4226

}

4227

4228

if (N0 == N1) {

4229

// The sext(setcc()) => setcc() optimization relies on the appropriate

4230

// constant being emitted.

4231

assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4232, __extension__ __PRETTY_FUNCTION__))

4232

"Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4232, __extension__ __PRETTY_FUNCTION__));

4233

4234

bool EqTrue = ISD::isTrueWhenEqual(Cond);

4235

unsigned UOF = ISD::getUnorderedFlavor(Cond);

4236

if (UOF == 2) // FP operators that are undefined on NaNs.

4237

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4238

if (UOF == unsigned(EqTrue))

4239

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4240

// Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO

4241

// if it is not already.

4242

ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;

4243

if (NewCond != Cond &&

4244

(DCI.isBeforeLegalizeOps() ||

4245

isCondCodeLegal(NewCond, N0.getSimpleValueType())))

4246

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4247

}

4248

4249

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4250

N0.getValueType().isInteger()) {

4251

if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||

4252

N0.getOpcode() == ISD::XOR) {

4253

// Simplify (X+Y) == (X+Z) --> Y == Z

4254

if (N0.getOpcode() == N1.getOpcode()) {

4255

if (N0.getOperand(0) == N1.getOperand(0))

4256

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);

4257

if (N0.getOperand(1) == N1.getOperand(1))

4258

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);

4259

if (isCommutativeBinOp(N0.getOpcode())) {

4260

// If X op Y == Y op X, try other combinations.

4261

if (N0.getOperand(0) == N1.getOperand(1))

4262

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),

4263

Cond);

4264

if (N0.getOperand(1) == N1.getOperand(0))

4265

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),

4266

Cond);

4267

}

4268

}

4269

4270

// If RHS is a legal immediate value for a compare instruction, we need

4271

// to be careful about increasing register pressure needlessly.

4272

bool LegalRHSImm = false;

4273

4274

if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {

4275

if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4276

// Turn (X+C1) == C2 --> X == C2-C1

4277

if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {

4278

return DAG.getSetCC(dl, VT, N0.getOperand(0),

4279

DAG.getConstant(RHSC->getAPIntValue()-

4280

LHSR->getAPIntValue(),

4281

dl, N0.getValueType()), Cond);

4282

}

4283

4284

// Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.

4285

if (N0.getOpcode() == ISD::XOR)

4286

// If we know that all of the inverted bits are zero, don't bother

4287

// performing the inversion.

4288

if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))

4289

return

4290

DAG.getSetCC(dl, VT, N0.getOperand(0),

4291

DAG.getConstant(LHSR->getAPIntValue() ^

4292

RHSC->getAPIntValue(),

4293

dl, N0.getValueType()),

4294

Cond);

4295

}

4296

4297

// Turn (C1-X) == C2 --> X == C1-C2

4298

if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {

4299

if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {

4300

return

4301

DAG.getSetCC(dl, VT, N0.getOperand(1),

4302

DAG.getConstant(SUBC->getAPIntValue() -

4303

RHSC->getAPIntValue(),

4304

dl, N0.getValueType()),

4305

Cond);

4306

}

4307

}

4308

4309

// Could RHSC fold directly into a compare?

4310

if (RHSC->getValueType(0).getSizeInBits() <= 64)

4311

LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());

4312

}

4313

4314

// (X+Y) == X --> Y == 0 and similar folds.

4315

// Don't do this if X is an immediate that can fold into a cmp

4316

// instruction and X+Y has other uses. It could be an induction variable

4317

// chain, and the transform would increase register pressure.

4318

if (!LegalRHSImm || N0.hasOneUse())

4319

if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))

4320

return V;

4321

}

4322

4323

if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||

4324

N1.getOpcode() == ISD::XOR)

4325

if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))

4326

return V;

4327

4328

if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))

4329

return V;

4330

}

4331

4332

// Fold remainder of division by a constant.

4333

if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&

4334

N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

4335

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

4336

4337

// When division is cheap or optimizing for minimum size,

4338

// fall through to DIVREM creation by skipping this fold.

4339

if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {

4340

if (N0.getOpcode() == ISD::UREM) {

4341

if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))

4342

return Folded;

4343

} else if (N0.getOpcode() == ISD::SREM) {

4344

if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))

4345

return Folded;

4346

}

4347

}

4348

}

4349

4350

// Fold away ALL boolean setcc's.

4351

if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {

4352

SDValue Temp;

4353

switch (Cond) {

4354

default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4354);

4355

case ISD::SETEQ: // X == Y -> ~(X^Y)

4356

Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4357

N0 = DAG.getNOT(dl, Temp, OpVT);

4358

if (!DCI.isCalledByLegalizer())

4359

DCI.AddToWorklist(Temp.getNode());

4360

break;

4361

case ISD::SETNE: // X != Y --> (X^Y)

4362

N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4363

break;

4364

case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y

4365

case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y

4366

Temp = DAG.getNOT(dl, N0, OpVT);

4367

N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);

4368

if (!DCI.isCalledByLegalizer())

4369

DCI.AddToWorklist(Temp.getNode());

4370

break;

4371

case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X

4372

case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X

4373

Temp = DAG.getNOT(dl, N1, OpVT);

4374

N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);

4375

if (!DCI.isCalledByLegalizer())

4376

DCI.AddToWorklist(Temp.getNode());

4377

break;

4378

case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y

4379

case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y

4380

Temp = DAG.getNOT(dl, N0, OpVT);

4381

N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);

4382

if (!DCI.isCalledByLegalizer())

4383

DCI.AddToWorklist(Temp.getNode());

4384

break;

4385

case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X

4386

case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X

4387

Temp = DAG.getNOT(dl, N1, OpVT);

4388

N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);

4389

break;

4390

}

4391

if (VT.getScalarType() != MVT::i1) {

4392

if (!DCI.isCalledByLegalizer())

4393

DCI.AddToWorklist(N0.getNode());

4394

// FIXME: If running after legalize, we probably can't do this.

4395

ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));

4396

N0 = DAG.getNode(ExtendCode, dl, VT, N0);

4397

}

4398

return N0;

4399

}

4400

4401

// Could not fold it.

4402

return SDValue();

4403

}

4404

4405

/// Returns true (and the GlobalValue and the offset) if the node is a

4406

/// GlobalAddress + offset.

4407

bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,

4408

int64_t &Offset) const {

4409

4410

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

4411

4412

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {

4413

GA = GASD->getGlobal();

4414

Offset += GASD->getOffset();

4415

return true;

4416

}

4417

4418

if (N->getOpcode() == ISD::ADD) {

4419

SDValue N1 = N->getOperand(0);

4420

SDValue N2 = N->getOperand(1);

4421

if (isGAPlusOffset(N1.getNode(), GA, Offset)) {

4422

if (auto *V = dyn_cast<ConstantSDNode>(N2)) {

4423

Offset += V->getSExtValue();

4424

return true;

4425

}

4426

} else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {

4427

if (auto *V = dyn_cast<ConstantSDNode>(N1)) {

4428

Offset += V->getSExtValue();

4429

return true;

4430

}

4431

}

4432

}

4433

4434

return false;

4435

}

4436

4437

SDValue TargetLowering::PerformDAGCombine(SDNode *N,

4438

DAGCombinerInfo &DCI) const {

4439

// Default implementation: no optimization.

4440

return SDValue();

4441

}

4442

4443

//===----------------------------------------------------------------------===//

4444

// Inline Assembler Implementation Methods

4445

//===----------------------------------------------------------------------===//

4446

4447

TargetLowering::ConstraintType

4448

TargetLowering::getConstraintType(StringRef Constraint) const {

4449

unsigned S = Constraint.size();

4450

4451

if (S == 1) {

4452

switch (Constraint[0]) {

4453

default: break;

4454

case 'r':

4455

return C_RegisterClass;

4456

case 'm': // memory

4457

case 'o': // offsetable

4458

case 'V': // not offsetable

4459

return C_Memory;

4460

case 'n': // Simple Integer

4461

case 'E': // Floating Point Constant

4462

case 'F': // Floating Point Constant

4463

return C_Immediate;

4464

case 'i': // Simple Integer or Relocatable Constant

4465

case 's': // Relocatable Constant

4466

case 'p': // Address.

4467

case 'X': // Allow ANY value.

4468

case 'I': // Target registers.

4469

case 'J':

4470

case 'K':

4471

case 'L':

4472

case 'M':

4473

case 'N':

4474

case 'O':

4475

case 'P':

4476

case '<':

4477

case '>':

4478

return C_Other;

4479

}

4480

}

4481

4482

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {

4483

if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"

4484

return C_Memory;

4485

return C_Register;

4486

}

4487

return C_Unknown;

4488

}

4489

4490

/// Try to replace an X constraint, which matches anything, with another that

4491

/// has more specific requirements based on the type of the corresponding

4492

/// operand.

4493

const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {

4494

if (ConstraintVT.isInteger())

4495

return "r";

4496

if (ConstraintVT.isFloatingPoint())

4497

return "f"; // works for many targets

4498

return nullptr;

4499

}

4500

4501

SDValue TargetLowering::LowerAsmOutputForConstraint(

4502

SDValue &Chain, SDValue &Flag, const SDLoc &DL,

4503

const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {

4504

return SDValue();

4505

}

4506

4507

/// Lower the specified operand into the Ops vector.

4508

/// If it is invalid, don't add anything to Ops.

4509

void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,

4510

std::string &Constraint,

4511

std::vector<SDValue> &Ops,

4512

SelectionDAG &DAG) const {

4513

4514

if (Constraint.length() > 1) return;

4515

4516

char ConstraintLetter = Constraint[0];

4517

switch (ConstraintLetter) {

4518

default: break;

4519

case 'X': // Allows any operand; labels (basic block) use this.

4520

if (Op.getOpcode() == ISD::BasicBlock ||

4521

Op.getOpcode() == ISD::TargetBlockAddress) {

4522

Ops.push_back(Op);

4523

return;

4524

}

4525

LLVM_FALLTHROUGH[[gnu::fallthrough]];

4526

case 'i': // Simple Integer or Relocatable Constant

4527

case 'n': // Simple Integer

4528

case 's': { // Relocatable Constant

4529

4530

GlobalAddressSDNode *GA;

4531

ConstantSDNode *C;

4532

BlockAddressSDNode *BA;

4533

uint64_t Offset = 0;

4534

4535

// Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),

4536

// etc., since getelementpointer is variadic. We can't use

4537

// SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible

4538

// while in this case the GA may be furthest from the root node which is

4539

// likely an ISD::ADD.

4540

while (1) {

4541

if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4542

Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),

4543

GA->getValueType(0),

4544

Offset + GA->getOffset()));

4545

return;

4546

}

4547

if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {

4548

// gcc prints these as sign extended. Sign extend value to 64 bits

4549

// now; without this it would get ZExt'd later in

4550

// ScheduleDAGSDNodes::EmitNode, which is very generic.

4551

bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;

4552

BooleanContent BCont = getBooleanContents(MVT::i64);

4553

ISD::NodeType ExtOpc =

4554

IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;

4555

int64_t ExtVal =

4556

ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();

4557

Ops.push_back(

4558

DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));

4559

return;

4560

}

4561

if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4562

Ops.push_back(DAG.getTargetBlockAddress(

4563

BA->getBlockAddress(), BA->getValueType(0),

4564

Offset + BA->getOffset(), BA->getTargetFlags()));

4565

return;

4566

}

4567

const unsigned OpCode = Op.getOpcode();

4568

if (OpCode == ISD::ADD || OpCode == ISD::SUB) {

4569

if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))

4570

Op = Op.getOperand(1);

4571

// Subtraction is not commutative.

4572

else if (OpCode == ISD::ADD &&

4573

(C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))

4574

Op = Op.getOperand(0);

4575

else

4576

return;

4577

Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();

4578

continue;

4579

}

4580

return;

4581

}

4582

break;

4583

}

4584

}

4585

}

4586

4587

std::pair<unsigned, const TargetRegisterClass *>

4588

TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,

4589

StringRef Constraint,

4590

MVT VT) const {

4591

if (Constraint.empty() || Constraint[0] != '{')

4592

return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));

4593

assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4593, __extension__ __PRETTY_FUNCTION__));

4594

4595

// Remove the braces from around the name.

4596

StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

4597

4598

std::pair<unsigned, const TargetRegisterClass *> R =

4599

std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

4600

4601

// Figure out which register class contains this reg.

4602

for (const TargetRegisterClass *RC : RI->regclasses()) {

4603

// If none of the value types for this register class are valid, we

4604

// can't use it. For example, 64-bit reg classes on 32-bit targets.

4605

if (!isLegalRC(*RI, *RC))

4606

continue;

4607

4608

for (const MCPhysReg &PR : *RC) {

4609

if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {

4610

std::pair<unsigned, const TargetRegisterClass *> S =

4611

std::make_pair(PR, RC);

4612

4613

// If this register class has the requested value type, return it,

4614

// otherwise keep searching and return the first class found

4615

// if no other is found which explicitly has the requested type.

4616

if (RI->isTypeLegalForClass(*RC, VT))

4617

return S;

4618

if (!R.second)

4619

R = S;

4620

}

4621

}

4622

}

4623

4624

return R;

4625

}

4626

4627

//===----------------------------------------------------------------------===//

4628

// Constraint Selection.

4629

4630

/// Return true of this is an input operand that is a matching constraint like

4631

/// "4".

4632

bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {

4633

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4633, __extension__ __PRETTY_FUNCTION__));

4634

return isdigit(static_cast<unsigned char>(ConstraintCode[0]));

4635

}

4636

4637

/// If this is an input matching constraint, this method returns the output

4638

/// operand it matches.

4639

unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {

4640

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4640, __extension__ __PRETTY_FUNCTION__));

4641

return atoi(ConstraintCode.c_str());

4642

}

4643

4644

/// Split up the constraint string from the inline assembly value into the

4645

/// specific constraints and their prefixes, and also tie in the associated

4646

/// operand values.

4647

/// If this returns an empty vector, and if the constraint string itself

4648

/// isn't empty, there was an error parsing.

4649

TargetLowering::AsmOperandInfoVector

4650

TargetLowering::ParseConstraints(const DataLayout &DL,

4651

const TargetRegisterInfo *TRI,

4652

const CallBase &Call) const {

4653

/// Information about all of the constraints.

4654

AsmOperandInfoVector ConstraintOperands;

4655

const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());

4656

unsigned maCount = 0; // Largest number of multiple alternative constraints.

4657

4658

// Do a prepass over the constraints, canonicalizing them, and building up the

4659

// ConstraintOperands list.

4660

unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.

4661

unsigned ResNo = 0; // ResNo - The result number of the next output.

4662

4663

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {

4664

ConstraintOperands.emplace_back(std::move(CI));

4665

AsmOperandInfo &OpInfo = ConstraintOperands.back();

4666

4667

// Update multiple alternative constraint count.

4668

if (OpInfo.multipleAlternatives.size() > maCount)

4669

maCount = OpInfo.multipleAlternatives.size();

4670

4671

OpInfo.ConstraintVT = MVT::Other;

4672

4673

// Compute the value type for each operand.

4674

switch (OpInfo.Type) {

4675

case InlineAsm::isOutput:

4676

// Indirect outputs just consume an argument.

4677

if (OpInfo.isIndirect) {

4678

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4679

break;

4680

}

4681

4682

// The return value of the call is this value. As such, there is no

4683

// corresponding argument.

4684

assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4684, __extension__ __PRETTY_FUNCTION__));

4685

if (StructType *STy = dyn_cast<StructType>(Call.getType())) {

4686

OpInfo.ConstraintVT =

4687

getSimpleValueType(DL, STy->getElementType(ResNo));

4688

} else {

4689

assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4689, __extension__ __PRETTY_FUNCTION__));

4690

OpInfo.ConstraintVT =

4691

getAsmOperandValueType(DL, Call.getType()).getSimpleVT();

4692

}

4693

++ResNo;

4694

break;

4695

case InlineAsm::isInput:

4696

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4697

break;

4698

case InlineAsm::isClobber:

4699

// Nothing to do.

4700

break;

4701

}

4702

4703

if (OpInfo.CallOperandVal) {

4704

llvm::Type *OpTy = OpInfo.CallOperandVal->getType();

4705

if (OpInfo.isIndirect) {

4706

llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);

4707

if (!PtrTy)

4708

report_fatal_error("Indirect operand for inline asm not a pointer!");

4709

OpTy = PtrTy->getElementType();

4710

}

4711

4712

// Look for vector wrapped in a struct. e.g. { <16 x i8> }.

4713

if (StructType *STy = dyn_cast<StructType>(OpTy))

4714

if (STy->getNumElements() == 1)

4715

OpTy = STy->getElementType(0);

4716

4717

// If OpTy is not a single value, it may be a struct/union that we

4718

// can tile with integers.

4719

if (!OpTy->isSingleValueType() && OpTy->isSized()) {

4720

unsigned BitSize = DL.getTypeSizeInBits(OpTy);

4721

switch (BitSize) {

4722

default: break;

4723

case 1:

4724

case 8:

4725

case 16:

4726

case 32:

4727

case 64:

4728

case 128:

4729

OpInfo.ConstraintVT =

4730

MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);

4731

break;

4732

}

4733

} else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {

4734

unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());

4735

OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);

4736

} else {

4737

OpInfo.ConstraintVT = MVT::getVT(OpTy, true);

4738

}

4739

}

4740

}

4741

4742

// If we have multiple alternative constraints, select the best alternative.

4743

if (!ConstraintOperands.empty()) {

4744

if (maCount) {

4745

unsigned bestMAIndex = 0;

4746

int bestWeight = -1;

4747

// weight: -1 = invalid match, and 0 = so-so match to 5 = good match.

4748

int weight = -1;

4749

unsigned maIndex;

4750

// Compute the sums of the weights for each alternative, keeping track

4751

// of the best (highest weight) one so far.

4752

for (maIndex = 0; maIndex < maCount; ++maIndex) {

4753

int weightSum = 0;

4754

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4755

cIndex != eIndex; ++cIndex) {

4756

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4757

if (OpInfo.Type == InlineAsm::isClobber)

4758

continue;

4759

4760

// If this is an output operand with a matching input operand,

4761

// look up the matching input. If their types mismatch, e.g. one

4762

// is an integer, the other is floating point, or their sizes are

4763

// different, flag it as an maCantMatch.

4764

if (OpInfo.hasMatchingInput()) {

4765

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4766

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4767

if ((OpInfo.ConstraintVT.isInteger() !=

4768

Input.ConstraintVT.isInteger()) ||

4769

(OpInfo.ConstraintVT.getSizeInBits() !=

4770

Input.ConstraintVT.getSizeInBits())) {

4771

weightSum = -1; // Can't match.

4772

break;

4773

}

4774

}

4775

}

4776

weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);

4777

if (weight == -1) {

4778

weightSum = -1;

4779

break;

4780

}

4781

weightSum += weight;

4782

}

4783

// Update best.

4784

if (weightSum > bestWeight) {

4785

bestWeight = weightSum;

4786

bestMAIndex = maIndex;

4787

}

4788

}

4789

4790

// Now select chosen alternative in each constraint.

4791

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4792

cIndex != eIndex; ++cIndex) {

4793

AsmOperandInfo &cInfo = ConstraintOperands[cIndex];

4794

if (cInfo.Type == InlineAsm::isClobber)

4795

continue;

4796

cInfo.selectAlternative(bestMAIndex);

4797

}

4798

}

4799

}

4800

4801

// Check and hook up tied operands, choose constraint code to use.

4802

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4803

cIndex != eIndex; ++cIndex) {

4804

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4805

4806

// If this is an output operand with a matching input operand, look up the

4807

// matching input. If their types mismatch, e.g. one is an integer, the

4808

// other is floating point, or their sizes are different, flag it as an

4809

// error.

4810

if (OpInfo.hasMatchingInput()) {

4811

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4812

4813

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4814

std::pair<unsigned, const TargetRegisterClass *> MatchRC =

4815

getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,

4816

OpInfo.ConstraintVT);

4817

std::pair<unsigned, const TargetRegisterClass *> InputRC =

4818

getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,

4819

Input.ConstraintVT);

4820

if ((OpInfo.ConstraintVT.isInteger() !=

4821

Input.ConstraintVT.isInteger()) ||

4822

(MatchRC.second != InputRC.second)) {

4823

report_fatal_error("Unsupported asm: input constraint"

4824

" with a matching output constraint of"

4825

" incompatible type!");

4826

}

4827

}

4828

}

4829

}

4830

4831

return ConstraintOperands;

4832

}

4833

4834

/// Return an integer indicating how general CT is.

4835

static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {

4836

switch (CT) {

4837

case TargetLowering::C_Immediate:

4838

case TargetLowering::C_Other:

4839

case TargetLowering::C_Unknown:

4840

return 0;

4841

case TargetLowering::C_Register:

4842

return 1;

4843

case TargetLowering::C_RegisterClass:

4844

return 2;

4845

case TargetLowering::C_Memory:

4846

return 3;

4847

}

4848

llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4848);

4849

}

4850

4851

/// Examine constraint type and operand type and determine a weight value.

4852

/// This object must already have been set up with the operand type

4853

/// and the current alternative constraint selected.

4854

TargetLowering::ConstraintWeight

4855

TargetLowering::getMultipleConstraintMatchWeight(

4856

AsmOperandInfo &info, int maIndex) const {

4857

InlineAsm::ConstraintCodeVector *rCodes;

4858

if (maIndex >= (int)info.multipleAlternatives.size())

4859

rCodes = &info.Codes;

4860

else

4861

rCodes = &info.multipleAlternatives[maIndex].Codes;

4862

ConstraintWeight BestWeight = CW_Invalid;

4863

4864

// Loop over the options, keeping track of the most general one.

4865

for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {

4866

ConstraintWeight weight =

4867

getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());

4868

if (weight > BestWeight)

4869

BestWeight = weight;

4870

}

4871

4872

return BestWeight;

4873

}

4874

4875

/// Examine constraint type and operand type and determine a weight value.

4876

/// This object must already have been set up with the operand type

4877

/// and the current alternative constraint selected.

4878

TargetLowering::ConstraintWeight

4879

TargetLowering::getSingleConstraintMatchWeight(

4880

AsmOperandInfo &info, const char *constraint) const {

4881

ConstraintWeight weight = CW_Invalid;

4882

Value *CallOperandVal = info.CallOperandVal;

4883

// If we don't have a value, we can't do a match,

4884

// but allow it at the lowest weight.

4885

if (!CallOperandVal)

4886

return CW_Default;

4887

// Look at the constraint type.

4888

switch (*constraint) {

4889

case 'i': // immediate integer.

4890

case 'n': // immediate integer with a known value.

4891

if (isa<ConstantInt>(CallOperandVal))

4892

weight = CW_Constant;

4893

break;

4894

case 's': // non-explicit intregal immediate.

4895

if (isa<GlobalValue>(CallOperandVal))

4896

weight = CW_Constant;

4897

break;

4898

case 'E': // immediate float if host format.

4899

case 'F': // immediate float.

4900

if (isa<ConstantFP>(CallOperandVal))

4901

weight = CW_Constant;

4902

break;

4903

case '<': // memory operand with autodecrement.

4904

case '>': // memory operand with autoincrement.

4905

case 'm': // memory operand.

4906

case 'o': // offsettable memory operand

4907

case 'V': // non-offsettable memory operand

4908

weight = CW_Memory;

4909

break;

4910

case 'r': // general register.

4911

case 'g': // general register, memory operand or immediate integer.

4912

// note: Clang converts "g" to "imr".

4913

if (CallOperandVal->getType()->isIntegerTy())

4914

weight = CW_Register;

4915

break;

4916

case 'X': // any operand.

4917

default:

4918

weight = CW_Default;

4919

break;

4920

}

4921

return weight;

4922

}

4923

4924

/// If there are multiple different constraints that we could pick for this

4925

/// operand (e.g. "imr") try to pick the 'best' one.

4926

/// This is somewhat tricky: constraints fall into four classes:

4927

/// Other -> immediates and magic values

4928

/// Register -> one specific register

4929

/// RegisterClass -> a group of regs

4930

/// Memory -> memory

4931

/// Ideally, we would pick the most specific constraint possible: if we have

4932

/// something that fits into a register, we would pick it. The problem here

4933

/// is that if we have something that could either be in a register or in

4934

/// memory that use of the register could cause selection of *other*

4935

/// operands to fail: they might only succeed if we pick memory. Because of

4936

/// this the heuristic we use is:

4937

///

4938

/// 1) If there is an 'other' constraint, and if the operand is valid for

4939

/// that constraint, use it. This makes us take advantage of 'i'

4940

/// constraints when available.

4941

/// 2) Otherwise, pick the most general constraint present. This prefers

4942

/// 'm' over 'r', for example.

4943

///

4944

static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,

4945

const TargetLowering &TLI,

4946

SDValue Op, SelectionDAG *DAG) {

4947

assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4947, __extension__ __PRETTY_FUNCTION__));

4948

unsigned BestIdx = 0;

4949

TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;

4950

int BestGenerality = -1;

4951

4952

// Loop over the options, keeping track of the most general one.

4953

for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {

4954

TargetLowering::ConstraintType CType =

4955

TLI.getConstraintType(OpInfo.Codes[i]);

4956

4957

// Indirect 'other' or 'immediate' constraints are not allowed.

4958

if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||

4959

CType == TargetLowering::C_Register ||

4960

CType == TargetLowering::C_RegisterClass))

4961

continue;

4962

4963

// If this is an 'other' or 'immediate' constraint, see if the operand is

4964

// valid for it. For example, on X86 we might have an 'rI' constraint. If

4965

// the operand is an integer in the range [0..31] we want to use I (saving a

4966

// load of a register), otherwise we must use 'r'.

4967

if ((CType == TargetLowering::C_Other ||

4968

CType == TargetLowering::C_Immediate) && Op.getNode()) {

4969

assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))

4970

"Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__));

4971

std::vector<SDValue> ResultOps;

4972

TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],

4973

ResultOps, *DAG);

4974

if (!ResultOps.empty()) {

4975

BestType = CType;

4976

BestIdx = i;

4977

break;

4978

}

4979

}

4980

4981

// Things with matching constraints can only be registers, per gcc

4982

// documentation. This mainly affects "g" constraints.

4983

if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())

4984

continue;

4985

4986

// This constraint letter is more general than the previous one, use it.

4987

int Generality = getConstraintGenerality(CType);

4988

if (Generality > BestGenerality) {

4989

BestType = CType;

4990

BestIdx = i;

4991

BestGenerality = Generality;

4992

}

4993

}

4994

4995

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];

4996

OpInfo.ConstraintType = BestType;

4997

}

4998

4999

/// Determines the constraint code and constraint type to use for the specific

5000

/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.

5001

void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

5002

SDValue Op,

5003

SelectionDAG *DAG) const {

5004

assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5004, __extension__ __PRETTY_FUNCTION__));

5005

5006

// Single-letter constraints ('r') are very common.

5007

if (OpInfo.Codes.size() == 1) {

5008

OpInfo.ConstraintCode = OpInfo.Codes[0];

5009

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5010

} else {

5011

ChooseConstraint(OpInfo, *this, Op, DAG);

5012

}

5013

5014

// 'X' matches anything.

5015

if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {

5016

// Labels and constants are handled elsewhere ('X' is the only thing

5017

// that matches labels). For Functions, the type here is the type of

5018

// the result, which is not what we want to look at; leave them alone.

5019

Value *v = OpInfo.CallOperandVal;

5020

if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {

5021

OpInfo.CallOperandVal = v;

5022

return;

5023

}

5024

5025

if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)

5026

return;

5027

5028

// Otherwise, try to resolve it to something we know about by looking at

5029

// the actual operand type.

5030

if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {

5031

OpInfo.ConstraintCode = Repl;

5032

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5033

}

5034

}

5035

}

5036

5037

/// Given an exact SDIV by a constant, create a multiplication

5038

/// with the multiplicative inverse of the constant.

5039

static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,

5040

const SDLoc &dl, SelectionDAG &DAG,

5041

SmallVectorImpl<SDNode *> &Created) {

5042

SDValue Op0 = N->getOperand(0);

5043

SDValue Op1 = N->getOperand(1);

5044

EVT VT = N->getValueType(0);

5045

EVT SVT = VT.getScalarType();

5046

EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());

5047

EVT ShSVT = ShVT.getScalarType();

5048

5049

bool UseSRA = false;

5050

SmallVector<SDValue, 16> Shifts, Factors;

5051

5052

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5053

if (C->isNullValue())

5054

return false;

5055

APInt Divisor = C->getAPIntValue();

5056

unsigned Shift = Divisor.countTrailingZeros();

5057

if (Shift) {

5058

Divisor.ashrInPlace(Shift);

5059

UseSRA = true;

5060

}

5061

// Calculate the multiplicative inverse, using Newton's method.

5062

APInt t;

5063

APInt Factor = Divisor;

5064

while ((t = Divisor * Factor) != 1)

5065

Factor *= APInt(Divisor.getBitWidth(), 2) - t;

5066

Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));

5067

Factors.push_back(DAG.getConstant(Factor, dl, SVT));

5068

return true;

5069

};

5070

5071

// Collect all magic values from the build vector.

5072

if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))

5073

return SDValue();

5074

5075

SDValue Shift, Factor;

5076

if (Op1.getOpcode() == ISD::BUILD_VECTOR) {

5077

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5078

Factor = DAG.getBuildVector(VT, dl, Factors);

5079

} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {

5080

assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__))

5081

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__))

5082

"vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__));

5083

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5084

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5085

} else {

5086

assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5086, __extension__ __PRETTY_FUNCTION__));

5087

Shift = Shifts[0];

5088

Factor = Factors[0];

5089

}

5090

5091

SDValue Res = Op0;

5092

5093

// Shift the value upfront if it is even, so the LSB is one.

5094

if (UseSRA) {

5095

// TODO: For UDIV use SRL instead of SRA.

5096

SDNodeFlags Flags;

5097

Flags.setExact(true);

5098

Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);

5099

Created.push_back(Res.getNode());

5100

}

5101

5102

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);

5103

}

5104

5105

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,

5106

SelectionDAG &DAG,

5107

SmallVectorImpl<SDNode *> &Created) const {

5108

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

5109

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5110

if (TLI.isIntDivCheap(N->getValueType(0), Attr))

5111

return SDValue(N, 0); // Lower SDIV as SDIV

5112

return SDValue();

5113

}

5114

5115

/// Given an ISD::SDIV node expressing a divide by constant,

5116

/// return a DAG expression to select that will generate the same value by

5117

/// multiplying by a magic number.

5118

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5119

SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,

5120

bool IsAfterLegalization,

5121

SmallVectorImpl<SDNode *> &Created) const {

5122

SDLoc dl(N);

5123

EVT VT = N->getValueType(0);

5124

EVT SVT = VT.getScalarType();

5125

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5126

EVT ShSVT = ShVT.getScalarType();

5127

unsigned EltBits = VT.getScalarSizeInBits();

5128

EVT MulVT;

5129

5130

// Check to see if we can do this.

5131

// FIXME: We should be more aggressive here.

5132

if (!isTypeLegal(VT)) {

5133

// Limit this to simple scalars for now.

5134

if (VT.isVector() || !VT.isSimple())

5135

return SDValue();

5136

5137

// If this type will be promoted to a large enough type with a legal

5138

// multiply operation, we can go ahead and do this transform.

5139

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5140

return SDValue();

5141

5142

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5143

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5144

!isOperationLegal(ISD::MUL, MulVT))

5145

return SDValue();

5146

}

5147

5148

// If the sdiv has an 'exact' bit we can use a simpler lowering.

5149

if (N->getFlags().hasExact())

5150

return BuildExactSDIV(*this, N, dl, DAG, Created);

5151

5152

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

5153

5154

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5155

if (C->isNullValue())

5156

return false;

5157

5158

const APInt &Divisor = C->getAPIntValue();

5159

APInt::ms magics = Divisor.magic();

5160

int NumeratorFactor = 0;

5161

int ShiftMask = -1;

5162

5163

if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {

5164

// If d is +1/-1, we just multiply the numerator by +1/-1.

5165

NumeratorFactor = Divisor.getSExtValue();

5166

magics.m = 0;

5167

magics.s = 0;

5168

ShiftMask = 0;

5169

} else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {

5170

// If d > 0 and m < 0, add the numerator.

5171

NumeratorFactor = 1;

5172

} else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {

5173

// If d < 0 and m > 0, subtract the numerator.

5174

NumeratorFactor = -1;

5175

}

5176

5177

MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));

5178

Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));

5179

Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));

5180

ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));

5181

return true;

5182

};

5183

5184

SDValue N0 = N->getOperand(0);

5185

SDValue N1 = N->getOperand(1);

5186

5187

// Collect the shifts / magic values from each element.

5188

if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))

5189

return SDValue();

5190

5191

SDValue MagicFactor, Factor, Shift, ShiftMask;

5192

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5193

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5194

Factor = DAG.getBuildVector(VT, dl, Factors);

5195

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5196

ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);

5197

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5198

assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5199

Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5200

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5201

"vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__));

5202

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5203

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5204

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5205

ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);

5206

} else {

5207

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5207, __extension__ __PRETTY_FUNCTION__));

5208

MagicFactor = MagicFactors[0];

5209

Factor = Factors[0];

5210

Shift = Shifts[0];

5211

ShiftMask = ShiftMasks[0];

5212

}

5213

5214

// Multiply the numerator (operand 0) by the magic value.

5215

// FIXME: We should support doing a MUL in a wider type.

5216

auto GetMULHS = [&](SDValue X, SDValue Y) {

5217

// If the type isn't legal, use a wider mul of the the type calculated

5218

// earlier.

5219

if (!isTypeLegal(VT)) {

5220

X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);

5221

Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);

5222

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5223

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5224

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5225

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5226

}

5227

5228

if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))

5229

return DAG.getNode(ISD::MULHS, dl, VT, X, Y);

5230

if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {

5231

SDValue LoHi =

5232

DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5233

return SDValue(LoHi.getNode(), 1);

5234

}

5235

return SDValue();

5236

};

5237

5238

SDValue Q = GetMULHS(N0, MagicFactor);

5239

if (!Q)

5240

return SDValue();

5241

5242

Created.push_back(Q.getNode());

5243

5244

// (Optionally) Add/subtract the numerator using Factor.

5245

Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);

5246

Created.push_back(Factor.getNode());

5247

Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);

5248

Created.push_back(Q.getNode());

5249

5250

// Shift right algebraic by shift value.

5251

Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);

5252

Created.push_back(Q.getNode());

5253

5254

// Extract the sign bit, mask it and add it to the quotient.

5255

SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);

5256

SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);

5257

Created.push_back(T.getNode());

5258

T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);

5259

Created.push_back(T.getNode());

5260

return DAG.getNode(ISD::ADD, dl, VT, Q, T);

5261

}

5262

5263

/// Given an ISD::UDIV node expressing a divide by constant,

5264

/// return a DAG expression to select that will generate the same value by

5265

/// multiplying by a magic number.

5266

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5267

SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,

5268

bool IsAfterLegalization,

5269

SmallVectorImpl<SDNode *> &Created) const {

5270

SDLoc dl(N);

5271

EVT VT = N->getValueType(0);

5272

EVT SVT = VT.getScalarType();

5273

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5274

EVT ShSVT = ShVT.getScalarType();

5275

unsigned EltBits = VT.getScalarSizeInBits();

5276

EVT MulVT;

5277

5278

// Check to see if we can do this.

5279

// FIXME: We should be more aggressive here.

5280

if (!isTypeLegal(VT)) {

5281

// Limit this to simple scalars for now.

5282

if (VT.isVector() || !VT.isSimple())

5283

return SDValue();

5284

5285

// If this type will be promoted to a large enough type with a legal

5286

// multiply operation, we can go ahead and do this transform.

5287

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5288

return SDValue();

5289

5290

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5291

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5292

!isOperationLegal(ISD::MUL, MulVT))

5293

return SDValue();

5294

}

5295

5296

bool UseNPQ = false;

5297

SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

5298

5299

auto BuildUDIVPattern = [&](ConstantSDNode *C) {

5300

if (C->isNullValue())

5301

return false;

5302

// FIXME: We should use a narrower constant when the upper

5303

// bits are known to be zero.

5304

const APInt& Divisor = C->getAPIntValue();

5305

APInt::mu magics = Divisor.magicu();

5306

unsigned PreShift = 0, PostShift = 0;

5307

5308

// If the divisor is even, we can avoid using the expensive fixup by

5309

// shifting the divided value upfront.

5310

if (magics.a != 0 && !Divisor[0]) {

5311

PreShift = Divisor.countTrailingZeros();

5312

// Get magic number for the shifted divisor.

5313

magics = Divisor.lshr(PreShift).magicu(PreShift);

5314

assert(magics.a == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.a == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5314, __extension__ __PRETTY_FUNCTION__));

5315

}

5316

5317

APInt Magic = magics.m;

5318

5319

unsigned SelNPQ;

5320

if (magics.a == 0 || Divisor.isOneValue()) {

5321

assert(magics.s < Divisor.getBitWidth() &&(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5322, __extension__ __PRETTY_FUNCTION__))

5322

"We shouldn't generate an undefined shift!")(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5322, __extension__ __PRETTY_FUNCTION__));

5323

PostShift = magics.s;

5324

SelNPQ = false;

5325

} else {

5326

PostShift = magics.s - 1;

5327

SelNPQ = true;

5328

}

5329

5330

PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));

5331

MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));

5332

NPQFactors.push_back(

5333

DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)

5334

: APInt::getNullValue(EltBits),

5335

dl, SVT));

5336

PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));

5337

UseNPQ |= SelNPQ;

5338

return true;

5339

};

5340

5341

SDValue N0 = N->getOperand(0);

5342

SDValue N1 = N->getOperand(1);

5343

5344

// Collect the shifts/magic values from each element.

5345

if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))

5346

return SDValue();

5347

5348

SDValue PreShift, PostShift, MagicFactor, NPQFactor;

5349

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5350

PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);

5351

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5352

NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);

5353

PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);

5354

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5355

assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__))

5356

NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__))

5357

"Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__));

5358

PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);

5359

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5360

NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);

5361

PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);

5362

} else {

5363

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5363, __extension__ __PRETTY_FUNCTION__));

5364

PreShift = PreShifts[0];

5365

MagicFactor = MagicFactors[0];

5366

PostShift = PostShifts[0];

5367

}

5368

5369

SDValue Q = N0;

5370

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);

5371

Created.push_back(Q.getNode());

5372

5373

// FIXME: We should support doing a MUL in a wider type.

5374

auto GetMULHU = [&](SDValue X, SDValue Y) {

5375

// If the type isn't legal, use a wider mul of the the type calculated

5376

// earlier.

5377

if (!isTypeLegal(VT)) {

5378

X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);

5379

Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);

5380

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5381

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5382

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5383

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5384

}

5385

5386

if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))

5387

return DAG.getNode(ISD::MULHU, dl, VT, X, Y);

5388

if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {

5389

SDValue LoHi =

5390

DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5391

return SDValue(LoHi.getNode(), 1);

5392

}

5393

return SDValue(); // No mulhu or equivalent

5394

};

5395

5396

// Multiply the numerator (operand 0) by the magic value.

5397

Q = GetMULHU(Q, MagicFactor);

5398

if (!Q)

5399

return SDValue();

5400

5401

Created.push_back(Q.getNode());

5402

5403

if (UseNPQ) {

5404

SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);

5405

Created.push_back(NPQ.getNode());

5406

5407

// For vectors we might have a mix of non-NPQ/NPQ paths, so use

5408

// MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.

5409

if (VT.isVector())

5410

NPQ = GetMULHU(NPQ, NPQFactor);

5411

else

5412

NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

5413

5414

Created.push_back(NPQ.getNode());

5415

5416

Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);

5417

Created.push_back(Q.getNode());

5418

}

5419

5420

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);

5421

Created.push_back(Q.getNode());

5422

5423

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5424

5425

SDValue One = DAG.getConstant(1, dl, VT);

5426

SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);

5427

return DAG.getSelect(dl, VT, IsOne, N0, Q);

5428

}

5429

5430

/// If all values in Values that *don't* match the predicate are same 'splat'

5431

/// value, then replace all values with that splat value.

5432

/// Else, if AlternativeReplacement was provided, then replace all values that

5433

/// do match predicate with AlternativeReplacement value.

5434

static void

5435

turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,

5436

std::function<bool(SDValue)> Predicate,

5437

SDValue AlternativeReplacement = SDValue()) {

5438

SDValue Replacement;

5439

// Is there a value for which the Predicate does *NOT* match? What is it?

5440

auto SplatValue = llvm::find_if_not(Values, Predicate);

5441

if (SplatValue != Values.end()) {

5442

// Does Values consist only of SplatValue's and values matching Predicate?

5443

if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {

5444

return Value == *SplatValue || Predicate(Value);

5445

})) // Then we shall replace values matching predicate with SplatValue.

5446

Replacement = *SplatValue;

5447

}

5448

if (!Replacement) {

5449

// Oops, we did not find the "baseline" splat value.

5450

if (!AlternativeReplacement)

5451

return; // Nothing to do.

5452

// Let's replace with provided value then.

5453

Replacement = AlternativeReplacement;

5454

}

5455

std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);

5456

}

5457

5458

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE

5459

/// where the divisor is constant and the comparison target is zero,

5460

/// return a DAG expression that will generate the same comparison result

5461

/// using only multiplications, additions and shifts/rotations.

5462

/// Ref: "Hacker's Delight" 10-17.

5463

SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,

5464

SDValue CompTargetNode,

5465

ISD::CondCode Cond,

5466

DAGCombinerInfo &DCI,

5467

const SDLoc &DL) const {

5468

SmallVector<SDNode *, 5> Built;

5469

if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5470

DCI, DL, Built)) {

5471

for (SDNode *N : Built)

5472

DCI.AddToWorklist(N);

5473

return Folded;

5474

}

5475

5476

return SDValue();

5477

}

5478

5479

SDValue

5480

TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,

5481

SDValue CompTargetNode, ISD::CondCode Cond,

5482

DAGCombinerInfo &DCI, const SDLoc &DL,

5483

SmallVectorImpl<SDNode *> &Created) const {

5484

// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)

5485

// - D must be constant, with D = D0 * 2^K where D0 is odd

5486

// - P is the multiplicative inverse of D0 modulo 2^W

5487

// - Q = floor(((2^W) - 1) / D)

5488

// where W is the width of the common type of N and D.

5489

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5490, __extension__ __PRETTY_FUNCTION__))

5490

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5490, __extension__ __PRETTY_FUNCTION__));

5491

5492

SelectionDAG &DAG = DCI.DAG;

5493

5494

EVT VT = REMNode.getValueType();

5495

EVT SVT = VT.getScalarType();

5496

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5497

EVT ShSVT = ShVT.getScalarType();

5498

5499

// If MUL is unavailable, we cannot proceed in any case.

5500

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5501

return SDValue();

5502

5503

bool ComparingWithAllZeros = true;

5504

bool AllComparisonsWithNonZerosAreTautological = true;

5505

bool HadTautologicalLanes = false;

5506

bool AllLanesAreTautological = true;

5507

bool HadEvenDivisor = false;

5508

bool AllDivisorsArePowerOfTwo = true;

5509

bool HadTautologicalInvertedLanes = false;

5510

SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

5511

5512

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {

5513

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5514

if (CDiv->isNullValue())

5515

return false;

5516

5517

const APInt &D = CDiv->getAPIntValue();

5518

const APInt &Cmp = CCmp->getAPIntValue();

5519

5520

ComparingWithAllZeros &= Cmp.isNullValue();

5521

5522

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5523

// if C2 is not less than C1, the comparison is always false.

5524

// But we will only be able to produce the comparison that will give the

5525

// opposive tautological answer. So this lane would need to be fixed up.

5526

bool TautologicalInvertedLane = D.ule(Cmp);

5527

HadTautologicalInvertedLanes |= TautologicalInvertedLane;

5528

5529

// If all lanes are tautological (either all divisors are ones, or divisor

5530

// is not greater than the constant we are comparing with),

5531

// we will prefer to avoid the fold.

5532

bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;

5533

HadTautologicalLanes |= TautologicalLane;

5534

AllLanesAreTautological &= TautologicalLane;

5535

5536

// If we are comparing with non-zero, we need'll need to subtract said

5537

// comparison value from the LHS. But there is no point in doing that if

5538

// every lane where we are comparing with non-zero is tautological..

5539

if (!Cmp.isNullValue())

5540

AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

5541

5542

// Decompose D into D0 * 2^K

5543

unsigned K = D.countTrailingZeros();

5544

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5544, __extension__ __PRETTY_FUNCTION__));

5545

APInt D0 = D.lshr(K);

5546

5547

// D is even if it has trailing zeros.

5548

HadEvenDivisor |= (K != 0);

5549

// D is a power-of-two if D0 is one.

5550

// If all divisors are power-of-two, we will prefer to avoid the fold.

5551

AllDivisorsArePowerOfTwo &= D0.isOneValue();

5552

5553

// P = inv(D0, 2^W)

5554

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5555

unsigned W = D.getBitWidth();

5556

APInt P = D0.zext(W + 1)

5557

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5558

.trunc(W);

5559

assert(!P.isNullValue() && "No multiplicative inverse!")(static_cast <bool> (!P.isNullValue() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5559, __extension__ __PRETTY_FUNCTION__)); // unreachable

5560

assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5560, __extension__ __PRETTY_FUNCTION__));

5561

5562

// Q = floor((2^W - 1) u/ D)

5563

// R = ((2^W - 1) u% D)

5564

APInt Q, R;

5565

APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);

5566

5567

// If we are comparing with zero, then that comparison constant is okay,

5568

// else it may need to be one less than that.

5569

if (Cmp.ugt(R))

5570

Q -= 1;

5571

5572

assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5573, __extension__ __PRETTY_FUNCTION__))

5573

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5573, __extension__ __PRETTY_FUNCTION__));

5574

5575

// If the lane is tautological the result can be constant-folded.

5576

if (TautologicalLane) {

5577

// Set P and K amount to a bogus values so we can try to splat them.

5578

P = 0;

5579

K = -1;

5580

// And ensure that comparison constant is tautological,

5581

// it will always compare true/false.

5582

Q = -1;

5583

}

5584

5585

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5586

KAmts.push_back(

5587

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5588

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5589

return true;

5590

};

5591

5592

SDValue N = REMNode.getOperand(0);

5593

SDValue D = REMNode.getOperand(1);

5594

5595

// Collect the values from each element.

5596

if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))

5597

return SDValue();

5598

5599

// If all lanes are tautological, the result can be constant-folded.

5600

if (AllLanesAreTautological)

5601

return SDValue();

5602

5603

// If this is a urem by a powers-of-two, avoid the fold since it can be

5604

// best implemented as a bit test.

5605

if (AllDivisorsArePowerOfTwo)

5606

return SDValue();

5607

5608

SDValue PVal, KVal, QVal;

5609

if (D.getOpcode() == ISD::BUILD_VECTOR) {

5610

if (HadTautologicalLanes) {

5611

// Try to turn PAmts into a splat, since we don't care about the values

5612

// that are currently '0'. If we can't, just keep '0'`s.

5613

turnVectorIntoSplatVector(PAmts, isNullConstant);

5614

// Try to turn KAmts into a splat, since we don't care about the values

5615

// that are currently '-1'. If we can't, change them to '0'`s.

5616

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5617

DAG.getConstant(0, DL, ShSVT));

5618

}

5619

5620

PVal = DAG.getBuildVector(VT, DL, PAmts);

5621

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5622

QVal = DAG.getBuildVector(VT, DL, QAmts);

5623

} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {

5624

assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__))

5625

"Expected matchBinaryPredicate to return one element for "(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__))

5626

"SPLAT_VECTORs")(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__));

5627

PVal = DAG.getSplatVector(VT, DL, PAmts[0]);

5628

KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);

5629

QVal = DAG.getSplatVector(VT, DL, QAmts[0]);

5630

} else {

5631

PVal = PAmts[0];

5632

KVal = KAmts[0];

5633

QVal = QAmts[0];

5634

}

5635

5636

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {

5637

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))

5638

return

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 7980, column 5 Value stored to 'IsScaledIndex' is never read

Bug Summary

Annotated Source Code