/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1

//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//

2

//

3

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4

// See https://llvm.org/LICENSE.txt for license information.

5

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6

//

7

//===----------------------------------------------------------------------===//

8

//

9

// This implements the TargetLowering class.

10

//

11

//===----------------------------------------------------------------------===//

12

13

#include "llvm/CodeGen/TargetLowering.h"

14

#include "llvm/ADT/STLExtras.h"

15

#include "llvm/CodeGen/CallingConvLower.h"

16

#include "llvm/CodeGen/MachineFrameInfo.h"

17

#include "llvm/CodeGen/MachineFunction.h"

18

#include "llvm/CodeGen/MachineJumpTableInfo.h"

19

#include "llvm/CodeGen/MachineRegisterInfo.h"

20

#include "llvm/CodeGen/SelectionDAG.h"

21

#include "llvm/CodeGen/TargetRegisterInfo.h"

22

#include "llvm/CodeGen/TargetSubtargetInfo.h"

23

#include "llvm/IR/DataLayout.h"

24

#include "llvm/IR/DerivedTypes.h"

25

#include "llvm/IR/GlobalVariable.h"

26

#include "llvm/IR/LLVMContext.h"

27

#include "llvm/MC/MCAsmInfo.h"

28

#include "llvm/MC/MCExpr.h"

29

#include "llvm/Support/ErrorHandling.h"

30

#include "llvm/Support/KnownBits.h"

31

#include "llvm/Support/MathExtras.h"

32

#include "llvm/Target/TargetLoweringObjectFile.h"

33

#include "llvm/Target/TargetMachine.h"

34

#include <cctype>

35

using namespace llvm;

36

37

/// NOTE: The TargetMachine owns TLOF.

38

TargetLowering::TargetLowering(const TargetMachine &tm)

39

: TargetLoweringBase(tm) {}

40

41

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {

42

return nullptr;

43

}

44

45

bool TargetLowering::isPositionIndependent() const {

46

return getTargetMachine().isPositionIndependent();

47

}

48

49

/// Check whether a given call node is in tail position within its function. If

50

/// so, it sets Chain to the input chain of the tail call.

51

bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,

52

SDValue &Chain) const {

53

const Function &F = DAG.getMachineFunction().getFunction();

54

55

// First, check if tail calls have been disabled in this function.

56

if (F.getFnAttribute("disable-tail-calls").getValueAsBool())

57

return false;

58

59

// Conservatively require the attributes of the call to match those of

60

// the return. Ignore following attributes because they don't affect the

61

// call sequence.

62

AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);

63

for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,

64

Attribute::DereferenceableOrNull, Attribute::NoAlias,

65

Attribute::NonNull})

66

CallerAttrs.removeAttribute(Attr);

67

68

if (CallerAttrs.hasAttributes())

69

return false;

70

71

// It's not safe to eliminate the sign / zero extension of the return value.

72

if (CallerAttrs.contains(Attribute::ZExt) ||

73

CallerAttrs.contains(Attribute::SExt))

74

return false;

75

76

// Check if the only use is a function return node.

77

return isUsedByReturnOnly(Node, Chain);

78

}

79

80

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,

81

const uint32_t *CallerPreservedMask,

82

const SmallVectorImpl<CCValAssign> &ArgLocs,

83

const SmallVectorImpl<SDValue> &OutVals) const {

84

for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {

85

const CCValAssign &ArgLoc = ArgLocs[I];

86

if (!ArgLoc.isRegLoc())

87

continue;

88

MCRegister Reg = ArgLoc.getLocReg();

89

// Only look at callee saved registers.

90

if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))

91

continue;

92

// Check that we pass the value used for the caller.

93

// (We look for a CopyFromReg reading a virtual register that is used

94

// for the function live-in value of register Reg)

95

SDValue Value = OutVals[I];

96

if (Value->getOpcode() != ISD::CopyFromReg)

97

return false;

98

Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();

99

if (MRI.getLiveInPhysReg(ArgReg) != Reg)

100

return false;

101

}

102

return true;

103

}

104

105

/// Set CallLoweringInfo attribute flags based on a call instruction

106

/// and called function attributes.

107

void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,

108

unsigned ArgIdx) {

109

IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);

110

IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);

111

IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);

112

IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);

113

IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);

114

IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);

115

IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);

116

IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);

117

IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);

118

IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);

119

IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);

120

IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);

121

Alignment = Call->getParamStackAlign(ArgIdx);

122

IndirectType = nullptr;

123

assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 124, __extension__ __PRETTY_FUNCTION__))

124

"multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 124, __extension__ __PRETTY_FUNCTION__));

125

if (IsByVal) {

126

IndirectType = Call->getParamByValType(ArgIdx);

127

if (!Alignment)

128

Alignment = Call->getParamAlign(ArgIdx);

129

}

130

if (IsPreallocated)

131

IndirectType = Call->getParamPreallocatedType(ArgIdx);

132

if (IsInAlloca)

133

IndirectType = Call->getParamInAllocaType(ArgIdx);

134

}

135

136

/// Generate a libcall taking the given operands as arguments and returning a

137

/// result of type RetVT.

138

std::pair<SDValue, SDValue>

139

TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,

140

ArrayRef<SDValue> Ops,

141

MakeLibCallOptions CallOptions,

142

const SDLoc &dl,

143

SDValue InChain) const {

144

if (!InChain)

145

InChain = DAG.getEntryNode();

146

147

TargetLowering::ArgListTy Args;

148

Args.reserve(Ops.size());

149

150

TargetLowering::ArgListEntry Entry;

151

for (unsigned i = 0; i < Ops.size(); ++i) {

152

SDValue NewOp = Ops[i];

153

Entry.Node = NewOp;

154

Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());

155

Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),

156

CallOptions.IsSExt);

157

Entry.IsZExt = !Entry.IsSExt;

158

159

if (CallOptions.IsSoften &&

160

!shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {

161

Entry.IsSExt = Entry.IsZExt = false;

162

}

163

Args.push_back(Entry);

164

}

165

166

if (LC == RTLIB::UNKNOWN_LIBCALL)

167

report_fatal_error("Unsupported library call operation!");

168

SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),

169

getPointerTy(DAG.getDataLayout()));

170

171

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());

172

TargetLowering::CallLoweringInfo CLI(DAG);

173

bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);

174

bool zeroExtend = !signExtend;

175

176

if (CallOptions.IsSoften &&

177

!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {

178

signExtend = zeroExtend = false;

179

}

180

181

CLI.setDebugLoc(dl)

182

.setChain(InChain)

183

.setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))

184

.setNoReturn(CallOptions.DoesNotReturn)

185

.setDiscardResult(!CallOptions.IsReturnValueUsed)

186

.setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)

187

.setSExtResult(signExtend)

188

.setZExtResult(zeroExtend);

189

return LowerCallTo(CLI);

190

}

191

192

bool TargetLowering::findOptimalMemOpLowering(

193

std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,

194

unsigned SrcAS, const AttributeList &FuncAttributes) const {

195

if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())

196

return false;

197

198

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

199

200

if (VT == MVT::Other) {

201

// Use the largest integer type whose alignment constraints are satisfied.

202

// We only need to check DstAlign here as SrcAlign is always greater or

203

// equal to DstAlign (or zero).

204

VT = MVT::i64;

205

if (Op.isFixedDstAlign())

206

while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&

207

!allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))

208

VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);

209

assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 209, __extension__ __PRETTY_FUNCTION__));

210

211

// Find the largest legal integer type.

212

MVT LVT = MVT::i64;

213

while (!isTypeLegal(LVT))

214

LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);

215

assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 215, __extension__ __PRETTY_FUNCTION__));

216

217

// If the type we've chosen is larger than the largest legal integer type

218

// then use that instead.

219

if (VT.bitsGT(LVT))

220

VT = LVT;

221

}

222

223

unsigned NumMemOps = 0;

224

uint64_t Size = Op.size();

225

while (Size) {

226

unsigned VTSize = VT.getSizeInBits() / 8;

227

while (VTSize > Size) {

228

// For now, only use non-vector load / store's for the left-over pieces.

229

EVT NewVT = VT;

230

unsigned NewVTSize;

231

232

bool Found = false;

233

if (VT.isVector() || VT.isFloatingPoint()) {

234

NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;

235

if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&

236

isSafeMemOpType(NewVT.getSimpleVT()))

237

Found = true;

238

else if (NewVT == MVT::i64 &&

239

isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&

240

isSafeMemOpType(MVT::f64)) {

241

// i64 is usually not legal on 32-bit targets, but f64 may be.

242

NewVT = MVT::f64;

243

Found = true;

244

}

245

}

246

247

if (!Found) {

248

do {

249

NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);

250

if (NewVT == MVT::i8)

251

break;

252

} while (!isSafeMemOpType(NewVT.getSimpleVT()));

253

}

254

NewVTSize = NewVT.getSizeInBits() / 8;

255

256

// If the new VT cannot cover all of the remaining bits, then consider

257

// issuing a (or a pair of) unaligned and overlapping load / store.

258

bool Fast;

259

if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&

260

allowsMisalignedMemoryAccesses(

261

VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),

262

MachineMemOperand::MONone, &Fast) &&

263

Fast)

264

VTSize = Size;

265

else {

266

VT = NewVT;

267

VTSize = NewVTSize;

268

}

269

}

270

271

if (++NumMemOps > Limit)

272

return false;

273

274

MemOps.push_back(VT);

275

Size -= VTSize;

276

}

277

278

return true;

279

}

280

281

/// Soften the operands of a comparison. This code is shared among BR_CC,

282

/// SELECT_CC, and SETCC handlers.

283

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

284

SDValue &NewLHS, SDValue &NewRHS,

285

ISD::CondCode &CCCode,

286

const SDLoc &dl, const SDValue OldLHS,

287

const SDValue OldRHS) const {

288

SDValue Chain;

289

return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,

290

OldRHS, Chain);

291

}

292

293

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,

294

SDValue &NewLHS, SDValue &NewRHS,

295

ISD::CondCode &CCCode,

296

const SDLoc &dl, const SDValue OldLHS,

297

const SDValue OldRHS,

298

SDValue &Chain,

299

bool IsSignaling) const {

300

// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc

301

// not supporting it. We can update this code when libgcc provides such

302

// functions.

303

304

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __extension__ __PRETTY_FUNCTION__))

305

&& "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __extension__ __PRETTY_FUNCTION__));

306

307

// Expand into one or more soft-fp libcall(s).

308

RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;

309

bool ShouldInvertCC = false;

310

switch (CCCode) {

311

case ISD::SETEQ:

312

case ISD::SETOEQ:

313

LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

314

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

315

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

316

break;

317

case ISD::SETNE:

318

case ISD::SETUNE:

319

LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :

320

(VT == MVT::f64) ? RTLIB::UNE_F64 :

321

(VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;

322

break;

323

case ISD::SETGE:

324

case ISD::SETOGE:

325

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

326

(VT == MVT::f64) ? RTLIB::OGE_F64 :

327

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

328

break;

329

case ISD::SETLT:

330

case ISD::SETOLT:

331

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

332

(VT == MVT::f64) ? RTLIB::OLT_F64 :

333

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

334

break;

335

case ISD::SETLE:

336

case ISD::SETOLE:

337

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

338

(VT == MVT::f64) ? RTLIB::OLE_F64 :

339

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

340

break;

341

case ISD::SETGT:

342

case ISD::SETOGT:

343

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

344

(VT == MVT::f64) ? RTLIB::OGT_F64 :

345

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

346

break;

347

case ISD::SETO:

348

ShouldInvertCC = true;

349

LLVM_FALLTHROUGH[[gnu::fallthrough]];

350

case ISD::SETUO:

351

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

352

(VT == MVT::f64) ? RTLIB::UO_F64 :

353

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

354

break;

355

case ISD::SETONE:

356

// SETONE = O && UNE

357

ShouldInvertCC = true;

358

LLVM_FALLTHROUGH[[gnu::fallthrough]];

359

case ISD::SETUEQ:

360

LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :

361

(VT == MVT::f64) ? RTLIB::UO_F64 :

362

(VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;

363

LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :

364

(VT == MVT::f64) ? RTLIB::OEQ_F64 :

365

(VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;

366

break;

367

default:

368

// Invert CC for unordered comparisons

369

ShouldInvertCC = true;

370

switch (CCCode) {

371

case ISD::SETULT:

372

LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :

373

(VT == MVT::f64) ? RTLIB::OGE_F64 :

374

(VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;

375

break;

376

case ISD::SETULE:

377

LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :

378

(VT == MVT::f64) ? RTLIB::OGT_F64 :

379

(VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;

380

break;

381

case ISD::SETUGT:

382

LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :

383

(VT == MVT::f64) ? RTLIB::OLE_F64 :

384

(VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;

385

break;

386

case ISD::SETUGE:

387

LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :

388

(VT == MVT::f64) ? RTLIB::OLT_F64 :

389

(VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;

390

break;

391

default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 391);

392

}

393

}

394

395

// Use the target specific return value for comparions lib calls.

396

EVT RetVT = getCmpLibcallReturnType();

397

SDValue Ops[2] = {NewLHS, NewRHS};

398

TargetLowering::MakeLibCallOptions CallOptions;

399

EVT OpsVT[2] = { OldLHS.getValueType(),

400

OldRHS.getValueType() };

401

CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);

402

auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);

403

NewLHS = Call.first;

404

NewRHS = DAG.getConstant(0, dl, RetVT);

405

406

CCCode = getCmpLibcallCC(LC1);

407

if (ShouldInvertCC) {

408

assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 408, __extension__ __PRETTY_FUNCTION__));

409

CCCode = getSetCCInverse(CCCode, RetVT);

410

}

411

412

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {

413

// Update Chain.

414

Chain = Call.second;

415

} else {

416

EVT SetCCVT =

417

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);

418

SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);

419

auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);

420

CCCode = getCmpLibcallCC(LC2);

421

if (ShouldInvertCC)

422

CCCode = getSetCCInverse(CCCode, RetVT);

423

NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);

424

if (Chain)

425

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,

426

Call2.second);

427

NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,

428

Tmp.getValueType(), Tmp, NewLHS);

429

NewRHS = SDValue();

430

}

431

}

432

433

/// Return the entry encoding for a jump table in the current function. The

434

/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.

435

unsigned TargetLowering::getJumpTableEncoding() const {

436

// In non-pic modes, just use the address of a block.

437

if (!isPositionIndependent())

438

return MachineJumpTableInfo::EK_BlockAddress;

439

440

// In PIC mode, if the target supports a GPRel32 directive, use it.

441

if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)

442

return MachineJumpTableInfo::EK_GPRel32BlockAddress;

443

444

// Otherwise, use a label difference.

445

return MachineJumpTableInfo::EK_LabelDifference32;

446

}

447

448

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,

449

SelectionDAG &DAG) const {

450

// If our PIC model is GP relative, use the global offset table as the base.

451

unsigned JTEncoding = getJumpTableEncoding();

452

453

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||

454

(JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))

455

return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

456

457

return Table;

458

}

459

460

/// This returns the relocation base for the given PIC jumptable, the same as

461

/// getPICJumpTableRelocBase, but as an MCExpr.

462

const MCExpr *

463

TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,

464

unsigned JTI,MCContext &Ctx) const{

465

// The normal PIC reloc base is the label at the start of the jump table.

466

return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);

467

}

468

469

bool

470

TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {

471

const TargetMachine &TM = getTargetMachine();

472

const GlobalValue *GV = GA->getGlobal();

473

474

// If the address is not even local to this DSO we will have to load it from

475

// a got and then add the offset.

476

if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))

477

return false;

478

479

// If the code is position independent we will have to add a base register.

480

if (isPositionIndependent())

481

return false;

482

483

// Otherwise we can do it.

484

return true;

485

}

486

487

//===----------------------------------------------------------------------===//

488

// Optimization Methods

489

//===----------------------------------------------------------------------===//

490

491

/// If the specified instruction has a constant integer operand and there are

492

/// bits set in that constant that are not demanded, then clear those bits and

493

/// return true.

494

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

495

const APInt &DemandedBits,

496

const APInt &DemandedElts,

497

TargetLoweringOpt &TLO) const {

498

SDLoc DL(Op);

499

unsigned Opcode = Op.getOpcode();

500

501

// Do target-specific constant optimization.

502

if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

503

return TLO.New.getNode();

504

505

// FIXME: ISD::SELECT, ISD::SELECT_CC

506

switch (Opcode) {

507

default:

508

break;

509

case ISD::XOR:

510

case ISD::AND:

511

case ISD::OR: {

512

auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));

513

if (!Op1C || Op1C->isOpaque())

514

return false;

515

516

// If this is a 'not' op, don't touch it because that's a canonical form.

517

const APInt &C = Op1C->getAPIntValue();

518

if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))

519

return false;

520

521

if (!C.isSubsetOf(DemandedBits)) {

522

EVT VT = Op.getValueType();

523

SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);

524

SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);

525

return TLO.CombineTo(Op, NewOp);

526

}

527

528

break;

529

}

530

}

531

532

return false;

533

}

534

535

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,

536

const APInt &DemandedBits,

537

TargetLoweringOpt &TLO) const {

538

EVT VT = Op.getValueType();

539

APInt DemandedElts = VT.isVector()

540

? APInt::getAllOnesValue(VT.getVectorNumElements())

541

: APInt(1, 1);

542

return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);

543

}

544

545

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.

546

/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be

547

/// generalized for targets with other types of implicit widening casts.

548

bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,

549

const APInt &Demanded,

550

TargetLoweringOpt &TLO) const {

551

assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))

552

"ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 552, __extension__ __PRETTY_FUNCTION__));

553

assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 554, __extension__ __PRETTY_FUNCTION__))

554

"ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 554, __extension__ __PRETTY_FUNCTION__));

555

556

SelectionDAG &DAG = TLO.DAG;

557

SDLoc dl(Op);

558

559

// Early return, as this function cannot handle vector types.

560

if (Op.getValueType().isVector())

561

return false;

562

563

// Don't do this if the node has another user, which may require the

564

// full value.

565

if (!Op.getNode()->hasOneUse())

566

return false;

567

568

// Search for the smallest integer type with free casts to and from

569

// Op's type. For expedience, just check power-of-2 integer types.

570

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

571

unsigned DemandedSize = Demanded.getActiveBits();

572

unsigned SmallVTBits = DemandedSize;

573

if (!isPowerOf2_32(SmallVTBits))

574

SmallVTBits = NextPowerOf2(SmallVTBits);

575

for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {

576

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);

577

if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&

578

TLI.isZExtFree(SmallVT, Op.getValueType())) {

579

// We found a type with free casts.

580

SDValue X = DAG.getNode(

581

Op.getOpcode(), dl, SmallVT,

582

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),

583

DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));

584

assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 584, __extension__ __PRETTY_FUNCTION__));

585

SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);

586

return TLO.CombineTo(Op, Z);

587

}

588

}

589

return false;

590

}

591

592

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

593

DAGCombinerInfo &DCI) const {

594

SelectionDAG &DAG = DCI.DAG;

595

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

596

!DCI.isBeforeLegalizeOps());

597

KnownBits Known;

598

599

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);

600

if (Simplified) {

601

DCI.AddToWorklist(Op.getNode());

602

DCI.CommitTargetLoweringOpt(TLO);

603

}

604

return Simplified;

605

}

606

607

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,

608

KnownBits &Known,

609

TargetLoweringOpt &TLO,

610

unsigned Depth,

611

bool AssumeSingleUse) const {

612

EVT VT = Op.getValueType();

613

614

// TODO: We can probably do more work on calculating the known bits and

615

// simplifying the operations for scalable vectors, but for now we just

616

// bail out.

617

if (VT.isScalableVector()) {

618

// Pretend we don't know anything for now.

619

Known = KnownBits(DemandedBits.getBitWidth());

620

return false;

621

}

622

623

APInt DemandedElts = VT.isVector()

624

? APInt::getAllOnesValue(VT.getVectorNumElements())

625

: APInt(1, 1);

626

return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,

627

AssumeSingleUse);

628

}

629

630

// TODO: Can we merge SelectionDAG::GetDemandedBits into this?

631

// TODO: Under what circumstances can we create nodes? Constant folding?

632

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

633

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

634

SelectionDAG &DAG, unsigned Depth) const {

635

// Limit search depth.

636

if (Depth >= SelectionDAG::MaxRecursionDepth)

637

return SDValue();

638

639

// Ignore UNDEFs.

640

if (Op.isUndef())

641

return SDValue();

642

643

// Not demanding any bits/elts from Op.

644

if (DemandedBits == 0 || DemandedElts == 0)

645

return DAG.getUNDEF(Op.getValueType());

646

647

unsigned NumElts = DemandedElts.getBitWidth();

648

unsigned BitWidth = DemandedBits.getBitWidth();

649

KnownBits LHSKnown, RHSKnown;

650

switch (Op.getOpcode()) {

651

case ISD::BITCAST: {

652

SDValue Src = peekThroughBitcasts(Op.getOperand(0));

653

EVT SrcVT = Src.getValueType();

654

EVT DstVT = Op.getValueType();

655

if (SrcVT == DstVT)

656

return Src;

657

658

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

659

unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

660

if (NumSrcEltBits == NumDstEltBits)

661

if (SDValue V = SimplifyMultipleUseDemandedBits(

662

Src, DemandedBits, DemandedElts, DAG, Depth + 1))

663

return DAG.getBitcast(DstVT, V);

664

665

// TODO - bigendian once we have test coverage.

666

if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&

667

DAG.getDataLayout().isLittleEndian()) {

668

unsigned Scale = NumDstEltBits / NumSrcEltBits;

669

unsigned NumSrcElts = SrcVT.getVectorNumElements();

670

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

671

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

672

for (unsigned i = 0; i != Scale; ++i) {

673

unsigned Offset = i * NumSrcEltBits;

674

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

675

if (!Sub.isNullValue()) {

676

DemandedSrcBits |= Sub;

677

for (unsigned j = 0; j != NumElts; ++j)

678

if (DemandedElts[j])

679

DemandedSrcElts.setBit((j * Scale) + i);

680

}

681

}

682

683

if (SDValue V = SimplifyMultipleUseDemandedBits(

684

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

685

return DAG.getBitcast(DstVT, V);

686

}

687

688

// TODO - bigendian once we have test coverage.

689

if ((NumSrcEltBits % NumDstEltBits) == 0 &&

690

DAG.getDataLayout().isLittleEndian()) {

691

unsigned Scale = NumSrcEltBits / NumDstEltBits;

692

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

693

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

694

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

695

for (unsigned i = 0; i != NumElts; ++i)

696

if (DemandedElts[i]) {

697

unsigned Offset = (i % Scale) * NumDstEltBits;

698

DemandedSrcBits.insertBits(DemandedBits, Offset);

699

DemandedSrcElts.setBit(i / Scale);

700

}

701

702

if (SDValue V = SimplifyMultipleUseDemandedBits(

703

Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))

704

return DAG.getBitcast(DstVT, V);

705

}

706

707

break;

708

}

709

case ISD::AND: {

710

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

711

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

712

713

// If all of the demanded bits are known 1 on one side, return the other.

714

// These bits cannot contribute to the result of the 'and' in this

715

// context.

716

if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))

717

return Op.getOperand(0);

718

if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))

719

return Op.getOperand(1);

720

break;

721

}

722

case ISD::OR: {

723

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

724

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

725

726

// If all of the demanded bits are known zero on one side, return the

727

// other. These bits cannot contribute to the result of the 'or' in this

728

// context.

729

if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))

730

return Op.getOperand(0);

731

if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))

732

return Op.getOperand(1);

733

break;

734

}

735

case ISD::XOR: {

736

LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);

737

RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

738

739

// If all of the demanded bits are known zero on one side, return the

740

// other.

741

if (DemandedBits.isSubsetOf(RHSKnown.Zero))

742

return Op.getOperand(0);

743

if (DemandedBits.isSubsetOf(LHSKnown.Zero))

744

return Op.getOperand(1);

745

break;

746

}

747

case ISD::SHL: {

748

// If we are only demanding sign bits then we can use the shift source

749

// directly.

750

if (const APInt *MaxSA =

751

DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

752

SDValue Op0 = Op.getOperand(0);

753

unsigned ShAmt = MaxSA->getZExtValue();

754

unsigned NumSignBits =

755

DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

756

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

757

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

758

return Op0;

759

}

760

break;

761

}

762

case ISD::SETCC: {

763

SDValue Op0 = Op.getOperand(0);

764

SDValue Op1 = Op.getOperand(1);

765

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

766

// If (1) we only need the sign-bit, (2) the setcc operands are the same

767

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

768

// -1, we may be able to bypass the setcc.

769

if (DemandedBits.isSignMask() &&

770

Op0.getScalarValueSizeInBits() == BitWidth &&

771

getBooleanContents(Op0.getValueType()) ==

772

BooleanContent::ZeroOrNegativeOneBooleanContent) {

773

// If we're testing X < 0, then this compare isn't needed - just use X!

774

// FIXME: We're limiting to integer types here, but this should also work

775

// if we don't care about FP signed-zero. The use of SETLT with FP means

776

// that we don't care about NaNs.

777

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

778

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

779

return Op0;

780

}

781

break;

782

}

783

case ISD::SIGN_EXTEND_INREG: {

784

// If none of the extended bits are demanded, eliminate the sextinreg.

785

SDValue Op0 = Op.getOperand(0);

786

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

787

unsigned ExBits = ExVT.getScalarSizeInBits();

788

if (DemandedBits.getActiveBits() <= ExBits)

789

return Op0;

790

// If the input is already sign extended, just drop the extension.

791

unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

792

if (NumSignBits >= (BitWidth - ExBits + 1))

793

return Op0;

794

break;

795

}

796

case ISD::ANY_EXTEND_VECTOR_INREG:

797

case ISD::SIGN_EXTEND_VECTOR_INREG:

798

case ISD::ZERO_EXTEND_VECTOR_INREG: {

799

// If we only want the lowest element and none of extended bits, then we can

800

// return the bitcasted source vector.

801

SDValue Src = Op.getOperand(0);

802

EVT SrcVT = Src.getValueType();

803

EVT DstVT = Op.getValueType();

804

if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&

805

DAG.getDataLayout().isLittleEndian() &&

806

DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {

807

return DAG.getBitcast(DstVT, Src);

808

}

809

break;

810

}

811

case ISD::INSERT_VECTOR_ELT: {

812

// If we don't demand the inserted element, return the base vector.

813

SDValue Vec = Op.getOperand(0);

814

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

815

EVT VecVT = Vec.getValueType();

816

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&

817

!DemandedElts[CIdx->getZExtValue()])

818

return Vec;

819

break;

820

}

821

case ISD::INSERT_SUBVECTOR: {

822

// If we don't demand the inserted subvector, return the base vector.

823

SDValue Vec = Op.getOperand(0);

824

SDValue Sub = Op.getOperand(1);

825

uint64_t Idx = Op.getConstantOperandVal(2);

826

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

827

if (DemandedElts.extractBits(NumSubElts, Idx) == 0)

828

return Vec;

829

break;

830

}

831

case ISD::VECTOR_SHUFFLE: {

832

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

833

834

// If all the demanded elts are from one operand and are inline,

835

// then we can use the operand directly.

836

bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;

837

for (unsigned i = 0; i != NumElts; ++i) {

838

int M = ShuffleMask[i];

839

if (M < 0 || !DemandedElts[i])

840

continue;

841

AllUndef = false;

842

IdentityLHS &= (M == (int)i);

843

IdentityRHS &= ((M - NumElts) == i);

844

}

845

846

if (AllUndef)

847

return DAG.getUNDEF(Op.getValueType());

848

if (IdentityLHS)

849

return Op.getOperand(0);

850

if (IdentityRHS)

851

return Op.getOperand(1);

852

break;

853

}

854

default:

855

if (Op.getOpcode() >= ISD::BUILTIN_OP_END)

856

if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(

857

Op, DemandedBits, DemandedElts, DAG, Depth))

858

return V;

859

break;

860

}

861

return SDValue();

862

}

863

864

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(

865

SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,

866

unsigned Depth) const {

867

EVT VT = Op.getValueType();

868

APInt DemandedElts = VT.isVector()

869

? APInt::getAllOnesValue(VT.getVectorNumElements())

870

: APInt(1, 1);

871

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

872

Depth);

873

}

874

875

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(

876

SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,

877

unsigned Depth) const {

878

APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());

879

return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,

880

Depth);

881

}

882

883

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the

884

/// result of Op are ever used downstream. If we can use this information to

885

/// simplify Op, create a new simplified DAG node and return true, returning the

886

/// original and new nodes in Old and New. Otherwise, analyze the expression and

887

/// return a mask of Known bits for the expression (used to simplify the

888

/// caller). The Known bits may only be accurate for those bits in the

889

/// OriginalDemandedBits and OriginalDemandedElts.

890

bool TargetLowering::SimplifyDemandedBits(

891

SDValue Op, const APInt &OriginalDemandedBits,

892

const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,

893

unsigned Depth, bool AssumeSingleUse) const {

894

unsigned BitWidth = OriginalDemandedBits.getBitWidth();

895

assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 896, __extension__ __PRETTY_FUNCTION__))

896

"Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 896, __extension__ __PRETTY_FUNCTION__));

897

898

// Don't know anything.

899

Known = KnownBits(BitWidth);

900

901

// TODO: We can probably do more work on calculating the known bits and

902

// simplifying the operations for scalable vectors, but for now we just

903

// bail out.

904

if (Op.getValueType().isScalableVector())

905

return false;

906

907

unsigned NumElts = OriginalDemandedElts.getBitWidth();

908

assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))

909

NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))

910

"Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 910, __extension__ __PRETTY_FUNCTION__));

911

912

APInt DemandedBits = OriginalDemandedBits;

913

APInt DemandedElts = OriginalDemandedElts;

914

SDLoc dl(Op);

915

auto &DL = TLO.DAG.getDataLayout();

916

917

// Undef operand.

918

if (Op.isUndef())

919

return false;

920

921

if (Op.getOpcode() == ISD::Constant) {

922

// We know all of the bits for a constant!

923

Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());

924

return false;

925

}

926

927

if (Op.getOpcode() == ISD::ConstantFP) {

928

// We know all of the bits for a floating point constant!

929

Known = KnownBits::makeConstant(

930

cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());

931

return false;

932

}

933

934

// Other users may use these bits.

935

EVT VT = Op.getValueType();

936

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {

937

if (Depth != 0) {

938

// If not at the root, Just compute the Known bits to

939

// simplify things downstream.

940

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

941

return false;

942

}

943

// If this is the root being simplified, allow it to have multiple uses,

944

// just set the DemandedBits/Elts to all bits.

945

DemandedBits = APInt::getAllOnesValue(BitWidth);

946

DemandedElts = APInt::getAllOnesValue(NumElts);

947

} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {

948

// Not demanding any bits/elts from Op.

949

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

950

} else if (Depth >= SelectionDAG::MaxRecursionDepth) {

951

// Limit search depth.

952

return false;

953

}

954

955

KnownBits Known2;

956

switch (Op.getOpcode()) {

957

case ISD::TargetConstant:

958

llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 958);

959

case ISD::SCALAR_TO_VECTOR: {

960

if (!DemandedElts[0])

961

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

962

963

KnownBits SrcKnown;

964

SDValue Src = Op.getOperand(0);

965

unsigned SrcBitWidth = Src.getScalarValueSizeInBits();

966

APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);

967

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))

968

return true;

969

970

// Upper elements are undef, so only get the knownbits if we just demand

971

// the bottom element.

972

if (DemandedElts == 1)

973

Known = SrcKnown.anyextOrTrunc(BitWidth);

974

break;

975

}

976

case ISD::BUILD_VECTOR:

977

// Collect the known bits that are shared by every demanded element.

978

// TODO: Call SimplifyDemandedBits for non-constant demanded elements.

979

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

980

return false; // Don't fall through, will infinitely loop.

981

case ISD::LOAD: {

982

auto *LD = cast<LoadSDNode>(Op);

983

if (getTargetConstantFromLoad(LD)) {

984

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

985

return false; // Don't fall through, will infinitely loop.

986

}

987

if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {

988

// If this is a ZEXTLoad and we are looking at the loaded value.

989

EVT MemVT = LD->getMemoryVT();

990

unsigned MemBits = MemVT.getScalarSizeInBits();

991

Known.Zero.setBitsFrom(MemBits);

992

return false; // Don't fall through, will infinitely loop.

993

}

994

break;

995

}

996

case ISD::INSERT_VECTOR_ELT: {

997

SDValue Vec = Op.getOperand(0);

998

SDValue Scl = Op.getOperand(1);

999

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

1000

EVT VecVT = Vec.getValueType();

1001

1002

// If index isn't constant, assume we need all vector elements AND the

1003

// inserted element.

1004

APInt DemandedVecElts(DemandedElts);

1005

if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {

1006

unsigned Idx = CIdx->getZExtValue();

1007

DemandedVecElts.clearBit(Idx);

1008

1009

// Inserted element is not required.

1010

if (!DemandedElts[Idx])

1011

return TLO.CombineTo(Op, Vec);

1012

}

1013

1014

KnownBits KnownScl;

1015

unsigned NumSclBits = Scl.getScalarValueSizeInBits();

1016

APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);

1017

if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))

1018

return true;

1019

1020

Known = KnownScl.anyextOrTrunc(BitWidth);

1021

1022

KnownBits KnownVec;

1023

if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,

1024

Depth + 1))

1025

return true;

1026

1027

if (!!DemandedVecElts)

1028

Known = KnownBits::commonBits(Known, KnownVec);

1029

1030

return false;

1031

}

1032

case ISD::INSERT_SUBVECTOR: {

1033

// Demand any elements from the subvector and the remainder from the src its

1034

// inserted into.

1035

SDValue Src = Op.getOperand(0);

1036

SDValue Sub = Op.getOperand(1);

1037

uint64_t Idx = Op.getConstantOperandVal(2);

1038

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

1039

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

1040

APInt DemandedSrcElts = DemandedElts;

1041

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

1042

1043

KnownBits KnownSub, KnownSrc;

1044

if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,

1045

Depth + 1))

1046

return true;

1047

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,

1048

Depth + 1))

1049

return true;

1050

1051

Known.Zero.setAllBits();

1052

Known.One.setAllBits();

1053

if (!!DemandedSubElts)

1054

Known = KnownBits::commonBits(Known, KnownSub);

1055

if (!!DemandedSrcElts)

1056

Known = KnownBits::commonBits(Known, KnownSrc);

1057

1058

// Attempt to avoid multi-use src if we don't need anything from it.

1059

if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||

1060

!DemandedSrcElts.isAllOnesValue()) {

1061

SDValue NewSub = SimplifyMultipleUseDemandedBits(

1062

Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);

1063

SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1064

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1065

if (NewSub || NewSrc) {

1066

NewSub = NewSub ? NewSub : Sub;

1067

NewSrc = NewSrc ? NewSrc : Src;

1068

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,

1069

Op.getOperand(2));

1070

return TLO.CombineTo(Op, NewOp);

1071

}

1072

}

1073

break;

1074

}

1075

case ISD::EXTRACT_SUBVECTOR: {

1076

// Offset the demanded elts by the subvector index.

1077

SDValue Src = Op.getOperand(0);

1078

if (Src.getValueType().isScalableVector())

1079

break;

1080

uint64_t Idx = Op.getConstantOperandVal(1);

1081

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

1082

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

1083

1084

if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,

1085

Depth + 1))

1086

return true;

1087

1088

// Attempt to avoid multi-use src if we don't need anything from it.

1089

if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {

1090

SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

1091

Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);

1092

if (DemandedSrc) {

1093

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,

1094

Op.getOperand(1));

1095

return TLO.CombineTo(Op, NewOp);

1096

}

1097

}

1098

break;

1099

}

1100

case ISD::CONCAT_VECTORS: {

1101

Known.Zero.setAllBits();

1102

Known.One.setAllBits();

1103

EVT SubVT = Op.getOperand(0).getValueType();

1104

unsigned NumSubVecs = Op.getNumOperands();

1105

unsigned NumSubElts = SubVT.getVectorNumElements();

1106

for (unsigned i = 0; i != NumSubVecs; ++i) {

1107

APInt DemandedSubElts =

1108

DemandedElts.extractBits(NumSubElts, i * NumSubElts);

1109

if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,

1110

Known2, TLO, Depth + 1))

1111

return true;

1112

// Known bits are shared by every demanded subvector element.

1113

if (!!DemandedSubElts)

1114

Known = KnownBits::commonBits(Known, Known2);

1115

}

1116

break;

1117

}

1118

case ISD::VECTOR_SHUFFLE: {

1119

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

1120

1121

// Collect demanded elements from shuffle operands..

1122

APInt DemandedLHS(NumElts, 0);

1123

APInt DemandedRHS(NumElts, 0);

1124

for (unsigned i = 0; i != NumElts; ++i) {

1125

if (!DemandedElts[i])

1126

continue;

1127

int M = ShuffleMask[i];

1128

if (M < 0) {

1129

// For UNDEF elements, we don't know anything about the common state of

1130

// the shuffle result.

1131

DemandedLHS.clearAllBits();

1132

DemandedRHS.clearAllBits();

1133

break;

1134

}

1135

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1135, __extension__ __PRETTY_FUNCTION__));

1136

if (M < (int)NumElts)

1137

DemandedLHS.setBit(M);

1138

else

1139

DemandedRHS.setBit(M - NumElts);

1140

}

1141

1142

if (!!DemandedLHS || !!DemandedRHS) {

1143

SDValue Op0 = Op.getOperand(0);

1144

SDValue Op1 = Op.getOperand(1);

1145

1146

Known.Zero.setAllBits();

1147

Known.One.setAllBits();

1148

if (!!DemandedLHS) {

1149

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,

1150

Depth + 1))

1151

return true;

1152

Known = KnownBits::commonBits(Known, Known2);

1153

}

1154

if (!!DemandedRHS) {

1155

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,

1156

Depth + 1))

1157

return true;

1158

Known = KnownBits::commonBits(Known, Known2);

1159

}

1160

1161

// Attempt to avoid multi-use ops if we don't need anything from them.

1162

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1163

Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);

1164

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1165

Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);

1166

if (DemandedOp0 || DemandedOp1) {

1167

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1168

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1169

SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);

1170

return TLO.CombineTo(Op, NewOp);

1171

}

1172

}

1173

break;

1174

}

1175

case ISD::AND: {

1176

SDValue Op0 = Op.getOperand(0);

1177

SDValue Op1 = Op.getOperand(1);

1178

1179

// If the RHS is a constant, check to see if the LHS would be zero without

1180

// using the bits from the RHS. Below, we use knowledge about the RHS to

1181

// simplify the LHS, here we're using information from the LHS to simplify

1182

// the RHS.

1183

if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {

1184

// Do not increment Depth here; that can cause an infinite loop.

1185

KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);

1186

// If the LHS already has zeros where RHSC does, this 'and' is dead.

1187

if ((LHSKnown.Zero & DemandedBits) ==

1188

(~RHSC->getAPIntValue() & DemandedBits))

1189

return TLO.CombineTo(Op, Op0);

1190

1191

// If any of the set bits in the RHS are known zero on the LHS, shrink

1192

// the constant.

1193

if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,

1194

DemandedElts, TLO))

1195

return true;

1196

1197

// Bitwise-not (xor X, -1) is a special case: we don't usually shrink its

1198

// constant, but if this 'and' is only clearing bits that were just set by

1199

// the xor, then this 'and' can be eliminated by shrinking the mask of

1200

// the xor. For example, for a 32-bit X:

1201

// and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1

1202

if (isBitwiseNot(Op0) && Op0.hasOneUse() &&

1203

LHSKnown.One == ~RHSC->getAPIntValue()) {

1204

SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);

1205

return TLO.CombineTo(Op, Xor);

1206

}

1207

}

1208

1209

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1210

Depth + 1))

1211

return true;

1212

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1212, __extension__ __PRETTY_FUNCTION__));

1213

if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,

1214

Known2, TLO, Depth + 1))

1215

return true;

1216

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1216, __extension__ __PRETTY_FUNCTION__));

1217

1218

// Attempt to avoid multi-use ops if we don't need anything from them.

1219

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1220

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1221

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1222

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1223

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1224

if (DemandedOp0 || DemandedOp1) {

1225

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1226

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1227

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1228

return TLO.CombineTo(Op, NewOp);

1229

}

1230

}

1231

1232

// If all of the demanded bits are known one on one side, return the other.

1233

// These bits cannot contribute to the result of the 'and'.

1234

if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))

1235

return TLO.CombineTo(Op, Op0);

1236

if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))

1237

return TLO.CombineTo(Op, Op1);

1238

// If all of the demanded bits in the inputs are known zeros, return zero.

1239

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1240

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));

1241

// If the RHS is a constant, see if we can simplify it.

1242

if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,

1243

TLO))

1244

return true;

1245

// If the operation can be done in a smaller type, do so.

1246

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1247

return true;

1248

1249

Known &= Known2;

1250

break;

1251

}

1252

case ISD::OR: {

1253

SDValue Op0 = Op.getOperand(0);

1254

SDValue Op1 = Op.getOperand(1);

1255

1256

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1257

Depth + 1))

1258

return true;

1259

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1259, __extension__ __PRETTY_FUNCTION__));

1260

if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,

1261

Known2, TLO, Depth + 1))

1262

return true;

1263

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1263, __extension__ __PRETTY_FUNCTION__));

1264

1265

// Attempt to avoid multi-use ops if we don't need anything from them.

1266

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1267

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1268

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1269

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1270

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1271

if (DemandedOp0 || DemandedOp1) {

1272

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1273

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1274

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1275

return TLO.CombineTo(Op, NewOp);

1276

}

1277

}

1278

1279

// If all of the demanded bits are known zero on one side, return the other.

1280

// These bits cannot contribute to the result of the 'or'.

1281

if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))

1282

return TLO.CombineTo(Op, Op0);

1283

if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))

1284

return TLO.CombineTo(Op, Op1);

1285

// If the RHS is a constant, see if we can simplify it.

1286

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1287

return true;

1288

// If the operation can be done in a smaller type, do so.

1289

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1290

return true;

1291

1292

Known |= Known2;

1293

break;

1294

}

1295

case ISD::XOR: {

1296

SDValue Op0 = Op.getOperand(0);

1297

SDValue Op1 = Op.getOperand(1);

1298

1299

if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,

1300

Depth + 1))

1301

return true;

1302

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1302, __extension__ __PRETTY_FUNCTION__));

1303

if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,

1304

Depth + 1))

1305

return true;

1306

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1306, __extension__ __PRETTY_FUNCTION__));

1307

1308

// Attempt to avoid multi-use ops if we don't need anything from them.

1309

if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1310

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1311

Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1312

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

1313

Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);

1314

if (DemandedOp0 || DemandedOp1) {

1315

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

1316

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

1317

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);

1318

return TLO.CombineTo(Op, NewOp);

1319

}

1320

}

1321

1322

// If all of the demanded bits are known zero on one side, return the other.

1323

// These bits cannot contribute to the result of the 'xor'.

1324

if (DemandedBits.isSubsetOf(Known.Zero))

1325

return TLO.CombineTo(Op, Op0);

1326

if (DemandedBits.isSubsetOf(Known2.Zero))

1327

return TLO.CombineTo(Op, Op1);

1328

// If the operation can be done in a smaller type, do so.

1329

if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1330

return true;

1331

1332

// If all of the unknown bits are known to be zero on one side or the other

1333

// turn this into an *inclusive* or.

1334

// e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0

1335

if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))

1336

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

1337

1338

ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);

1339

if (C) {

1340

// If one side is a constant, and all of the set bits in the constant are

1341

// also known set on the other side, turn this into an AND, as we know

1342

// the bits will be cleared.

1343

// e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2

1344

// NB: it is okay if more bits are known than are requested

1345

if (C->getAPIntValue() == Known2.One) {

1346

SDValue ANDC =

1347

TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);

1348

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));

1349

}

1350

1351

// If the RHS is a constant, see if we can change it. Don't alter a -1

1352

// constant because that's a 'not' op, and that is better for combining

1353

// and codegen.

1354

if (!C->isAllOnesValue() &&

1355

DemandedBits.isSubsetOf(C->getAPIntValue())) {

1356

// We're flipping all demanded bits. Flip the undemanded bits too.

1357

SDValue New = TLO.DAG.getNOT(dl, Op0, VT);

1358

return TLO.CombineTo(Op, New);

1359

}

1360

}

1361

1362

// If we can't turn this into a 'not', try to shrink the constant.

1363

if (!C || !C->isAllOnesValue())

1364

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1365

return true;

1366

1367

Known ^= Known2;

1368

break;

1369

}

1370

case ISD::SELECT:

1371

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,

1372

Depth + 1))

1373

return true;

1374

if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,

1375

Depth + 1))

1376

return true;

1377

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1377, __extension__ __PRETTY_FUNCTION__));

1378

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1378, __extension__ __PRETTY_FUNCTION__));

1379

1380

// If the operands are constants, see if we can simplify them.

1381

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1382

return true;

1383

1384

// Only known if known in both the LHS and RHS.

1385

Known = KnownBits::commonBits(Known, Known2);

1386

break;

1387

case ISD::SELECT_CC:

1388

if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,

1389

Depth + 1))

1390

return true;

1391

if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,

1392

Depth + 1))

1393

return true;

1394

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1394, __extension__ __PRETTY_FUNCTION__));

1395

assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1395, __extension__ __PRETTY_FUNCTION__));

1396

1397

// If the operands are constants, see if we can simplify them.

1398

if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))

1399

return true;

1400

1401

// Only known if known in both the LHS and RHS.

1402

Known = KnownBits::commonBits(Known, Known2);

1403

break;

1404

case ISD::SETCC: {

1405

SDValue Op0 = Op.getOperand(0);

1406

SDValue Op1 = Op.getOperand(1);

1407

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

1408

// If (1) we only need the sign-bit, (2) the setcc operands are the same

1409

// width as the setcc result, and (3) the result of a setcc conforms to 0 or

1410

// -1, we may be able to bypass the setcc.

1411

if (DemandedBits.isSignMask() &&

1412

Op0.getScalarValueSizeInBits() == BitWidth &&

1413

getBooleanContents(Op0.getValueType()) ==

1414

BooleanContent::ZeroOrNegativeOneBooleanContent) {

1415

// If we're testing X < 0, then this compare isn't needed - just use X!

1416

// FIXME: We're limiting to integer types here, but this should also work

1417

// if we don't care about FP signed-zero. The use of SETLT with FP means

1418

// that we don't care about NaNs.

1419

if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&

1420

(isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))

1421

return TLO.CombineTo(Op, Op0);

1422

1423

// TODO: Should we check for other forms of sign-bit comparisons?

1424

// Examples: X <= -1, X >= 0

1425

}

1426

if (getBooleanContents(Op0.getValueType()) ==

1427

TargetLowering::ZeroOrOneBooleanContent &&

1428

BitWidth > 1)

1429

Known.Zero.setBitsFrom(1);

1430

break;

1431

}

1432

case ISD::SHL: {

1433

SDValue Op0 = Op.getOperand(0);

1434

SDValue Op1 = Op.getOperand(1);

1435

EVT ShiftVT = Op1.getValueType();

1436

1437

if (const APInt *SA =

1438

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1439

unsigned ShAmt = SA->getZExtValue();

1440

if (ShAmt == 0)

1441

return TLO.CombineTo(Op, Op0);

1442

1443

// If this is ((X >>u C1) << ShAmt), see if we can simplify this into a

1444

// single shift. We can do this if the bottom bits (which are shifted

1445

// out) are never demanded.

1446

// TODO - support non-uniform vector amounts.

1447

if (Op0.getOpcode() == ISD::SRL) {

1448

if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {

1449

if (const APInt *SA2 =

1450

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1451

unsigned C1 = SA2->getZExtValue();

1452

unsigned Opc = ISD::SHL;

1453

int Diff = ShAmt - C1;

1454

if (Diff < 0) {

1455

Diff = -Diff;

1456

Opc = ISD::SRL;

1457

}

1458

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1459

return TLO.CombineTo(

1460

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1461

}

1462

}

1463

}

1464

1465

// Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits

1466

// are not demanded. This will likely allow the anyext to be folded away.

1467

// TODO - support non-uniform vector amounts.

1468

if (Op0.getOpcode() == ISD::ANY_EXTEND) {

1469

SDValue InnerOp = Op0.getOperand(0);

1470

EVT InnerVT = InnerOp.getValueType();

1471

unsigned InnerBits = InnerVT.getScalarSizeInBits();

1472

if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&

1473

isTypeDesirableForOp(ISD::SHL, InnerVT)) {

1474

EVT ShTy = getShiftAmountTy(InnerVT, DL);

1475

if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))

1476

ShTy = InnerVT;

1477

SDValue NarrowShl =

1478

TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,

1479

TLO.DAG.getConstant(ShAmt, dl, ShTy));

1480

return TLO.CombineTo(

1481

Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));

1482

}

1483

1484

// Repeat the SHL optimization above in cases where an extension

1485

// intervenes: (shl (anyext (shr x, c1)), c2) to

1486

// (shl (anyext x), c2-c1). This requires that the bottom c1 bits

1487

// aren't demanded (as above) and that the shifted upper c1 bits of

1488

// x aren't demanded.

1489

// TODO - support non-uniform vector amounts.

1490

if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&

1491

InnerOp.hasOneUse()) {

1492

if (const APInt *SA2 =

1493

TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {

1494

unsigned InnerShAmt = SA2->getZExtValue();

1495

if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&

1496

DemandedBits.getActiveBits() <=

1497

(InnerBits - InnerShAmt + ShAmt) &&

1498

DemandedBits.countTrailingZeros() >= ShAmt) {

1499

SDValue NewSA =

1500

TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);

1501

SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,

1502

InnerOp.getOperand(0));

1503

return TLO.CombineTo(

1504

Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));

1505

}

1506

}

1507

}

1508

}

1509

1510

APInt InDemandedMask = DemandedBits.lshr(ShAmt);

1511

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1512

Depth + 1))

1513

return true;

1514

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1514, __extension__ __PRETTY_FUNCTION__));

1515

Known.Zero <<= ShAmt;

1516

Known.One <<= ShAmt;

1517

// low bits known zero.

1518

Known.Zero.setLowBits(ShAmt);

1519

1520

// Try shrinking the operation as long as the shift amount will still be

1521

// in range.

1522

if ((ShAmt < DemandedBits.getActiveBits()) &&

1523

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))

1524

return true;

1525

}

1526

1527

// If we are only demanding sign bits then we can use the shift source

1528

// directly.

1529

if (const APInt *MaxSA =

1530

TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {

1531

unsigned ShAmt = MaxSA->getZExtValue();

1532

unsigned NumSignBits =

1533

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1534

unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1535

if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))

1536

return TLO.CombineTo(Op, Op0);

1537

}

1538

break;

1539

}

1540

case ISD::SRL: {

1541

SDValue Op0 = Op.getOperand(0);

1542

SDValue Op1 = Op.getOperand(1);

1543

EVT ShiftVT = Op1.getValueType();

1544

1545

if (const APInt *SA =

1546

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1547

unsigned ShAmt = SA->getZExtValue();

1548

if (ShAmt == 0)

1549

return TLO.CombineTo(Op, Op0);

1550

1551

// If this is ((X << C1) >>u ShAmt), see if we can simplify this into a

1552

// single shift. We can do this if the top bits (which are shifted out)

1553

// are never demanded.

1554

// TODO - support non-uniform vector amounts.

1555

if (Op0.getOpcode() == ISD::SHL) {

1556

if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {

1557

if (const APInt *SA2 =

1558

TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {

1559

unsigned C1 = SA2->getZExtValue();

1560

unsigned Opc = ISD::SRL;

1561

int Diff = ShAmt - C1;

1562

if (Diff < 0) {

1563

Diff = -Diff;

1564

Opc = ISD::SHL;

1565

}

1566

SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);

1567

return TLO.CombineTo(

1568

Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));

1569

}

1570

}

1571

}

1572

1573

APInt InDemandedMask = (DemandedBits << ShAmt);

1574

1575

// If the shift is exact, then it does demand the low bits (and knows that

1576

// they are zero).

1577

if (Op->getFlags().hasExact())

1578

InDemandedMask.setLowBits(ShAmt);

1579

1580

// Compute the new bits that are at the top now.

1581

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1582

Depth + 1))

1583

return true;

1584

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1584, __extension__ __PRETTY_FUNCTION__));

1585

Known.Zero.lshrInPlace(ShAmt);

1586

Known.One.lshrInPlace(ShAmt);

1587

// High bits known zero.

1588

Known.Zero.setHighBits(ShAmt);

1589

}

1590

break;

1591

}

1592

case ISD::SRA: {

1593

SDValue Op0 = Op.getOperand(0);

1594

SDValue Op1 = Op.getOperand(1);

1595

EVT ShiftVT = Op1.getValueType();

1596

1597

// If we only want bits that already match the signbit then we don't need

1598

// to shift.

1599

unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();

1600

if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=

1601

NumHiDemandedBits)

1602

return TLO.CombineTo(Op, Op0);

1603

1604

// If this is an arithmetic shift right and only the low-bit is set, we can

1605

// always convert this into a logical shr, even if the shift amount is

1606

// variable. The low bit of the shift cannot be an input sign bit unless

1607

// the shift amount is >= the size of the datatype, which is undefined.

1608

if (DemandedBits.isOneValue())

1609

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

1610

1611

if (const APInt *SA =

1612

TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {

1613

unsigned ShAmt = SA->getZExtValue();

1614

if (ShAmt == 0)

1615

return TLO.CombineTo(Op, Op0);

1616

1617

APInt InDemandedMask = (DemandedBits << ShAmt);

1618

1619

// If the shift is exact, then it does demand the low bits (and knows that

1620

// they are zero).

1621

if (Op->getFlags().hasExact())

1622

InDemandedMask.setLowBits(ShAmt);

1623

1624

// If any of the demanded bits are produced by the sign extension, we also

1625

// demand the input sign bit.

1626

if (DemandedBits.countLeadingZeros() < ShAmt)

1627

InDemandedMask.setSignBit();

1628

1629

if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,

1630

Depth + 1))

1631

return true;

1632

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__));

1633

Known.Zero.lshrInPlace(ShAmt);

1634

Known.One.lshrInPlace(ShAmt);

1635

1636

// If the input sign bit is known to be zero, or if none of the top bits

1637

// are demanded, turn this into an unsigned shift right.

1638

if (Known.Zero[BitWidth - ShAmt - 1] ||

1639

DemandedBits.countLeadingZeros() >= ShAmt) {

1640

SDNodeFlags Flags;

1641

Flags.setExact(Op->getFlags().hasExact());

1642

return TLO.CombineTo(

1643

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));

1644

}

1645

1646

int Log2 = DemandedBits.exactLogBase2();

1647

if (Log2 >= 0) {

1648

// The bit must come from the sign.

1649

SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);

1650

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));

1651

}

1652

1653

if (Known.One[BitWidth - ShAmt - 1])

1654

// New bits are known one.

1655

Known.One.setHighBits(ShAmt);

1656

1657

// Attempt to avoid multi-use ops if we don't need anything from them.

1658

if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

1659

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

1660

Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);

1661

if (DemandedOp0) {

1662

SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);

1663

return TLO.CombineTo(Op, NewOp);

1664

}

1665

}

1666

}

1667

break;

1668

}

1669

case ISD::FSHL:

1670

case ISD::FSHR: {

1671

SDValue Op0 = Op.getOperand(0);

1672

SDValue Op1 = Op.getOperand(1);

1673

SDValue Op2 = Op.getOperand(2);

1674

bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

1675

1676

if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {

1677

unsigned Amt = SA->getAPIntValue().urem(BitWidth);

1678

1679

// For fshl, 0-shift returns the 1st arg.

1680

// For fshr, 0-shift returns the 2nd arg.

1681

if (Amt == 0) {

1682

if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,

1683

Known, TLO, Depth + 1))

1684

return true;

1685

break;

1686

}

1687

1688

// fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))

1689

// fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)

1690

APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));

1691

APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);

1692

if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,

1693

Depth + 1))

1694

return true;

1695

if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,

1696

Depth + 1))

1697

return true;

1698

1699

Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));

1700

Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));

1701

Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1702

Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);

1703

Known.One |= Known2.One;

1704

Known.Zero |= Known2.Zero;

1705

}

1706

1707

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1708

if (isPowerOf2_32(BitWidth)) {

1709

APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);

1710

if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,

1711

Known2, TLO, Depth + 1))

1712

return true;

1713

}

1714

break;

1715

}

1716

case ISD::ROTL:

1717

case ISD::ROTR: {

1718

SDValue Op0 = Op.getOperand(0);

1719

SDValue Op1 = Op.getOperand(1);

1720

1721

// If we're rotating an 0/-1 value, then it stays an 0/-1 value.

1722

if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))

1723

return TLO.CombineTo(Op, Op0);

1724

1725

// For pow-2 bitwidths we only demand the bottom modulo amt bits.

1726

if (isPowerOf2_32(BitWidth)) {

1727

APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);

1728

if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,

1729

Depth + 1))

1730

return true;

1731

}

1732

break;

1733

}

1734

case ISD::UMIN: {

1735

// Check if one arg is always less than (or equal) to the other arg.

1736

SDValue Op0 = Op.getOperand(0);

1737

SDValue Op1 = Op.getOperand(1);

1738

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1739

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1740

Known = KnownBits::umin(Known0, Known1);

1741

if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))

1742

return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);

1743

if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))

1744

return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);

1745

break;

1746

}

1747

case ISD::UMAX: {

1748

// Check if one arg is always greater than (or equal) to the other arg.

1749

SDValue Op0 = Op.getOperand(0);

1750

SDValue Op1 = Op.getOperand(1);

1751

KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);

1752

KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);

1753

Known = KnownBits::umax(Known0, Known1);

1754

if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))

1755

return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);

1756

if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))

1757

return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);

1758

break;

1759

}

1760

case ISD::BITREVERSE: {

1761

SDValue Src = Op.getOperand(0);

1762

APInt DemandedSrcBits = DemandedBits.reverseBits();

1763

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1764

Depth + 1))

1765

return true;

1766

Known.One = Known2.One.reverseBits();

1767

Known.Zero = Known2.Zero.reverseBits();

1768

break;

1769

}

1770

case ISD::BSWAP: {

1771

SDValue Src = Op.getOperand(0);

1772

APInt DemandedSrcBits = DemandedBits.byteSwap();

1773

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,

1774

Depth + 1))

1775

return true;

1776

Known.One = Known2.One.byteSwap();

1777

Known.Zero = Known2.Zero.byteSwap();

1778

break;

1779

}

1780

case ISD::CTPOP: {

1781

// If only 1 bit is demanded, replace with PARITY as long as we're before

1782

// op legalization.

1783

// FIXME: Limit to scalars for now.

1784

if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())

1785

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,

1786

Op.getOperand(0)));

1787

1788

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

1789

break;

1790

}

1791

case ISD::SIGN_EXTEND_INREG: {

1792

SDValue Op0 = Op.getOperand(0);

1793

EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

1794

unsigned ExVTBits = ExVT.getScalarSizeInBits();

1795

1796

// If we only care about the highest bit, don't bother shifting right.

1797

if (DemandedBits.isSignMask()) {

1798

unsigned NumSignBits =

1799

TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);

1800

bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;

1801

// However if the input is already sign extended we expect the sign

1802

// extension to be dropped altogether later and do not simplify.

1803

if (!AlreadySignExtended) {

1804

// Compute the correct shift amount type, which must be getShiftAmountTy

1805

// for scalar types after legalization.

1806

EVT ShiftAmtTy = VT;

1807

if (TLO.LegalTypes() && !ShiftAmtTy.isVector())

1808

ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

1809

1810

SDValue ShiftAmt =

1811

TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);

1812

return TLO.CombineTo(Op,

1813

TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));

1814

}

1815

}

1816

1817

// If none of the extended bits are demanded, eliminate the sextinreg.

1818

if (DemandedBits.getActiveBits() <= ExVTBits)

1819

return TLO.CombineTo(Op, Op0);

1820

1821

APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

1822

1823

// Since the sign extended bits are demanded, we know that the sign

1824

// bit is demanded.

1825

InputDemandedBits.setBit(ExVTBits - 1);

1826

1827

if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))

1828

return true;

1829

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1829, __extension__ __PRETTY_FUNCTION__));

1830

1831

// If the sign bit of the input is known set or clear, then we know the

1832

// top bits of the result.

1833

1834

// If the input sign bit is known zero, convert this into a zero extension.

1835

if (Known.Zero[ExVTBits - 1])

1836

return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

1837

1838

APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);

1839

if (Known.One[ExVTBits - 1]) { // Input sign bit known set

1840

Known.One.setBitsFrom(ExVTBits);

1841

Known.Zero &= Mask;

1842

} else { // Input sign bit unknown

1843

Known.Zero &= Mask;

1844

Known.One &= Mask;

1845

}

1846

break;

1847

}

1848

case ISD::BUILD_PAIR: {

1849

EVT HalfVT = Op.getOperand(0).getValueType();

1850

unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

1851

1852

APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);

1853

APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

1854

1855

KnownBits KnownLo, KnownHi;

1856

1857

if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))

1858

return true;

1859

1860

if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))

1861

return true;

1862

1863

Known.Zero = KnownLo.Zero.zext(BitWidth) |

1864

KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

1865

1866

Known.One = KnownLo.One.zext(BitWidth) |

1867

KnownHi.One.zext(BitWidth).shl(HalfBitWidth);

1868

break;

1869

}

1870

case ISD::ZERO_EXTEND:

1871

case ISD::ZERO_EXTEND_VECTOR_INREG: {

1872

SDValue Src = Op.getOperand(0);

1873

EVT SrcVT = Src.getValueType();

1874

unsigned InBits = SrcVT.getScalarSizeInBits();

1875

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1876

bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

1877

1878

// If none of the top bits are demanded, convert this into an any_extend.

1879

if (DemandedBits.getActiveBits() <= InBits) {

1880

// If we only need the non-extended bits of the bottom element

1881

// then we can just bitcast to the result.

1882

if (IsVecInReg && DemandedElts == 1 &&

1883

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1884

TLO.DAG.getDataLayout().isLittleEndian())

1885

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1886

1887

unsigned Opc =

1888

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1889

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1890

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1891

}

1892

1893

APInt InDemandedBits = DemandedBits.trunc(InBits);

1894

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1895

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1896

Depth + 1))

1897

return true;

1898

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1898, __extension__ __PRETTY_FUNCTION__));

1899

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1899, __extension__ __PRETTY_FUNCTION__));

1900

Known = Known.zext(BitWidth);

1901

1902

// Attempt to avoid multi-use ops if we don't need anything from them.

1903

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1904

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1905

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1906

break;

1907

}

1908

case ISD::SIGN_EXTEND:

1909

case ISD::SIGN_EXTEND_VECTOR_INREG: {

1910

SDValue Src = Op.getOperand(0);

1911

EVT SrcVT = Src.getValueType();

1912

unsigned InBits = SrcVT.getScalarSizeInBits();

1913

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1914

bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

1915

1916

// If none of the top bits are demanded, convert this into an any_extend.

1917

if (DemandedBits.getActiveBits() <= InBits) {

1918

// If we only need the non-extended bits of the bottom element

1919

// then we can just bitcast to the result.

1920

if (IsVecInReg && DemandedElts == 1 &&

1921

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1922

TLO.DAG.getDataLayout().isLittleEndian())

1923

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1924

1925

unsigned Opc =

1926

IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;

1927

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1928

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1929

}

1930

1931

APInt InDemandedBits = DemandedBits.trunc(InBits);

1932

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1933

1934

// Since some of the sign extended bits are demanded, we know that the sign

1935

// bit is demanded.

1936

InDemandedBits.setBit(InBits - 1);

1937

1938

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1939

Depth + 1))

1940

return true;

1941

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1941, __extension__ __PRETTY_FUNCTION__));

1942

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1942, __extension__ __PRETTY_FUNCTION__));

1943

1944

// If the sign bit is known one, the top bits match.

1945

Known = Known.sext(BitWidth);

1946

1947

// If the sign bit is known zero, convert this to a zero extend.

1948

if (Known.isNonNegative()) {

1949

unsigned Opc =

1950

IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;

1951

if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))

1952

return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));

1953

}

1954

1955

// Attempt to avoid multi-use ops if we don't need anything from them.

1956

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1957

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1958

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1959

break;

1960

}

1961

case ISD::ANY_EXTEND:

1962

case ISD::ANY_EXTEND_VECTOR_INREG: {

1963

SDValue Src = Op.getOperand(0);

1964

EVT SrcVT = Src.getValueType();

1965

unsigned InBits = SrcVT.getScalarSizeInBits();

1966

unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

1967

bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

1968

1969

// If we only need the bottom element then we can just bitcast.

1970

// TODO: Handle ANY_EXTEND?

1971

if (IsVecInReg && DemandedElts == 1 &&

1972

VT.getSizeInBits() == SrcVT.getSizeInBits() &&

1973

TLO.DAG.getDataLayout().isLittleEndian())

1974

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

1975

1976

APInt InDemandedBits = DemandedBits.trunc(InBits);

1977

APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

1978

if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,

1979

Depth + 1))

1980

return true;

1981

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1981, __extension__ __PRETTY_FUNCTION__));

1982

assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1982, __extension__ __PRETTY_FUNCTION__));

1983

Known = Known.anyext(BitWidth);

1984

1985

// Attempt to avoid multi-use ops if we don't need anything from them.

1986

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

1987

Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))

1988

return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));

1989

break;

1990

}

1991

case ISD::TRUNCATE: {

1992

SDValue Src = Op.getOperand(0);

1993

1994

// Simplify the input, using demanded bit information, and compute the known

1995

// zero/one bits live out.

1996

unsigned OperandBitWidth = Src.getScalarValueSizeInBits();

1997

APInt TruncMask = DemandedBits.zext(OperandBitWidth);

1998

if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,

1999

Depth + 1))

2000

return true;

2001

Known = Known.trunc(BitWidth);

2002

2003

// Attempt to avoid multi-use ops if we don't need anything from them.

2004

if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(

2005

Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))

2006

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

2007

2008

// If the input is only used by this truncate, see if we can shrink it based

2009

// on the known demanded bits.

2010

if (Src.getNode()->hasOneUse()) {

2011

switch (Src.getOpcode()) {

2012

default:

2013

break;

2014

case ISD::SRL:

2015

// Shrink SRL by a constant if none of the high bits shifted in are

2016

// demanded.

2017

if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))

2018

// Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is

2019

// undesirable.

2020

break;

2021

2022

const APInt *ShAmtC =

2023

TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);

2024

if (!ShAmtC || ShAmtC->uge(BitWidth))

2025

break;

2026

uint64_t ShVal = ShAmtC->getZExtValue();

2027

2028

APInt HighBits =

2029

APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);

2030

HighBits.lshrInPlace(ShVal);

2031

HighBits = HighBits.trunc(BitWidth);

2032

2033

if (!(HighBits & DemandedBits)) {

2034

// None of the shifted in bits are needed. Add a truncate of the

2035

// shift input, then shift it.

2036

SDValue NewShAmt = TLO.DAG.getConstant(

2037

ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));

2038

SDValue NewTrunc =

2039

TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));

2040

return TLO.CombineTo(

2041

Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));

2042

}

2043

break;

2044

}

2045

}

2046

2047

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2047, __extension__ __PRETTY_FUNCTION__));

2048

break;

2049

}

2050

case ISD::AssertZext: {

2051

// AssertZext demands all of the high bits, plus any of the low bits

2052

// demanded by its users.

2053

EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

2054

APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());

2055

if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,

2056

TLO, Depth + 1))

2057

return true;

2058

assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2058, __extension__ __PRETTY_FUNCTION__));

2059

2060

Known.Zero |= ~InMask;

2061

break;

2062

}

2063

case ISD::EXTRACT_VECTOR_ELT: {

2064

SDValue Src = Op.getOperand(0);

2065

SDValue Idx = Op.getOperand(1);

2066

ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();

2067

unsigned EltBitWidth = Src.getScalarValueSizeInBits();

2068

2069

if (SrcEltCnt.isScalable())

2070

return false;

2071

2072

// Demand the bits from every vector element without a constant index.

2073

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2074

APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);

2075

if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))

2076

if (CIdx->getAPIntValue().ult(NumSrcElts))

2077

DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

2078

2079

// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know

2080

// anything about the extended bits.

2081

APInt DemandedSrcBits = DemandedBits;

2082

if (BitWidth > EltBitWidth)

2083

DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

2084

2085

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,

2086

Depth + 1))

2087

return true;

2088

2089

// Attempt to avoid multi-use ops if we don't need anything from them.

2090

if (!DemandedSrcBits.isAllOnesValue() ||

2091

!DemandedSrcElts.isAllOnesValue()) {

2092

if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(

2093

Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {

2094

SDValue NewOp =

2095

TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);

2096

return TLO.CombineTo(Op, NewOp);

2097

}

2098

}

2099

2100

Known = Known2;

2101

if (BitWidth > EltBitWidth)

2102

Known = Known.anyext(BitWidth);

2103

break;

2104

}

2105

case ISD::BITCAST: {

2106

SDValue Src = Op.getOperand(0);

2107

EVT SrcVT = Src.getValueType();

2108

unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

2109

2110

// If this is an FP->Int bitcast and if the sign bit is the only

2111

// thing demanded, turn this into a FGETSIGN.

2112

if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&

2113

DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&

2114

SrcVT.isFloatingPoint()) {

2115

bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);

2116

bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);

2117

if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&

2118

SrcVT != MVT::f128) {

2119

// Cannot eliminate/lower SHL for f128 yet.

2120

EVT Ty = OpVTLegal ? VT : MVT::i32;

2121

// Make a FGETSIGN + SHL to move the sign bit into the appropriate

2122

// place. We expect the SHL to be eliminated by other optimizations.

2123

SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);

2124

unsigned OpVTSizeInBits = Op.getValueSizeInBits();

2125

if (!OpVTLegal && OpVTSizeInBits > 32)

2126

Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);

2127

unsigned ShVal = Op.getValueSizeInBits() - 1;

2128

SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);

2129

return TLO.CombineTo(Op,

2130

TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));

2131

}

2132

}

2133

2134

// Bitcast from a vector using SimplifyDemanded Bits/VectorElts.

2135

// Demand the elt/bit if any of the original elts/bits are demanded.

2136

// TODO - bigendian once we have test coverage.

2137

if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&

2138

TLO.DAG.getDataLayout().isLittleEndian()) {

2139

unsigned Scale = BitWidth / NumSrcEltBits;

2140

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2141

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2142

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2143

for (unsigned i = 0; i != Scale; ++i) {

2144

unsigned Offset = i * NumSrcEltBits;

2145

APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);

2146

if (!Sub.isNullValue()) {

2147

DemandedSrcBits |= Sub;

2148

for (unsigned j = 0; j != NumElts; ++j)

2149

if (DemandedElts[j])

2150

DemandedSrcElts.setBit((j * Scale) + i);

2151

}

2152

}

2153

2154

APInt KnownSrcUndef, KnownSrcZero;

2155

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2156

KnownSrcZero, TLO, Depth + 1))

2157

return true;

2158

2159

KnownBits KnownSrcBits;

2160

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2161

KnownSrcBits, TLO, Depth + 1))

2162

return true;

2163

} else if ((NumSrcEltBits % BitWidth) == 0 &&

2164

TLO.DAG.getDataLayout().isLittleEndian()) {

2165

unsigned Scale = NumSrcEltBits / BitWidth;

2166

unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;

2167

APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);

2168

APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);

2169

for (unsigned i = 0; i != NumElts; ++i)

2170

if (DemandedElts[i]) {

2171

unsigned Offset = (i % Scale) * BitWidth;

2172

DemandedSrcBits.insertBits(DemandedBits, Offset);

2173

DemandedSrcElts.setBit(i / Scale);

2174

}

2175

2176

if (SrcVT.isVector()) {

2177

APInt KnownSrcUndef, KnownSrcZero;

2178

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,

2179

KnownSrcZero, TLO, Depth + 1))

2180

return true;

2181

}

2182

2183

KnownBits KnownSrcBits;

2184

if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,

2185

KnownSrcBits, TLO, Depth + 1))

2186

return true;

2187

}

2188

2189

// If this is a bitcast, let computeKnownBits handle it. Only do this on a

2190

// recursive call where Known may be useful to the caller.

2191

if (Depth > 0) {

2192

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2193

return false;

2194

}

2195

break;

2196

}

2197

case ISD::ADD:

2198

case ISD::MUL:

2199

case ISD::SUB: {

2200

// Add, Sub, and Mul don't demand any bits in positions beyond that

2201

// of the highest bit demanded of them.

2202

SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);

2203

SDNodeFlags Flags = Op.getNode()->getFlags();

2204

unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();

2205

APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

2206

if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,

2207

Depth + 1) ||

2208

SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,

2209

Depth + 1) ||

2210

// See if the operation should be performed at a smaller bit width.

2211

ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {

2212

if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {

2213

// Disable the nsw and nuw flags. We can no longer guarantee that we

2214

// won't wrap after simplification.

2215

Flags.setNoSignedWrap(false);

2216

Flags.setNoUnsignedWrap(false);

2217

SDValue NewOp =

2218

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2219

return TLO.CombineTo(Op, NewOp);

2220

}

2221

return true;

2222

}

2223

2224

// Attempt to avoid multi-use ops if we don't need anything from them.

2225

if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {

2226

SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(

2227

Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2228

SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(

2229

Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);

2230

if (DemandedOp0 || DemandedOp1) {

2231

Flags.setNoSignedWrap(false);

2232

Flags.setNoUnsignedWrap(false);

2233

Op0 = DemandedOp0 ? DemandedOp0 : Op0;

2234

Op1 = DemandedOp1 ? DemandedOp1 : Op1;

2235

SDValue NewOp =

2236

TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);

2237

return TLO.CombineTo(Op, NewOp);

2238

}

2239

}

2240

2241

// If we have a constant operand, we may be able to turn it into -1 if we

2242

// do not demand the high bits. This can make the constant smaller to

2243

// encode, allow more general folding, or match specialized instruction

2244

// patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that

2245

// is probably not useful (and could be detrimental).

2246

ConstantSDNode *C = isConstOrConstSplat(Op1);

2247

APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);

2248

if (C && !C->isAllOnesValue() && !C->isOne() &&

2249

(C->getAPIntValue() | HighMask).isAllOnesValue()) {

2250

SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);

2251

// Disable the nsw and nuw flags. We can no longer guarantee that we

2252

// won't wrap after simplification.

2253

Flags.setNoSignedWrap(false);

2254

Flags.setNoUnsignedWrap(false);

2255

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);

2256

return TLO.CombineTo(Op, NewOp);

2257

}

2258

2259

LLVM_FALLTHROUGH[[gnu::fallthrough]];

2260

}

2261

default:

2262

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2263

if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,

2264

Known, TLO, Depth))

2265

return true;

2266

break;

2267

}

2268

2269

// Just use computeKnownBits to compute output bits.

2270

Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);

2271

break;

2272

}

2273

2274

// If we know the value of all of the demanded bits, return this as a

2275

// constant.

2276

if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {

2277

// Avoid folding to a constant if any OpaqueConstant is involved.

2278

const SDNode *N = Op.getNode();

2279

for (SDNode *Op :

2280

llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {

2281

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))

2282

if (C->isOpaque())

2283

return false;

2284

}

2285

if (VT.isInteger())

2286

return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));

2287

if (VT.isFloatingPoint())

2288

return TLO.CombineTo(

2289

Op,

2290

TLO.DAG.getConstantFP(

2291

APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));

2292

}

2293

2294

return false;

2295

}

2296

2297

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,

2298

const APInt &DemandedElts,

2299

APInt &KnownUndef,

2300

APInt &KnownZero,

2301

DAGCombinerInfo &DCI) const {

2302

SelectionDAG &DAG = DCI.DAG;

2303

TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),

2304

!DCI.isBeforeLegalizeOps());

2305

2306

bool Simplified =

2307

SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);

2308

if (Simplified) {

2309

DCI.AddToWorklist(Op.getNode());

2310

DCI.CommitTargetLoweringOpt(TLO);

2311

}

2312

2313

return Simplified;

2314

}

2315

2316

/// Given a vector binary operation and known undefined elements for each input

2317

/// operand, compute whether each element of the output is undefined.

2318

static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,

2319

const APInt &UndefOp0,

2320

const APInt &UndefOp1) {

2321

EVT VT = BO.getValueType();

2322

assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2323, __extension__ __PRETTY_FUNCTION__))

2323

"Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2323, __extension__ __PRETTY_FUNCTION__));

2324

2325

EVT EltVT = VT.getVectorElementType();

2326

unsigned NumElts = VT.getVectorNumElements();

2327

assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__))

2328

UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__));

2329

2330

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,

2331

const APInt &UndefVals) {

2332

if (UndefVals[Index])

2333

return DAG.getUNDEF(EltVT);

2334

2335

if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {

2336

// Try hard to make sure that the getNode() call is not creating temporary

2337

// nodes. Ignore opaque integers because they do not constant fold.

2338

SDValue Elt = BV->getOperand(Index);

2339

auto *C = dyn_cast<ConstantSDNode>(Elt);

2340

if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))

2341

return Elt;

2342

}

2343

2344

return SDValue();

2345

};

2346

2347

APInt KnownUndef = APInt::getNullValue(NumElts);

2348

for (unsigned i = 0; i != NumElts; ++i) {

2349

// If both inputs for this element are either constant or undef and match

2350

// the element type, compute the constant/undef result for this element of

2351

// the vector.

2352

// TODO: Ideally we would use FoldConstantArithmetic() here, but that does

2353

// not handle FP constants. The code within getNode() should be refactored

2354

// to avoid the danger of creating a bogus temporary node here.

2355

SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);

2356

SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);

2357

if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)

2358

if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())

2359

KnownUndef.setBit(i);

2360

}

2361

return KnownUndef;

2362

}

2363

2364

bool TargetLowering::SimplifyDemandedVectorElts(

2365

SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,

2366

APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,

2367

bool AssumeSingleUse) const {

2368

EVT VT = Op.getValueType();

2369

unsigned Opcode = Op.getOpcode();

2370

APInt DemandedElts = OriginalDemandedElts;

2371

unsigned NumElts = DemandedElts.getBitWidth();

2372

assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2372, __extension__ __PRETTY_FUNCTION__));

2373

2374

KnownUndef = KnownZero = APInt::getNullValue(NumElts);

2375

2376

// TODO: For now we assume we know nothing about scalable vectors.

2377

if (VT.isScalableVector())

2378

return false;

2379

2380

assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2381, __extension__ __PRETTY_FUNCTION__))

2381

"Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2381, __extension__ __PRETTY_FUNCTION__));

2382

2383

// Undef operand.

2384

if (Op.isUndef()) {

2385

KnownUndef.setAllBits();

2386

return false;

2387

}

2388

2389

// If Op has other users, assume that all elements are needed.

2390

if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)

2391

DemandedElts.setAllBits();

2392

2393

// Not demanding any elements from Op.

2394

if (DemandedElts == 0) {

2395

KnownUndef.setAllBits();

2396

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2397

}

2398

2399

// Limit search depth.

2400

if (Depth >= SelectionDAG::MaxRecursionDepth)

2401

return false;

2402

2403

SDLoc DL(Op);

2404

unsigned EltSizeInBits = VT.getScalarSizeInBits();

2405

2406

// Helper for demanding the specified elements and all the bits of both binary

2407

// operands.

2408

auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {

2409

SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,

2410

TLO.DAG, Depth + 1);

2411

SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,

2412

TLO.DAG, Depth + 1);

2413

if (NewOp0 || NewOp1) {

2414

SDValue NewOp = TLO.DAG.getNode(

2415

Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);

2416

return TLO.CombineTo(Op, NewOp);

2417

}

2418

return false;

2419

};

2420

2421

switch (Opcode) {

2422

case ISD::SCALAR_TO_VECTOR: {

2423

if (!DemandedElts[0]) {

2424

KnownUndef.setAllBits();

2425

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2426

}

2427

SDValue ScalarSrc = Op.getOperand(0);

2428

if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {

2429

SDValue Src = ScalarSrc.getOperand(0);

2430

SDValue Idx = ScalarSrc.getOperand(1);

2431

EVT SrcVT = Src.getValueType();

2432

2433

ElementCount SrcEltCnt = SrcVT.getVectorElementCount();

2434

2435

if (SrcEltCnt.isScalable())

2436

return false;

2437

2438

unsigned NumSrcElts = SrcEltCnt.getFixedValue();

2439

if (isNullConstant(Idx)) {

2440

APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);

2441

APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);

2442

APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);

2443

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2444

TLO, Depth + 1))

2445

return true;

2446

}

2447

}

2448

KnownUndef.setHighBits(NumElts - 1);

2449

break;

2450

}

2451

case ISD::BITCAST: {

2452

SDValue Src = Op.getOperand(0);

2453

EVT SrcVT = Src.getValueType();

2454

2455

// We only handle vectors here.

2456

// TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?

2457

if (!SrcVT.isVector())

2458

break;

2459

2460

// Fast handling of 'identity' bitcasts.

2461

unsigned NumSrcElts = SrcVT.getVectorNumElements();

2462

if (NumSrcElts == NumElts)

2463

return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,

2464

KnownZero, TLO, Depth + 1);

2465

2466

APInt SrcZero, SrcUndef;

2467

APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);

2468

2469

// Bitcast from 'large element' src vector to 'small element' vector, we

2470

// must demand a source element if any DemandedElt maps to it.

2471

if ((NumElts % NumSrcElts) == 0) {

2472

unsigned Scale = NumElts / NumSrcElts;

2473

for (unsigned i = 0; i != NumElts; ++i)

2474

if (DemandedElts[i])

2475

SrcDemandedElts.setBit(i / Scale);

2476

2477

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2478

TLO, Depth + 1))

2479

return true;

2480

2481

// Try calling SimplifyDemandedBits, converting demanded elts to the bits

2482

// of the large element.

2483

// TODO - bigendian once we have test coverage.

2484

if (TLO.DAG.getDataLayout().isLittleEndian()) {

2485

unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();

2486

APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);

2487

for (unsigned i = 0; i != NumElts; ++i)

2488

if (DemandedElts[i]) {

2489

unsigned Ofs = (i % Scale) * EltSizeInBits;

2490

SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);

2491

}

2492

2493

KnownBits Known;

2494

if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,

2495

TLO, Depth + 1))

2496

return true;

2497

}

2498

2499

// If the src element is zero/undef then all the output elements will be -

2500

// only demanded elements are guaranteed to be correct.

2501

for (unsigned i = 0; i != NumSrcElts; ++i) {

2502

if (SrcDemandedElts[i]) {

2503

if (SrcZero[i])

2504

KnownZero.setBits(i * Scale, (i + 1) * Scale);

2505

if (SrcUndef[i])

2506

KnownUndef.setBits(i * Scale, (i + 1) * Scale);

2507

}

2508

}

2509

}

2510

2511

// Bitcast from 'small element' src vector to 'large element' vector, we

2512

// demand all smaller source elements covered by the larger demanded element

2513

// of this vector.

2514

if ((NumSrcElts % NumElts) == 0) {

2515

unsigned Scale = NumSrcElts / NumElts;

2516

for (unsigned i = 0; i != NumElts; ++i)

2517

if (DemandedElts[i])

2518

SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);

2519

2520

if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,

2521

TLO, Depth + 1))

2522

return true;

2523

2524

// If all the src elements covering an output element are zero/undef, then

2525

// the output element will be as well, assuming it was demanded.

2526

for (unsigned i = 0; i != NumElts; ++i) {

2527

if (DemandedElts[i]) {

2528

if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())

2529

KnownZero.setBit(i);

2530

if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())

2531

KnownUndef.setBit(i);

2532

}

2533

}

2534

}

2535

break;

2536

}

2537

case ISD::BUILD_VECTOR: {

2538

// Check all elements and simplify any unused elements with UNDEF.

2539

if (!DemandedElts.isAllOnesValue()) {

2540

// Don't simplify BROADCASTS.

2541

if (llvm::any_of(Op->op_values(),

2542

[&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {

2543

SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());

2544

bool Updated = false;

2545

for (unsigned i = 0; i != NumElts; ++i) {

2546

if (!DemandedElts[i] && !Ops[i].isUndef()) {

2547

Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());

2548

KnownUndef.setBit(i);

2549

Updated = true;

2550

}

2551

}

2552

if (Updated)

2553

return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));

2554

}

2555

}

2556

for (unsigned i = 0; i != NumElts; ++i) {

2557

SDValue SrcOp = Op.getOperand(i);

2558

if (SrcOp.isUndef()) {

2559

KnownUndef.setBit(i);

2560

} else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&

2561

(isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {

2562

KnownZero.setBit(i);

2563

}

2564

}

2565

break;

2566

}

2567

case ISD::CONCAT_VECTORS: {

2568

EVT SubVT = Op.getOperand(0).getValueType();

2569

unsigned NumSubVecs = Op.getNumOperands();

2570

unsigned NumSubElts = SubVT.getVectorNumElements();

2571

for (unsigned i = 0; i != NumSubVecs; ++i) {

2572

SDValue SubOp = Op.getOperand(i);

2573

APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);

2574

APInt SubUndef, SubZero;

2575

if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,

2576

Depth + 1))

2577

return true;

2578

KnownUndef.insertBits(SubUndef, i * NumSubElts);

2579

KnownZero.insertBits(SubZero, i * NumSubElts);

2580

}

2581

break;

2582

}

2583

case ISD::INSERT_SUBVECTOR: {

2584

// Demand any elements from the subvector and the remainder from the src its

2585

// inserted into.

2586

SDValue Src = Op.getOperand(0);

2587

SDValue Sub = Op.getOperand(1);

2588

uint64_t Idx = Op.getConstantOperandVal(2);

2589

unsigned NumSubElts = Sub.getValueType().getVectorNumElements();

2590

APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);

2591

APInt DemandedSrcElts = DemandedElts;

2592

DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);

2593

2594

APInt SubUndef, SubZero;

2595

if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,

2596

Depth + 1))

2597

return true;

2598

2599

// If none of the src operand elements are demanded, replace it with undef.

2600

if (!DemandedSrcElts && !Src.isUndef())

2601

return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,

2602

TLO.DAG.getUNDEF(VT), Sub,

2603

Op.getOperand(2)));

2604

2605

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,

2606

TLO, Depth + 1))

2607

return true;

2608

KnownUndef.insertBits(SubUndef, Idx);

2609

KnownZero.insertBits(SubZero, Idx);

2610

2611

// Attempt to avoid multi-use ops if we don't need anything from them.

2612

if (!DemandedSrcElts.isAllOnesValue() ||

2613

!DemandedSubElts.isAllOnesValue()) {

2614

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2615

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2616

SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(

2617

Sub, DemandedSubElts, TLO.DAG, Depth + 1);

2618

if (NewSrc || NewSub) {

2619

NewSrc = NewSrc ? NewSrc : Src;

2620

NewSub = NewSub ? NewSub : Sub;

2621

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2622

NewSub, Op.getOperand(2));

2623

return TLO.CombineTo(Op, NewOp);

2624

}

2625

}

2626

break;

2627

}

2628

case ISD::EXTRACT_SUBVECTOR: {

2629

// Offset the demanded elts by the subvector index.

2630

SDValue Src = Op.getOperand(0);

2631

if (Src.getValueType().isScalableVector())

2632

break;

2633

uint64_t Idx = Op.getConstantOperandVal(1);

2634

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2635

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

2636

2637

APInt SrcUndef, SrcZero;

2638

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2639

Depth + 1))

2640

return true;

2641

KnownUndef = SrcUndef.extractBits(NumElts, Idx);

2642

KnownZero = SrcZero.extractBits(NumElts, Idx);

2643

2644

// Attempt to avoid multi-use ops if we don't need anything from them.

2645

if (!DemandedElts.isAllOnesValue()) {

2646

SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(

2647

Src, DemandedSrcElts, TLO.DAG, Depth + 1);

2648

if (NewSrc) {

2649

SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,

2650

Op.getOperand(1));

2651

return TLO.CombineTo(Op, NewOp);

2652

}

2653

}

2654

break;

2655

}

2656

case ISD::INSERT_VECTOR_ELT: {

2657

SDValue Vec = Op.getOperand(0);

2658

SDValue Scl = Op.getOperand(1);

2659

auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

2660

2661

// For a legal, constant insertion index, if we don't need this insertion

2662

// then strip it, else remove it from the demanded elts.

2663

if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {

2664

unsigned Idx = CIdx->getZExtValue();

2665

if (!DemandedElts[Idx])

2666

return TLO.CombineTo(Op, Vec);

2667

2668

APInt DemandedVecElts(DemandedElts);

2669

DemandedVecElts.clearBit(Idx);

2670

if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,

2671

KnownZero, TLO, Depth + 1))

2672

return true;

2673

2674

KnownUndef.setBitVal(Idx, Scl.isUndef());

2675

2676

KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));

2677

break;

2678

}

2679

2680

APInt VecUndef, VecZero;

2681

if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,

2682

Depth + 1))

2683

return true;

2684

// Without knowing the insertion index we can't set KnownUndef/KnownZero.

2685

break;

2686

}

2687

case ISD::VSELECT: {

2688

// Try to transform the select condition based on the current demanded

2689

// elements.

2690

// TODO: If a condition element is undef, we can choose from one arm of the

2691

// select (and if one arm is undef, then we can propagate that to the

2692

// result).

2693

// TODO - add support for constant vselect masks (see IR version of this).

2694

APInt UnusedUndef, UnusedZero;

2695

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,

2696

UnusedZero, TLO, Depth + 1))

2697

return true;

2698

2699

// See if we can simplify either vselect operand.

2700

APInt DemandedLHS(DemandedElts);

2701

APInt DemandedRHS(DemandedElts);

2702

APInt UndefLHS, ZeroLHS;

2703

APInt UndefRHS, ZeroRHS;

2704

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,

2705

ZeroLHS, TLO, Depth + 1))

2706

return true;

2707

if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,

2708

ZeroRHS, TLO, Depth + 1))

2709

return true;

2710

2711

KnownUndef = UndefLHS & UndefRHS;

2712

KnownZero = ZeroLHS & ZeroRHS;

2713

break;

2714

}

2715

case ISD::VECTOR_SHUFFLE: {

2716

ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

2717

2718

// Collect demanded elements from shuffle operands..

2719

APInt DemandedLHS(NumElts, 0);

2720

APInt DemandedRHS(NumElts, 0);

2721

for (unsigned i = 0; i != NumElts; ++i) {

2722

int M = ShuffleMask[i];

2723

if (M < 0 || !DemandedElts[i])

2724

continue;

2725

assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2725, __extension__ __PRETTY_FUNCTION__));

2726

if (M < (int)NumElts)

2727

DemandedLHS.setBit(M);

2728

else

2729

DemandedRHS.setBit(M - NumElts);

2730

}

2731

2732

// See if we can simplify either shuffle operand.

2733

APInt UndefLHS, ZeroLHS;

2734

APInt UndefRHS, ZeroRHS;

2735

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,

2736

ZeroLHS, TLO, Depth + 1))

2737

return true;

2738

if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,

2739

ZeroRHS, TLO, Depth + 1))

2740

return true;

2741

2742

// Simplify mask using undef elements from LHS/RHS.

2743

bool Updated = false;

2744

bool IdentityLHS = true, IdentityRHS = true;

2745

SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());

2746

for (unsigned i = 0; i != NumElts; ++i) {

2747

int &M = NewMask[i];

2748

if (M < 0)

2749

continue;

2750

if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||

2751

(M >= (int)NumElts && UndefRHS[M - NumElts])) {

2752

Updated = true;

2753

M = -1;

2754

}

2755

IdentityLHS &= (M < 0) || (M == (int)i);

2756

IdentityRHS &= (M < 0) || ((M - NumElts) == i);

2757

}

2758

2759

// Update legal shuffle masks based on demanded elements if it won't reduce

2760

// to Identity which can cause premature removal of the shuffle mask.

2761

if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {

2762

SDValue LegalShuffle =

2763

buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),

2764

NewMask, TLO.DAG);

2765

if (LegalShuffle)

2766

return TLO.CombineTo(Op, LegalShuffle);

2767

}

2768

2769

// Propagate undef/zero elements from LHS/RHS.

2770

for (unsigned i = 0; i != NumElts; ++i) {

2771

int M = ShuffleMask[i];

2772

if (M < 0) {

2773

KnownUndef.setBit(i);

2774

} else if (M < (int)NumElts) {

2775

if (UndefLHS[M])

2776

KnownUndef.setBit(i);

2777

if (ZeroLHS[M])

2778

KnownZero.setBit(i);

2779

} else {

2780

if (UndefRHS[M - NumElts])

2781

KnownUndef.setBit(i);

2782

if (ZeroRHS[M - NumElts])

2783

KnownZero.setBit(i);

2784

}

2785

}

2786

break;

2787

}

2788

case ISD::ANY_EXTEND_VECTOR_INREG:

2789

case ISD::SIGN_EXTEND_VECTOR_INREG:

2790

case ISD::ZERO_EXTEND_VECTOR_INREG: {

2791

APInt SrcUndef, SrcZero;

2792

SDValue Src = Op.getOperand(0);

2793

unsigned NumSrcElts = Src.getValueType().getVectorNumElements();

2794

APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);

2795

if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,

2796

Depth + 1))

2797

return true;

2798

KnownZero = SrcZero.zextOrTrunc(NumElts);

2799

KnownUndef = SrcUndef.zextOrTrunc(NumElts);

2800

2801

if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&

2802

Op.getValueSizeInBits() == Src.getValueSizeInBits() &&

2803

DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {

2804

// aext - if we just need the bottom element then we can bitcast.

2805

return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

2806

}

2807

2808

if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {

2809

// zext(undef) upper bits are guaranteed to be zero.

2810

if (DemandedElts.isSubsetOf(KnownUndef))

2811

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2812

KnownUndef.clearAllBits();

2813

}

2814

break;

2815

}

2816

2817

// TODO: There are more binop opcodes that could be handled here - MIN,

2818

// MAX, saturated math, etc.

2819

case ISD::OR:

2820

case ISD::XOR:

2821

case ISD::ADD:

2822

case ISD::SUB:

2823

case ISD::FADD:

2824

case ISD::FSUB:

2825

case ISD::FMUL:

2826

case ISD::FDIV:

2827

case ISD::FREM: {

2828

SDValue Op0 = Op.getOperand(0);

2829

SDValue Op1 = Op.getOperand(1);

2830

2831

APInt UndefRHS, ZeroRHS;

2832

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2833

Depth + 1))

2834

return true;

2835

APInt UndefLHS, ZeroLHS;

2836

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2837

Depth + 1))

2838

return true;

2839

2840

KnownZero = ZeroLHS & ZeroRHS;

2841

KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

2842

2843

// Attempt to avoid multi-use ops if we don't need anything from them.

2844

// TODO - use KnownUndef to relax the demandedelts?

2845

if (!DemandedElts.isAllOnesValue())

2846

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2847

return true;

2848

break;

2849

}

2850

case ISD::SHL:

2851

case ISD::SRL:

2852

case ISD::SRA:

2853

case ISD::ROTL:

2854

case ISD::ROTR: {

2855

SDValue Op0 = Op.getOperand(0);

2856

SDValue Op1 = Op.getOperand(1);

2857

2858

APInt UndefRHS, ZeroRHS;

2859

if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,

2860

Depth + 1))

2861

return true;

2862

APInt UndefLHS, ZeroLHS;

2863

if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,

2864

Depth + 1))

2865

return true;

2866

2867

KnownZero = ZeroLHS;

2868

KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

2869

2870

// Attempt to avoid multi-use ops if we don't need anything from them.

2871

// TODO - use KnownUndef to relax the demandedelts?

2872

if (!DemandedElts.isAllOnesValue())

2873

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2874

return true;

2875

break;

2876

}

2877

case ISD::MUL:

2878

case ISD::AND: {

2879

SDValue Op0 = Op.getOperand(0);

2880

SDValue Op1 = Op.getOperand(1);

2881

2882

APInt SrcUndef, SrcZero;

2883

if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,

2884

Depth + 1))

2885

return true;

2886

if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,

2887

TLO, Depth + 1))

2888

return true;

2889

2890

// If either side has a zero element, then the result element is zero, even

2891

// if the other is an UNDEF.

2892

// TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros

2893

// and then handle 'and' nodes with the rest of the binop opcodes.

2894

KnownZero |= SrcZero;

2895

KnownUndef &= SrcUndef;

2896

KnownUndef &= ~KnownZero;

2897

2898

// Attempt to avoid multi-use ops if we don't need anything from them.

2899

// TODO - use KnownUndef to relax the demandedelts?

2900

if (!DemandedElts.isAllOnesValue())

2901

if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))

2902

return true;

2903

break;

2904

}

2905

case ISD::TRUNCATE:

2906

case ISD::SIGN_EXTEND:

2907

case ISD::ZERO_EXTEND:

2908

if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,

2909

KnownZero, TLO, Depth + 1))

2910

return true;

2911

2912

if (Op.getOpcode() == ISD::ZERO_EXTEND) {

2913

// zext(undef) upper bits are guaranteed to be zero.

2914

if (DemandedElts.isSubsetOf(KnownUndef))

2915

return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

2916

KnownUndef.clearAllBits();

2917

}

2918

break;

2919

default: {

2920

if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {

2921

if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,

2922

KnownZero, TLO, Depth))

2923

return true;

2924

} else {

2925

KnownBits Known;

2926

APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);

2927

if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,

2928

TLO, Depth, AssumeSingleUse))

2929

return true;

2930

}

2931

break;

2932

}

2933

}

2934

assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2934, __extension__ __PRETTY_FUNCTION__));

2935

2936

// Constant fold all undef cases.

2937

// TODO: Handle zero cases as well.

2938

if (DemandedElts.isSubsetOf(KnownUndef))

2939

return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

2940

2941

return false;

2942

}

2943

2944

/// Determine which of the bits specified in Mask are known to be either zero or

2945

/// one and return them in the Known.

2946

void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,

2947

KnownBits &Known,

2948

const APInt &DemandedElts,

2949

const SelectionDAG &DAG,

2950

unsigned Depth) const {

2951

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2952

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2953

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2954

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2955

"Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__))

2956

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2956, __extension__ __PRETTY_FUNCTION__));

2957

Known.resetAll();

2958

}

2959

2960

void TargetLowering::computeKnownBitsForTargetInstr(

2961

GISelKnownBits &Analysis, Register R, KnownBits &Known,

2962

const APInt &DemandedElts, const MachineRegisterInfo &MRI,

2963

unsigned Depth) const {

2964

Known.resetAll();

2965

}

2966

2967

void TargetLowering::computeKnownBitsForFrameIndex(

2968

const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {

2969

// The low bits are known zero if the pointer is aligned.

2970

Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));

2971

}

2972

2973

Align TargetLowering::computeKnownAlignForTargetInstr(

2974

GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,

2975

unsigned Depth) const {

2976

return Align(1);

2977

}

2978

2979

/// This method can be implemented by targets that want to expose additional

2980

/// information about sign bits to the DAG Combiner.

2981

unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,

2982

const APInt &,

2983

const SelectionDAG &,

2984

unsigned Depth) const {

2985

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2986

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2987

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2988

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2989

"Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__))

2990

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __extension__ __PRETTY_FUNCTION__));

2991

return 1;

2992

}

2993

2994

unsigned TargetLowering::computeNumSignBitsForTargetInstr(

2995

GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,

2996

const MachineRegisterInfo &MRI, unsigned Depth) const {

2997

return 1;

2998

}

2999

3000

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(

3001

SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,

3002

TargetLoweringOpt &TLO, unsigned Depth) const {

3003

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3004

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3005

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3006

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3007

"Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__))

3008

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __extension__ __PRETTY_FUNCTION__));

3009

return false;

3010

}

3011

3012

bool TargetLowering::SimplifyDemandedBitsForTargetNode(

3013

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3014

KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {

3015

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3016

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3017

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3018

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3019

"Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__))

3020

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3020, __extension__ __PRETTY_FUNCTION__));

3021

computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);

3022

return false;

3023

}

3024

3025

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(

3026

SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,

3027

SelectionDAG &DAG, unsigned Depth) const {

3028

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3029

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3030

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3031

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3032

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3033

"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__))

3034

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3034, __extension__ __PRETTY_FUNCTION__));

3035

return SDValue();

3036

}

3037

3038

SDValue

3039

TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,

3040

SDValue N1, MutableArrayRef<int> Mask,

3041

SelectionDAG &DAG) const {

3042

bool LegalMask = isShuffleMaskLegal(Mask, VT);

3043

if (!LegalMask) {

3044

std::swap(N0, N1);

3045

ShuffleVectorSDNode::commuteMask(Mask);

3046

LegalMask = isShuffleMaskLegal(Mask, VT);

3047

}

3048

3049

if (!LegalMask)

3050

return SDValue();

3051

3052

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);

3053

}

3054

3055

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {

3056

return nullptr;

3057

}

3058

3059

bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(

3060

SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,

3061

bool PoisonOnly, unsigned Depth) const {

3062

assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3063

(Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3064

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3065

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3066

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3067

"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__))

3068

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3068, __extension__ __PRETTY_FUNCTION__));

3069

return false;

3070

}

3071

3072

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,

3073

const SelectionDAG &DAG,

3074

bool SNaN,

3075

unsigned Depth) const {

3076

assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3077

Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3078

Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3079

Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3080

"Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__))

3081

" is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __extension__ __PRETTY_FUNCTION__));

3082

return false;

3083

}

3084

3085

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must

3086

// work with truncating build vectors and vectors with elements of less than

3087

// 8 bits.

3088

bool TargetLowering::isConstTrueVal(const SDNode *N) const {

3089

if (!N)

3090

return false;

3091

3092

APInt CVal;

3093

if (auto *CN = dyn_cast<ConstantSDNode>(N)) {

3094

CVal = CN->getAPIntValue();

3095

} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {

3096

auto *CN = BV->getConstantSplatNode();

3097

if (!CN)

3098

return false;

3099

3100

// If this is a truncating build vector, truncate the splat value.

3101

// Otherwise, we may fail to match the expected values below.

3102

unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();

3103

CVal = CN->getAPIntValue();

3104

if (BVEltWidth < CVal.getBitWidth())

3105

CVal = CVal.trunc(BVEltWidth);

3106

} else {

3107

return false;

3108

}

3109

3110

switch (getBooleanContents(N->getValueType(0))) {

3111

case UndefinedBooleanContent:

3112

return CVal[0];

3113

case ZeroOrOneBooleanContent:

3114

return CVal.isOneValue();

3115

case ZeroOrNegativeOneBooleanContent:

3116

return CVal.isAllOnesValue();

3117

}

3118

3119

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3119);

3120

}

3121

3122

bool TargetLowering::isConstFalseVal(const SDNode *N) const {

3123

if (!N)

3124

return false;

3125

3126

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);

3127

if (!CN) {

3128

const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);

3129

if (!BV)

3130

return false;

3131

3132

// Only interested in constant splats, we don't care about undef

3133

// elements in identifying boolean constants and getConstantSplatNode

3134

// returns NULL if all ops are undef;

3135

CN = BV->getConstantSplatNode();

3136

if (!CN)

3137

return false;

3138

}

3139

3140

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)

3141

return !CN->getAPIntValue()[0];

3142

3143

return CN->isNullValue();

3144

}

3145

3146

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,

3147

bool SExt) const {

3148

if (VT == MVT::i1)

3149

return N->isOne();

3150

3151

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);

3152

switch (Cnt) {

3153

case TargetLowering::ZeroOrOneBooleanContent:

3154

// An extended value of 1 is always true, unless its original type is i1,

3155

// in which case it will be sign extended to -1.

3156

return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));

3157

case TargetLowering::UndefinedBooleanContent:

3158

case TargetLowering::ZeroOrNegativeOneBooleanContent:

3159

return N->isAllOnesValue() && SExt;

3160

}

3161

llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3161);

3162

}

3163

3164

/// This helper function of SimplifySetCC tries to optimize the comparison when

3165

/// either operand of the SetCC node is a bitwise-and instruction.

3166

SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,

3167

ISD::CondCode Cond, const SDLoc &DL,

3168

DAGCombinerInfo &DCI) const {

3169

// Match these patterns in any of their permutations:

3170

// (X & Y) == Y

3171

// (X & Y) != Y

3172

if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)

3173

std::swap(N0, N1);

3174

3175

EVT OpVT = N0.getValueType();

3176

if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||

3177

(Cond != ISD::SETEQ && Cond != ISD::SETNE))

3178

return SDValue();

3179

3180

SDValue X, Y;

3181

if (N0.getOperand(0) == N1) {

3182

X = N0.getOperand(1);

3183

Y = N0.getOperand(0);

3184

} else if (N0.getOperand(1) == N1) {

3185

X = N0.getOperand(0);

3186

Y = N0.getOperand(1);

3187

} else {

3188

return SDValue();

3189

}

3190

3191

SelectionDAG &DAG = DCI.DAG;

3192

SDValue Zero = DAG.getConstant(0, DL, OpVT);

3193

if (DAG.isKnownToBeAPowerOfTwo(Y)) {

3194

// Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.

3195

// Note that where Y is variable and is known to have at most one bit set

3196

// (for example, if it is Z & 1) we cannot do this; the expressions are not

3197

// equivalent when Y == 0.

3198

assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3198, __extension__ __PRETTY_FUNCTION__));

3199

Cond = ISD::getSetCCInverse(Cond, OpVT);

3200

if (DCI.isBeforeLegalizeOps() ||

3201

isCondCodeLegal(Cond, N0.getSimpleValueType()))

3202

return DAG.getSetCC(DL, VT, N0, Zero, Cond);

3203

} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {

3204

// If the target supports an 'and-not' or 'and-complement' logic operation,

3205

// try to use that to make a comparison operation more efficient.

3206

// But don't do this transform if the mask is a single bit because there are

3207

// more efficient ways to deal with that case (for example, 'bt' on x86 or

3208

// 'rlwinm' on PPC).

3209

3210

// Bail out if the compare operand that we want to turn into a zero is

3211

// already a zero (otherwise, infinite loop).

3212

auto *YConst = dyn_cast<ConstantSDNode>(Y);

3213

if (YConst && YConst->isNullValue())

3214

return SDValue();

3215

3216

// Transform this into: ~X & Y == 0.

3217

SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);

3218

SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);

3219

return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);

3220

}

3221

3222

return SDValue();

3223

}

3224

3225

/// There are multiple IR patterns that could be checking whether certain

3226

/// truncation of a signed number would be lossy or not. The pattern which is

3227

/// best at IR level, may not lower optimally. Thus, we want to unfold it.

3228

/// We are looking for the following pattern: (KeptBits is a constant)

3229

/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)

3230

/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.

3231

/// KeptBits also can't be 1, that would have been folded to %x dstcond 0

3232

/// We will unfold it into the natural trunc+sext pattern:

3233

/// ((%x << C) a>> C) dstcond %x

3234

/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)

3235

SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(

3236

EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,

3237

const SDLoc &DL) const {

3238

// We must be comparing with a constant.

3239

ConstantSDNode *C1;

3240

if (!(C1 = dyn_cast<ConstantSDNode>(N1)))

3241

return SDValue();

3242

3243

// N0 should be: add %x, (1 << (KeptBits-1))

3244

if (N0->getOpcode() != ISD::ADD)

3245

return SDValue();

3246

3247

// And we must be 'add'ing a constant.

3248

ConstantSDNode *C01;

3249

if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))

3250

return SDValue();

3251

3252

SDValue X = N0->getOperand(0);

3253

EVT XVT = X.getValueType();

3254

3255

// Validate constants ...

3256

3257

APInt I1 = C1->getAPIntValue();

3258

3259

ISD::CondCode NewCond;

3260

if (Cond == ISD::CondCode::SETULT) {

3261

NewCond = ISD::CondCode::SETEQ;

3262

} else if (Cond == ISD::CondCode::SETULE) {

3263

NewCond = ISD::CondCode::SETEQ;

3264

// But need to 'canonicalize' the constant.

3265

I1 += 1;

3266

} else if (Cond == ISD::CondCode::SETUGT) {

3267

NewCond = ISD::CondCode::SETNE;

3268

// But need to 'canonicalize' the constant.

3269

I1 += 1;

3270

} else if (Cond == ISD::CondCode::SETUGE) {

3271

NewCond = ISD::CondCode::SETNE;

3272

} else

3273

return SDValue();

3274

3275

APInt I01 = C01->getAPIntValue();

3276

3277

auto checkConstants = [&I1, &I01]() -> bool {

3278

// Both of them must be power-of-two, and the constant from setcc is bigger.

3279

return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();

3280

};

3281

3282

if (checkConstants()) {

3283

// Great, e.g. got icmp ult i16 (add i16 %x, 128), 256

3284

} else {

3285

// What if we invert constants? (and the target predicate)

3286

I1.negate();

3287

I01.negate();

3288

assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3288, __extension__ __PRETTY_FUNCTION__));

3289

NewCond = getSetCCInverse(NewCond, XVT);

3290

if (!checkConstants())

3291

return SDValue();

3292

// Great, e.g. got icmp uge i16 (add i16 %x, -128), -256

3293

}

3294

3295

// They are power-of-two, so which bit is set?

3296

const unsigned KeptBits = I1.logBase2();

3297

const unsigned KeptBitsMinusOne = I01.logBase2();

3298

3299

// Magic!

3300

if (KeptBits != (KeptBitsMinusOne + 1))

3301

return SDValue();

3302

assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3302, __extension__ __PRETTY_FUNCTION__));

3303

3304

// We don't want to do this in every single case.

3305

SelectionDAG &DAG = DCI.DAG;

3306

if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(

3307

XVT, KeptBits))

3308

return SDValue();

3309

3310

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;

3311

assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3311, __extension__ __PRETTY_FUNCTION__));

3312

3313

// Unfold into: ((%x << C) a>> C) cond %x

3314

// Where 'cond' will be either 'eq' or 'ne'.

3315

SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);

3316

SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);

3317

SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);

3318

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

3319

3320

return T2;

3321

}

3322

3323

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

3324

SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(

3325

EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,

3326

DAGCombinerInfo &DCI, const SDLoc &DL) const {

3327

assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__))

3328

isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__))

3329

"Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3329, __extension__ __PRETTY_FUNCTION__));

3330

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__))

3331

"Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3331, __extension__ __PRETTY_FUNCTION__));

3332

3333

unsigned NewShiftOpcode;

3334

SDValue X, C, Y;

3335

3336

SelectionDAG &DAG = DCI.DAG;

3337

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

3338

3339

// Look for '(C l>>/<< Y)'.

3340

auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {

3341

// The shift should be one-use.

3342

if (!V.hasOneUse())

3343

return false;

3344

unsigned OldShiftOpcode = V.getOpcode();

3345

switch (OldShiftOpcode) {

3346

case ISD::SHL:

3347

NewShiftOpcode = ISD::SRL;

3348

break;

3349

case ISD::SRL:

3350

NewShiftOpcode = ISD::SHL;

3351

break;

3352

default:

3353

return false; // must be a logical shift.

3354

}

3355

// We should be shifting a constant.

3356

// FIXME: best to use isConstantOrConstantVector().

3357

C = V.getOperand(0);

3358

ConstantSDNode *CC =

3359

isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3360

if (!CC)

3361

return false;

3362

Y = V.getOperand(1);

3363

3364

ConstantSDNode *XC =

3365

isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);

3366

return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(

3367

X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);

3368

};

3369

3370

// LHS of comparison should be an one-use 'and'.

3371

if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())

3372

return SDValue();

3373

3374

X = N0.getOperand(0);

3375

SDValue Mask = N0.getOperand(1);

3376

3377

// 'and' is commutative!

3378

if (!Match(Mask)) {

3379

std::swap(X, Mask);

3380

if (!Match(Mask))

3381

return SDValue();

3382

}

3383

3384

EVT VT = X.getValueType();

3385

3386

// Produce:

3387

// ((X 'OppositeShiftOpcode' Y) & C) Cond 0

3388

SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);

3389

SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);

3390

SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);

3391

return T2;

3392

}

3393

3394

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as

3395

/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to

3396

/// handle the commuted versions of these patterns.

3397

SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,

3398

ISD::CondCode Cond, const SDLoc &DL,

3399

DAGCombinerInfo &DCI) const {

3400

unsigned BOpcode = N0.getOpcode();

3401

assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__))

3402

"Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__));

3403

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3403, __extension__ __PRETTY_FUNCTION__));

3404

3405

// (X + Y) == X --> Y == 0

3406

// (X - Y) == X --> Y == 0

3407

// (X ^ Y) == X --> Y == 0

3408

SelectionDAG &DAG = DCI.DAG;

3409

EVT OpVT = N0.getValueType();

3410

SDValue X = N0.getOperand(0);

3411

SDValue Y = N0.getOperand(1);

3412

if (X == N1)

3413

return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

3414

3415

if (Y != N1)

3416

return SDValue();

3417

3418

// (X + Y) == Y --> X == 0

3419

// (X ^ Y) == Y --> X == 0

3420

if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)

3421

return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

3422

3423

// The shift would not be valid if the operands are boolean (i1).

3424

if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)

3425

return SDValue();

3426

3427

// (X - Y) == Y --> X == Y << 1

3428

EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),

3429

!DCI.isBeforeLegalize());

3430

SDValue One = DAG.getConstant(1, DL, ShiftVT);

3431

SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);

3432

if (!DCI.isCalledByLegalizer())

3433

DCI.AddToWorklist(YShl1.getNode());

3434

return DAG.getSetCC(DL, VT, X, YShl1, Cond);

3435

}

3436

3437

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,

3438

SDValue N0, const APInt &C1,

3439

ISD::CondCode Cond, const SDLoc &dl,

3440

SelectionDAG &DAG) {

3441

// Look through truncs that don't change the value of a ctpop.

3442

// FIXME: Add vector support? Need to be careful with setcc result type below.

3443

SDValue CTPOP = N0;

3444

if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&

3445

N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))

3446

CTPOP = N0.getOperand(0);

3447

3448

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())

3449

return SDValue();

3450

3451

EVT CTVT = CTPOP.getValueType();

3452

SDValue CTOp = CTPOP.getOperand(0);

3453

3454

// If this is a vector CTPOP, keep the CTPOP if it is legal.

3455

// TODO: Should we check if CTPOP is legal(or custom) for scalars?

3456

if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))

3457

return SDValue();

3458

3459

// (ctpop x) u< 2 -> (x & x-1) == 0

3460

// (ctpop x) u> 1 -> (x & x-1) != 0

3461

if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {

3462

unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);

3463

if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))

3464

return SDValue();

3465

if (C1 == 0 && (Cond == ISD::SETULT))

3466

return SDValue(); // This is handled elsewhere.

3467

3468

unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

3469

3470

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3471

SDValue Result = CTOp;

3472

for (unsigned i = 0; i < Passes; i++) {

3473

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);

3474

Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);

3475

}

3476

ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;

3477

return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);

3478

}

3479

3480

// If ctpop is not supported, expand a power-of-2 comparison based on it.

3481

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {

3482

// For scalars, keep CTPOP if it is legal or custom.

3483

if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))

3484

return SDValue();

3485

// This is based on X86's custom lowering for CTPOP which produces more

3486

// instructions than the expansion here.

3487

3488

// (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)

3489

// (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)

3490

SDValue Zero = DAG.getConstant(0, dl, CTVT);

3491

SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);

3492

assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3492, __extension__ __PRETTY_FUNCTION__));

3493

ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);

3494

SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

3495

SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);

3496

SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);

3497

SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);

3498

unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;

3499

return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);

3500

}

3501

3502

return SDValue();

3503

}

3504

3505

/// Try to simplify a setcc built with the specified operands and cc. If it is

3506

/// unable to simplify it, return a null SDValue.

3507

SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,

3508

ISD::CondCode Cond, bool foldBooleans,

3509

DAGCombinerInfo &DCI,

3510

const SDLoc &dl) const {

3511

SelectionDAG &DAG = DCI.DAG;

3512

const DataLayout &Layout = DAG.getDataLayout();

3513

EVT OpVT = N0.getValueType();

3514

3515

// Constant fold or commute setcc.

3516

if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))

3517

return Fold;

3518

3519

// Ensure that the constant occurs on the RHS and fold constant comparisons.

3520

// TODO: Handle non-splat vector constants. All undef causes trouble.

3521

// FIXME: We can't yet fold constant scalable vector splats, so avoid an

3522

// infinite loop here when we encounter one.

3523

ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);

3524

if (isConstOrConstSplat(N0) &&

3525

(!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&

3526

(DCI.isBeforeLegalizeOps() ||

3527

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))

3528

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3529

3530

// If we have a subtract with the same 2 non-constant operands as this setcc

3531

// -- but in reverse order -- then try to commute the operands of this setcc

3532

// to match. A matching pair of setcc (cmp) and sub may be combined into 1

3533

// instruction on some targets.

3534

if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&

3535

(DCI.isBeforeLegalizeOps() ||

3536

isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&

3537

DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&

3538

!DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))

3539

return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

3540

3541

if (auto *N1C = isConstOrConstSplat(N1)) {

3542

const APInt &C1 = N1C->getAPIntValue();

3543

3544

// Optimize some CTPOP cases.

3545

if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))

3546

return V;

3547

3548

// If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an

3549

// equality comparison, then we're just comparing whether X itself is

3550

// zero.

3551

if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&

3552

N0.getOperand(0).getOpcode() == ISD::CTLZ &&

3553

isPowerOf2_32(N0.getScalarValueSizeInBits())) {

3554

if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {

3555

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3556

ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {

3557

if ((C1 == 0) == (Cond == ISD::SETEQ)) {

3558

// (srl (ctlz x), 5) == 0 -> X != 0

3559

// (srl (ctlz x), 5) != 1 -> X != 0

3560

Cond = ISD::SETNE;

3561

} else {

3562

// (srl (ctlz x), 5) != 0 -> X == 0

3563

// (srl (ctlz x), 5) == 1 -> X == 0

3564

Cond = ISD::SETEQ;

3565

}

3566

SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());

3567

return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,

3568

Cond);

3569

}

3570

}

3571

}

3572

}

3573

3574

// FIXME: Support vectors.

3575

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

3576

const APInt &C1 = N1C->getAPIntValue();

3577

3578

// (zext x) == C --> x == (trunc C)

3579

// (sext x) == C --> x == (trunc C)

3580

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3581

DCI.isBeforeLegalize() && N0->hasOneUse()) {

3582

unsigned MinBits = N0.getValueSizeInBits();

3583

SDValue PreExt;

3584

bool Signed = false;

3585

if (N0->getOpcode() == ISD::ZERO_EXTEND) {

3586

// ZExt

3587

MinBits = N0->getOperand(0).getValueSizeInBits();

3588

PreExt = N0->getOperand(0);

3589

} else if (N0->getOpcode() == ISD::AND) {

3590

// DAGCombine turns costly ZExts into ANDs

3591

if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))

3592

if ((C->getAPIntValue()+1).isPowerOf2()) {

3593

MinBits = C->getAPIntValue().countTrailingOnes();

3594

PreExt = N0->getOperand(0);

3595

}

3596

} else if (N0->getOpcode() == ISD::SIGN_EXTEND) {

3597

// SExt

3598

MinBits = N0->getOperand(0).getValueSizeInBits();

3599

PreExt = N0->getOperand(0);

3600

Signed = true;

3601

} else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {

3602

// ZEXTLOAD / SEXTLOAD

3603

if (LN0->getExtensionType() == ISD::ZEXTLOAD) {

3604

MinBits = LN0->getMemoryVT().getSizeInBits();

3605

PreExt = N0;

3606

} else if (LN0->getExtensionType() == ISD::SEXTLOAD) {

3607

Signed = true;

3608

MinBits = LN0->getMemoryVT().getSizeInBits();

3609

PreExt = N0;

3610

}

3611

}

3612

3613

// Figure out how many bits we need to preserve this constant.

3614

unsigned ReqdBits = Signed ?

3615

C1.getBitWidth() - C1.getNumSignBits() + 1 :

3616

C1.getActiveBits();

3617

3618

// Make sure we're not losing bits from the constant.

3619

if (MinBits > 0 &&

3620

MinBits < C1.getBitWidth() &&

3621

MinBits >= ReqdBits) {

3622

EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);

3623

if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {

3624

// Will get folded away.

3625

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);

3626

if (MinBits == 1 && C1 == 1)

3627

// Invert the condition.

3628

return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),

3629

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3630

SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);

3631

return DAG.getSetCC(dl, VT, Trunc, C, Cond);

3632

}

3633

3634

// If truncating the setcc operands is not desirable, we can still

3635

// simplify the expression in some cases:

3636

// setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)

3637

// setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))

3638

// setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))

3639

// setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)

3640

// setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))

3641

// setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)

3642

SDValue TopSetCC = N0->getOperand(0);

3643

unsigned N0Opc = N0->getOpcode();

3644

bool SExt = (N0Opc == ISD::SIGN_EXTEND);

3645

if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&

3646

TopSetCC.getOpcode() == ISD::SETCC &&

3647

(N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&

3648

(isConstFalseVal(N1C) ||

3649

isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

3650

3651

bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||

3652

(!N1C->isNullValue() && Cond == ISD::SETNE);

3653

3654

if (!Inverse)

3655

return TopSetCC;

3656

3657

ISD::CondCode InvCond = ISD::getSetCCInverse(

3658

cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),

3659

TopSetCC.getOperand(0).getValueType());

3660

return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),

3661

TopSetCC.getOperand(1),

3662

InvCond);

3663

}

3664

}

3665

}

3666

3667

// If the LHS is '(and load, const)', the RHS is 0, the test is for

3668

// equality or unsigned, and all 1 bits of the const are in the same

3669

// partial word, see if we can shorten the load.

3670

if (DCI.isBeforeLegalize() &&

3671

!ISD::isSignedIntSetCC(Cond) &&

3672

N0.getOpcode() == ISD::AND && C1 == 0 &&

3673

N0.getNode()->hasOneUse() &&

3674

isa<LoadSDNode>(N0.getOperand(0)) &&

3675

N0.getOperand(0).getNode()->hasOneUse() &&

3676

isa<ConstantSDNode>(N0.getOperand(1))) {

3677

LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));

3678

APInt bestMask;

3679

unsigned bestWidth = 0, bestOffset = 0;

3680

if (Lod->isSimple() && Lod->isUnindexed()) {

3681

unsigned origWidth = N0.getValueSizeInBits();

3682

unsigned maskWidth = origWidth;

3683

// We can narrow (e.g.) 16-bit extending loads on 32-bit target to

3684

// 8 bits, but have to be careful...

3685

if (Lod->getExtensionType() != ISD::NON_EXTLOAD)

3686

origWidth = Lod->getMemoryVT().getSizeInBits();

3687

const APInt &Mask = N0.getConstantOperandAPInt(1);

3688

for (unsigned width = origWidth / 2; width>=8; width /= 2) {

3689

APInt newMask = APInt::getLowBitsSet(maskWidth, width);

3690

for (unsigned offset=0; offset<origWidth/width; offset++) {

3691

if (Mask.isSubsetOf(newMask)) {

3692

if (Layout.isLittleEndian())

3693

bestOffset = (uint64_t)offset * (width/8);

3694

else

3695

bestOffset = (origWidth/width - offset - 1) * (width/8);

3696

bestMask = Mask.lshr(offset * (width/8) * 8);

3697

bestWidth = width;

3698

break;

3699

}

3700

newMask <<= width;

3701

}

3702

}

3703

}

3704

if (bestWidth) {

3705

EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);

3706

if (newVT.isRound() &&

3707

shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {

3708

SDValue Ptr = Lod->getBasePtr();

3709

if (bestOffset != 0)

3710

Ptr =

3711

DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);

3712

SDValue NewLoad =

3713

DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,

3714

Lod->getPointerInfo().getWithOffset(bestOffset),

3715

Lod->getOriginalAlign());

3716

return DAG.getSetCC(dl, VT,

3717

DAG.getNode(ISD::AND, dl, newVT, NewLoad,

3718

DAG.getConstant(bestMask.trunc(bestWidth),

3719

dl, newVT)),

3720

DAG.getConstant(0LL, dl, newVT), Cond);

3721

}

3722

}

3723

}

3724

3725

// If the LHS is a ZERO_EXTEND, perform the comparison on the input.

3726

if (N0.getOpcode() == ISD::ZERO_EXTEND) {

3727

unsigned InSize = N0.getOperand(0).getValueSizeInBits();

3728

3729

// If the comparison constant has bits in the upper part, the

3730

// zero-extended value could never match.

3731

if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),

3732

C1.getBitWidth() - InSize))) {

3733

switch (Cond) {

3734

case ISD::SETUGT:

3735

case ISD::SETUGE:

3736

case ISD::SETEQ:

3737

return DAG.getConstant(0, dl, VT);

3738

case ISD::SETULT:

3739

case ISD::SETULE:

3740

case ISD::SETNE:

3741

return DAG.getConstant(1, dl, VT);

3742

case ISD::SETGT:

3743

case ISD::SETGE:

3744

// True if the sign bit of C1 is set.

3745

return DAG.getConstant(C1.isNegative(), dl, VT);

3746

case ISD::SETLT:

3747

case ISD::SETLE:

3748

// True if the sign bit of C1 isn't set.

3749

return DAG.getConstant(C1.isNonNegative(), dl, VT);

3750

default:

3751

break;

3752

}

3753

}

3754

3755

// Otherwise, we can perform the comparison with the low bits.

3756

switch (Cond) {

3757

case ISD::SETEQ:

3758

case ISD::SETNE:

3759

case ISD::SETUGT:

3760

case ISD::SETUGE:

3761

case ISD::SETULT:

3762

case ISD::SETULE: {

3763

EVT newVT = N0.getOperand(0).getValueType();

3764

if (DCI.isBeforeLegalizeOps() ||

3765

(isOperationLegal(ISD::SETCC, newVT) &&

3766

isCondCodeLegal(Cond, newVT.getSimpleVT()))) {

3767

EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);

3768

SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

3769

3770

SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),

3771

NewConst, Cond);

3772

return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());

3773

}

3774

break;

3775

}

3776

default:

3777

break; // todo, be more careful with signed comparisons

3778

}

3779

} else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&

3780

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

3781

!isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),

3782

OpVT)) {

3783

EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();

3784

unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();

3785

EVT ExtDstTy = N0.getValueType();

3786

unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

3787

3788

// If the constant doesn't fit into the number of bits for the source of

3789

// the sign extension, it is impossible for both sides to be equal.

3790

if (C1.getMinSignedBits() > ExtSrcTyBits)

3791

return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

3792

3793

assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__))

3794

ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__));

3795

APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);

3796

SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),

3797

DAG.getConstant(Imm, dl, ExtDstTy));

3798

if (!DCI.isCalledByLegalizer())

3799

DCI.AddToWorklist(ZextOp.getNode());

3800

// Otherwise, make this a use of a zext.

3801

return DAG.getSetCC(dl, VT, ZextOp,

3802

DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);

3803

} else if ((N1C->isNullValue() || N1C->isOne()) &&

3804

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3805

// SETCC (SETCC), [0|1], [EQ|NE] -> SETCC

3806

if (N0.getOpcode() == ISD::SETCC &&

3807

isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&

3808

(N0.getValueType() == MVT::i1 ||

3809

getBooleanContents(N0.getOperand(0).getValueType()) ==

3810

ZeroOrOneBooleanContent)) {

3811

bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());

3812

if (TrueWhenTrue)

3813

return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);

3814

// Invert the condition.

3815

ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();

3816

CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());

3817

if (DCI.isBeforeLegalizeOps() ||

3818

isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))

3819

return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);

3820

}

3821

3822

if ((N0.getOpcode() == ISD::XOR ||

3823

(N0.getOpcode() == ISD::AND &&

3824

N0.getOperand(0).getOpcode() == ISD::XOR &&

3825

N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&

3826

isOneConstant(N0.getOperand(1))) {

3827

// If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We

3828

// can only do this if the top bits are known zero.

3829

unsigned BitWidth = N0.getValueSizeInBits();

3830

if (DAG.MaskedValueIsZero(N0,

3831

APInt::getHighBitsSet(BitWidth,

3832

BitWidth-1))) {

3833

// Okay, get the un-inverted input value.

3834

SDValue Val;

3835

if (N0.getOpcode() == ISD::XOR) {

3836

Val = N0.getOperand(0);

3837

} else {

3838

assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3839, __extension__ __PRETTY_FUNCTION__))

3839

N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3839, __extension__ __PRETTY_FUNCTION__));

3840

// ((X^1)&1)^1 -> X & 1

3841

Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),

3842

N0.getOperand(0).getOperand(0),

3843

N0.getOperand(1));

3844

}

3845

3846

return DAG.getSetCC(dl, VT, Val, N1,

3847

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3848

}

3849

} else if (N1C->isOne()) {

3850

SDValue Op0 = N0;

3851

if (Op0.getOpcode() == ISD::TRUNCATE)

3852

Op0 = Op0.getOperand(0);

3853

3854

if ((Op0.getOpcode() == ISD::XOR) &&

3855

Op0.getOperand(0).getOpcode() == ISD::SETCC &&

3856

Op0.getOperand(1).getOpcode() == ISD::SETCC) {

3857

SDValue XorLHS = Op0.getOperand(0);

3858

SDValue XorRHS = Op0.getOperand(1);

3859

// Ensure that the input setccs return an i1 type or 0/1 value.

3860

if (Op0.getValueType() == MVT::i1 ||

3861

(getBooleanContents(XorLHS.getOperand(0).getValueType()) ==

3862

ZeroOrOneBooleanContent &&

3863

getBooleanContents(XorRHS.getOperand(0).getValueType()) ==

3864

ZeroOrOneBooleanContent)) {

3865

// (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)

3866

Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;

3867

return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);

3868

}

3869

}

3870

if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {

3871

// If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.

3872

if (Op0.getValueType().bitsGT(VT))

3873

Op0 = DAG.getNode(ISD::AND, dl, VT,

3874

DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),

3875

DAG.getConstant(1, dl, VT));

3876

else if (Op0.getValueType().bitsLT(VT))

3877

Op0 = DAG.getNode(ISD::AND, dl, VT,

3878

DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),

3879

DAG.getConstant(1, dl, VT));

3880

3881

return DAG.getSetCC(dl, VT, Op0,

3882

DAG.getConstant(0, dl, Op0.getValueType()),

3883

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3884

}

3885

if (Op0.getOpcode() == ISD::AssertZext &&

3886

cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)

3887

return DAG.getSetCC(dl, VT, Op0,

3888

DAG.getConstant(0, dl, Op0.getValueType()),

3889

Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);

3890

}

3891

}

3892

3893

// Given:

3894

// icmp eq/ne (urem %x, %y), 0

3895

// Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':

3896

// icmp eq/ne %x, 0

3897

if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&

3898

(Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

3899

KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));

3900

KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));

3901

if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)

3902

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);

3903

}

3904

3905

if (SDValue V =

3906

optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))

3907

return V;

3908

}

3909

3910

// These simplifications apply to splat vectors as well.

3911

// TODO: Handle more splat vector cases.

3912

if (auto *N1C = isConstOrConstSplat(N1)) {

3913

const APInt &C1 = N1C->getAPIntValue();

3914

3915

APInt MinVal, MaxVal;

3916

unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();

3917

if (ISD::isSignedIntSetCC(Cond)) {

3918

MinVal = APInt::getSignedMinValue(OperandBitSize);

3919

MaxVal = APInt::getSignedMaxValue(OperandBitSize);

3920

} else {

3921

MinVal = APInt::getMinValue(OperandBitSize);

3922

MaxVal = APInt::getMaxValue(OperandBitSize);

3923

}

3924

3925

// Canonicalize GE/LE comparisons to use GT/LT comparisons.

3926

if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {

3927

// X >= MIN --> true

3928

if (C1 == MinVal)

3929

return DAG.getBoolConstant(true, dl, VT, OpVT);

3930

3931

if (!VT.isVector()) { // TODO: Support this for vectors.

3932

// X >= C0 --> X > (C0 - 1)

3933

APInt C = C1 - 1;

3934

ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;

3935

if ((DCI.isBeforeLegalizeOps() ||

3936

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3937

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3938

isLegalICmpImmediate(C.getSExtValue())))) {

3939

return DAG.getSetCC(dl, VT, N0,

3940

DAG.getConstant(C, dl, N1.getValueType()),

3941

NewCC);

3942

}

3943

}

3944

}

3945

3946

if (Cond == ISD::SETLE || Cond == ISD::SETULE) {

3947

// X <= MAX --> true

3948

if (C1 == MaxVal)

3949

return DAG.getBoolConstant(true, dl, VT, OpVT);

3950

3951

// X <= C0 --> X < (C0 + 1)

3952

if (!VT.isVector()) { // TODO: Support this for vectors.

3953

APInt C = C1 + 1;

3954

ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;

3955

if ((DCI.isBeforeLegalizeOps() ||

3956

isCondCodeLegal(NewCC, VT.getSimpleVT())) &&

3957

(!N1C->isOpaque() || (C.getBitWidth() <= 64 &&

3958

isLegalICmpImmediate(C.getSExtValue())))) {

3959

return DAG.getSetCC(dl, VT, N0,

3960

DAG.getConstant(C, dl, N1.getValueType()),

3961

NewCC);

3962

}

3963

}

3964

}

3965

3966

if (Cond == ISD::SETLT || Cond == ISD::SETULT) {

3967

if (C1 == MinVal)

3968

return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

3969

3970

// TODO: Support this for vectors after legalize ops.

3971

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3972

// Canonicalize setlt X, Max --> setne X, Max

3973

if (C1 == MaxVal)

3974

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3975

3976

// If we have setult X, 1, turn it into seteq X, 0

3977

if (C1 == MinVal+1)

3978

return DAG.getSetCC(dl, VT, N0,

3979

DAG.getConstant(MinVal, dl, N0.getValueType()),

3980

ISD::SETEQ);

3981

}

3982

}

3983

3984

if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {

3985

if (C1 == MaxVal)

3986

return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

3987

3988

// TODO: Support this for vectors after legalize ops.

3989

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

3990

// Canonicalize setgt X, Min --> setne X, Min

3991

if (C1 == MinVal)

3992

return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

3993

3994

// If we have setugt X, Max-1, turn it into seteq X, Max

3995

if (C1 == MaxVal-1)

3996

return DAG.getSetCC(dl, VT, N0,

3997

DAG.getConstant(MaxVal, dl, N0.getValueType()),

3998

ISD::SETEQ);

3999

}

4000

}

4001

4002

if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {

4003

// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0

4004

if (C1.isNullValue())

4005

if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(

4006

VT, N0, N1, Cond, DCI, dl))

4007

return CC;

4008

4009

// For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).

4010

// For example, when high 32-bits of i64 X are known clear:

4011

// all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0

4012

// all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1

4013

bool CmpZero = N1C->getAPIntValue().isNullValue();

4014

bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();

4015

if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {

4016

// Match or(lo,shl(hi,bw/2)) pattern.

4017

auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {

4018

unsigned EltBits = V.getScalarValueSizeInBits();

4019

if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)

4020

return false;

4021

SDValue LHS = V.getOperand(0);

4022

SDValue RHS = V.getOperand(1);

4023

APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);

4024

// Unshifted element must have zero upperbits.

4025

if (RHS.getOpcode() == ISD::SHL &&

4026

isa<ConstantSDNode>(RHS.getOperand(1)) &&

4027

RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4028

DAG.MaskedValueIsZero(LHS, HiBits)) {

4029

Lo = LHS;

4030

Hi = RHS.getOperand(0);

4031

return true;

4032

}

4033

if (LHS.getOpcode() == ISD::SHL &&

4034

isa<ConstantSDNode>(LHS.getOperand(1)) &&

4035

LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&

4036

DAG.MaskedValueIsZero(RHS, HiBits)) {

4037

Lo = RHS;

4038

Hi = LHS.getOperand(0);

4039

return true;

4040

}

4041

return false;

4042

};

4043

4044

auto MergeConcat = [&](SDValue Lo, SDValue Hi) {

4045

unsigned EltBits = N0.getScalarValueSizeInBits();

4046

unsigned HalfBits = EltBits / 2;

4047

APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);

4048

SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);

4049

SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);

4050

SDValue NewN0 =

4051

DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);

4052

SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;

4053

return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);

4054

};

4055

4056

SDValue Lo, Hi;

4057

if (IsConcat(N0, Lo, Hi))

4058

return MergeConcat(Lo, Hi);

4059

4060

if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {

4061

SDValue Lo0, Lo1, Hi0, Hi1;

4062

if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&

4063

IsConcat(N0.getOperand(1), Lo1, Hi1)) {

4064

return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),

4065

DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));

4066

}

4067

}

4068

}

4069

}

4070

4071

// If we have "setcc X, C0", check to see if we can shrink the immediate

4072

// by changing cc.

4073

// TODO: Support this for vectors after legalize ops.

4074

if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {

4075

// SETUGT X, SINTMAX -> SETLT X, 0

4076

// SETUGE X, SINTMIN -> SETLT X, 0

4077

if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||

4078

(Cond == ISD::SETUGE && C1.isMinSignedValue()))

4079

return DAG.getSetCC(dl, VT, N0,

4080

DAG.getConstant(0, dl, N1.getValueType()),

4081

ISD::SETLT);

4082

4083

// SETULT X, SINTMIN -> SETGT X, -1

4084

// SETULE X, SINTMAX -> SETGT X, -1

4085

if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||

4086

(Cond == ISD::SETULE && C1.isMaxSignedValue()))

4087

return DAG.getSetCC(dl, VT, N0,

4088

DAG.getAllOnesConstant(dl, N1.getValueType()),

4089

ISD::SETGT);

4090

}

4091

}

4092

4093

// Back to non-vector simplifications.

4094

// TODO: Can we do these for vector splats?

4095

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {

4096

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

4097

const APInt &C1 = N1C->getAPIntValue();

4098

EVT ShValTy = N0.getValueType();

4099

4100

// Fold bit comparisons when we can. This will result in an

4101

// incorrect value when boolean false is negative one, unless

4102

// the bitsize is 1 in which case the false value is the same

4103

// in practice regardless of the representation.

4104

if ((VT.getSizeInBits() == 1 ||

4105

getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&

4106

(Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4107

(VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&

4108

N0.getOpcode() == ISD::AND) {

4109

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4110

EVT ShiftTy =

4111

getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4112

if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3

4113

// Perform the xform if the AND RHS is a single bit.

4114

unsigned ShCt = AndRHS->getAPIntValue().logBase2();

4115

if (AndRHS->getAPIntValue().isPowerOf2() &&

4116

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4117

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4118

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4119

DAG.getConstant(ShCt, dl, ShiftTy)));

4120

}

4121

} else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {

4122

// (X & 8) == 8 --> (X & 8) >> 3

4123

// Perform the xform if C1 is a single bit.

4124

unsigned ShCt = C1.logBase2();

4125

if (C1.isPowerOf2() &&

4126

!TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {

4127

return DAG.getNode(ISD::TRUNCATE, dl, VT,

4128

DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4129

DAG.getConstant(ShCt, dl, ShiftTy)));

4130

}

4131

}

4132

}

4133

}

4134

4135

if (C1.getMinSignedBits() <= 64 &&

4136

!isLegalICmpImmediate(C1.getSExtValue())) {

4137

EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());

4138

// (X & -256) == 256 -> (X >> 8) == 1

4139

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4140

N0.getOpcode() == ISD::AND && N0.hasOneUse()) {

4141

if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4142

const APInt &AndRHSC = AndRHS->getAPIntValue();

4143

if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {

4144

unsigned ShiftBits = AndRHSC.countTrailingZeros();

4145

if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4146

SDValue Shift =

4147

DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),

4148

DAG.getConstant(ShiftBits, dl, ShiftTy));

4149

SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);

4150

return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);

4151

}

4152

}

4153

}

4154

} else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||

4155

Cond == ISD::SETULE || Cond == ISD::SETUGT) {

4156

bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);

4157

// X < 0x100000000 -> (X >> 32) < 1

4158

// X >= 0x100000000 -> (X >> 32) >= 1

4159

// X <= 0x0ffffffff -> (X >> 32) < 1

4160

// X > 0x0ffffffff -> (X >> 32) >= 1

4161

unsigned ShiftBits;

4162

APInt NewC = C1;

4163

ISD::CondCode NewCond = Cond;

4164

if (AdjOne) {

4165

ShiftBits = C1.countTrailingOnes();

4166

NewC = NewC + 1;

4167

NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;

4168

} else {

4169

ShiftBits = C1.countTrailingZeros();

4170

}

4171

NewC.lshrInPlace(ShiftBits);

4172

if (ShiftBits && NewC.getMinSignedBits() <= 64 &&

4173

isLegalICmpImmediate(NewC.getSExtValue()) &&

4174

!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {

4175

SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,

4176

DAG.getConstant(ShiftBits, dl, ShiftTy));

4177

SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);

4178

return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);

4179

}

4180

}

4181

}

4182

}

4183

4184

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {

4185

auto *CFP = cast<ConstantFPSDNode>(N1);

4186

assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4186, __extension__ __PRETTY_FUNCTION__));

4187

4188

// Otherwise, we know the RHS is not a NaN. Simplify the node to drop the

4189

// constant if knowing that the operand is non-nan is enough. We prefer to

4190

// have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to

4191

// materialize 0.0.

4192

if (Cond == ISD::SETO || Cond == ISD::SETUO)

4193

return DAG.getSetCC(dl, VT, N0, N0, Cond);

4194

4195

// setcc (fneg x), C -> setcc swap(pred) x, -C

4196

if (N0.getOpcode() == ISD::FNEG) {

4197

ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);

4198

if (DCI.isBeforeLegalizeOps() ||

4199

isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {

4200

SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);

4201

return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);

4202

}

4203

}

4204

4205

// If the condition is not legal, see if we can find an equivalent one

4206

// which is legal.

4207

if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {

4208

// If the comparison was an awkward floating-point == or != and one of

4209

// the comparison operands is infinity or negative infinity, convert the

4210

// condition to a less-awkward <= or >=.

4211

if (CFP->getValueAPF().isInfinity()) {

4212

bool IsNegInf = CFP->getValueAPF().isNegative();

4213

ISD::CondCode NewCond = ISD::SETCC_INVALID;

4214

switch (Cond) {

4215

case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;

4216

case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;

4217

case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;

4218

case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;

4219

default: break;

4220

}

4221

if (NewCond != ISD::SETCC_INVALID &&

4222

isCondCodeLegal(NewCond, N0.getSimpleValueType()))

4223

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4224

}

4225

}

4226

}

4227

4228

if (N0 == N1) {

4229

// The sext(setcc()) => setcc() optimization relies on the appropriate

4230

// constant being emitted.

4231

assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4232, __extension__ __PRETTY_FUNCTION__))

4232

"Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4232, __extension__ __PRETTY_FUNCTION__));

4233

4234

bool EqTrue = ISD::isTrueWhenEqual(Cond);

4235

unsigned UOF = ISD::getUnorderedFlavor(Cond);

4236

if (UOF == 2) // FP operators that are undefined on NaNs.

4237

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4238

if (UOF == unsigned(EqTrue))

4239

return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);

4240

// Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO

4241

// if it is not already.

4242

ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;

4243

if (NewCond != Cond &&

4244

(DCI.isBeforeLegalizeOps() ||

4245

isCondCodeLegal(NewCond, N0.getSimpleValueType())))

4246

return DAG.getSetCC(dl, VT, N0, N1, NewCond);

4247

}

4248

4249

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&

4250

N0.getValueType().isInteger()) {

4251

if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||

4252

N0.getOpcode() == ISD::XOR) {

4253

// Simplify (X+Y) == (X+Z) --> Y == Z

4254

if (N0.getOpcode() == N1.getOpcode()) {

4255

if (N0.getOperand(0) == N1.getOperand(0))

4256

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);

4257

if (N0.getOperand(1) == N1.getOperand(1))

4258

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);

4259

if (isCommutativeBinOp(N0.getOpcode())) {

4260

// If X op Y == Y op X, try other combinations.

4261

if (N0.getOperand(0) == N1.getOperand(1))

4262

return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),

4263

Cond);

4264

if (N0.getOperand(1) == N1.getOperand(0))

4265

return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),

4266

Cond);

4267

}

4268

}

4269

4270

// If RHS is a legal immediate value for a compare instruction, we need

4271

// to be careful about increasing register pressure needlessly.

4272

bool LegalRHSImm = false;

4273

4274

if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {

4275

if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {

4276

// Turn (X+C1) == C2 --> X == C2-C1

4277

if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {

4278

return DAG.getSetCC(dl, VT, N0.getOperand(0),

4279

DAG.getConstant(RHSC->getAPIntValue()-

4280

LHSR->getAPIntValue(),

4281

dl, N0.getValueType()), Cond);

4282

}

4283

4284

// Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.

4285

if (N0.getOpcode() == ISD::XOR)

4286

// If we know that all of the inverted bits are zero, don't bother

4287

// performing the inversion.

4288

if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))

4289

return

4290

DAG.getSetCC(dl, VT, N0.getOperand(0),

4291

DAG.getConstant(LHSR->getAPIntValue() ^

4292

RHSC->getAPIntValue(),

4293

dl, N0.getValueType()),

4294

Cond);

4295

}

4296

4297

// Turn (C1-X) == C2 --> X == C1-C2

4298

if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {

4299

if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {

4300

return

4301

DAG.getSetCC(dl, VT, N0.getOperand(1),

4302

DAG.getConstant(SUBC->getAPIntValue() -

4303

RHSC->getAPIntValue(),

4304

dl, N0.getValueType()),

4305

Cond);

4306

}

4307

}

4308

4309

// Could RHSC fold directly into a compare?

4310

if (RHSC->getValueType(0).getSizeInBits() <= 64)

4311

LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());

4312

}

4313

4314

// (X+Y) == X --> Y == 0 and similar folds.

4315

// Don't do this if X is an immediate that can fold into a cmp

4316

// instruction and X+Y has other uses. It could be an induction variable

4317

// chain, and the transform would increase register pressure.

4318

if (!LegalRHSImm || N0.hasOneUse())

4319

if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))

4320

return V;

4321

}

4322

4323

if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||

4324

N1.getOpcode() == ISD::XOR)

4325

if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))

4326

return V;

4327

4328

if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))

4329

return V;

4330

}

4331

4332

// Fold remainder of division by a constant.

4333

if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&

4334

N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {

4335

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

4336

4337

// When division is cheap or optimizing for minimum size,

4338

// fall through to DIVREM creation by skipping this fold.

4339

if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {

4340

if (N0.getOpcode() == ISD::UREM) {

4341

if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))

4342

return Folded;

4343

} else if (N0.getOpcode() == ISD::SREM) {

4344

if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))

4345

return Folded;

4346

}

4347

}

4348

}

4349

4350

// Fold away ALL boolean setcc's.

4351

if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {

4352

SDValue Temp;

4353

switch (Cond) {

4354

default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4354);

4355

case ISD::SETEQ: // X == Y -> ~(X^Y)

4356

Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4357

N0 = DAG.getNOT(dl, Temp, OpVT);

4358

if (!DCI.isCalledByLegalizer())

4359

DCI.AddToWorklist(Temp.getNode());

4360

break;

4361

case ISD::SETNE: // X != Y --> (X^Y)

4362

N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);

4363

break;

4364

case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y

4365

case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y

4366

Temp = DAG.getNOT(dl, N0, OpVT);

4367

N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);

4368

if (!DCI.isCalledByLegalizer())

4369

DCI.AddToWorklist(Temp.getNode());

4370

break;

4371

case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X

4372

case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X

4373

Temp = DAG.getNOT(dl, N1, OpVT);

4374

N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);

4375

if (!DCI.isCalledByLegalizer())

4376

DCI.AddToWorklist(Temp.getNode());

4377

break;

4378

case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y

4379

case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y

4380

Temp = DAG.getNOT(dl, N0, OpVT);

4381

N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);

4382

if (!DCI.isCalledByLegalizer())

4383

DCI.AddToWorklist(Temp.getNode());

4384

break;

4385

case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X

4386

case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X

4387

Temp = DAG.getNOT(dl, N1, OpVT);

4388

N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);

4389

break;

4390

}

4391

if (VT.getScalarType() != MVT::i1) {

4392

if (!DCI.isCalledByLegalizer())

4393

DCI.AddToWorklist(N0.getNode());

4394

// FIXME: If running after legalize, we probably can't do this.

4395

ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));

4396

N0 = DAG.getNode(ExtendCode, dl, VT, N0);

4397

}

4398

return N0;

4399

}

4400

4401

// Could not fold it.

4402

return SDValue();

4403

}

4404

4405

/// Returns true (and the GlobalValue and the offset) if the node is a

4406

/// GlobalAddress + offset.

4407

bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,

4408

int64_t &Offset) const {

4409

4410

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

4411

4412

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {

4413

GA = GASD->getGlobal();

4414

Offset += GASD->getOffset();

4415

return true;

4416

}

4417

4418

if (N->getOpcode() == ISD::ADD) {

4419

SDValue N1 = N->getOperand(0);

4420

SDValue N2 = N->getOperand(1);

4421

if (isGAPlusOffset(N1.getNode(), GA, Offset)) {

4422

if (auto *V = dyn_cast<ConstantSDNode>(N2)) {

4423

Offset += V->getSExtValue();

4424

return true;

4425

}

4426

} else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {

4427

if (auto *V = dyn_cast<ConstantSDNode>(N1)) {

4428

Offset += V->getSExtValue();

4429

return true;

4430

}

4431

}

4432

}

4433

4434

return false;

4435

}

4436

4437

SDValue TargetLowering::PerformDAGCombine(SDNode *N,

4438

DAGCombinerInfo &DCI) const {

4439

// Default implementation: no optimization.

4440

return SDValue();

4441

}

4442

4443

//===----------------------------------------------------------------------===//

4444

// Inline Assembler Implementation Methods

4445

//===----------------------------------------------------------------------===//

4446

4447

TargetLowering::ConstraintType

4448

TargetLowering::getConstraintType(StringRef Constraint) const {

4449

unsigned S = Constraint.size();

4450

4451

if (S == 1) {

4452

switch (Constraint[0]) {

4453

default: break;

4454

case 'r':

4455

return C_RegisterClass;

4456

case 'm': // memory

4457

case 'o': // offsetable

4458

case 'V': // not offsetable

4459

return C_Memory;

4460

case 'n': // Simple Integer

4461

case 'E': // Floating Point Constant

4462

case 'F': // Floating Point Constant

4463

return C_Immediate;

4464

case 'i': // Simple Integer or Relocatable Constant

4465

case 's': // Relocatable Constant

4466

case 'p': // Address.

4467

case 'X': // Allow ANY value.

4468

case 'I': // Target registers.

4469

case 'J':

4470

case 'K':

4471

case 'L':

4472

case 'M':

4473

case 'N':

4474

case 'O':

4475

case 'P':

4476

case '<':

4477

case '>':

4478

return C_Other;

4479

}

4480

}

4481

4482

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {

4483

if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"

4484

return C_Memory;

4485

return C_Register;

4486

}

4487

return C_Unknown;

4488

}

4489

4490

/// Try to replace an X constraint, which matches anything, with another that

4491

/// has more specific requirements based on the type of the corresponding

4492

/// operand.

4493

const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {

4494

if (ConstraintVT.isInteger())

4495

return "r";

4496

if (ConstraintVT.isFloatingPoint())

4497

return "f"; // works for many targets

4498

return nullptr;

4499

}

4500

4501

SDValue TargetLowering::LowerAsmOutputForConstraint(

4502

SDValue &Chain, SDValue &Flag, const SDLoc &DL,

4503

const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {

4504

return SDValue();

4505

}

4506

4507

/// Lower the specified operand into the Ops vector.

4508

/// If it is invalid, don't add anything to Ops.

4509

void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,

4510

std::string &Constraint,

4511

std::vector<SDValue> &Ops,

4512

SelectionDAG &DAG) const {

4513

4514

if (Constraint.length() > 1) return;

4515

4516

char ConstraintLetter = Constraint[0];

4517

switch (ConstraintLetter) {

4518

default: break;

4519

case 'X': // Allows any operand; labels (basic block) use this.

4520

if (Op.getOpcode() == ISD::BasicBlock ||

4521

Op.getOpcode() == ISD::TargetBlockAddress) {

4522

Ops.push_back(Op);

4523

return;

4524

}

4525

LLVM_FALLTHROUGH[[gnu::fallthrough]];

4526

case 'i': // Simple Integer or Relocatable Constant

4527

case 'n': // Simple Integer

4528

case 's': { // Relocatable Constant

4529

4530

GlobalAddressSDNode *GA;

4531

ConstantSDNode *C;

4532

BlockAddressSDNode *BA;

4533

uint64_t Offset = 0;

4534

4535

// Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),

4536

// etc., since getelementpointer is variadic. We can't use

4537

// SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible

4538

// while in this case the GA may be furthest from the root node which is

4539

// likely an ISD::ADD.

4540

while (1) {

4541

if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4542

Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),

4543

GA->getValueType(0),

4544

Offset + GA->getOffset()));

4545

return;

4546

}

4547

if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {

4548

// gcc prints these as sign extended. Sign extend value to 64 bits

4549

// now; without this it would get ZExt'd later in

4550

// ScheduleDAGSDNodes::EmitNode, which is very generic.

4551

bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;

4552

BooleanContent BCont = getBooleanContents(MVT::i64);

4553

ISD::NodeType ExtOpc =

4554

IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;

4555

int64_t ExtVal =

4556

ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();

4557

Ops.push_back(

4558

DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));

4559

return;

4560

}

4561

if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {

4562

Ops.push_back(DAG.getTargetBlockAddress(

4563

BA->getBlockAddress(), BA->getValueType(0),

4564

Offset + BA->getOffset(), BA->getTargetFlags()));

4565

return;

4566

}

4567

const unsigned OpCode = Op.getOpcode();

4568

if (OpCode == ISD::ADD || OpCode == ISD::SUB) {

4569

if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))

4570

Op = Op.getOperand(1);

4571

// Subtraction is not commutative.

4572

else if (OpCode == ISD::ADD &&

4573

(C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))

4574

Op = Op.getOperand(0);

4575

else

4576

return;

4577

Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();

4578

continue;

4579

}

4580

return;

4581

}

4582

break;

4583

}

4584

}

4585

}

4586

4587

std::pair<unsigned, const TargetRegisterClass *>

4588

TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,

4589

StringRef Constraint,

4590

MVT VT) const {

4591

if (Constraint.empty() || Constraint[0] != '{')

4592

return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));

4593

assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4593, __extension__ __PRETTY_FUNCTION__));

4594

4595

// Remove the braces from around the name.

4596

StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

4597

4598

std::pair<unsigned, const TargetRegisterClass *> R =

4599

std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

4600

4601

// Figure out which register class contains this reg.

4602

for (const TargetRegisterClass *RC : RI->regclasses()) {

4603

// If none of the value types for this register class are valid, we

4604

// can't use it. For example, 64-bit reg classes on 32-bit targets.

4605

if (!isLegalRC(*RI, *RC))

4606

continue;

4607

4608

for (const MCPhysReg &PR : *RC) {

4609

if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {

4610

std::pair<unsigned, const TargetRegisterClass *> S =

4611

std::make_pair(PR, RC);

4612

4613

// If this register class has the requested value type, return it,

4614

// otherwise keep searching and return the first class found

4615

// if no other is found which explicitly has the requested type.

4616

if (RI->isTypeLegalForClass(*RC, VT))

4617

return S;

4618

if (!R.second)

4619

R = S;

4620

}

4621

}

4622

}

4623

4624

return R;

4625

}

4626

4627

//===----------------------------------------------------------------------===//

4628

// Constraint Selection.

4629

4630

/// Return true of this is an input operand that is a matching constraint like

4631

/// "4".

4632

bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {

4633

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4633, __extension__ __PRETTY_FUNCTION__));

4634

return isdigit(static_cast<unsigned char>(ConstraintCode[0]));

4635

}

4636

4637

/// If this is an input matching constraint, this method returns the output

4638

/// operand it matches.

4639

unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {

4640

assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4640, __extension__ __PRETTY_FUNCTION__));

4641

return atoi(ConstraintCode.c_str());

4642

}

4643

4644

/// Split up the constraint string from the inline assembly value into the

4645

/// specific constraints and their prefixes, and also tie in the associated

4646

/// operand values.

4647

/// If this returns an empty vector, and if the constraint string itself

4648

/// isn't empty, there was an error parsing.

4649

TargetLowering::AsmOperandInfoVector

4650

TargetLowering::ParseConstraints(const DataLayout &DL,

4651

const TargetRegisterInfo *TRI,

4652

const CallBase &Call) const {

4653

/// Information about all of the constraints.

4654

AsmOperandInfoVector ConstraintOperands;

4655

const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());

4656

unsigned maCount = 0; // Largest number of multiple alternative constraints.

4657

4658

// Do a prepass over the constraints, canonicalizing them, and building up the

4659

// ConstraintOperands list.

4660

unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.

4661

unsigned ResNo = 0; // ResNo - The result number of the next output.

4662

4663

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {

4664

ConstraintOperands.emplace_back(std::move(CI));

4665

AsmOperandInfo &OpInfo = ConstraintOperands.back();

4666

4667

// Update multiple alternative constraint count.

4668

if (OpInfo.multipleAlternatives.size() > maCount)

4669

maCount = OpInfo.multipleAlternatives.size();

4670

4671

OpInfo.ConstraintVT = MVT::Other;

4672

4673

// Compute the value type for each operand.

4674

switch (OpInfo.Type) {

4675

case InlineAsm::isOutput:

4676

// Indirect outputs just consume an argument.

4677

if (OpInfo.isIndirect) {

4678

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4679

break;

4680

}

4681

4682

// The return value of the call is this value. As such, there is no

4683

// corresponding argument.

4684

assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4684, __extension__ __PRETTY_FUNCTION__));

4685

if (StructType *STy = dyn_cast<StructType>(Call.getType())) {

4686

OpInfo.ConstraintVT =

4687

getSimpleValueType(DL, STy->getElementType(ResNo));

4688

} else {

4689

assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4689, __extension__ __PRETTY_FUNCTION__));

4690

OpInfo.ConstraintVT =

4691

getAsmOperandValueType(DL, Call.getType()).getSimpleVT();

4692

}

4693

++ResNo;

4694

break;

4695

case InlineAsm::isInput:

4696

OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);

4697

break;

4698

case InlineAsm::isClobber:

4699

// Nothing to do.

4700

break;

4701

}

4702

4703

if (OpInfo.CallOperandVal) {

4704

llvm::Type *OpTy = OpInfo.CallOperandVal->getType();

4705

if (OpInfo.isIndirect) {

4706

llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);

4707

if (!PtrTy)

4708

report_fatal_error("Indirect operand for inline asm not a pointer!");

4709

OpTy = PtrTy->getElementType();

4710

}

4711

4712

// Look for vector wrapped in a struct. e.g. { <16 x i8> }.

4713

if (StructType *STy = dyn_cast<StructType>(OpTy))

4714

if (STy->getNumElements() == 1)

4715

OpTy = STy->getElementType(0);

4716

4717

// If OpTy is not a single value, it may be a struct/union that we

4718

// can tile with integers.

4719

if (!OpTy->isSingleValueType() && OpTy->isSized()) {

4720

unsigned BitSize = DL.getTypeSizeInBits(OpTy);

4721

switch (BitSize) {

4722

default: break;

4723

case 1:

4724

case 8:

4725

case 16:

4726

case 32:

4727

case 64:

4728

case 128:

4729

OpInfo.ConstraintVT =

4730

MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);

4731

break;

4732

}

4733

} else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {

4734

unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());

4735

OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);

4736

} else {

4737

OpInfo.ConstraintVT = MVT::getVT(OpTy, true);

4738

}

4739

}

4740

}

4741

4742

// If we have multiple alternative constraints, select the best alternative.

4743

if (!ConstraintOperands.empty()) {

4744

if (maCount) {

4745

unsigned bestMAIndex = 0;

4746

int bestWeight = -1;

4747

// weight: -1 = invalid match, and 0 = so-so match to 5 = good match.

4748

int weight = -1;

4749

unsigned maIndex;

4750

// Compute the sums of the weights for each alternative, keeping track

4751

// of the best (highest weight) one so far.

4752

for (maIndex = 0; maIndex < maCount; ++maIndex) {

4753

int weightSum = 0;

4754

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4755

cIndex != eIndex; ++cIndex) {

4756

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4757

if (OpInfo.Type == InlineAsm::isClobber)

4758

continue;

4759

4760

// If this is an output operand with a matching input operand,

4761

// look up the matching input. If their types mismatch, e.g. one

4762

// is an integer, the other is floating point, or their sizes are

4763

// different, flag it as an maCantMatch.

4764

if (OpInfo.hasMatchingInput()) {

4765

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4766

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4767

if ((OpInfo.ConstraintVT.isInteger() !=

4768

Input.ConstraintVT.isInteger()) ||

4769

(OpInfo.ConstraintVT.getSizeInBits() !=

4770

Input.ConstraintVT.getSizeInBits())) {

4771

weightSum = -1; // Can't match.

4772

break;

4773

}

4774

}

4775

}

4776

weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);

4777

if (weight == -1) {

4778

weightSum = -1;

4779

break;

4780

}

4781

weightSum += weight;

4782

}

4783

// Update best.

4784

if (weightSum > bestWeight) {

4785

bestWeight = weightSum;

4786

bestMAIndex = maIndex;

4787

}

4788

}

4789

4790

// Now select chosen alternative in each constraint.

4791

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4792

cIndex != eIndex; ++cIndex) {

4793

AsmOperandInfo &cInfo = ConstraintOperands[cIndex];

4794

if (cInfo.Type == InlineAsm::isClobber)

4795

continue;

4796

cInfo.selectAlternative(bestMAIndex);

4797

}

4798

}

4799

}

4800

4801

// Check and hook up tied operands, choose constraint code to use.

4802

for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();

4803

cIndex != eIndex; ++cIndex) {

4804

AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

4805

4806

// If this is an output operand with a matching input operand, look up the

4807

// matching input. If their types mismatch, e.g. one is an integer, the

4808

// other is floating point, or their sizes are different, flag it as an

4809

// error.

4810

if (OpInfo.hasMatchingInput()) {

4811

AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

4812

4813

if (OpInfo.ConstraintVT != Input.ConstraintVT) {

4814

std::pair<unsigned, const TargetRegisterClass *> MatchRC =

4815

getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,

4816

OpInfo.ConstraintVT);

4817

std::pair<unsigned, const TargetRegisterClass *> InputRC =

4818

getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,

4819

Input.ConstraintVT);

4820

if ((OpInfo.ConstraintVT.isInteger() !=

4821

Input.ConstraintVT.isInteger()) ||

4822

(MatchRC.second != InputRC.second)) {

4823

report_fatal_error("Unsupported asm: input constraint"

4824

" with a matching output constraint of"

4825

" incompatible type!");

4826

}

4827

}

4828

}

4829

}

4830

4831

return ConstraintOperands;

4832

}

4833

4834

/// Return an integer indicating how general CT is.

4835

static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {

4836

switch (CT) {

4837

case TargetLowering::C_Immediate:

4838

case TargetLowering::C_Other:

4839

case TargetLowering::C_Unknown:

4840

return 0;

4841

case TargetLowering::C_Register:

4842

return 1;

4843

case TargetLowering::C_RegisterClass:

4844

return 2;

4845

case TargetLowering::C_Memory:

4846

return 3;

4847

}

4848

llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4848);

4849

}

4850

4851

/// Examine constraint type and operand type and determine a weight value.

4852

/// This object must already have been set up with the operand type

4853

/// and the current alternative constraint selected.

4854

TargetLowering::ConstraintWeight

4855

TargetLowering::getMultipleConstraintMatchWeight(

4856

AsmOperandInfo &info, int maIndex) const {

4857

InlineAsm::ConstraintCodeVector *rCodes;

4858

if (maIndex >= (int)info.multipleAlternatives.size())

4859

rCodes = &info.Codes;

4860

else

4861

rCodes = &info.multipleAlternatives[maIndex].Codes;

4862

ConstraintWeight BestWeight = CW_Invalid;

4863

4864

// Loop over the options, keeping track of the most general one.

4865

for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {

4866

ConstraintWeight weight =

4867

getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());

4868

if (weight > BestWeight)

4869

BestWeight = weight;

4870

}

4871

4872

return BestWeight;

4873

}

4874

4875

/// Examine constraint type and operand type and determine a weight value.

4876

/// This object must already have been set up with the operand type

4877

/// and the current alternative constraint selected.

4878

TargetLowering::ConstraintWeight

4879

TargetLowering::getSingleConstraintMatchWeight(

4880

AsmOperandInfo &info, const char *constraint) const {

4881

ConstraintWeight weight = CW_Invalid;

4882

Value *CallOperandVal = info.CallOperandVal;

4883

// If we don't have a value, we can't do a match,

4884

// but allow it at the lowest weight.

4885

if (!CallOperandVal)

4886

return CW_Default;

4887

// Look at the constraint type.

4888

switch (*constraint) {

4889

case 'i': // immediate integer.

4890

case 'n': // immediate integer with a known value.

4891

if (isa<ConstantInt>(CallOperandVal))

4892

weight = CW_Constant;

4893

break;

4894

case 's': // non-explicit intregal immediate.

4895

if (isa<GlobalValue>(CallOperandVal))

4896

weight = CW_Constant;

4897

break;

4898

case 'E': // immediate float if host format.

4899

case 'F': // immediate float.

4900

if (isa<ConstantFP>(CallOperandVal))

4901

weight = CW_Constant;

4902

break;

4903

case '<': // memory operand with autodecrement.

4904

case '>': // memory operand with autoincrement.

4905

case 'm': // memory operand.

4906

case 'o': // offsettable memory operand

4907

case 'V': // non-offsettable memory operand

4908

weight = CW_Memory;

4909

break;

4910

case 'r': // general register.

4911

case 'g': // general register, memory operand or immediate integer.

4912

// note: Clang converts "g" to "imr".

4913

if (CallOperandVal->getType()->isIntegerTy())

4914

weight = CW_Register;

4915

break;

4916

case 'X': // any operand.

4917

default:

4918

weight = CW_Default;

4919

break;

4920

}

4921

return weight;

4922

}

4923

4924

/// If there are multiple different constraints that we could pick for this

4925

/// operand (e.g. "imr") try to pick the 'best' one.

4926

/// This is somewhat tricky: constraints fall into four classes:

4927

/// Other -> immediates and magic values

4928

/// Register -> one specific register

4929

/// RegisterClass -> a group of regs

4930

/// Memory -> memory

4931

/// Ideally, we would pick the most specific constraint possible: if we have

4932

/// something that fits into a register, we would pick it. The problem here

4933

/// is that if we have something that could either be in a register or in

4934

/// memory that use of the register could cause selection of *other*

4935

/// operands to fail: they might only succeed if we pick memory. Because of

4936

/// this the heuristic we use is:

4937

///

4938

/// 1) If there is an 'other' constraint, and if the operand is valid for

4939

/// that constraint, use it. This makes us take advantage of 'i'

4940

/// constraints when available.

4941

/// 2) Otherwise, pick the most general constraint present. This prefers

4942

/// 'm' over 'r', for example.

4943

///

4944

static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,

4945

const TargetLowering &TLI,

4946

SDValue Op, SelectionDAG *DAG) {

4947

assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4947, __extension__ __PRETTY_FUNCTION__));

4948

unsigned BestIdx = 0;

4949

TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;

4950

int BestGenerality = -1;

4951

4952

// Loop over the options, keeping track of the most general one.

4953

for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {

4954

TargetLowering::ConstraintType CType =

4955

TLI.getConstraintType(OpInfo.Codes[i]);

4956

4957

// Indirect 'other' or 'immediate' constraints are not allowed.

4958

if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||

4959

CType == TargetLowering::C_Register ||

4960

CType == TargetLowering::C_RegisterClass))

4961

continue;

4962

4963

// If this is an 'other' or 'immediate' constraint, see if the operand is

4964

// valid for it. For example, on X86 we might have an 'rI' constraint. If

4965

// the operand is an integer in the range [0..31] we want to use I (saving a

4966

// load of a register), otherwise we must use 'r'.

4967

if ((CType == TargetLowering::C_Other ||

4968

CType == TargetLowering::C_Immediate) && Op.getNode()) {

4969

assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__))

4970

"Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4970, __extension__ __PRETTY_FUNCTION__));

4971

std::vector<SDValue> ResultOps;

4972

TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],

4973

ResultOps, *DAG);

4974

if (!ResultOps.empty()) {

4975

BestType = CType;

4976

BestIdx = i;

4977

break;

4978

}

4979

}

4980

4981

// Things with matching constraints can only be registers, per gcc

4982

// documentation. This mainly affects "g" constraints.

4983

if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())

4984

continue;

4985

4986

// This constraint letter is more general than the previous one, use it.

4987

int Generality = getConstraintGenerality(CType);

4988

if (Generality > BestGenerality) {

4989

BestType = CType;

4990

BestIdx = i;

4991

BestGenerality = Generality;

4992

}

4993

}

4994

4995

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];

4996

OpInfo.ConstraintType = BestType;

4997

}

4998

4999

/// Determines the constraint code and constraint type to use for the specific

5000

/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.

5001

void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,

5002

SDValue Op,

5003

SelectionDAG *DAG) const {

5004

assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5004, __extension__ __PRETTY_FUNCTION__));

5005

5006

// Single-letter constraints ('r') are very common.

5007

if (OpInfo.Codes.size() == 1) {

5008

OpInfo.ConstraintCode = OpInfo.Codes[0];

5009

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5010

} else {

5011

ChooseConstraint(OpInfo, *this, Op, DAG);

5012

}

5013

5014

// 'X' matches anything.

5015

if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {

5016

// Labels and constants are handled elsewhere ('X' is the only thing

5017

// that matches labels). For Functions, the type here is the type of

5018

// the result, which is not what we want to look at; leave them alone.

5019

Value *v = OpInfo.CallOperandVal;

5020

if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {

5021

OpInfo.CallOperandVal = v;

5022

return;

5023

}

5024

5025

if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)

5026

return;

5027

5028

// Otherwise, try to resolve it to something we know about by looking at

5029

// the actual operand type.

5030

if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {

5031

OpInfo.ConstraintCode = Repl;

5032

OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);

5033

}

5034

}

5035

}

5036

5037

/// Given an exact SDIV by a constant, create a multiplication

5038

/// with the multiplicative inverse of the constant.

5039

static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,

5040

const SDLoc &dl, SelectionDAG &DAG,

5041

SmallVectorImpl<SDNode *> &Created) {

5042

SDValue Op0 = N->getOperand(0);

5043

SDValue Op1 = N->getOperand(1);

5044

EVT VT = N->getValueType(0);

5045

EVT SVT = VT.getScalarType();

5046

EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());

5047

EVT ShSVT = ShVT.getScalarType();

5048

5049

bool UseSRA = false;

5050

SmallVector<SDValue, 16> Shifts, Factors;

5051

5052

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5053

if (C->isNullValue())

5054

return false;

5055

APInt Divisor = C->getAPIntValue();

5056

unsigned Shift = Divisor.countTrailingZeros();

5057

if (Shift) {

5058

Divisor.ashrInPlace(Shift);

5059

UseSRA = true;

5060

}

5061

// Calculate the multiplicative inverse, using Newton's method.

5062

APInt t;

5063

APInt Factor = Divisor;

5064

while ((t = Divisor * Factor) != 1)

5065

Factor *= APInt(Divisor.getBitWidth(), 2) - t;

5066

Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));

5067

Factors.push_back(DAG.getConstant(Factor, dl, SVT));

5068

return true;

5069

};

5070

5071

// Collect all magic values from the build vector.

5072

if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))

5073

return SDValue();

5074

5075

SDValue Shift, Factor;

5076

if (Op1.getOpcode() == ISD::BUILD_VECTOR) {

5077

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5078

Factor = DAG.getBuildVector(VT, dl, Factors);

5079

} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {

5080

assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__))

5081

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__))

5082

"vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __extension__ __PRETTY_FUNCTION__));

5083

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5084

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5085

} else {

5086

assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5086, __extension__ __PRETTY_FUNCTION__));

5087

Shift = Shifts[0];

5088

Factor = Factors[0];

5089

}

5090

5091

SDValue Res = Op0;

5092

5093

// Shift the value upfront if it is even, so the LSB is one.

5094

if (UseSRA) {

5095

// TODO: For UDIV use SRL instead of SRA.

5096

SDNodeFlags Flags;

5097

Flags.setExact(true);

5098

Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);

5099

Created.push_back(Res.getNode());

5100

}

5101

5102

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);

5103

}

5104

5105

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,

5106

SelectionDAG &DAG,

5107

SmallVectorImpl<SDNode *> &Created) const {

5108

AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

5109

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

5110

if (TLI.isIntDivCheap(N->getValueType(0), Attr))

5111

return SDValue(N, 0); // Lower SDIV as SDIV

5112

return SDValue();

5113

}

5114

5115

/// Given an ISD::SDIV node expressing a divide by constant,

5116

/// return a DAG expression to select that will generate the same value by

5117

/// multiplying by a magic number.

5118

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5119

SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,

5120

bool IsAfterLegalization,

5121

SmallVectorImpl<SDNode *> &Created) const {

5122

SDLoc dl(N);

5123

EVT VT = N->getValueType(0);

5124

EVT SVT = VT.getScalarType();

5125

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5126

EVT ShSVT = ShVT.getScalarType();

5127

unsigned EltBits = VT.getScalarSizeInBits();

5128

EVT MulVT;

5129

5130

// Check to see if we can do this.

5131

// FIXME: We should be more aggressive here.

5132

if (!isTypeLegal(VT)) {

5133

// Limit this to simple scalars for now.

5134

if (VT.isVector() || !VT.isSimple())

5135

return SDValue();

5136

5137

// If this type will be promoted to a large enough type with a legal

5138

// multiply operation, we can go ahead and do this transform.

5139

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5140

return SDValue();

5141

5142

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5143

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5144

!isOperationLegal(ISD::MUL, MulVT))

5145

return SDValue();

5146

}

5147

5148

// If the sdiv has an 'exact' bit we can use a simpler lowering.

5149

if (N->getFlags().hasExact())

5150

return BuildExactSDIV(*this, N, dl, DAG, Created);

5151

5152

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

5153

5154

auto BuildSDIVPattern = [&](ConstantSDNode *C) {

5155

if (C->isNullValue())

5156

return false;

5157

5158

const APInt &Divisor = C->getAPIntValue();

5159

APInt::ms magics = Divisor.magic();

5160

int NumeratorFactor = 0;

5161

int ShiftMask = -1;

5162

5163

if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {

5164

// If d is +1/-1, we just multiply the numerator by +1/-1.

5165

NumeratorFactor = Divisor.getSExtValue();

5166

magics.m = 0;

5167

magics.s = 0;

5168

ShiftMask = 0;

5169

} else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {

5170

// If d > 0 and m < 0, add the numerator.

5171

NumeratorFactor = 1;

5172

} else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {

5173

// If d < 0 and m > 0, subtract the numerator.

5174

NumeratorFactor = -1;

5175

}

5176

5177

MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));

5178

Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));

5179

Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));

5180

ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));

5181

return true;

5182

};

5183

5184

SDValue N0 = N->getOperand(0);

5185

SDValue N1 = N->getOperand(1);

5186

5187

// Collect the shifts / magic values from each element.

5188

if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))

5189

return SDValue();

5190

5191

SDValue MagicFactor, Factor, Shift, ShiftMask;

5192

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5193

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5194

Factor = DAG.getBuildVector(VT, dl, Factors);

5195

Shift = DAG.getBuildVector(ShVT, dl, Shifts);

5196

ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);

5197

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5198

assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5199

Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5200

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))

5201

"vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__));

5202

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5203

Factor = DAG.getSplatVector(VT, dl, Factors[0]);

5204

Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);

5205

ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);

5206

} else {

5207

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5207, __extension__ __PRETTY_FUNCTION__));

5208

MagicFactor = MagicFactors[0];

5209

Factor = Factors[0];

5210

Shift = Shifts[0];

5211

ShiftMask = ShiftMasks[0];

5212

}

5213

5214

// Multiply the numerator (operand 0) by the magic value.

5215

// FIXME: We should support doing a MUL in a wider type.

5216

auto GetMULHS = [&](SDValue X, SDValue Y) {

5217

// If the type isn't legal, use a wider mul of the the type calculated

5218

// earlier.

5219

if (!isTypeLegal(VT)) {

5220

X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);

5221

Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);

5222

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5223

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5224

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5225

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5226

}

5227

5228

if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))

5229

return DAG.getNode(ISD::MULHS, dl, VT, X, Y);

5230

if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {

5231

SDValue LoHi =

5232

DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5233

return SDValue(LoHi.getNode(), 1);

5234

}

5235

return SDValue();

5236

};

5237

5238

SDValue Q = GetMULHS(N0, MagicFactor);

5239

if (!Q)

5240

return SDValue();

5241

5242

Created.push_back(Q.getNode());

5243

5244

// (Optionally) Add/subtract the numerator using Factor.

5245

Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);

5246

Created.push_back(Factor.getNode());

5247

Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);

5248

Created.push_back(Q.getNode());

5249

5250

// Shift right algebraic by shift value.

5251

Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);

5252

Created.push_back(Q.getNode());

5253

5254

// Extract the sign bit, mask it and add it to the quotient.

5255

SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);

5256

SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);

5257

Created.push_back(T.getNode());

5258

T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);

5259

Created.push_back(T.getNode());

5260

return DAG.getNode(ISD::ADD, dl, VT, Q, T);

5261

}

5262

5263

/// Given an ISD::UDIV node expressing a divide by constant,

5264

/// return a DAG expression to select that will generate the same value by

5265

/// multiplying by a magic number.

5266

/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".

5267

SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,

5268

bool IsAfterLegalization,

5269

SmallVectorImpl<SDNode *> &Created) const {

5270

SDLoc dl(N);

5271

EVT VT = N->getValueType(0);

5272

EVT SVT = VT.getScalarType();

5273

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

5274

EVT ShSVT = ShVT.getScalarType();

5275

unsigned EltBits = VT.getScalarSizeInBits();

5276

EVT MulVT;

5277

5278

// Check to see if we can do this.

5279

// FIXME: We should be more aggressive here.

5280

if (!isTypeLegal(VT)) {

5281

// Limit this to simple scalars for now.

5282

if (VT.isVector() || !VT.isSimple())

5283

return SDValue();

5284

5285

// If this type will be promoted to a large enough type with a legal

5286

// multiply operation, we can go ahead and do this transform.

5287

if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)

5288

return SDValue();

5289

5290

MulVT = getTypeToTransformTo(*DAG.getContext(), VT);

5291

if (MulVT.getSizeInBits() < (2 * EltBits) ||

5292

!isOperationLegal(ISD::MUL, MulVT))

5293

return SDValue();

5294

}

5295

5296

bool UseNPQ = false;

5297

SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

5298

5299

auto BuildUDIVPattern = [&](ConstantSDNode *C) {

5300

if (C->isNullValue())

5301

return false;

5302

// FIXME: We should use a narrower constant when the upper

5303

// bits are known to be zero.

5304

const APInt& Divisor = C->getAPIntValue();

5305

APInt::mu magics = Divisor.magicu();

5306

unsigned PreShift = 0, PostShift = 0;

5307

5308

// If the divisor is even, we can avoid using the expensive fixup by

5309

// shifting the divided value upfront.

5310

if (magics.a != 0 && !Divisor[0]) {

5311

PreShift = Divisor.countTrailingZeros();

5312

// Get magic number for the shifted divisor.

5313

magics = Divisor.lshr(PreShift).magicu(PreShift);

5314

assert(magics.a == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.a == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5314, __extension__ __PRETTY_FUNCTION__));

5315

}

5316

5317

APInt Magic = magics.m;

5318

5319

unsigned SelNPQ;

5320

if (magics.a == 0 || Divisor.isOneValue()) {

5321

assert(magics.s < Divisor.getBitWidth() &&(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5322, __extension__ __PRETTY_FUNCTION__))

5322

"We shouldn't generate an undefined shift!")(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5322, __extension__ __PRETTY_FUNCTION__));

5323

PostShift = magics.s;

5324

SelNPQ = false;

5325

} else {

5326

PostShift = magics.s - 1;

5327

SelNPQ = true;

5328

}

5329

5330

PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));

5331

MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));

5332

NPQFactors.push_back(

5333

DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)

5334

: APInt::getNullValue(EltBits),

5335

dl, SVT));

5336

PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));

5337

UseNPQ |= SelNPQ;

5338

return true;

5339

};

5340

5341

SDValue N0 = N->getOperand(0);

5342

SDValue N1 = N->getOperand(1);

5343

5344

// Collect the shifts/magic values from each element.

5345

if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))

5346

return SDValue();

5347

5348

SDValue PreShift, PostShift, MagicFactor, NPQFactor;

5349

if (N1.getOpcode() == ISD::BUILD_VECTOR) {

5350

PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);

5351

MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);

5352

NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);

5353

PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);

5354

} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {

5355

assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__))

5356

NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__))

5357

"Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5357, __extension__ __PRETTY_FUNCTION__));

5358

PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);

5359

MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);

5360

NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);

5361

PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);

5362

} else {

5363

assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5363, __extension__ __PRETTY_FUNCTION__));

5364

PreShift = PreShifts[0];

5365

MagicFactor = MagicFactors[0];

5366

PostShift = PostShifts[0];

5367

}

5368

5369

SDValue Q = N0;

5370

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);

5371

Created.push_back(Q.getNode());

5372

5373

// FIXME: We should support doing a MUL in a wider type.

5374

auto GetMULHU = [&](SDValue X, SDValue Y) {

5375

// If the type isn't legal, use a wider mul of the the type calculated

5376

// earlier.

5377

if (!isTypeLegal(VT)) {

5378

X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);

5379

Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);

5380

Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);

5381

Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,

5382

DAG.getShiftAmountConstant(EltBits, MulVT, dl));

5383

return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);

5384

}

5385

5386

if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))

5387

return DAG.getNode(ISD::MULHU, dl, VT, X, Y);

5388

if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {

5389

SDValue LoHi =

5390

DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);

5391

return SDValue(LoHi.getNode(), 1);

5392

}

5393

return SDValue(); // No mulhu or equivalent

5394

};

5395

5396

// Multiply the numerator (operand 0) by the magic value.

5397

Q = GetMULHU(Q, MagicFactor);

5398

if (!Q)

5399

return SDValue();

5400

5401

Created.push_back(Q.getNode());

5402

5403

if (UseNPQ) {

5404

SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);

5405

Created.push_back(NPQ.getNode());

5406

5407

// For vectors we might have a mix of non-NPQ/NPQ paths, so use

5408

// MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.

5409

if (VT.isVector())

5410

NPQ = GetMULHU(NPQ, NPQFactor);

5411

else

5412

NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

5413

5414

Created.push_back(NPQ.getNode());

5415

5416

Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);

5417

Created.push_back(Q.getNode());

5418

}

5419

5420

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);

5421

Created.push_back(Q.getNode());

5422

5423

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5424

5425

SDValue One = DAG.getConstant(1, dl, VT);

5426

SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);

5427

return DAG.getSelect(dl, VT, IsOne, N0, Q);

5428

}

5429

5430

/// If all values in Values that *don't* match the predicate are same 'splat'

5431

/// value, then replace all values with that splat value.

5432

/// Else, if AlternativeReplacement was provided, then replace all values that

5433

/// do match predicate with AlternativeReplacement value.

5434

static void

5435

turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,

5436

std::function<bool(SDValue)> Predicate,

5437

SDValue AlternativeReplacement = SDValue()) {

5438

SDValue Replacement;

5439

// Is there a value for which the Predicate does *NOT* match? What is it?

5440

auto SplatValue = llvm::find_if_not(Values, Predicate);

5441

if (SplatValue != Values.end()) {

5442

// Does Values consist only of SplatValue's and values matching Predicate?

5443

if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {

5444

return Value == *SplatValue || Predicate(Value);

5445

})) // Then we shall replace values matching predicate with SplatValue.

5446

Replacement = *SplatValue;

5447

}

5448

if (!Replacement) {

5449

// Oops, we did not find the "baseline" splat value.

5450

if (!AlternativeReplacement)

5451

return; // Nothing to do.

5452

// Let's replace with provided value then.

5453

Replacement = AlternativeReplacement;

5454

}

5455

std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);

5456

}

5457

5458

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE

5459

/// where the divisor is constant and the comparison target is zero,

5460

/// return a DAG expression that will generate the same comparison result

5461

/// using only multiplications, additions and shifts/rotations.

5462

/// Ref: "Hacker's Delight" 10-17.

5463

SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,

5464

SDValue CompTargetNode,

5465

ISD::CondCode Cond,

5466

DAGCombinerInfo &DCI,

5467

const SDLoc &DL) const {

5468

SmallVector<SDNode *, 5> Built;

5469

if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5470

DCI, DL, Built)) {

5471

for (SDNode *N : Built)

5472

DCI.AddToWorklist(N);

5473

return Folded;

5474

}

5475

5476

return SDValue();

5477

}

5478

5479

SDValue

5480

TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,

5481

SDValue CompTargetNode, ISD::CondCode Cond,

5482

DAGCombinerInfo &DCI, const SDLoc &DL,

5483

SmallVectorImpl<SDNode *> &Created) const {

5484

// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)

5485

// - D must be constant, with D = D0 * 2^K where D0 is odd

5486

// - P is the multiplicative inverse of D0 modulo 2^W

5487

// - Q = floor(((2^W) - 1) / D)

5488

// where W is the width of the common type of N and D.

5489

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5490, __extension__ __PRETTY_FUNCTION__))

5490

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5490, __extension__ __PRETTY_FUNCTION__));

5491

5492

SelectionDAG &DAG = DCI.DAG;

5493

5494

EVT VT = REMNode.getValueType();

5495

EVT SVT = VT.getScalarType();

5496

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5497

EVT ShSVT = ShVT.getScalarType();

5498

5499

// If MUL is unavailable, we cannot proceed in any case.

5500

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5501

return SDValue();

5502

5503

bool ComparingWithAllZeros = true;

5504

bool AllComparisonsWithNonZerosAreTautological = true;

5505

bool HadTautologicalLanes = false;

5506

bool AllLanesAreTautological = true;

5507

bool HadEvenDivisor = false;

5508

bool AllDivisorsArePowerOfTwo = true;

5509

bool HadTautologicalInvertedLanes = false;

5510

SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

5511

5512

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {

5513

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5514

if (CDiv->isNullValue())

5515

return false;

5516

5517

const APInt &D = CDiv->getAPIntValue();

5518

const APInt &Cmp = CCmp->getAPIntValue();

5519

5520

ComparingWithAllZeros &= Cmp.isNullValue();

5521

5522

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5523

// if C2 is not less than C1, the comparison is always false.

5524

// But we will only be able to produce the comparison that will give the

5525

// opposive tautological answer. So this lane would need to be fixed up.

5526

bool TautologicalInvertedLane = D.ule(Cmp);

5527

HadTautologicalInvertedLanes |= TautologicalInvertedLane;

5528

5529

// If all lanes are tautological (either all divisors are ones, or divisor

5530

// is not greater than the constant we are comparing with),

5531

// we will prefer to avoid the fold.

5532

bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;

5533

HadTautologicalLanes |= TautologicalLane;

5534

AllLanesAreTautological &= TautologicalLane;

5535

5536

// If we are comparing with non-zero, we need'll need to subtract said

5537

// comparison value from the LHS. But there is no point in doing that if

5538

// every lane where we are comparing with non-zero is tautological..

5539

if (!Cmp.isNullValue())

5540

AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

5541

5542

// Decompose D into D0 * 2^K

5543

unsigned K = D.countTrailingZeros();

5544

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5544, __extension__ __PRETTY_FUNCTION__));

5545

APInt D0 = D.lshr(K);

5546

5547

// D is even if it has trailing zeros.

5548

HadEvenDivisor |= (K != 0);

5549

// D is a power-of-two if D0 is one.

5550

// If all divisors are power-of-two, we will prefer to avoid the fold.

5551

AllDivisorsArePowerOfTwo &= D0.isOneValue();

5552

5553

// P = inv(D0, 2^W)

5554

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5555

unsigned W = D.getBitWidth();

5556

APInt P = D0.zext(W + 1)

5557

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5558

.trunc(W);

5559

assert(!P.isNullValue() && "No multiplicative inverse!")(static_cast <bool> (!P.isNullValue() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5559, __extension__ __PRETTY_FUNCTION__)); // unreachable

5560

assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5560, __extension__ __PRETTY_FUNCTION__));

5561

5562

// Q = floor((2^W - 1) u/ D)

5563

// R = ((2^W - 1) u% D)

5564

APInt Q, R;

5565

APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);

5566

5567

// If we are comparing with zero, then that comparison constant is okay,

5568

// else it may need to be one less than that.

5569

if (Cmp.ugt(R))

5570

Q -= 1;

5571

5572

assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5573, __extension__ __PRETTY_FUNCTION__))

5573

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5573, __extension__ __PRETTY_FUNCTION__));

5574

5575

// If the lane is tautological the result can be constant-folded.

5576

if (TautologicalLane) {

5577

// Set P and K amount to a bogus values so we can try to splat them.

5578

P = 0;

5579

K = -1;

5580

// And ensure that comparison constant is tautological,

5581

// it will always compare true/false.

5582

Q = -1;

5583

}

5584

5585

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5586

KAmts.push_back(

5587

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5588

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5589

return true;

5590

};

5591

5592

SDValue N = REMNode.getOperand(0);

5593

SDValue D = REMNode.getOperand(1);

5594

5595

// Collect the values from each element.

5596

if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))

5597

return SDValue();

5598

5599

// If all lanes are tautological, the result can be constant-folded.

5600

if (AllLanesAreTautological)

5601

return SDValue();

5602

5603

// If this is a urem by a powers-of-two, avoid the fold since it can be

5604

// best implemented as a bit test.

5605

if (AllDivisorsArePowerOfTwo)

5606

return SDValue();

5607

5608

SDValue PVal, KVal, QVal;

5609

if (D.getOpcode() == ISD::BUILD_VECTOR) {

5610

if (HadTautologicalLanes) {

5611

// Try to turn PAmts into a splat, since we don't care about the values

5612

// that are currently '0'. If we can't, just keep '0'`s.

5613

turnVectorIntoSplatVector(PAmts, isNullConstant);

5614

// Try to turn KAmts into a splat, since we don't care about the values

5615

// that are currently '-1'. If we can't, change them to '0'`s.

5616

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5617

DAG.getConstant(0, DL, ShSVT));

5618

}

5619

5620

PVal = DAG.getBuildVector(VT, DL, PAmts);

5621

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5622

QVal = DAG.getBuildVector(VT, DL, QAmts);

5623

} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {

5624

assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__))

5625

"Expected matchBinaryPredicate to return one element for "(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__))

5626

"SPLAT_VECTORs")(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
"SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__));

5627

PVal = DAG.getSplatVector(VT, DL, PAmts[0]);

5628

KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);

5629

QVal = DAG.getSplatVector(VT, DL, QAmts[0]);

5630

} else {

5631

PVal = PAmts[0];

5632

KVal = KAmts[0];

5633

QVal = QAmts[0];

5634

}

5635

5636

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {

5637

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))

5638

return SDValue(); // FIXME: Could/should use `ISD::ADD`?

5639

assert(CompTargetNode.getValueType() == N.getValueType() &&(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5640, __extension__ __PRETTY_FUNCTION__))

5640

"Expecting that the types on LHS and RHS of comparisons match.")(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5640, __extension__ __PRETTY_FUNCTION__));

5641

N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);

5642

}

5643

5644

// (mul N, P)

5645

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5646

Created.push_back(Op0.getNode());

5647

5648

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5649

// divisors as a performance improvement, since rotating by 0 is a no-op.

5650

if (HadEvenDivisor) {

5651

// We need ROTR to do this.

5652

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5653

return SDValue();

5654

// UREM: (rotr (mul N, P), K)

5655

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);

5656

Created.push_back(Op0.getNode());

5657

}

5658

5659

// UREM: (setule/setugt (rotr (mul N, P), K), Q)

5660

SDValue NewCC =

5661

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5662

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5663

if (!HadTautologicalInvertedLanes)

5664

return NewCC;

5665

5666

// If any lanes previously compared always-false, the NewCC will give

5667

// always-true result for them, so we need to fixup those lanes.

5668

// Or the other way around for inequality predicate.

5669

assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5669, __extension__ __PRETTY_FUNCTION__));

5670

Created.push_back(NewCC.getNode());

5671

5672

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,

5673

// if C2 is not less than C1, the comparison is always false.

5674

// But we have produced the comparison that will give the

5675

// opposive tautological answer. So these lanes would need to be fixed up.

5676

SDValue TautologicalInvertedChannels =

5677

DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);

5678

Created.push_back(TautologicalInvertedChannels.getNode());

5679

5680

// NOTE: we avoid letting illegal types through even if we're before legalize

5681

// ops – legalization has a hard time producing good code for this.

5682

if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {

5683

// If we have a vector select, let's replace the comparison results in the

5684

// affected lanes with the correct tautological result.

5685

SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,

5686

DL, SETCCVT, SETCCVT);

5687

return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,

5688

Replacement, NewCC);

5689

}

5690

5691

// Else, we can just invert the comparison result in the appropriate lanes.

5692

//

5693

// NOTE: see the note above VSELECT above.

5694

if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))

5695

return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,

5696

TautologicalInvertedChannels);

5697

5698

return SDValue(); // Don't know how to lower.

5699

}

5700

5701

/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE

5702

/// where the divisor is constant and the comparison target is zero,

5703

/// return a DAG expression that will generate the same comparison result

5704

/// using only multiplications, additions and shifts/rotations.

5705

/// Ref: "Hacker's Delight" 10-17.

5706

SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,

5707

SDValue CompTargetNode,

5708

ISD::CondCode Cond,

5709

DAGCombinerInfo &DCI,

5710

const SDLoc &DL) const {

5711

SmallVector<SDNode *, 7> Built;

5712

if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,

5713

DCI, DL, Built)) {

5714

assert(Built.size() <= 7 && "Max size prediction failed.")(static_cast <bool> (Built.size() <= 7 && "Max size prediction failed."
) ? void (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5714, __extension__ __PRETTY_FUNCTION__));

5715

for (SDNode *N : Built)

5716

DCI.AddToWorklist(N);

5717

return Folded;

5718

}

5719

5720

return SDValue();

5721

}

5722

5723

SDValue

5724

TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,

5725

SDValue CompTargetNode, ISD::CondCode Cond,

5726

DAGCombinerInfo &DCI, const SDLoc &DL,

5727

SmallVectorImpl<SDNode *> &Created) const {

5728

// Fold:

5729

// (seteq/ne (srem N, D), 0)

5730

// To:

5731

// (setule/ugt (rotr (add (mul N, P), A), K), Q)

5732

//

5733

// - D must be constant, with D = D0 * 2^K where D0 is odd

5734

// - P is the multiplicative inverse of D0 modulo 2^W

5735

// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))

5736

// - Q = floor((2 * A) / (2^K))

5737

// where W is the width of the common type of N and D.

5738

assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5739, __extension__ __PRETTY_FUNCTION__))

5739

"Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5739, __extension__ __PRETTY_FUNCTION__));

5740

5741

SelectionDAG &DAG = DCI.DAG;

5742

5743

EVT VT = REMNode.getValueType();

5744

EVT SVT = VT.getScalarType();

5745

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());

5746

EVT ShSVT = ShVT.getScalarType();

5747

5748

// If we are after ops legalization, and MUL is unavailable, we can not

5749

// proceed.

5750

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))

5751

return SDValue();

5752

5753

// TODO: Could support comparing with non-zero too.

5754

ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);

5755

if (!CompTarget || !CompTarget->isNullValue())

5756

return SDValue();

5757

5758

bool HadIntMinDivisor = false;

5759

bool HadOneDivisor = false;

5760

bool AllDivisorsAreOnes = true;

5761

bool HadEvenDivisor = false;

5762

bool NeedToApplyOffset = false;

5763

bool AllDivisorsArePowerOfTwo = true;

5764

SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

5765

5766

auto BuildSREMPattern = [&](ConstantSDNode *C) {

5767

// Division by 0 is UB. Leave it to be constant-folded elsewhere.

5768

if (C->isNullValue())

5769

return false;

5770

5771

// FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

5772

5773

// WARNING: this fold is only valid for positive divisors!

5774

APInt D = C->getAPIntValue();

5775

if (D.isNegative())

5776

D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`

5777

5778

HadIntMinDivisor |= D.isMinSignedValue();

5779

5780

// If all divisors are ones, we will prefer to avoid the fold.

5781

HadOneDivisor |= D.isOneValue();

5782

AllDivisorsAreOnes &= D.isOneValue();

5783

5784

// Decompose D into D0 * 2^K

5785

unsigned K = D.countTrailingZeros();

5786

assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5786, __extension__ __PRETTY_FUNCTION__));

5787

APInt D0 = D.lshr(K);

5788

5789

if (!D.isMinSignedValue()) {

5790

// D is even if it has trailing zeros; unless it's INT_MIN, in which case

5791

// we don't care about this lane in this fold, we'll special-handle it.

5792

HadEvenDivisor |= (K != 0);

5793

}

5794

5795

// D is a power-of-two if D0 is one. This includes INT_MIN.

5796

// If all divisors are power-of-two, we will prefer to avoid the fold.

5797

AllDivisorsArePowerOfTwo &= D0.isOneValue();

5798

5799

// P = inv(D0, 2^W)

5800

// 2^W requires W + 1 bits, so we have to extend and then truncate.

5801

unsigned W = D.getBitWidth();

5802

APInt P = D0.zext(W + 1)

5803

.multiplicativeInverse(APInt::getSignedMinValue(W + 1))

5804

.trunc(W);

5805

assert(!P.isNullValue() && "No multiplicative inverse!")(static_cast <bool> (!P.isNullValue() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5805, __extension__ __PRETTY_FUNCTION__)); // unreachable

5806

assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5806, __extension__ __PRETTY_FUNCTION__));

5807

5808

// A = floor((2^(W - 1) - 1) / D0) & -2^K

5809

APInt A = APInt::getSignedMaxValue(W).udiv(D0);

5810

A.clearLowBits(K);

5811

5812

if (!D.isMinSignedValue()) {

5813

// If divisor INT_MIN, then we don't care about this lane in this fold,

5814

// we'll special-handle it.

5815

NeedToApplyOffset |= A != 0;

5816

}

5817

5818

// Q = floor((2 * A) / (2^K))

5819

APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));

5820

5821

assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&(static_cast <bool> (APInt::getAllOnesValue(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5822, __extension__ __PRETTY_FUNCTION__))

5822

"We are expecting that A is always less than all-ones for SVT")(static_cast <bool> (APInt::getAllOnesValue(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5822, __extension__ __PRETTY_FUNCTION__));

5823

assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5824, __extension__ __PRETTY_FUNCTION__))

5824

"We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5824, __extension__ __PRETTY_FUNCTION__));

5825

5826

// If the divisor is 1 the result can be constant-folded. Likewise, we

5827

// don't care about INT_MIN lanes, those can be set to undef if appropriate.

5828

if (D.isOneValue()) {

5829

// Set P, A and K to a bogus values so we can try to splat them.

5830

P = 0;

5831

A = -1;

5832

K = -1;

5833

5834

// x ?% 1 == 0 <--> true <--> x u<= -1

5835

Q = -1;

5836

}

5837

5838

PAmts.push_back(DAG.getConstant(P, DL, SVT));

5839

AAmts.push_back(DAG.getConstant(A, DL, SVT));

5840

KAmts.push_back(

5841

DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));

5842

QAmts.push_back(DAG.getConstant(Q, DL, SVT));

5843

return true;

5844

};

5845

5846

SDValue N = REMNode.getOperand(0);

5847

SDValue D = REMNode.getOperand(1);

5848

5849

// Collect the values from each element.

5850

if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))

5851

return SDValue();

5852

5853

// If this is a srem by a one, avoid the fold since it can be constant-folded.

5854

if (AllDivisorsAreOnes)

5855

return SDValue();

5856

5857

// If this is a srem by a powers-of-two (including INT_MIN), avoid the fold

5858

// since it can be best implemented as a bit test.

5859

if (AllDivisorsArePowerOfTwo)

5860

return SDValue();

5861

5862

SDValue PVal, AVal, KVal, QVal;

5863

if (D.getOpcode() == ISD::BUILD_VECTOR) {

5864

if (HadOneDivisor) {

5865

// Try to turn PAmts into a splat, since we don't care about the values

5866

// that are currently '0'. If we can't, just keep '0'`s.

5867

turnVectorIntoSplatVector(PAmts, isNullConstant);

5868

// Try to turn AAmts into a splat, since we don't care about the

5869

// values that are currently '-1'. If we can't, change them to '0'`s.

5870

turnVectorIntoSplatVector(AAmts, isAllOnesConstant,

5871

DAG.getConstant(0, DL, SVT));

5872

// Try to turn KAmts into a splat, since we don't care about the values

5873

// that are currently '-1'. If we can't, change them to '0'`s.

5874

turnVectorIntoSplatVector(KAmts, isAllOnesConstant,

5875

DAG.getConstant(0, DL, ShSVT));

5876

}

5877

5878

PVal = DAG.getBuildVector(VT, DL, PAmts);

5879

AVal = DAG.getBuildVector(VT, DL, AAmts);

5880

KVal = DAG.getBuildVector(ShVT, DL, KAmts);

5881

QVal = DAG.getBuildVector(VT, DL, QAmts);

5882

} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {

5883

assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5886, __extension__ __PRETTY_FUNCTION__))

5884

QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5886, __extension__ __PRETTY_FUNCTION__))

5885

"Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5886, __extension__ __PRETTY_FUNCTION__))

5886

"vectors")(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5886, __extension__ __PRETTY_FUNCTION__));

5887

PVal = DAG.getSplatVector(VT, DL, PAmts[0]);

5888

AVal = DAG.getSplatVector(VT, DL, AAmts[0]);

5889

KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);

5890

QVal = DAG.getSplatVector(VT, DL, QAmts[0]);

5891

} else {

5892

assert(isa<ConstantSDNode>(D) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(D) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(D) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5892, __extension__ __PRETTY_FUNCTION__));

5893

PVal = PAmts[0];

5894

AVal = AAmts[0];

5895

KVal = KAmts[0];

5896

QVal = QAmts[0];

5897

}

5898

5899

// (mul N, P)

5900

SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);

5901

Created.push_back(Op0.getNode());

5902

5903

if (NeedToApplyOffset) {

5904

// We need ADD to do this.

5905

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))

5906

return SDValue();

5907

5908

// (add (mul N, P), A)

5909

Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);

5910

Created.push_back(Op0.getNode());

5911

}

5912

5913

// Rotate right only if any divisor was even. We avoid rotates for all-odd

5914

// divisors as a performance improvement, since rotating by 0 is a no-op.

5915

if (HadEvenDivisor) {

5916

// We need ROTR to do this.

5917

if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))

5918

return SDValue();

5919

// SREM: (rotr (add (mul N, P), A), K)

5920

Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);

5921

Created.push_back(Op0.getNode());

5922

}

5923

5924

// SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)

5925

SDValue Fold =

5926

DAG.getSetCC(DL, SETCCVT, Op0, QVal,

5927

((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

5928

5929

// If we didn't have lanes with INT_MIN divisor, then we're done.

5930

if (!HadIntMinDivisor)

5931

return Fold;

5932

5933

// That fold is only valid for positive divisors. Which effectively means,

5934

// it is invalid for INT_MIN divisors. So if we have such a lane,

5935

// we must fix-up results for said lanes.

5936

assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5936, __extension__ __PRETTY_FUNCTION__));

5937

5938

// NOTE: we avoid letting illegal types through even if we're before legalize

5939

// ops – legalization has a hard time producing good code for the code that

5940

// follows.

5941

if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||

5942

!isOperationLegalOrCustom(ISD::AND, VT) ||

5943

!isOperationLegalOrCustom(Cond, VT) ||

5944

!isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))

5945

return SDValue();

5946

5947

Created.push_back(Fold.getNode());

5948

5949

SDValue IntMin = DAG.getConstant(

5950

APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);

5951

SDValue IntMax = DAG.getConstant(

5952

APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);

5953

SDValue Zero =

5954

DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);

5955

5956

// Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.

5957

SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);

5958

Created.push_back(DivisorIsIntMin.getNode());

5959

5960

// (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0

5961

SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);

5962

Created.push_back(Masked.getNode());

5963

SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);

5964

Created.push_back(MaskedIsZero.getNode());

5965

5966

// To produce final result we need to blend 2 vectors: 'SetCC' and

5967

// 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick

5968

// from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is

5969

// constant-folded, select can get lowered to a shuffle with constant mask.

5970

SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,

5971

MaskedIsZero, Fold);

5972

5973

return Blended;

5974

}

5975

5976

bool TargetLowering::

5977

verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {

5978

if (!isa<ConstantSDNode>(Op.getOperand(0))) {

5979

DAG.getContext()->emitError("argument to '__builtin_return_address' must "

5980

"be a constant integer");

5981

return true;

5982

}

5983

5984

return false;

5985

}

5986

5987

SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,

5988

const DenormalMode &Mode) const {

5989

SDLoc DL(Op);

5990

EVT VT = Op.getValueType();

5991

EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

5992

SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);

5993

// Testing it with denormal inputs to avoid wrong estimate.

5994

if (Mode.Input == DenormalMode::IEEE) {

5995

// This is specifically a check for the handling of denormal inputs,

5996

// not the result.

5997

5998

// Test = fabs(X) < SmallestNormal

5999

const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);

6000

APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);

6001

SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);

6002

SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);

6003

return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);

6004

}

6005

// Test = X == 0.0

6006

return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);

6007

}

6008

6009

SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,

6010

bool LegalOps, bool OptForSize,

6011

NegatibleCost &Cost,

6012

unsigned Depth) const {

6013

// fneg is removable even if it has multiple uses.

6014

if (Op.getOpcode() == ISD::FNEG) {

6015

Cost = NegatibleCost::Cheaper;

6016

return Op.getOperand(0);

6017

}

6018

6019

// Don't recurse exponentially.

6020

if (Depth > SelectionDAG::MaxRecursionDepth)

6021

return SDValue();

6022

6023

// Pre-increment recursion depth for use in recursive calls.

6024

++Depth;

6025

const SDNodeFlags Flags = Op->getFlags();

6026

const TargetOptions &Options = DAG.getTarget().Options;

6027

EVT VT = Op.getValueType();

6028

unsigned Opcode = Op.getOpcode();

6029

6030

// Don't allow anything with multiple uses unless we know it is free.

6031

if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {

6032

bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&

6033

isFPExtFree(VT, Op.getOperand(0).getValueType());

6034

if (!IsFreeExtend)

6035

return SDValue();

6036

}

6037

6038

auto RemoveDeadNode = [&](SDValue N) {

6039

if (N && N.getNode()->use_empty())

6040

DAG.RemoveDeadNode(N.getNode());

6041

};

6042

6043

SDLoc DL(Op);

6044

6045

// Because getNegatedExpression can delete nodes we need a handle to keep

6046

// temporary nodes alive in case the recursion manages to create an identical

6047

// node.

6048

std::list<HandleSDNode> Handles;

6049

6050

switch (Opcode) {

6051

case ISD::ConstantFP: {

6052

// Don't invert constant FP values after legalization unless the target says

6053

// the negated constant is legal.

6054

bool IsOpLegal =

6055

isOperationLegal(ISD::ConstantFP, VT) ||

6056

isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,

6057

OptForSize);

6058

6059

if (LegalOps && !IsOpLegal)

6060

break;

6061

6062

APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();

6063

V.changeSign();

6064

SDValue CFP = DAG.getConstantFP(V, DL, VT);

6065

6066

// If we already have the use of the negated floating constant, it is free

6067

// to negate it even it has multiple uses.

6068

if (!Op.hasOneUse() && CFP.use_empty())

6069

break;

6070

Cost = NegatibleCost::Neutral;

6071

return CFP;

6072

}

6073

case ISD::BUILD_VECTOR: {

6074

// Only permit BUILD_VECTOR of constants.

6075

if (llvm::any_of(Op->op_values(), [&](SDValue N) {

6076

return !N.isUndef() && !isa<ConstantFPSDNode>(N);

6077

}))

6078

break;

6079

6080

bool IsOpLegal =

6081

(isOperationLegal(ISD::ConstantFP, VT) &&

6082

isOperationLegal(ISD::BUILD_VECTOR, VT)) ||

6083

llvm::all_of(Op->op_values(), [&](SDValue N) {

6084

return N.isUndef() ||

6085

isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,

6086

OptForSize);

6087

});

6088

6089

if (LegalOps && !IsOpLegal)

6090

break;

6091

6092

SmallVector<SDValue, 4> Ops;

6093

for (SDValue C : Op->op_values()) {

6094

if (C.isUndef()) {

6095

Ops.push_back(C);

6096

continue;

6097

}

6098

APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();

6099

V.changeSign();

6100

Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));

6101

}

6102

Cost = NegatibleCost::Neutral;

6103

return DAG.getBuildVector(VT, DL, Ops);

6104

}

6105

case ISD::FADD: {

6106

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6107

break;

6108

6109

// After operation legalization, it might not be legal to create new FSUBs.

6110

if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))

6111

break;

6112

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6113

6114

// fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)

6115

NegatibleCost CostX = NegatibleCost::Expensive;

6116

SDValue NegX =

6117

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6118

// Prevent this node from being deleted by the next call.

6119

if (NegX)

6120

Handles.emplace_back(NegX);

6121

6122

// fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)

6123

NegatibleCost CostY = NegatibleCost::Expensive;

6124

SDValue NegY =

6125

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6126

6127

// We're done with the handles.

6128

Handles.clear();

6129

6130

// Negate the X if its cost is less or equal than Y.

6131

if (NegX && (CostX <= CostY)) {

6132

Cost = CostX;

6133

SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);

6134

if (NegY != N)

6135

RemoveDeadNode(NegY);

6136

return N;

6137

}

6138

6139

// Negate the Y if it is not expensive.

6140

if (NegY) {

6141

Cost = CostY;

6142

SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);

6143

if (NegX != N)

6144

RemoveDeadNode(NegX);

6145

return N;

6146

}

6147

break;

6148

}

6149

case ISD::FSUB: {

6150

// We can't turn -(A-B) into B-A when we honor signed zeros.

6151

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6152

break;

6153

6154

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6155

// fold (fneg (fsub 0, Y)) -> Y

6156

if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))

6157

if (C->isZero()) {

6158

Cost = NegatibleCost::Cheaper;

6159

return Y;

6160

}

6161

6162

// fold (fneg (fsub X, Y)) -> (fsub Y, X)

6163

Cost = NegatibleCost::Neutral;

6164

return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);

6165

}

6166

case ISD::FMUL:

6167

case ISD::FDIV: {

6168

SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

6169

6170

// fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)

6171

NegatibleCost CostX = NegatibleCost::Expensive;

6172

SDValue NegX =

6173

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6174

// Prevent this node from being deleted by the next call.

6175

if (NegX)

6176

Handles.emplace_back(NegX);

6177

6178

// fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))

6179

NegatibleCost CostY = NegatibleCost::Expensive;

6180

SDValue NegY =

6181

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6182

6183

// We're done with the handles.

6184

Handles.clear();

6185

6186

// Negate the X if its cost is less or equal than Y.

6187

if (NegX && (CostX <= CostY)) {

6188

Cost = CostX;

6189

SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);

6190

if (NegY != N)

6191

RemoveDeadNode(NegY);

6192

return N;

6193

}

6194

6195

// Ignore X * 2.0 because that is expected to be canonicalized to X + X.

6196

if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))

6197

if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)

6198

break;

6199

6200

// Negate the Y if it is not expensive.

6201

if (NegY) {

6202

Cost = CostY;

6203

SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);

6204

if (NegX != N)

6205

RemoveDeadNode(NegX);

6206

return N;

6207

}

6208

break;

6209

}

6210

case ISD::FMA:

6211

case ISD::FMAD: {

6212

if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())

6213

break;

6214

6215

SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);

6216

NegatibleCost CostZ = NegatibleCost::Expensive;

6217

SDValue NegZ =

6218

getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);

6219

// Give up if fail to negate the Z.

6220

if (!NegZ)

6221

break;

6222

6223

// Prevent this node from being deleted by the next two calls.

6224

Handles.emplace_back(NegZ);

6225

6226

// fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))

6227

NegatibleCost CostX = NegatibleCost::Expensive;

6228

SDValue NegX =

6229

getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);

6230

// Prevent this node from being deleted by the next call.

6231

if (NegX)

6232

Handles.emplace_back(NegX);

6233

6234

// fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))

6235

NegatibleCost CostY = NegatibleCost::Expensive;

6236

SDValue NegY =

6237

getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

6238

6239

// We're done with the handles.

6240

Handles.clear();

6241

6242

// Negate the X if its cost is less or equal than Y.

6243

if (NegX && (CostX <= CostY)) {

6244

Cost = std::min(CostX, CostZ);

6245

SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);

6246

if (NegY != N)

6247

RemoveDeadNode(NegY);

6248

return N;

6249

}

6250

6251

// Negate the Y if it is not expensive.

6252

if (NegY) {

6253

Cost = std::min(CostY, CostZ);

6254

SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);

6255

if (NegX != N)

6256

RemoveDeadNode(NegX);

6257

return N;

6258

}

6259

break;

6260

}

6261

6262

case ISD::FP_EXTEND:

6263

case ISD::FSIN:

6264

if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,

6265

OptForSize, Cost, Depth))

6266

return DAG.getNode(Opcode, DL, VT, NegV);

6267

break;

6268

case ISD::FP_ROUND:

6269

if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,

6270

OptForSize, Cost, Depth))

6271

return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));

6272

break;

6273

}

6274

6275

return SDValue();

6276

}

6277

6278

//===----------------------------------------------------------------------===//

6279

// Legalization Utilities

6280

//===----------------------------------------------------------------------===//

6281

6282

bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,

6283

SDValue LHS, SDValue RHS,

6284

SmallVectorImpl<SDValue> &Result,

6285

EVT HiLoVT, SelectionDAG &DAG,

6286

MulExpansionKind Kind, SDValue LL,

6287

SDValue LH, SDValue RL, SDValue RH) const {

6288

assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6289, __extension__ __PRETTY_FUNCTION__))

6289

Opcode == ISD::SMUL_LOHI)(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6289, __extension__ __PRETTY_FUNCTION__));

6290

6291

bool HasMULHS = (Kind == MulExpansionKind::Always) ||

6292

isOperationLegalOrCustom(ISD::MULHS, HiLoVT);

6293

bool HasMULHU = (Kind == MulExpansionKind::Always) ||

6294

isOperationLegalOrCustom(ISD::MULHU, HiLoVT);

6295

bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||

6296

isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);

6297

bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||

6298

isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);

6299

6300

if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)

6301

return false;

6302

6303

unsigned OuterBitSize = VT.getScalarSizeInBits();

6304

unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();

6305

6306

// LL, LH, RL, and RH must be either all NULL or all set to a value.

6307

assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
!RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6308, __extension__ __PRETTY_FUNCTION__))

6308

(!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()))(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
!RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6308, __extension__ __PRETTY_FUNCTION__));

6309

6310

SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);

6311

auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,

6312

bool Signed) -> bool {

6313

if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {

6314

Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);

6315

Hi = SDValue(Lo.getNode(), 1);

6316

return true;

6317

}

6318

if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {

6319

Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);

6320

Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);

6321

return true;

6322

}

6323

return false;

6324

};

6325

6326

SDValue Lo, Hi;

6327

6328

if (!LL.getNode() && !RL.getNode() &&

6329

isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {

6330

LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);

6331

RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);

6332

}

6333

6334

if (!LL.getNode())

6335

return false;

6336

6337

APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);

6338

if (DAG.MaskedValueIsZero(LHS, HighMask) &&

6339

DAG.MaskedValueIsZero(RHS, HighMask)) {

6340

// The inputs are both zero-extended.

6341

if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {

6342

Result.push_back(Lo);

6343

Result.push_back(Hi);

6344

if (Opcode != ISD::MUL) {

6345

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);

6346

Result.push_back(Zero);

6347

Result.push_back(Zero);

6348

}

6349

return true;

6350

}

6351

}

6352

6353

if (!VT.isVector() && Opcode == ISD::MUL &&

6354

DAG.ComputeNumSignBits(LHS) > InnerBitSize &&

6355

DAG.ComputeNumSignBits(RHS) > InnerBitSize) {

6356

// The input values are both sign-extended.

6357

// TODO non-MUL case?

6358

if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {

6359

Result.push_back(Lo);

6360

Result.push_back(Hi);

6361

return true;

6362

}

6363

}

6364

6365

unsigned ShiftAmount = OuterBitSize - InnerBitSize;

6366

EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());

6367

if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {

6368

// FIXME getShiftAmountTy does not always return a sensible result when VT

6369

// is an illegal type, and so the type may be too small to fit the shift

6370

// amount. Override it with i32. The shift will have to be legalized.

6371

ShiftAmountTy = MVT::i32;

6372

}

6373

SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);

6374

6375

if (!LH.getNode() && !RH.getNode() &&

6376

isOperationLegalOrCustom(ISD::SRL, VT) &&

6377

isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {

6378

LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);

6379

LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);

6380

RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);

6381

RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);

6382

}

6383

6384

if (!LH.getNode())

6385

return false;

6386

6387

if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))

6388

return false;

6389

6390

Result.push_back(Lo);

6391

6392

if (Opcode == ISD::MUL) {

6393

RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);

6394

LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);

6395

Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);

6396

Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);

6397

Result.push_back(Hi);

6398

return true;

6399

}

6400

6401

// Compute the full width result.

6402

auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {

6403

Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);

6404

Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);

6405

Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);

6406

return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);

6407

};

6408

6409

SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);

6410

if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))

6411

return false;

6412

6413

// This is effectively the add part of a multiply-add of half-sized operands,

6414

// so it cannot overflow.

6415

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

6416

6417

if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))

6418

return false;

6419

6420

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);

6421

EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

6422

6423

bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&

6424

isOperationLegalOrCustom(ISD::ADDE, VT));

6425

if (UseGlue)

6426

Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,

6427

Merge(Lo, Hi));

6428

else

6429

Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,

6430

Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));

6431

6432

SDValue Carry = Next.getValue(1);

6433

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6434

Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

6435

6436

if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))

6437

return false;

6438

6439

if (UseGlue)

6440

Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,

6441

Carry);

6442

else

6443

Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,

6444

Zero, Carry);

6445

6446

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

6447

6448

if (Opcode == ISD::SMUL_LOHI) {

6449

SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,

6450

DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));

6451

Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);

6452

6453

NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,

6454

DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));

6455

Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);

6456

}

6457

6458

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6459

Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

6460

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));

6461

return true;

6462

}

6463

6464

bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,

6465

SelectionDAG &DAG, MulExpansionKind Kind,

6466

SDValue LL, SDValue LH, SDValue RL,

6467

SDValue RH) const {

6468

SmallVector<SDValue, 2> Result;

6469

bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),

6470

N->getOperand(0), N->getOperand(1), Result, HiLoVT,

6471

DAG, Kind, LL, LH, RL, RH);

6472

if (Ok) {

6473

assert(Result.size() == 2)(static_cast <bool> (Result.size() == 2) ? void (0) : __assert_fail
("Result.size() == 2", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6473, __extension__ __PRETTY_FUNCTION__));

6474

Lo = Result[0];

6475

Hi = Result[1];

6476

}

6477

return Ok;

6478

}

6479

6480

// Check that (every element of) Z is undef or not an exact multiple of BW.

6481

static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {

6482

return ISD::matchUnaryPredicate(

6483

Z,

6484

[=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },

6485

true);

6486

}

6487

6488

bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,

6489

SelectionDAG &DAG) const {

6490

EVT VT = Node->getValueType(0);

6491

6492

if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||

6493

!isOperationLegalOrCustom(ISD::SRL, VT) ||

6494

!isOperationLegalOrCustom(ISD::SUB, VT) ||

6495

!isOperationLegalOrCustomOrPromote(ISD::OR, VT)))

6496

return false;

6497

6498

SDValue X = Node->getOperand(0);

6499

SDValue Y = Node->getOperand(1);

6500

SDValue Z = Node->getOperand(2);

6501

6502

unsigned BW = VT.getScalarSizeInBits();

6503

bool IsFSHL = Node->getOpcode() == ISD::FSHL;

6504

SDLoc DL(SDValue(Node, 0));

6505

6506

EVT ShVT = Z.getValueType();

6507

6508

// If a funnel shift in the other direction is more supported, use it.

6509

unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;

6510

if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&

6511

isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {

6512

if (isNonZeroModBitWidthOrUndef(Z, BW)) {

6513

// fshl X, Y, Z -> fshr X, Y, -Z

6514

// fshr X, Y, Z -> fshl X, Y, -Z

6515

SDValue Zero = DAG.getConstant(0, DL, ShVT);

6516

Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);

6517

} else {

6518

// fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z

6519

// fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z

6520

SDValue One = DAG.getConstant(1, DL, ShVT);

6521

if (IsFSHL) {

6522

Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);

6523

X = DAG.getNode(ISD::SRL, DL, VT, X, One);

6524

} else {

6525

X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);

6526

Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);

6527

}

6528

Z = DAG.getNOT(DL, Z, ShVT);

6529

}

6530

Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);

6531

return true;

6532

}

6533

6534

SDValue ShX, ShY;

6535

SDValue ShAmt, InvShAmt;

6536

if (isNonZeroModBitWidthOrUndef(Z, BW)) {

6537

// fshl: X << C | Y >> (BW - C)

6538

// fshr: X << (BW - C) | Y >> C

6539

// where C = Z % BW is not zero

6540

SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);

6541

ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);

6542

InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);

6543

ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);

6544

ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);

6545

} else {

6546

// fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))

6547

// fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)

6548

SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);

6549

if (isPowerOf2_32(BW)) {

6550

// Z % BW -> Z & (BW - 1)

6551

ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);

6552

// (BW - 1) - (Z % BW) -> ~Z & (BW - 1)

6553

InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);

6554

} else {

6555

SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);

6556

ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);

6557

InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);

6558

}

6559

6560

SDValue One = DAG.getConstant(1, DL, ShVT);

6561

if (IsFSHL) {

6562

ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);

6563

SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);

6564

ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);

6565

} else {

6566

SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);

6567

ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);

6568

ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);

6569

}

6570

}

6571

Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);

6572

return true;

6573

}

6574

6575

// TODO: Merge with expandFunnelShift.

6576

bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,

6577

SDValue &Result, SelectionDAG &DAG) const {

6578

EVT VT = Node->getValueType(0);

6579

unsigned EltSizeInBits = VT.getScalarSizeInBits();

6580

bool IsLeft = Node->getOpcode() == ISD::ROTL;

6581

SDValue Op0 = Node->getOperand(0);

6582

SDValue Op1 = Node->getOperand(1);

6583

SDLoc DL(SDValue(Node, 0));

6584

6585

EVT ShVT = Op1.getValueType();

6586

SDValue Zero = DAG.getConstant(0, DL, ShVT);

6587

6588

// If a rotate in the other direction is supported, use it.

6589

unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;

6590

if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {

6591

SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);

6592

Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);

6593

return true;

6594

}

6595

6596

if (!AllowVectorOps && VT.isVector() &&

6597

(!isOperationLegalOrCustom(ISD::SHL, VT) ||

6598

!isOperationLegalOrCustom(ISD::SRL, VT) ||

6599

!isOperationLegalOrCustom(ISD::SUB, VT) ||

6600

!isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||

6601

!isOperationLegalOrCustomOrPromote(ISD::AND, VT)))

6602

return false;

6603

6604

unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;

6605

unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;

6606

SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);

6607

SDValue ShVal;

6608

SDValue HsVal;

6609

if (isPowerOf2_32(EltSizeInBits)) {

6610

// (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))

6611

// (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))

6612

SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);

6613

SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);

6614

ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);

6615

SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);

6616

HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);

6617

} else {

6618

// (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))

6619

// (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))

6620

SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);

6621

SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);

6622

ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);

6623

SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);

6624

SDValue One = DAG.getConstant(1, DL, ShVT);

6625

HsVal =

6626

DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);

6627

}

6628

Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);

6629

return true;

6630

}

6631

6632

void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,

6633

SelectionDAG &DAG) const {

6634

assert(Node->getNumOperands() == 3 && "Not a double-shift!")(static_cast <bool> (Node->getNumOperands() == 3 &&
"Not a double-shift!") ? void (0) : __assert_fail ("Node->getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6634, __extension__ __PRETTY_FUNCTION__));

6635

EVT VT = Node->getValueType(0);

6636

unsigned VTBits = VT.getScalarSizeInBits();

6637

assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected")(static_cast <bool> (isPowerOf2_32(VTBits) && "Power-of-two integer type expected"
) ? void (0) : __assert_fail ("isPowerOf2_32(VTBits) && \"Power-of-two integer type expected\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6637, __extension__ __PRETTY_FUNCTION__));

6638

6639

bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;

6640

bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;

6641

SDValue ShOpLo = Node->getOperand(0);

6642

SDValue ShOpHi = Node->getOperand(1);

6643

SDValue ShAmt = Node->getOperand(2);

6644

EVT ShAmtVT = ShAmt.getValueType();

6645

EVT ShAmtCCVT =

6646

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);

6647

SDLoc dl(Node);

6648

6649

// ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and

6650

// ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized

6651

// away during isel.

6652

SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,

6653

DAG.getConstant(VTBits - 1, dl, ShAmtVT));

6654

SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,

6655

DAG.getConstant(VTBits - 1, dl, ShAmtVT))

6656

: DAG.getConstant(0, dl, VT);

6657

6658

SDValue Tmp2, Tmp3;

6659

if (IsSHL) {

6660

Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);

6661

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);

6662

} else {

6663

Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);

6664

Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);

6665

}

6666

6667

// If the shift amount is larger or equal than the width of a part we don't

6668

// use the result from the FSHL/FSHR. Insert a test and select the appropriate

6669

// values for large shift amounts.

6670

SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,

6671

DAG.getConstant(VTBits, dl, ShAmtVT));

6672

SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,

6673

DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);

6674

6675

if (IsSHL) {

6676

Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);

6677

Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);

6678

} else {

6679

Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);

6680

Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);

6681

}

6682

}

6683

6684

bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,

6685

SelectionDAG &DAG) const {

6686

unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;

6687

SDValue Src = Node->getOperand(OpNo);

6688

EVT SrcVT = Src.getValueType();

6689

EVT DstVT = Node->getValueType(0);

6690

SDLoc dl(SDValue(Node, 0));

6691

6692

// FIXME: Only f32 to i64 conversions are supported.

6693

if (SrcVT != MVT::f32 || DstVT != MVT::i64)

6694

return false;

6695

6696

if (Node->isStrictFPOpcode())

6697

// When a NaN is converted to an integer a trap is allowed. We can't

6698

// use this expansion here because it would eliminate that trap. Other

6699

// traps are also allowed and cannot be eliminated. See

6700

// IEEE 754-2008 sec 5.8.

6701

return false;

6702

6703

// Expand f32 -> i64 conversion

6704

// This algorithm comes from compiler-rt's implementation of fixsfdi:

6705

// https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c

6706

unsigned SrcEltBits = SrcVT.getScalarSizeInBits();

6707

EVT IntVT = SrcVT.changeTypeToInteger();

6708

EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());

6709

6710

SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);

6711

SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);

6712

SDValue Bias = DAG.getConstant(127, dl, IntVT);

6713

SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);

6714

SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);

6715

SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);

6716

6717

SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);

6718

6719

SDValue ExponentBits = DAG.getNode(

6720

ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),

6721

DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));

6722

SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);

6723

6724

SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,

6725

DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),

6726

DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));

6727

Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);

6728

6729

SDValue R = DAG.getNode(ISD::OR, dl, IntVT,

6730

DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),

6731

DAG.getConstant(0x00800000, dl, IntVT));

6732

6733

R = DAG.getZExtOrTrunc(R, dl, DstVT);

6734

6735

R = DAG.getSelectCC(

6736

dl, Exponent, ExponentLoBit,

6737

DAG.getNode(ISD::SHL, dl, DstVT, R,

6738

DAG.getZExtOrTrunc(

6739

DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),

6740

dl, IntShVT)),

6741

DAG.getNode(ISD::SRL, dl, DstVT, R,

6742

DAG.getZExtOrTrunc(

6743

DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),

6744

dl, IntShVT)),

6745

ISD::SETGT);

6746

6747

SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,

6748

DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);

6749

6750

Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),

6751

DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);

6752

return true;

6753

}

6754

6755

bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,

6756

SDValue &Chain,

6757

SelectionDAG &DAG) const {

6758

SDLoc dl(SDValue(Node, 0));

6759

unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;

6760

SDValue Src = Node->getOperand(OpNo);

6761

6762

EVT SrcVT = Src.getValueType();

6763

EVT DstVT = Node->getValueType(0);

6764

EVT SetCCVT =

6765

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);

6766

EVT DstSetCCVT =

6767

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);

6768

6769

// Only expand vector types if we have the appropriate vector bit operations.

6770

unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :

6771

ISD::FP_TO_SINT;

6772

if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||

6773

!isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))

6774

return false;

6775

6776

// If the maximum float value is smaller then the signed integer range,

6777

// the destination signmask can't be represented by the float, so we can

6778

// just use FP_TO_SINT directly.

6779

const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);

6780

APFloat APF(APFSem, APInt::getNullValue(SrcVT.getScalarSizeInBits()));

6781

APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());

6782

if (APFloat::opOverflow &

6783

APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {

6784

if (Node->isStrictFPOpcode()) {

6785

Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },

6786

{ Node->getOperand(0), Src });

6787

Chain = Result.getValue(1);

6788

} else

6789

Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);

6790

return true;

6791

}

6792

6793

// Don't expand it if there isn't cheap fsub instruction.

6794

if (!isOperationLegalOrCustom(

6795

Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))

6796

return false;

6797

6798

SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);

6799

SDValue Sel;

6800

6801

if (Node->isStrictFPOpcode()) {

6802

Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,

6803

Node->getOperand(0), /*IsSignaling*/ true);

6804

Chain = Sel.getValue(1);

6805

} else {

6806

Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);

6807

}

6808

6809

bool Strict = Node->isStrictFPOpcode() ||

6810

shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);

6811

6812

if (Strict) {

6813

// Expand based on maximum range of FP_TO_SINT, if the value exceeds the

6814

// signmask then offset (the result of which should be fully representable).

6815

// Sel = Src < 0x8000000000000000

6816

// FltOfs = select Sel, 0, 0x8000000000000000

6817

// IntOfs = select Sel, 0, 0x8000000000000000

6818

// Result = fp_to_sint(Src - FltOfs) ^ IntOfs

6819

6820

// TODO: Should any fast-math-flags be set for the FSUB?

6821

SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,

6822

DAG.getConstantFP(0.0, dl, SrcVT), Cst);

6823

Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);

6824

SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,

6825

DAG.getConstant(0, dl, DstVT),

6826

DAG.getConstant(SignMask, dl, DstVT));

6827

SDValue SInt;

6828

if (Node->isStrictFPOpcode()) {

6829

SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },

6830

{ Chain, Src, FltOfs });

6831

SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },

6832

{ Val.getValue(1), Val });

6833

Chain = SInt.getValue(1);

6834

} else {

6835

SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);

6836

SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);

6837

}

6838

Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);

6839

} else {

6840

// Expand based on maximum range of FP_TO_SINT:

6841

// True = fp_to_sint(Src)

6842

// False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)

6843

// Result = select (Src < 0x8000000000000000), True, False

6844

6845

SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);

6846

// TODO: Should any fast-math-flags be set for the FSUB?

6847

SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,

6848

DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));

6849

False = DAG.getNode(ISD::XOR, dl, DstVT, False,

6850

DAG.getConstant(SignMask, dl, DstVT));

6851

Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);

6852

Result = DAG.getSelect(dl, DstVT, Sel, True, False);

6853

}

6854

return true;

6855

}

6856

6857

bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,

6858

SDValue &Chain,

6859

SelectionDAG &DAG) const {

6860

// This transform is not correct for converting 0 when rounding mode is set

6861

// to round toward negative infinity which will produce -0.0. So disable under

6862

// strictfp.

6863

if (Node->isStrictFPOpcode())

6864

return false;

6865

6866

SDValue Src = Node->getOperand(0);

6867

EVT SrcVT = Src.getValueType();

6868

EVT DstVT = Node->getValueType(0);

6869

6870

if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)

6871

return false;

6872

6873

// Only expand vector types if we have the appropriate vector bit operations.

6874

if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||

6875

!isOperationLegalOrCustom(ISD::FADD, DstVT) ||

6876

!isOperationLegalOrCustom(ISD::FSUB, DstVT) ||

6877

!isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||

6878

!isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))

6879

return false;

6880

6881

SDLoc dl(SDValue(Node, 0));

6882

EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());

6883

6884

// Implementation of unsigned i64 to f64 following the algorithm in

6885

// __floatundidf in compiler_rt. This implementation performs rounding

6886

// correctly in all rounding modes with the exception of converting 0

6887

// when rounding toward negative infinity. In that case the fsub will produce

6888

// -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.

6889

SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, SrcVT);

6890

SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(

6891

BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl, DstVT);

6892

SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, SrcVT);

6893

SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF)0x00000000FFFFFFFFUL, dl, SrcVT);

6894

SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);

6895

6896

SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);

6897

SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);

6898

SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);

6899

SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);

6900

SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);

6901

SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);

6902

SDValue HiSub =

6903

DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);

6904

Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);

6905

return true;

6906

}

6907

6908

SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,

6909

SelectionDAG &DAG) const {

6910

SDLoc dl(Node);

6911

unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?

6912

ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;

6913

EVT VT = Node->getValueType(0);

6914

6915

if (VT.isScalableVector())

6916

report_fatal_error(

6917

"Expanding fminnum/fmaxnum for scalable vectors is undefined.");

6918

6919

if (isOperationLegalOrCustom(NewOp, VT)) {

6920

SDValue Quiet0 = Node->getOperand(0);

6921

SDValue Quiet1 = Node->getOperand(1);

6922

6923

if (!Node->getFlags().hasNoNaNs()) {

6924

// Insert canonicalizes if it's possible we need to quiet to get correct

6925

// sNaN behavior.

6926

if (!DAG.isKnownNeverSNaN(Quiet0)) {

6927

Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,

6928

Node->getFlags());

6929

}

6930

if (!DAG.isKnownNeverSNaN(Quiet1)) {

6931

Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,

6932

Node->getFlags());

6933

}

6934

}

6935

6936

return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());

6937

}

6938

6939

// If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that

6940

// instead if there are no NaNs.

6941

if (Node->getFlags().hasNoNaNs()) {

6942

unsigned IEEE2018Op =

6943

Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;

6944

if (isOperationLegalOrCustom(IEEE2018Op, VT)) {

6945

return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),

6946

Node->getOperand(1), Node->getFlags());

6947

}

6948

}

6949

6950

// If none of the above worked, but there are no NaNs, then expand to

6951

// a compare/select sequence. This is required for correctness since

6952

// InstCombine might have canonicalized a fcmp+select sequence to a

6953

// FMINNUM/FMAXNUM node. If we were to fall through to the default

6954

// expansion to libcall, we might introduce a link-time dependency

6955

// on libm into a file that originally did not have one.

6956

if (Node->getFlags().hasNoNaNs()) {

6957

ISD::CondCode Pred =

6958

Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;

6959

SDValue Op1 = Node->getOperand(0);

6960

SDValue Op2 = Node->getOperand(1);

6961

SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);

6962

// Copy FMF flags, but always set the no-signed-zeros flag

6963

// as this is implied by the FMINNUM/FMAXNUM semantics.

6964

SDNodeFlags Flags = Node->getFlags();

6965

Flags.setNoSignedZeros(true);

6966

SelCC->setFlags(Flags);

6967

return SelCC;

6968

}

6969

6970

return SDValue();

6971

}

6972

6973

SDValue TargetLowering::expandISNAN(EVT ResultVT, SDValue Op, SDNodeFlags Flags,

6974

const SDLoc &DL, SelectionDAG &DAG) const {

6975

EVT OperandVT = Op.getValueType();

6976

assert(OperandVT.isFloatingPoint())(static_cast <bool> (OperandVT.isFloatingPoint()) ? void
(0) : __assert_fail ("OperandVT.isFloatingPoint()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6976, __extension__ __PRETTY_FUNCTION__));

6977

6978

// If floating point exceptions are ignored, expand to unordered comparison.

6979

if ((Flags.hasNoFPExcept() &&

6980

isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) ||

6981

OperandVT == MVT::ppcf128)

6982

return DAG.getSetCC(DL, ResultVT, Op, DAG.getConstantFP(0.0, DL, OperandVT),

6983

ISD::SETUO);

6984

6985

// In general case use integer operations to avoid traps if argument is SNaN.

6986

6987

// NaN has all exp bits set and a non zero significand. Therefore:

6988

// isnan(V) == exp mask < abs(V)

6989

unsigned BitSize = OperandVT.getScalarSizeInBits();

6990

EVT IntVT = OperandVT.changeTypeToInteger();

6991

SDValue ArgV = DAG.getBitcast(IntVT, Op);

6992

APInt AndMask = APInt::getSignedMaxValue(BitSize);

6993

SDValue AndMaskV = DAG.getConstant(AndMask, DL, IntVT);

6994

SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, ArgV, AndMaskV);

6995

EVT ScalarFloatVT = OperandVT.getScalarType();

6996

const Type *FloatTy = ScalarFloatVT.getTypeForEVT(*DAG.getContext());

6997

const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();

6998

APInt ExpMask = APFloat::getInf(Semantics).bitcastToAPInt();

6999

SDValue ExpMaskV = DAG.getConstant(ExpMask, DL, IntVT);

7000

return DAG.getSetCC(DL, ResultVT, ExpMaskV, AbsV, ISD::SETLT);

7001

}

7002

7003

bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,

7004

SelectionDAG &DAG) const {

7005

SDLoc dl(Node);

7006

EVT VT = Node->getValueType(0);

7007

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

7008

SDValue Op = Node->getOperand(0);

7009

unsigned Len = VT.getScalarSizeInBits();

7010

assert(VT.isInteger() && "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7010, __extension__ __PRETTY_FUNCTION__));

7011

7012

// TODO: Add support for irregular type lengths.

7013

if (!(Len <= 128 && Len % 8 == 0))

7014

return false;

7015

7016

// Only expand vector types if we have the appropriate vector bit operations.

7017

if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||

7018

!isOperationLegalOrCustom(ISD::SUB, VT) ||

7019

!isOperationLegalOrCustom(ISD::SRL, VT) ||

7020

(Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||

7021

!isOperationLegalOrCustomOrPromote(ISD::AND, VT)))

7022

return false;

7023

7024

// This is the "best" algorithm from

7025

// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel

7026

SDValue Mask55 =

7027

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);

7028

SDValue Mask33 =

7029

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);

7030

SDValue Mask0F =

7031

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);

7032

SDValue Mask01 =

7033

DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);

7034

7035

// v = v - ((v >> 1) & 0x55555555...)

7036

Op = DAG.getNode(ISD::SUB, dl, VT, Op,

7037

DAG.getNode(ISD::AND, dl, VT,

7038

DAG.getNode(ISD::SRL, dl, VT, Op,

7039

DAG.getConstant(1, dl, ShVT)),

7040

Mask55));

7041

// v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)

7042

Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),

7043

DAG.getNode(ISD::AND, dl, VT,

7044

DAG.getNode(ISD::SRL, dl, VT, Op,

7045

DAG.getConstant(2, dl, ShVT)),

7046

Mask33));

7047

// v = (v + (v >> 4)) & 0x0F0F0F0F...

7048

Op = DAG.getNode(ISD::AND, dl, VT,

7049

DAG.getNode(ISD::ADD, dl, VT, Op,

7050

DAG.getNode(ISD::SRL, dl, VT, Op,

7051

DAG.getConstant(4, dl, ShVT))),

7052

Mask0F);

7053

// v = (v * 0x01010101...) >> (Len - 8)

7054

if (Len > 8)

7055

Op =

7056

DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),

7057

DAG.getConstant(Len - 8, dl, ShVT));

7058

7059

Result = Op;

7060

return true;

7061

}

7062

7063

bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,

7064

SelectionDAG &DAG) const {

7065

SDLoc dl(Node);

7066

EVT VT = Node->getValueType(0);

7067

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

7068

SDValue Op = Node->getOperand(0);

7069

unsigned NumBitsPerElt = VT.getScalarSizeInBits();

7070

7071

// If the non-ZERO_UNDEF version is supported we can use that instead.

7072

if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&

7073

isOperationLegalOrCustom(ISD::CTLZ, VT)) {

7074

Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);

7075

return true;

7076

}

7077

7078

// If the ZERO_UNDEF version is supported use that and handle the zero case.

7079

if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {

7080

EVT SetCCVT =

7081

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

7082

SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);

7083

SDValue Zero = DAG.getConstant(0, dl, VT);

7084

SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);

7085

Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,

7086

DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);

7087

return true;

7088

}

7089

7090

// Only expand vector types if we have the appropriate vector bit operations.

7091

if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||

7092

!isOperationLegalOrCustom(ISD::CTPOP, VT) ||

7093

!isOperationLegalOrCustom(ISD::SRL, VT) ||

7094

!isOperationLegalOrCustomOrPromote(ISD::OR, VT)))

7095

return false;

7096

7097

// for now, we do this:

7098

// x = x | (x >> 1);

7099

// x = x | (x >> 2);

7100

// ...

7101

// x = x | (x >>16);

7102

// x = x | (x >>32); // for 64-bit input

7103

// return popcount(~x);

7104

//

7105

// Ref: "Hacker's Delight" by Henry Warren

7106

for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {

7107

SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);

7108

Op = DAG.getNode(ISD::OR, dl, VT, Op,

7109

DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));

7110

}

7111

Op = DAG.getNOT(dl, Op, VT);

7112

Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);

7113

return true;

7114

}

7115

7116

bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,

7117

SelectionDAG &DAG) const {

7118

SDLoc dl(Node);

7119

EVT VT = Node->getValueType(0);

7120

SDValue Op = Node->getOperand(0);

7121

unsigned NumBitsPerElt = VT.getScalarSizeInBits();

7122

7123

// If the non-ZERO_UNDEF version is supported we can use that instead.

7124

if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&

7125

isOperationLegalOrCustom(ISD::CTTZ, VT)) {

7126

Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);

7127

return true;

7128

}

7129

7130

// If the ZERO_UNDEF version is supported use that and handle the zero case.

7131

if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {

7132

EVT SetCCVT =

7133

getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

7134

SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);

7135

SDValue Zero = DAG.getConstant(0, dl, VT);

7136

SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);

7137

Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,

7138

DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);

7139

return true;

7140

}

7141

7142

// Only expand vector types if we have the appropriate vector bit operations.

7143

if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||

7144

(!isOperationLegalOrCustom(ISD::CTPOP, VT) &&

7145

!isOperationLegalOrCustom(ISD::CTLZ, VT)) ||

7146

!isOperationLegalOrCustom(ISD::SUB, VT) ||

7147

!isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||

7148

!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))

7149

return false;

7150

7151

// for now, we use: { return popcount(~x & (x - 1)); }

7152

// unless the target has ctlz but not ctpop, in which case we use:

7153

// { return 32 - nlz(~x & (x-1)); }

7154

// Ref: "Hacker's Delight" by Henry Warren

7155

SDValue Tmp = DAG.getNode(

7156

ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),

7157

DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));

7158

7159

// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.

7160

if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {

7161

Result =

7162

DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),

7163

DAG.getNode(ISD::CTLZ, dl, VT, Tmp));

7164

return true;

7165

}

7166

7167

Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);

7168

return true;

7169

}

7170

7171

bool TargetLowering::expandABS(SDNode *N, SDValue &Result,

7172

SelectionDAG &DAG, bool IsNegative) const {

7173

SDLoc dl(N);

7174

EVT VT = N->getValueType(0);

7175

EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());

7176

SDValue Op = N->getOperand(0);

7177

7178

// abs(x) -> smax(x,sub(0,x))

7179

if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&

7180

isOperationLegal(ISD::SMAX, VT)) {

7181

SDValue Zero = DAG.getConstant(0, dl, VT);

7182

Result = DAG.getNode(ISD::SMAX, dl, VT, Op,

7183

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7184

return true;

7185

}

7186

7187

// abs(x) -> umin(x,sub(0,x))

7188

if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&

7189

isOperationLegal(ISD::UMIN, VT)) {

7190

SDValue Zero = DAG.getConstant(0, dl, VT);

7191

Result = DAG.getNode(ISD::UMIN, dl, VT, Op,

7192

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7193

return true;

7194

}

7195

7196

// 0 - abs(x) -> smin(x, sub(0,x))

7197

if (IsNegative && isOperationLegal(ISD::SUB, VT) &&

7198

isOperationLegal(ISD::SMIN, VT)) {

7199

SDValue Zero = DAG.getConstant(0, dl, VT);

7200

Result = DAG.getNode(ISD::SMIN, dl, VT, Op,

7201

DAG.getNode(ISD::SUB, dl, VT, Zero, Op));

7202

return true;

7203

}

7204

7205

// Only expand vector types if we have the appropriate vector operations.

7206

if (VT.isVector() &&

7207

(!isOperationLegalOrCustom(ISD::SRA, VT) ||

7208

(!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||

7209

(IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||

7210

!isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))

7211

return false;

7212

7213

SDValue Shift =

7214

DAG.getNode(ISD::SRA, dl, VT, Op,

7215

DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));

7216

if (!IsNegative) {

7217

SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);

7218

Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);

7219

} else {

7220

// 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))

7221

SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);

7222

Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);

7223

}

7224

return true;

7225

}

7226

7227

SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {

7228

SDLoc dl(N);

7229

EVT VT = N->getValueType(0);

7230

SDValue Op = N->getOperand(0);

7231

7232

if (!VT.isSimple())

7233

return SDValue();

7234

7235

EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());

7236

SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;

7237

switch (VT.getSimpleVT().getScalarType().SimpleTy) {

7238

default:

7239

return SDValue();

7240

case MVT::i16:

7241

// Use a rotate by 8. This can be further expanded if necessary.

7242

return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7243

case MVT::i32:

7244

Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7245

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7246

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7247

Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7248

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,

7249

DAG.getConstant(0xFF0000, dl, VT));

7250

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));

7251

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);

7252

Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);

7253

return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);

7254

case MVT::i64:

7255

Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));

7256

Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));

7257

Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7258

Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7259

Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));

7260

Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));

7261

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));

7262

Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));

7263

Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,

7264

DAG.getConstant(255ULL<<48, dl, VT));

7265

Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,

7266

DAG.getConstant(255ULL<<40, dl, VT));

7267

Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,

7268

DAG.getConstant(255ULL<<32, dl, VT));

7269

Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,

7270

DAG.getConstant(255ULL<<24, dl, VT));

7271

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,

7272

DAG.getConstant(255ULL<<16, dl, VT));

7273

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,

7274

DAG.getConstant(255ULL<<8 , dl, VT));

7275

Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);

7276

Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);

7277

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);

7278

Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);

7279

Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);

7280

Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);

7281

return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);

7282

}

7283

}

7284

7285

SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {

7286

SDLoc dl(N);

7287

EVT VT = N->getValueType(0);

7288

SDValue Op = N->getOperand(0);

7289

EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());

7290

unsigned Sz = VT.getScalarSizeInBits();

7291

7292

SDValue Tmp, Tmp2, Tmp3;

7293

7294

// If we can, perform BSWAP first and then the mask+swap the i4, then i2

7295

// and finally the i1 pairs.

7296

// TODO: We can easily support i4/i2 legal types if any target ever does.

7297

if (Sz >= 8 && isPowerOf2_32(Sz)) {

7298

// Create the masks - repeating the pattern every byte.

7299

APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));

7300

APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));

7301

APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));

7302

7303

// BSWAP if the type is wider than a single byte.

7304

Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);

7305

7306

// swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)

7307

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));

7308

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));

7309

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));

7310

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));

7311

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7312

7313

// swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)

7314

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));

7315

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));

7316

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));

7317

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));

7318

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7319

7320

// swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)

7321

Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));

7322

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));

7323

Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));

7324

Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));

7325

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

7326

return Tmp;

7327

}

7328

7329

Tmp = DAG.getConstant(0, dl, VT);

7330

for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {

7331

if (I < J)

7332

Tmp2 =

7333

DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));

7334

else

7335

Tmp2 =

7336

DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));

7337

7338

APInt Shift(Sz, 1);

7339

Shift <<= J;

7340

Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));

7341

Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);

7342

}

7343

7344

return Tmp;

7345

}

7346

7347

std::pair<SDValue, SDValue>

7348

TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,

7349

SelectionDAG &DAG) const {

7350

SDLoc SL(LD);

7351

SDValue Chain = LD->getChain();

7352

SDValue BasePTR = LD->getBasePtr();

7353

EVT SrcVT = LD->getMemoryVT();

7354

EVT DstVT = LD->getValueType(0);

7355

ISD::LoadExtType ExtType = LD->getExtensionType();

7356

7357

if (SrcVT.isScalableVector())

7358

report_fatal_error("Cannot scalarize scalable vector loads");

7359

7360

unsigned NumElem = SrcVT.getVectorNumElements();

7361

7362

EVT SrcEltVT = SrcVT.getScalarType();

7363

EVT DstEltVT = DstVT.getScalarType();

7364

7365

// A vector must always be stored in memory as-is, i.e. without any padding

7366

// between the elements, since various code depend on it, e.g. in the

7367

// handling of a bitcast of a vector type to int, which may be done with a

7368

// vector store followed by an integer load. A vector that does not have

7369

// elements that are byte-sized must therefore be stored as an integer

7370

// built out of the extracted vector elements.

7371

if (!SrcEltVT.isByteSized()) {

7372

unsigned NumLoadBits = SrcVT.getStoreSizeInBits();

7373

EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);

7374

7375

unsigned NumSrcBits = SrcVT.getSizeInBits();

7376

EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);

7377

7378

unsigned SrcEltBits = SrcEltVT.getSizeInBits();

7379

SDValue SrcEltBitMask = DAG.getConstant(

7380

APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);

7381

7382

// Load the whole vector and avoid masking off the top bits as it makes

7383

// the codegen worse.

7384

SDValue Load =

7385

DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,

7386

LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),

7387

LD->getMemOperand()->getFlags(), LD->getAAInfo());

7388

7389

SmallVector<SDValue, 8> Vals;

7390

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7391

unsigned ShiftIntoIdx =

7392

(DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);

7393

SDValue ShiftAmount =

7394

DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),

7395

LoadVT, SL, /*LegalTypes=*/false);

7396

SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);

7397

SDValue Elt =

7398

DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);

7399

SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);

7400

7401

if (ExtType != ISD::NON_EXTLOAD) {

7402

unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);

7403

Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);

7404

}

7405

7406

Vals.push_back(Scalar);

7407

}

7408

7409

SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

7410

return std::make_pair(Value, Load.getValue(1));

7411

}

7412

7413

unsigned Stride = SrcEltVT.getSizeInBits() / 8;

7414

assert(SrcEltVT.isByteSized())(static_cast <bool> (SrcEltVT.isByteSized()) ? void (0)
: __assert_fail ("SrcEltVT.isByteSized()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7414, __extension__ __PRETTY_FUNCTION__));

7415

7416

SmallVector<SDValue, 8> Vals;

7417

SmallVector<SDValue, 8> LoadChains;

7418

7419

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7420

SDValue ScalarLoad =

7421

DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,

7422

LD->getPointerInfo().getWithOffset(Idx * Stride),

7423

SrcEltVT, LD->getOriginalAlign(),

7424

LD->getMemOperand()->getFlags(), LD->getAAInfo());

7425

7426

BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));

7427

7428

Vals.push_back(ScalarLoad.getValue(0));

7429

LoadChains.push_back(ScalarLoad.getValue(1));

7430

}

7431

7432

SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);

7433

SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

7434

7435

return std::make_pair(Value, NewChain);

7436

}

7437

7438

SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,

7439

SelectionDAG &DAG) const {

7440

SDLoc SL(ST);

7441

7442

SDValue Chain = ST->getChain();

7443

SDValue BasePtr = ST->getBasePtr();

7444

SDValue Value = ST->getValue();

7445

EVT StVT = ST->getMemoryVT();

7446

7447

if (StVT.isScalableVector())

7448

report_fatal_error("Cannot scalarize scalable vector stores");

7449

7450

// The type of the data we want to save

7451

EVT RegVT = Value.getValueType();

7452

EVT RegSclVT = RegVT.getScalarType();

7453

7454

// The type of data as saved in memory.

7455

EVT MemSclVT = StVT.getScalarType();

7456

7457

unsigned NumElem = StVT.getVectorNumElements();

7458

7459

// A vector must always be stored in memory as-is, i.e. without any padding

7460

// between the elements, since various code depend on it, e.g. in the

7461

// handling of a bitcast of a vector type to int, which may be done with a

7462

// vector store followed by an integer load. A vector that does not have

7463

// elements that are byte-sized must therefore be stored as an integer

7464

// built out of the extracted vector elements.

7465

if (!MemSclVT.isByteSized()) {

7466

unsigned NumBits = StVT.getSizeInBits();

7467

EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);

7468

7469

SDValue CurrVal = DAG.getConstant(0, SL, IntVT);

7470

7471

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7472

SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,

7473

DAG.getVectorIdxConstant(Idx, SL));

7474

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);

7475

SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);

7476

unsigned ShiftIntoIdx =

7477

(DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);

7478

SDValue ShiftAmount =

7479

DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);

7480

SDValue ShiftedElt =

7481

DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);

7482

CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);

7483

}

7484

7485

return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),

7486

ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),

7487

ST->getAAInfo());

7488

}

7489

7490

// Store Stride in bytes

7491

unsigned Stride = MemSclVT.getSizeInBits() / 8;

7492

assert(Stride && "Zero stride!")(static_cast <bool> (Stride && "Zero stride!") ?
void (0) : __assert_fail ("Stride && \"Zero stride!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7492, __extension__ __PRETTY_FUNCTION__));

7493

// Extract each of the elements from the original vector and save them into

7494

// memory individually.

7495

SmallVector<SDValue, 8> Stores;

7496

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {

7497

SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,

7498

DAG.getVectorIdxConstant(Idx, SL));

7499

7500

SDValue Ptr =

7501

DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));

7502

7503

// This scalar TruncStore may be illegal, but we legalize it later.

7504

SDValue Store = DAG.getTruncStore(

7505

Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),

7506

MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),

7507

ST->getAAInfo());

7508

7509

Stores.push_back(Store);

7510

}

7511

7512

return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);

7513

}

7514

7515

std::pair<SDValue, SDValue>

7516

TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {

7517

assert(LD->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7518, __extension__ __PRETTY_FUNCTION__))

7518

"unaligned indexed loads not implemented!")(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7518, __extension__ __PRETTY_FUNCTION__));

7519

SDValue Chain = LD->getChain();

7520

SDValue Ptr = LD->getBasePtr();

7521

EVT VT = LD->getValueType(0);

7522

EVT LoadedVT = LD->getMemoryVT();

7523

SDLoc dl(LD);

7524

auto &MF = DAG.getMachineFunction();

7525

7526

if (VT.isFloatingPoint() || VT.isVector()) {

7527

EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());

7528

if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {

7529

if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&

7530

LoadedVT.isVector()) {

7531

// Scalarize the load and let the individual components be handled.

7532

return scalarizeVectorLoad(LD, DAG);

7533

}

7534

7535

// Expand to a (misaligned) integer load of the same size,

7536

// then bitconvert to floating point or vector.

7537

SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,

7538

LD->getMemOperand());

7539

SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);

7540

if (LoadedVT != VT)

7541

Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :

7542

ISD::ANY_EXTEND, dl, VT, Result);

7543

7544

return std::make_pair(Result, newLoad.getValue(1));

7545

}

7546

7547

// Copy the value to a (aligned) stack slot using (unaligned) integer

7548

// loads and stores, then do a (aligned) load from the stack slot.

7549

MVT RegVT = getRegisterType(*DAG.getContext(), intVT);

7550

unsigned LoadedBytes = LoadedVT.getStoreSize();

7551

unsigned RegBytes = RegVT.getSizeInBits() / 8;

7552

unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;

7553

7554

// Make sure the stack slot is also aligned for the register type.

7555

SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);

7556

auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();

7557

SmallVector<SDValue, 8> Stores;

7558

SDValue StackPtr = StackBase;

7559

unsigned Offset = 0;

7560

7561

EVT PtrVT = Ptr.getValueType();

7562

EVT StackPtrVT = StackPtr.getValueType();

7563

7564

SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);

7565

SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

7566

7567

// Do all but one copies using the full register width.

7568

for (unsigned i = 1; i < NumRegs; i++) {

7569

// Load one integer register's worth from the original location.

7570

SDValue Load = DAG.getLoad(

7571

RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),

7572

LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),

7573

LD->getAAInfo());

7574

// Follow the load with a store to the stack slot. Remember the store.

7575

Stores.push_back(DAG.getStore(

7576

Load.getValue(1), dl, Load, StackPtr,

7577

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));

7578

// Increment the pointers.

7579

Offset += RegBytes;

7580

7581

Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);

7582

StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);

7583

}

7584

7585

// The last copy may be partial. Do an extending load.

7586

EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),

7587

8 * (LoadedBytes - Offset));

7588

SDValue Load =

7589

DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,

7590

LD->getPointerInfo().getWithOffset(Offset), MemVT,

7591

LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),

7592

LD->getAAInfo());

7593

// Follow the load with a store to the stack slot. Remember the store.

7594

// On big-endian machines this requires a truncating store to ensure

7595

// that the bits end up in the right place.

7596

Stores.push_back(DAG.getTruncStore(

7597

Load.getValue(1), dl, Load, StackPtr,

7598

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));

7599

7600

// The order of the stores doesn't matter - say it with a TokenFactor.

7601

SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

7602

7603

// Finally, perform the original load only redirected to the stack slot.

7604

Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,

7605

MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),

7606

LoadedVT);

7607

7608

// Callers expect a MERGE_VALUES node.

7609

return std::make_pair(Load, TF);

7610

}

7611

7612

assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7613, __extension__ __PRETTY_FUNCTION__))

7613

"Unaligned load of unsupported type.")(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7613, __extension__ __PRETTY_FUNCTION__));

7614

7615

// Compute the new VT that is half the size of the old one. This is an

7616

// integer MVT.

7617

unsigned NumBits = LoadedVT.getSizeInBits();

7618

EVT NewLoadedVT;

7619

NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);

7620

NumBits >>= 1;

7621

7622

Align Alignment = LD->getOriginalAlign();

7623

unsigned IncrementSize = NumBits / 8;

7624

ISD::LoadExtType HiExtType = LD->getExtensionType();

7625

7626

// If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.

7627

if (HiExtType == ISD::NON_EXTLOAD)

7628

HiExtType = ISD::ZEXTLOAD;

7629

7630

// Load the value in two parts

7631

SDValue Lo, Hi;

7632

if (DAG.getDataLayout().isLittleEndian()) {

7633

Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),

7634

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7635

LD->getAAInfo());

7636

7637

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7638

Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,

7639

LD->getPointerInfo().getWithOffset(IncrementSize),

7640

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7641

LD->getAAInfo());

7642

} else {

7643

Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),

7644

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7645

LD->getAAInfo());

7646

7647

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7648

Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,

7649

LD->getPointerInfo().getWithOffset(IncrementSize),

7650

NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),

7651

LD->getAAInfo());

7652

}

7653

7654

// aggregate the two parts

7655

SDValue ShiftAmount =

7656

DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),

7657

DAG.getDataLayout()));

7658

SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);

7659

Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);

7660

7661

SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),

7662

Hi.getValue(1));

7663

7664

return std::make_pair(Result, TF);

7665

}

7666

7667

SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,

7668

SelectionDAG &DAG) const {

7669

assert(ST->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7670, __extension__ __PRETTY_FUNCTION__))

7670

"unaligned indexed stores not implemented!")(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7670, __extension__ __PRETTY_FUNCTION__));

7671

SDValue Chain = ST->getChain();

7672

SDValue Ptr = ST->getBasePtr();

7673

SDValue Val = ST->getValue();

7674

EVT VT = Val.getValueType();

7675

Align Alignment = ST->getOriginalAlign();

7676

auto &MF = DAG.getMachineFunction();

7677

EVT StoreMemVT = ST->getMemoryVT();

7678

7679

SDLoc dl(ST);

7680

if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {

7681

EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());

7682

if (isTypeLegal(intVT)) {

7683

if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&

7684

StoreMemVT.isVector()) {

7685

// Scalarize the store and let the individual components be handled.

7686

SDValue Result = scalarizeVectorStore(ST, DAG);

7687

return Result;

7688

}

7689

// Expand to a bitconvert of the value to the integer type of the

7690

// same size, then a (misaligned) int store.

7691

// FIXME: Does not handle truncating floating point stores!

7692

SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);

7693

Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),

7694

Alignment, ST->getMemOperand()->getFlags());

7695

return Result;

7696

}

7697

// Do a (aligned) store to a stack slot, then copy from the stack slot

7698

// to the final destination using (unaligned) integer loads and stores.

7699

MVT RegVT = getRegisterType(

7700

*DAG.getContext(),

7701

EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));

7702

EVT PtrVT = Ptr.getValueType();

7703

unsigned StoredBytes = StoreMemVT.getStoreSize();

7704

unsigned RegBytes = RegVT.getSizeInBits() / 8;

7705

unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;

7706

7707

// Make sure the stack slot is also aligned for the register type.

7708

SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);

7709

auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

7710

7711

// Perform the original store, only redirected to the stack slot.

7712

SDValue Store = DAG.getTruncStore(

7713

Chain, dl, Val, StackPtr,

7714

MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);

7715

7716

EVT StackPtrVT = StackPtr.getValueType();

7717

7718

SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);

7719

SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

7720

SmallVector<SDValue, 8> Stores;

7721

unsigned Offset = 0;

7722

7723

// Do all but one copies using the full register width.

7724

for (unsigned i = 1; i < NumRegs; i++) {

7725

// Load one integer register's worth from the stack slot.

7726

SDValue Load = DAG.getLoad(

7727

RegVT, dl, Store, StackPtr,

7728

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));

7729

// Store it to the final location. Remember the store.

7730

Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,

7731

ST->getPointerInfo().getWithOffset(Offset),

7732

ST->getOriginalAlign(),

7733

ST->getMemOperand()->getFlags()));

7734

// Increment the pointers.

7735

Offset += RegBytes;

7736

StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);

7737

Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);

7738

}

7739

7740

// The last store may be partial. Do a truncating store. On big-endian

7741

// machines this requires an extending load from the stack slot to ensure

7742

// that the bits are in the right place.

7743

EVT LoadMemVT =

7744

EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));

7745

7746

// Load from the stack slot.

7747

SDValue Load = DAG.getExtLoad(

7748

ISD::EXTLOAD, dl, RegVT, Store, StackPtr,

7749

MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);

7750

7751

Stores.push_back(

7752

DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,

7753

ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,

7754

ST->getOriginalAlign(),

7755

ST->getMemOperand()->getFlags(), ST->getAAInfo()));

7756

// The order of the stores doesn't matter - say it with a TokenFactor.

7757

SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

7758

return Result;

7759

}

7760

7761

assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7762, __extension__ __PRETTY_FUNCTION__))

7762

"Unaligned store of unknown type.")(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7762, __extension__ __PRETTY_FUNCTION__));

7763

// Get the half-size VT

7764

EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());

7765

unsigned NumBits = NewStoredVT.getFixedSizeInBits();

7766

unsigned IncrementSize = NumBits / 8;

7767

7768

// Divide the stored value in two parts.

7769

SDValue ShiftAmount = DAG.getConstant(

7770

NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));

7771

SDValue Lo = Val;

7772

SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);

7773

7774

// Store the two parts

7775

SDValue Store1, Store2;

7776

Store1 = DAG.getTruncStore(Chain, dl,

7777

DAG.getDataLayout().isLittleEndian() ? Lo : Hi,

7778

Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,

7779

ST->getMemOperand()->getFlags());

7780

7781

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));

7782

Store2 = DAG.getTruncStore(

7783

Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,

7784

ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,

7785

ST->getMemOperand()->getFlags(), ST->getAAInfo());

7786

7787

SDValue Result =

7788

DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);

7789

return Result;

7790

}

7791

7792

SDValue

7793

TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,

7794

const SDLoc &DL, EVT DataVT,

7795

SelectionDAG &DAG,

7796

bool IsCompressedMemory) const {

7797

SDValue Increment;

7798

EVT AddrVT = Addr.getValueType();

7799

EVT MaskVT = Mask.getValueType();

7800

assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7801, __extension__ __PRETTY_FUNCTION__))

7801

"Incompatible types of Data and Mask")(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7801, __extension__ __PRETTY_FUNCTION__));

7802

if (IsCompressedMemory) {

7803

if (DataVT.isScalableVector())

7804

report_fatal_error(

7805

"Cannot currently handle compressed memory with scalable vectors");

7806

// Incrementing the pointer according to number of '1's in the mask.

7807

EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());

7808

SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);

7809

if (MaskIntVT.getSizeInBits() < 32) {

7810

MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);

7811

MaskIntVT = MVT::i32;

7812

}

7813

7814

// Count '1's with POPCNT.

7815

Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);

7816

Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);

7817

// Scale is an element size in bytes.

7818

SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,

7819

AddrVT);

7820

Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);

7821

} else if (DataVT.isScalableVector()) {

7822

Increment = DAG.getVScale(DL, AddrVT,

7823

APInt(AddrVT.getFixedSizeInBits(),

7824

DataVT.getStoreSize().getKnownMinSize()));

7825

} else

7826

Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);

7827

7828

return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);

7829

}

7830

7831

static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,

7832

EVT VecVT, const SDLoc &dl,

7833

unsigned NumSubElts) {

7834

if (!VecVT.isScalableVector() && isa<ConstantSDNode>(Idx))

7835

return Idx;

7836

7837

EVT IdxVT = Idx.getValueType();

7838

unsigned NElts = VecVT.getVectorMinNumElements();

7839

if (VecVT.isScalableVector()) {

7840

// If this is a constant index and we know the value plus the number of the

7841

// elements in the subvector minus one is less than the minimum number of

7842

// elements then it's safe to return Idx.

7843

if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))

7844

if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)

7845

return Idx;

7846

SDValue VS =

7847

DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));

7848

unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;

7849

SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,

7850

DAG.getConstant(NumSubElts, dl, IdxVT));

7851

return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);

7852

}

7853

if (isPowerOf2_32(NElts) && NumSubElts == 1) {

7854

APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));

7855

return DAG.getNode(ISD::AND, dl, IdxVT, Idx,

7856

DAG.getConstant(Imm, dl, IdxVT));

7857

}

7858

unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;

7859

return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,

7860

DAG.getConstant(MaxIndex, dl, IdxVT));

7861

}

7862

7863

SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,

7864

SDValue VecPtr, EVT VecVT,

7865

SDValue Index) const {

7866

return getVectorSubVecPointer(

7867

DAG, VecPtr, VecVT,

7868

EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),

7869

Index);

7870

}

7871

7872

SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG,

7873

SDValue VecPtr, EVT VecVT,

7874

EVT SubVecVT,

7875

SDValue Index) const {

7876

SDLoc dl(Index);

7877

// Make sure the index type is big enough to compute in.

7878

Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());

7879

7880

EVT EltVT = VecVT.getVectorElementType();

7881

7882

// Calculate the element offset and add it to the pointer.

7883

unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.

7884

assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
(0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7885, __extension__ __PRETTY_FUNCTION__))

7885

"Converting bits to bytes lost precision")(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
(0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7885, __extension__ __PRETTY_FUNCTION__));

7886

7887

// Scalable vectors don't need clamping as these are checked at compile time

7888

if (SubVecVT.isFixedLengthVector()) {

7889

assert(SubVecVT.getVectorElementType() == EltVT &&(static_cast <bool> (SubVecVT.getVectorElementType() ==
EltVT && "Sub-vector must be a fixed vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a fixed vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7890, __extension__ __PRETTY_FUNCTION__))

7890

"Sub-vector must be a fixed vector with matching element type")(static_cast <bool> (SubVecVT.getVectorElementType() ==
EltVT && "Sub-vector must be a fixed vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a fixed vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7890, __extension__ __PRETTY_FUNCTION__));

7891

Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,

7892

SubVecVT.getVectorNumElements());

7893

}

7894

7895

EVT IdxVT = Index.getValueType();

7896

7897

Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,

7898

DAG.getConstant(EltSize, dl, IdxVT));

7899

return DAG.getMemBasePlusOffset(VecPtr, Index, dl);

7900

}

7901

7902

//===----------------------------------------------------------------------===//

7903

// Implementation of Emulated TLS Model

7904

//===----------------------------------------------------------------------===//

7905

7906

SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,

7907

SelectionDAG &DAG) const {

7908

// Access to address of TLS varialbe xyz is lowered to a function call:

7909

// __emutls_get_address( address of global variable named "__emutls_v.xyz" )

7910

EVT PtrVT = getPointerTy(DAG.getDataLayout());

7911

PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());

7912

SDLoc dl(GA);

7913

7914

ArgListTy Args;

7915

ArgListEntry Entry;

7916

std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();

7917

Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());

7918

StringRef EmuTlsVarName(NameString);

7919

GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);

7920

assert(EmuTlsVar && "Cannot find EmuTlsVar ")(static_cast <bool> (EmuTlsVar && "Cannot find EmuTlsVar "
) ? void (0) : __assert_fail ("EmuTlsVar && \"Cannot find EmuTlsVar \""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7920, __extension__ __PRETTY_FUNCTION__));

7921

Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);

7922

Entry.Ty = VoidPtrType;

7923

Args.push_back(Entry);

7924

7925

SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);

7926

7927

TargetLowering::CallLoweringInfo CLI(DAG);

7928

CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());

7929

CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));

7930

std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);

7931

7932

// TLSADDR will be codegen'ed as call. Inform MFI that function has calls.

7933

// At last for X86 targets, maybe good for other targets too?

7934

MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();

7935

MFI.setAdjustsStack(true); // Is this only for X86 target?

7936

MFI.setHasCalls(true);

7937

7938

assert((GA->getOffset() == 0) &&(static_cast <bool> ((GA->getOffset() == 0) &&
"Emulated TLS must have zero offset in GlobalAddressSDNode")
? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7939, __extension__ __PRETTY_FUNCTION__))

7939

"Emulated TLS must have zero offset in GlobalAddressSDNode")(static_cast <bool> ((GA->getOffset() == 0) &&
"Emulated TLS must have zero offset in GlobalAddressSDNode")
? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7939, __extension__ __PRETTY_FUNCTION__));

7940

return CallResult.first;

7941

}

7942

7943

SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,

7944

SelectionDAG &DAG) const {

7945

assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.")(static_cast <bool> ((Op->getOpcode() == ISD::SETCC)
&& "Input has to be a SETCC node.") ? void (0) : __assert_fail
("(Op->getOpcode() == ISD::SETCC) && \"Input has to be a SETCC node.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7945, __extension__ __PRETTY_FUNCTION__));

7946

if (!isCtlzFast())

7947

return SDValue();

7948

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

7949

SDLoc dl(Op);

7950

if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {

7951

if (C->isNullValue() && CC == ISD::SETEQ) {

7952

EVT VT = Op.getOperand(0).getValueType();

7953

SDValue Zext = Op.getOperand(0);

7954

if (VT.bitsLT(MVT::i32)) {

7955

VT = MVT::i32;

7956

Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));

7957

}

7958

unsigned Log2b = Log2_32(VT.getSizeInBits());

7959

SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);

7960

SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,

7961

DAG.getConstant(Log2b, dl, MVT::i32));

7962

return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);

7963

}

7964

}

7965

return SDValue();

7966

}

7967

7968

// Convert redundant addressing modes (e.g. scaling is redundant

7969

// when accessing bytes).

7970

ISD::MemIndexType

7971

TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,

7972

SDValue Offsets) const {

7973

bool IsScaledIndex =

7974

(IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);

7975

bool IsSignedIndex =

7976

(IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);

7977

7978

// Scaling is unimportant for bytes, canonicalize to unscaled.

7979

if (IsScaledIndex && MemVT.getScalarType() == MVT::i8) {

7980

IsScaledIndex = false;

7981

IndexType = IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;

7982

}

7983

7984

return IndexType;

7985

}

7986

7987

SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {

7988

SDValue Op0 = Node->getOperand(0);

7989

SDValue Op1 = Node->getOperand(1);

7990

EVT VT = Op0.getValueType();

7991

unsigned Opcode = Node->getOpcode();

7992

SDLoc DL(Node);

7993

7994

// umin(x,y) -> sub(x,usubsat(x,y))

7995

if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&

7996

isOperationLegal(ISD::USUBSAT, VT)) {

7997

return DAG.getNode(ISD::SUB, DL, VT, Op0,

7998

DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));

7999

}

8000

8001

// umax(x,y) -> add(x,usubsat(y,x))

8002

if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&

8003

isOperationLegal(ISD::USUBSAT, VT)) {

8004

return DAG.getNode(ISD::ADD, DL, VT, Op0,

8005

DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));

8006

}

8007

8008

// Expand Y = MAX(A, B) -> Y = (A > B) ? A : B

8009

ISD::CondCode CC;

8010

switch (Opcode) {

8011

default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8011);

8012

case ISD::SMAX: CC = ISD::SETGT; break;

8013

case ISD::SMIN: CC = ISD::SETLT; break;

8014

case ISD::UMAX: CC = ISD::SETUGT; break;

8015

case ISD::UMIN: CC = ISD::SETULT; break;

8016

}

8017

8018

// FIXME: Should really try to split the vector in case it's legal on a

8019

// subvector.

8020

if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))

8021

return DAG.UnrollVectorOp(Node);

8022

8023

SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);

8024

return DAG.getSelect(DL, VT, Cond, Op0, Op1);

8025

}

8026

8027

SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {

8028

unsigned Opcode = Node->getOpcode();

8029

SDValue LHS = Node->getOperand(0);

8030

SDValue RHS = Node->getOperand(1);

8031

EVT VT = LHS.getValueType();

8032

SDLoc dl(Node);

8033

8034

assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
"Expected operands to be the same type") ? void (0) : __assert_fail
("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8034, __extension__ __PRETTY_FUNCTION__));

8035

assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8035, __extension__ __PRETTY_FUNCTION__));

8036

8037

// usub.sat(a, b) -> umax(a, b) - b

8038

if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {

8039

SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);

8040

return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);

8041

}

8042

8043

// uadd.sat(a, b) -> umin(a, ~b) + b

8044

if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {

8045

SDValue InvRHS = DAG.getNOT(dl, RHS, VT);

8046

SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);

8047

return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);

8048

}

8049

8050

unsigned OverflowOp;

8051

switch (Opcode) {

8052

case ISD::SADDSAT:

8053

OverflowOp = ISD::SADDO;

8054

break;

8055

case ISD::UADDSAT:

8056

OverflowOp = ISD::UADDO;

8057

break;

8058

case ISD::SSUBSAT:

8059

OverflowOp = ISD::SSUBO;

8060

break;

8061

case ISD::USUBSAT:

8062

OverflowOp = ISD::USUBO;

8063

break;

8064

default:

8065

llvm_unreachable("Expected method to receive signed or unsigned saturation "::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
"addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8066)

8066

"addition or subtraction node.")::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
"addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8066);

8067

}

8068

8069

// FIXME: Should really try to split the vector in case it's legal on a

8070

// subvector.

8071

if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))

8072

return DAG.UnrollVectorOp(Node);

8073

8074

unsigned BitWidth = LHS.getScalarValueSizeInBits();

8075

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8076

SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8077

SDValue SumDiff = Result.getValue(0);

8078

SDValue Overflow = Result.getValue(1);

8079

SDValue Zero = DAG.getConstant(0, dl, VT);

8080

SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);

8081

8082

if (Opcode == ISD::UADDSAT) {

8083

if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {

8084

// (LHS + RHS) | OverflowMask

8085

SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);

8086

return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);

8087

}

8088

// Overflow ? 0xffff.... : (LHS + RHS)

8089

return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);

8090

}

8091

8092

if (Opcode == ISD::USUBSAT) {

8093

if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {

8094

// (LHS - RHS) & ~OverflowMask

8095

SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);

8096

SDValue Not = DAG.getNOT(dl, OverflowMask, VT);

8097

return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);

8098

}

8099

// Overflow ? 0 : (LHS - RHS)

8100

return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);

8101

}

8102

8103

// Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff

8104

APInt MinVal = APInt::getSignedMinValue(BitWidth);

8105

SDValue SatMin = DAG.getConstant(MinVal, dl, VT);

8106

SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,

8107

DAG.getConstant(BitWidth - 1, dl, VT));

8108

Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);

8109

return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);

8110

}

8111

8112

SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {

8113

unsigned Opcode = Node->getOpcode();

8114

bool IsSigned = Opcode == ISD::SSHLSAT;

8115

SDValue LHS = Node->getOperand(0);

8116

SDValue RHS = Node->getOperand(1);

8117

EVT VT = LHS.getValueType();

8118

SDLoc dl(Node);

8119

8120

assert((Node->getOpcode() == ISD::SSHLSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8122, __extension__ __PRETTY_FUNCTION__))

8121

Node->getOpcode() == ISD::USHLSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8122, __extension__ __PRETTY_FUNCTION__))

8122

"Expected a SHLSAT opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
|| Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8122, __extension__ __PRETTY_FUNCTION__));

8123

assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
"Expected operands to be the same type") ? void (0) : __assert_fail
("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8123, __extension__ __PRETTY_FUNCTION__));

8124

assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8124, __extension__ __PRETTY_FUNCTION__));

8125

8126

// If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.

8127

8128

unsigned BW = VT.getScalarSizeInBits();

8129

SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);

8130

SDValue Orig =

8131

DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);

8132

8133

SDValue SatVal;

8134

if (IsSigned) {

8135

SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);

8136

SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);

8137

SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),

8138

SatMin, SatMax, ISD::SETLT);

8139

} else {

8140

SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);

8141

}

8142

Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);

8143

8144

return Result;

8145

}

8146

8147

SDValue

8148

TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {

8149

assert((Node->getOpcode() == ISD::SMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8153, __extension__ __PRETTY_FUNCTION__))

8150

Node->getOpcode() == ISD::UMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8153, __extension__ __PRETTY_FUNCTION__))

8151

Node->getOpcode() == ISD::SMULFIXSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8153, __extension__ __PRETTY_FUNCTION__))

8152

Node->getOpcode() == ISD::UMULFIXSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8153, __extension__ __PRETTY_FUNCTION__))

8153

"Expected a fixed point multiplication opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
|| Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8153, __extension__ __PRETTY_FUNCTION__));

8154

8155

SDLoc dl(Node);

8156

SDValue LHS = Node->getOperand(0);

8157

SDValue RHS = Node->getOperand(1);

8158

EVT VT = LHS.getValueType();

8159

unsigned Scale = Node->getConstantOperandVal(2);

8160

bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||

8161

Node->getOpcode() == ISD::UMULFIXSAT);

8162

bool Signed = (Node->getOpcode() == ISD::SMULFIX ||

8163

Node->getOpcode() == ISD::SMULFIXSAT);

8164

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8165

unsigned VTSize = VT.getScalarSizeInBits();

8166

8167

if (!Scale) {

8168

// [us]mul.fix(a, b, 0) -> mul(a, b)

8169

if (!Saturating) {

8170

if (isOperationLegalOrCustom(ISD::MUL, VT))

8171

return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8172

} else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {

8173

SDValue Result =

8174

DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8175

SDValue Product = Result.getValue(0);

8176

SDValue Overflow = Result.getValue(1);

8177

SDValue Zero = DAG.getConstant(0, dl, VT);

8178

8179

APInt MinVal = APInt::getSignedMinValue(VTSize);

8180

APInt MaxVal = APInt::getSignedMaxValue(VTSize);

8181

SDValue SatMin = DAG.getConstant(MinVal, dl, VT);

8182

SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);

8183

SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);

8184

Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);

8185

return DAG.getSelect(dl, VT, Overflow, Result, Product);

8186

} else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {

8187

SDValue Result =

8188

DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);

8189

SDValue Product = Result.getValue(0);

8190

SDValue Overflow = Result.getValue(1);

8191

8192

APInt MaxVal = APInt::getMaxValue(VTSize);

8193

SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);

8194

return DAG.getSelect(dl, VT, Overflow, SatMax, Product);

8195

}

8196

}

8197

8198

assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8200, __extension__ __PRETTY_FUNCTION__))

8199

"Expected scale to be less than the number of bits if signed or at "(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8200, __extension__ __PRETTY_FUNCTION__))

8200

"most the number of bits if unsigned.")(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
"most the number of bits if unsigned.") ? void (0) : __assert_fail
("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8200, __extension__ __PRETTY_FUNCTION__));

8201

assert(LHS.getValueType() == RHS.getValueType() &&(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8202, __extension__ __PRETTY_FUNCTION__))

8202

"Expected both operands to be the same type")(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8202, __extension__ __PRETTY_FUNCTION__));

8203

8204

// Get the upper and lower bits of the result.

8205

SDValue Lo, Hi;

8206

unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;

8207

unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;

8208

if (isOperationLegalOrCustom(LoHiOp, VT)) {

8209

SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);

8210

Lo = Result.getValue(0);

8211

Hi = Result.getValue(1);

8212

} else if (isOperationLegalOrCustom(HiOp, VT)) {

8213

Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8214

Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);

8215

} else if (VT.isVector()) {

8216

return SDValue();

8217

} else {

8218

report_fatal_error("Unable to expand fixed point multiplication.");

8219

}

8220

8221

if (Scale == VTSize)

8222

// Result is just the top half since we'd be shifting by the width of the

8223

// operand. Overflow impossible so this works for both UMULFIX and

8224

// UMULFIXSAT.

8225

return Hi;

8226

8227

// The result will need to be shifted right by the scale since both operands

8228

// are scaled. The result is given to us in 2 halves, so we only want part of

8229

// both in the result.

8230

EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());

8231

SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,

8232

DAG.getConstant(Scale, dl, ShiftTy));

8233

if (!Saturating)

8234

return Result;

8235

8236

if (!Signed) {

8237

// Unsigned overflow happened if the upper (VTSize - Scale) bits (of the

8238

// widened multiplication) aren't all zeroes.

8239

8240

// Saturate to max if ((Hi >> Scale) != 0),

8241

// which is the same as if (Hi > ((1 << Scale) - 1))

8242

APInt MaxVal = APInt::getMaxValue(VTSize);

8243

SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),

8244

dl, VT);

8245

Result = DAG.getSelectCC(dl, Hi, LowMask,

8246

DAG.getConstant(MaxVal, dl, VT), Result,

8247

ISD::SETUGT);

8248

8249

return Result;

8250

}

8251

8252

// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the

8253

// widened multiplication) aren't all ones or all zeroes.

8254

8255

SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);

8256

SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);

8257

8258

if (Scale == 0) {

8259

SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,

8260

DAG.getConstant(VTSize - 1, dl, ShiftTy));

8261

SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);

8262

// Saturated to SatMin if wide product is negative, and SatMax if wide

8263

// product is positive ...

8264

SDValue Zero = DAG.getConstant(0, dl, VT);

8265

SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,

8266

ISD::SETLT);

8267

// ... but only if we overflowed.

8268

return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);

8269

}

8270

8271

// We handled Scale==0 above so all the bits to examine is in Hi.

8272

8273

// Saturate to max if ((Hi >> (Scale - 1)) > 0),

8274

// which is the same as if (Hi > (1 << (Scale - 1)) - 1)

8275

SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),

8276

dl, VT);

8277

Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);

8278

// Saturate to min if (Hi >> (Scale - 1)) < -1),

8279

// which is the same as if (HI < (-1 << (Scale - 1))

8280

SDValue HighMask =

8281

DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),

8282

dl, VT);

8283

Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);

8284

return Result;

8285

}

8286

8287

SDValue

8288

TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,

8289

SDValue LHS, SDValue RHS,

8290

unsigned Scale, SelectionDAG &DAG) const {

8291

assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8293, __extension__ __PRETTY_FUNCTION__))

8292

Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8293, __extension__ __PRETTY_FUNCTION__))

8293

"Expected a fixed point division opcode")(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8293, __extension__ __PRETTY_FUNCTION__));

8294

8295

EVT VT = LHS.getValueType();

8296

bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;

8297

bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;

8298

EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8299

8300

// If there is enough room in the type to upscale the LHS or downscale the

8301

// RHS before the division, we can perform it in this type without having to

8302

// resize. For signed operations, the LHS headroom is the number of

8303

// redundant sign bits, and for unsigned ones it is the number of zeroes.

8304

// The headroom for the RHS is the number of trailing zeroes.

8305

unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1

8306

: DAG.computeKnownBits(LHS).countMinLeadingZeros();

8307

unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();

8308

8309

// For signed saturating operations, we need to be able to detect true integer

8310

// division overflow; that is, when you have MIN / -EPS. However, this

8311

// is undefined behavior and if we emit divisions that could take such

8312

// values it may cause undesired behavior (arithmetic exceptions on x86, for

8313

// example).

8314

// Avoid this by requiring an extra bit so that we never get this case.

8315

// FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale

8316

// signed saturating division, we need to emit a whopping 32-bit division.

8317

if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))

8318

return SDValue();

8319

8320

unsigned LHSShift = std::min(LHSLead, Scale);

8321

unsigned RHSShift = Scale - LHSShift;

8322

8323

// At this point, we know that if we shift the LHS up by LHSShift and the

8324

// RHS down by RHSShift, we can emit a regular division with a final scaling

8325

// factor of Scale.

8326

8327

EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());

8328

if (LHSShift)

8329

LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,

8330

DAG.getConstant(LHSShift, dl, ShiftTy));

8331

if (RHSShift)

8332

RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,

8333

DAG.getConstant(RHSShift, dl, ShiftTy));

8334

8335

SDValue Quot;

8336

if (Signed) {

8337

// For signed operations, if the resulting quotient is negative and the

8338

// remainder is nonzero, subtract 1 from the quotient to round towards

8339

// negative infinity.

8340

SDValue Rem;

8341

// FIXME: Ideally we would always produce an SDIVREM here, but if the

8342

// type isn't legal, SDIVREM cannot be expanded. There is no reason why

8343

// we couldn't just form a libcall, but the type legalizer doesn't do it.

8344

if (isTypeLegal(VT) &&

8345

isOperationLegalOrCustom(ISD::SDIVREM, VT)) {

8346

Quot = DAG.getNode(ISD::SDIVREM, dl,

8347

DAG.getVTList(VT, VT),

8348

LHS, RHS);

8349

Rem = Quot.getValue(1);

8350

Quot = Quot.getValue(0);

8351

} else {

8352

Quot = DAG.getNode(ISD::SDIV, dl, VT,

8353

LHS, RHS);

8354

Rem = DAG.getNode(ISD::SREM, dl, VT,

8355

LHS, RHS);

8356

}

8357

SDValue Zero = DAG.getConstant(0, dl, VT);

8358

SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);

8359

SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);

8360

SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);

8361

SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);

8362

SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,

8363

DAG.getConstant(1, dl, VT));

8364

Quot = DAG.getSelect(dl, VT,

8365

DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),

8366

Sub1, Quot);

8367

} else

8368

Quot = DAG.getNode(ISD::UDIV, dl, VT,

8369

LHS, RHS);

8370

8371

return Quot;

8372

}

8373

8374

void TargetLowering::expandUADDSUBO(

8375

SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {

8376

SDLoc dl(Node);

8377

SDValue LHS = Node->getOperand(0);

8378

SDValue RHS = Node->getOperand(1);

8379

bool IsAdd = Node->getOpcode() == ISD::UADDO;

8380

8381

// If ADD/SUBCARRY is legal, use that instead.

8382

unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;

8383

if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {

8384

SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));

8385

SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),

8386

{ LHS, RHS, CarryIn });

8387

Result = SDValue(NodeCarry.getNode(), 0);

8388

Overflow = SDValue(NodeCarry.getNode(), 1);

8389

return;

8390

}

8391

8392

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,

8393

LHS.getValueType(), LHS, RHS);

8394

8395

EVT ResultType = Node->getValueType(1);

8396

EVT SetCCType = getSetCCResultType(

8397

DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

8398

ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;

8399

SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);

8400

Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);

8401

}

8402

8403

void TargetLowering::expandSADDSUBO(

8404

SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {

8405

SDLoc dl(Node);

8406

SDValue LHS = Node->getOperand(0);

8407

SDValue RHS = Node->getOperand(1);

8408

bool IsAdd = Node->getOpcode() == ISD::SADDO;

8409

8410

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,

8411

LHS.getValueType(), LHS, RHS);

8412

8413

EVT ResultType = Node->getValueType(1);

8414

EVT OType = getSetCCResultType(

8415

DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

8416

8417

// If SADDSAT/SSUBSAT is legal, compare results to detect overflow.

8418

unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;

8419

if (isOperationLegal(OpcSat, LHS.getValueType())) {

8420

SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);

8421

SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);

8422

Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);

8423

return;

8424

}

8425

8426

SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());

8427

8428

// For an addition, the result should be less than one of the operands (LHS)

8429

// if and only if the other operand (RHS) is negative, otherwise there will

8430

// be overflow.

8431

// For a subtraction, the result should be less than one of the operands

8432

// (LHS) if and only if the other operand (RHS) is (non-zero) positive,

8433

// otherwise there will be overflow.

8434

SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);

8435

SDValue ConditionRHS =

8436

DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);

8437

8438

Overflow = DAG.getBoolExtOrTrunc(

8439

DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,

8440

ResultType, ResultType);

8441

}

8442

8443

bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,

8444

SDValue &Overflow, SelectionDAG &DAG) const {

8445

SDLoc dl(Node);

8446

EVT VT = Node->getValueType(0);

8447

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

8448

SDValue LHS = Node->getOperand(0);

8449

SDValue RHS = Node->getOperand(1);

8450

bool isSigned = Node->getOpcode() == ISD::SMULO;

8451

8452

// For power-of-two multiplications we can use a simpler shift expansion.

8453

if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {

8454

const APInt &C = RHSC->getAPIntValue();

8455

// mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }

8456

if (C.isPowerOf2()) {

8457

// smulo(x, signed_min) is same as umulo(x, signed_min).

8458

bool UseArithShift = isSigned && !C.isMinSignedValue();

8459

EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());

8460

SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);

8461

Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);

8462

Overflow = DAG.getSetCC(dl, SetCCVT,

8463

DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,

8464

dl, VT, Result, ShiftAmt),

8465

LHS, ISD::SETNE);

8466

return true;

8467

}

8468

}

8469

8470

EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);

8471

if (VT.isVector())

8472

WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,

8473

VT.getVectorNumElements());

8474

8475

SDValue BottomHalf;

8476

SDValue TopHalf;

8477

static const unsigned Ops[2][3] =

8478

{ { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },

8479

{ ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};

8480

if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {

8481

BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);

8482

TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);

8483

} else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {

8484

BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,

8485

RHS);

8486

TopHalf = BottomHalf.getValue(1);

8487

} else if (isTypeLegal(WideVT)) {

8488

LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);

8489

RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);

8490

SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);

8491

BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);

8492

SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,

8493

getShiftAmountTy(WideVT, DAG.getDataLayout()));

8494

TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,

8495

DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));

8496

} else {

8497

if (VT.isVector())

8498

return false;

8499

8500

// We can fall back to a libcall with an illegal type for the MUL if we

8501

// have a libcall big enough.

8502

// Also, we can fall back to a division in some cases, but that's a big

8503

// performance hit in the general case.

8504

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;

8505

if (WideVT == MVT::i16)

8506

LC = RTLIB::MUL_I16;

8507

else if (WideVT == MVT::i32)

8508

LC = RTLIB::MUL_I32;

8509

else if (WideVT == MVT::i64)

8510

LC = RTLIB::MUL_I64;

8511

else if (WideVT == MVT::i128)

8512

LC = RTLIB::MUL_I128;

8513

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8513, __extension__ __PRETTY_FUNCTION__));

8514

8515

SDValue HiLHS;

8516

SDValue HiRHS;

8517

if (isSigned) {

8518

// The high part is obtained by SRA'ing all but one of the bits of low

8519

// part.

8520

unsigned LoSize = VT.getFixedSizeInBits();

8521

HiLHS =

8522

DAG.getNode(ISD::SRA, dl, VT, LHS,

8523

DAG.getConstant(LoSize - 1, dl,

8524

getPointerTy(DAG.getDataLayout())));

8525

HiRHS =

8526

DAG.getNode(ISD::SRA, dl, VT, RHS,

8527

DAG.getConstant(LoSize - 1, dl,

8528

getPointerTy(DAG.getDataLayout())));

8529

} else {

8530

HiLHS = DAG.getConstant(0, dl, VT);

8531

HiRHS = DAG.getConstant(0, dl, VT);

8532

}

8533

8534

// Here we're passing the 2 arguments explicitly as 4 arguments that are

8535

// pre-lowered to the correct types. This all depends upon WideVT not

8536

// being a legal type for the architecture and thus has to be split to

8537

// two arguments.

8538

SDValue Ret;

8539

TargetLowering::MakeLibCallOptions CallOptions;

8540

CallOptions.setSExt(isSigned);

8541

CallOptions.setIsPostTypeLegalization(true);

8542

if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {

8543

// Halves of WideVT are packed into registers in different order

8544

// depending on platform endianness. This is usually handled by

8545

// the C calling convention, but we can't defer to it in

8546

// the legalizer.

8547

SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };

8548

Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;

8549

} else {

8550

SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };

8551

Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;

8552

}

8553

assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8554, __extension__ __PRETTY_FUNCTION__))

8554

"Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
&& "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8554, __extension__ __PRETTY_FUNCTION__));

8555

if (DAG.getDataLayout().isLittleEndian()) {

8556

// Same as above.

8557

BottomHalf = Ret.getOperand(0);

8558

TopHalf = Ret.getOperand(1);

8559

} else {

8560

BottomHalf = Ret.getOperand(1);

8561

TopHalf = Ret.getOperand(0);

8562

}

8563

}

8564

8565

Result = BottomHalf;

8566

if (isSigned) {

8567

SDValue ShiftAmt = DAG.getConstant(

8568

VT.getScalarSizeInBits() - 1, dl,

8569

getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));

8570

SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);

8571

Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);

8572

} else {

8573

Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,

8574

DAG.getConstant(0, dl, VT), ISD::SETNE);

8575

}

8576

8577

// Truncate the result if SetCC returns a larger type than needed.

8578

EVT RType = Node->getValueType(1);

8579

if (RType.bitsLT(Overflow.getValueType()))

8580

Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);

8581

8582

assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8583, __extension__ __PRETTY_FUNCTION__))

8583

"Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8583, __extension__ __PRETTY_FUNCTION__));

8584

return true;

8585

}

8586

8587

SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {

8588

SDLoc dl(Node);

8589

unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

8590

SDValue Op = Node->getOperand(0);

8591

EVT VT = Op.getValueType();

8592

8593

if (VT.isScalableVector())

8594

report_fatal_error(

8595

"Expanding reductions for scalable vectors is undefined.");

8596

8597

// Try to use a shuffle reduction for power of two vectors.

8598

if (VT.isPow2VectorType()) {

8599

while (VT.getVectorNumElements() > 1) {

8600

EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());

8601

if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))

8602

break;

8603

8604

SDValue Lo, Hi;

8605

std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);

8606

Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);

8607

VT = HalfVT;

8608

}

8609

}

8610

8611

EVT EltVT = VT.getVectorElementType();

8612

unsigned NumElts = VT.getVectorNumElements();

8613

8614

SmallVector<SDValue, 8> Ops;

8615

DAG.ExtractVectorElements(Op, Ops, 0, NumElts);

8616

8617

SDValue Res = Ops[0];

8618

for (unsigned i = 1; i < NumElts; i++)

8619

Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());

8620

8621

// Result type may be wider than element type.

8622

if (EltVT != Node->getValueType(0))

8623

Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);

8624

return Res;

8625

}

8626

8627

SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {

8628

SDLoc dl(Node);

8629

SDValue AccOp = Node->getOperand(0);

8630

SDValue VecOp = Node->getOperand(1);

8631

SDNodeFlags Flags = Node->getFlags();

8632

8633

EVT VT = VecOp.getValueType();

8634

EVT EltVT = VT.getVectorElementType();

8635

8636

if (VT.isScalableVector())

8637

report_fatal_error(

8638

"Expanding reductions for scalable vectors is undefined.");

8639

8640

unsigned NumElts = VT.getVectorNumElements();

8641

8642

SmallVector<SDValue, 8> Ops;

8643

DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);

8644

8645

unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

8646

8647

SDValue Res = AccOp;

8648

for (unsigned i = 0; i < NumElts; i++)

8649

Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);

8650

8651

return Res;

8652

}

8653

8654

bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,

8655

SelectionDAG &DAG) const {

8656

EVT VT = Node->getValueType(0);

8657

SDLoc dl(Node);

8658

bool isSigned = Node->getOpcode() == ISD::SREM;

8659

unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;

8660

unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;

8661

SDValue Dividend = Node->getOperand(0);

8662

SDValue Divisor = Node->getOperand(1);

8663

if (isOperationLegalOrCustom(DivRemOpc, VT)) {

8664

SDVTList VTs = DAG.getVTList(VT, VT);

8665

Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);

8666

return true;

8667

}

8668

if (isOperationLegalOrCustom(DivOpc, VT)) {

8669

// X % Y -> X-X/Y*Y

8670

SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);

8671

SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);

8672

Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);

8673

return true;

8674

}

8675

return false;

8676

}

8677

8678

SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,

8679

SelectionDAG &DAG) const {

8680

bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;

8681

SDLoc dl(SDValue(Node, 0));

8682

SDValue Src = Node->getOperand(0);

8683

8684

// DstVT is the result type, while SatVT is the size to which we saturate

8685

EVT SrcVT = Src.getValueType();

8686

EVT DstVT = Node->getValueType(0);

8687

8688

EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();

8689

unsigned SatWidth = SatVT.getScalarSizeInBits();

8690

unsigned DstWidth = DstVT.getScalarSizeInBits();

8691

assert(SatWidth <= DstWidth &&(static_cast <bool> (SatWidth <= DstWidth &&
"Expected saturation width smaller than result width") ? void
(0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8692, __extension__ __PRETTY_FUNCTION__))

8692

"Expected saturation width smaller than result width")(static_cast <bool> (SatWidth <= DstWidth &&
"Expected saturation width smaller than result width") ? void
(0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8692, __extension__ __PRETTY_FUNCTION__));

8693

8694

// Determine minimum and maximum integer values and their corresponding

8695

// floating-point values.

8696

APInt MinInt, MaxInt;

8697

if (IsSigned) {

8698

MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);

8699

MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);

8700

} else {

8701

MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);

8702

MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);

8703

}

8704

8705

// We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as

8706

// libcall emission cannot handle this. Large result types will fail.

8707

if (SrcVT == MVT::f16) {

8708

Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);

8709

SrcVT = Src.getValueType();

8710

}

8711

8712

APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));

8713

APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));

8714

8715

APFloat::opStatus MinStatus =

8716

MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);

8717

APFloat::opStatus MaxStatus =

8718

MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);

8719

bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&

8720

!(MaxStatus & APFloat::opStatus::opInexact);

8721

8722

SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);

8723

SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);

8724

8725

// If the integer bounds are exactly representable as floats and min/max are

8726

// legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence

8727

// of comparisons and selects.

8728

bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&

8729

isOperationLegal(ISD::FMAXNUM, SrcVT);

8730

if (AreExactFloatBounds && MinMaxLegal) {

8731

SDValue Clamped = Src;

8732

8733

// Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.

8734

Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);

8735

// Clamp by MaxFloat from above. NaN cannot occur.

8736

Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);

8737

// Convert clamped value to integer.

8738

SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,

8739

dl, DstVT, Clamped);

8740

8741

// In the unsigned case we're done, because we mapped NaN to MinFloat,

8742

// which will cast to zero.

8743

if (!IsSigned)

8744

return FpToInt;

8745

8746

// Otherwise, select 0 if Src is NaN.

8747

SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);

8748

return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,

8749

ISD::CondCode::SETUO);

8750

}

8751

8752

SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);

8753

SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);

8754

8755

// Result of direct conversion. The assumption here is that the operation is

8756

// non-trapping and it's fine to apply it to an out-of-range value if we

8757

// select it away later.

8758

SDValue FpToInt =

8759

DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);

8760

8761

SDValue Select = FpToInt;

8762

8763

// If Src ULT MinFloat, select MinInt. In particular, this also selects

8764

// MinInt if Src is NaN.

8765

Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,

8766

ISD::CondCode::SETULT);

8767

// If Src OGT MaxFloat, select MaxInt.

8768

Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,

8769

ISD::CondCode::SETOGT);

8770

8771

// In the unsigned case we are done, because we mapped NaN to MinInt, which

8772

// is already zero.

8773

if (!IsSigned)

8774

return Select;

8775

8776

// Otherwise, select 0 if Src is NaN.

8777

SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);

8778

return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);

8779

}

8780

8781

SDValue TargetLowering::expandVectorSplice(SDNode *Node,

8782

SelectionDAG &DAG) const {

8783

assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!")(static_cast <bool> (Node->getOpcode() == ISD::VECTOR_SPLICE
&& "Unexpected opcode!") ? void (0) : __assert_fail (
"Node->getOpcode() == ISD::VECTOR_SPLICE && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8783, __extension__ __PRETTY_FUNCTION__));

8784

assert(Node->getValueType(0).isScalableVector() &&(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8785, __extension__ __PRETTY_FUNCTION__))

8785

"Fixed length vector types expected to use SHUFFLE_VECTOR!")(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8785, __extension__ __PRETTY_FUNCTION__));

8786

8787

EVT VT = Node->getValueType(0);

8788

SDValue V1 = Node->getOperand(0);

8789

SDValue V2 = Node->getOperand(1);

8790

int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();

8791

SDLoc DL(Node);

8792

8793

// Expand through memory thusly:

8794

// Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr

8795

// Store V1, Ptr

8796

// Store V2, Ptr + sizeof(V1)

8797

// If (Imm < 0)

8798

// TrailingElts = -Imm

8799

// Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))

8800

// else

8801

// Ptr = Ptr + (Imm * sizeof(VT.Elt))

8802

// Res = Load Ptr

8803

8804

Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);

8805

8806

EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),

8807

VT.getVectorElementCount() * 2);

8808

SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);

8809

EVT PtrVT = StackPtr.getValueType();

8810

auto &MF = DAG.getMachineFunction();

8811

auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

8812

auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);

8813

8814

// Store the lo part of CONCAT_VECTORS(V1, V2)

8815

SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);

8816

// Store the hi part of CONCAT_VECTORS(V1, V2)

8817

SDValue OffsetToV2 = DAG.getVScale(

8818

DL, PtrVT,

8819

APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));

8820

SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);

8821

SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);

8822

8823

if (Imm >= 0) {

8824

// Load back the required element. getVectorElementPointer takes care of

8825

// clamping the index if it's out-of-bounds.

8826

StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));

8827

// Load the spliced result

8828

return DAG.getLoad(VT, DL, StoreV2, StackPtr,

8829

MachinePointerInfo::getUnknownStack(MF));

8830

}

8831

8832

uint64_t TrailingElts = -Imm;

8833

8834

// NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.

8835

TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();

8836

SDValue TrailingBytes =

8837

DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);

8838

8839

if (TrailingElts > VT.getVectorMinNumElements()) {

8840

SDValue VLBytes = DAG.getVScale(

8841

DL, PtrVT,

8842

APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));

8843

TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);

8844

}

8845

8846

// Calculate the start address of the spliced result.

8847

StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);

8848

8849

// Load the spliced result

8850

return DAG.getLoad(VT, DL, StoreV2, StackPtr2,

8851

MachinePointerInfo::getUnknownStack(MF));

8852

}

8853

8854

bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,

8855

SDValue &LHS, SDValue &RHS,

8856

SDValue &CC, bool &NeedInvert,

8857

const SDLoc &dl, SDValue &Chain,

8858

bool IsSignaling) const {

8859

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

8860

MVT OpVT = LHS.getSimpleValueType();

8861

ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();

8862

NeedInvert = false;

8863

switch (TLI.getCondCodeAction(CCCode, OpVT)) {

8864

default:

8865

llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8865);

8866

case TargetLowering::Legal:

8867

// Nothing to do.

8868

break;

8869

case TargetLowering::Expand: {

8870

ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);

8871

if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8872

std::swap(LHS, RHS);

8873

CC = DAG.getCondCode(InvCC);

8874

return true;

8875

}

8876

// Swapping operands didn't work. Try inverting the condition.

8877

bool NeedSwap = false;

8878

InvCC = getSetCCInverse(CCCode, OpVT);

8879

if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8880

// If inverting the condition is not enough, try swapping operands

8881

// on top of it.

8882

InvCC = ISD::getSetCCSwappedOperands(InvCC);

8883

NeedSwap = true;

8884

}

8885

if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {

8886

CC = DAG.getCondCode(InvCC);

8887

NeedInvert = true;

8888

if (NeedSwap)

8889

std::swap(LHS, RHS);

8890

return true;

8891

}

8892

8893

ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;

8894

unsigned Opc = 0;

8895

switch (CCCode) {

8896

default:

8897

llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8897);

8898

case ISD::SETUO:

8899

if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {

8900

CC1 = ISD::SETUNE;

8901

CC2 = ISD::SETUNE;

8902

Opc = ISD::OR;

8903

break;

8904

}

8905

assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8906, __extension__ __PRETTY_FUNCTION__))

8906

"If SETUE is expanded, SETOEQ or SETUNE must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8906, __extension__ __PRETTY_FUNCTION__));

8907

NeedInvert = true;

8908

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8909

case ISD::SETO:

8910

assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8911, __extension__ __PRETTY_FUNCTION__))

8911

"If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
(0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8911, __extension__ __PRETTY_FUNCTION__));

8912

CC1 = ISD::SETOEQ;

8913

CC2 = ISD::SETOEQ;

8914

Opc = ISD::AND;

8915

break;

8916

case ISD::SETONE:

8917

case ISD::SETUEQ:

8918

// If the SETUO or SETO CC isn't legal, we might be able to use

8919

// SETOGT || SETOLT, inverting the result for SETUEQ. We only need one

8920

// of SETOGT/SETOLT to be legal, the other can be emulated by swapping

8921

// the operands.

8922

CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;

8923

if (!TLI.isCondCodeLegal(CC2, OpVT) &&

8924

(TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||

8925

TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {

8926

CC1 = ISD::SETOGT;

8927

CC2 = ISD::SETOLT;

8928

Opc = ISD::OR;

8929

NeedInvert = ((unsigned)CCCode & 0x8U);

8930

break;

8931

}

8932

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8933

case ISD::SETOEQ:

8934

case ISD::SETOGT:

8935

case ISD::SETOGE:

8936

case ISD::SETOLT:

8937

case ISD::SETOLE:

8938

case ISD::SETUNE:

8939

case ISD::SETUGT:

8940

case ISD::SETUGE:

8941

case ISD::SETULT:

8942

case ISD::SETULE:

8943

// If we are floating point, assign and break, otherwise fall through.

8944

if (!OpVT.isInteger()) {

8945

// We can use the 4th bit to tell if we are the unordered

8946

// or ordered version of the opcode.

8947

CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;

8948

Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;

8949

CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);

8950

break;

8951

}

8952

// Fallthrough if we are unsigned integer.

8953

LLVM_FALLTHROUGH[[gnu::fallthrough]];

8954

case ISD::SETLE:

8955

case ISD::SETGT:

8956

case ISD::SETGE:

8957

case ISD::SETLT:

8958

case ISD::SETNE:

8959

case ISD::SETEQ:

8960

// If all combinations of inverting the condition and swapping operands

8961

// didn't work then we have no means to expand the condition.

8962

llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8962);

8963

}

8964

8965

SDValue SetCC1, SetCC2;

8966

if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {

8967

// If we aren't the ordered or unorder operation,

8968

// then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).

8969

SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);

8970

SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);

8971

} else {

8972

// Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)

8973

SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);

8974

SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);

8975

}

8976

if (Chain)

8977

Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),

8978

SetCC2.getValue(1));

8979

LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);

8980

RHS = SDValue();

8981

CC = SDValue();

8982

return true;

8983

}

8984

}

8985

return false;

8986

}

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 7980, column 5 Value stored to 'IsScaledIndex' is never read

Bug Summary

Annotated Source Code