1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/Analysis/VectorUtils.h"
30#include "llvm/BinaryFormat/Dwarf.h"
31#include "llvm/CodeGen/Analysis.h"
32#include "llvm/CodeGen/FunctionLoweringInfo.h"
33#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineConstantPool.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/RuntimeLibcalls.h"
40#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
41#include "llvm/CodeGen/SelectionDAGNodes.h"
42#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
43#include "llvm/CodeGen/TargetFrameLowering.h"
44#include "llvm/CodeGen/TargetLowering.h"
45#include "llvm/CodeGen/TargetRegisterInfo.h"
46#include "llvm/CodeGen/TargetSubtargetInfo.h"
47#include "llvm/CodeGen/ValueTypes.h"
48#include "llvm/CodeGenTypes/MachineValueType.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/ConstantRange.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DataLayout.h"
53#include "llvm/IR/DebugInfoMetadata.h"
54#include "llvm/IR/DebugLoc.h"
55#include "llvm/IR/DerivedTypes.h"
56#include "llvm/IR/Function.h"
57#include "llvm/IR/GlobalValue.h"
58#include "llvm/IR/Metadata.h"
59#include "llvm/IR/Type.h"
60#include "llvm/Support/Casting.h"
61#include "llvm/Support/CodeGen.h"
62#include "llvm/Support/Compiler.h"
63#include "llvm/Support/Debug.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/KnownBits.h"
66#include "llvm/Support/MathExtras.h"
67#include "llvm/Support/Mutex.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Utils/SizeOpts.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <cstdlib>
77#include <limits>
78#include <set>
79#include <string>
80#include <utility>
81#include <vector>
82
83using namespace llvm;
84
85/// makeVTList - Return an instance of the SDVTList struct initialized with the
86/// specified members.
87static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
88 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
89 return Res;
90}
91
92// Default null implementations of the callbacks.
93void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
94void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
95void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
96
97void SelectionDAG::DAGNodeDeletedListener::anchor() {}
98void SelectionDAG::DAGNodeInsertedListener::anchor() {}
99
100#define DEBUG_TYPE "selectiondag"
101
102static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
103 cl::Hidden, cl::init(Val: true),
104 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
105
106static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
107 cl::desc("Number limit for gluing ld/st of memcpy."),
108 cl::Hidden, cl::init(Val: 0));
109
110static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
111 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
112}
113
114//===----------------------------------------------------------------------===//
115// ConstantFPSDNode Class
116//===----------------------------------------------------------------------===//
117
118/// isExactlyValue - We don't rely on operator== working on double values, as
119/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
120/// As such, this method can be used to do an exact bit-for-bit comparison of
121/// two floating point values.
122bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
123 return getValueAPF().bitwiseIsEqual(RHS: V);
124}
125
126bool ConstantFPSDNode::isValueValidForType(EVT VT,
127 const APFloat& Val) {
128 assert(VT.isFloatingPoint() && "Can only convert between FP types");
129
130 // convert modifies in place, so make a copy.
131 APFloat Val2 = APFloat(Val);
132 bool losesInfo;
133 (void) Val2.convert(ToSemantics: SelectionDAG::EVTToAPFloatSemantics(VT),
134 RM: APFloat::rmNearestTiesToEven,
135 losesInfo: &losesInfo);
136 return !losesInfo;
137}
138
139//===----------------------------------------------------------------------===//
140// ISD Namespace
141//===----------------------------------------------------------------------===//
142
143bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
144 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
145 unsigned EltSize =
146 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
147 if (auto *Op0 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
148 SplatVal = Op0->getAPIntValue().trunc(width: EltSize);
149 return true;
150 }
151 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0))) {
152 SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(width: EltSize);
153 return true;
154 }
155 }
156
157 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
158 if (!BV)
159 return false;
160
161 APInt SplatUndef;
162 unsigned SplatBitSize;
163 bool HasUndefs;
164 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
165 // Endianness does not matter here. We are checking for a splat given the
166 // element size of the vector, and if we find such a splat for little endian
167 // layout, then that should be valid also for big endian (as the full vector
168 // size is known to be a multiple of the element size).
169 const bool IsBigEndian = false;
170 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
171 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
172 EltSize == SplatBitSize;
173}
174
175// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
176// specializations of the more general isConstantSplatVector()?
177
178bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
179 // Look through a bit convert.
180 while (N->getOpcode() == ISD::BITCAST)
181 N = N->getOperand(Num: 0).getNode();
182
183 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
184 APInt SplatVal;
185 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
186 }
187
188 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
189
190 unsigned i = 0, e = N->getNumOperands();
191
192 // Skip over all of the undef values.
193 while (i != e && N->getOperand(Num: i).isUndef())
194 ++i;
195
196 // Do not accept an all-undef vector.
197 if (i == e) return false;
198
199 // Do not accept build_vectors that aren't all constants or which have non-~0
200 // elements. We have to be a bit careful here, as the type of the constant
201 // may not be the same as the type of the vector elements due to type
202 // legalization (the elements are promoted to a legal type for the target and
203 // a vector of a type may be legal when the base element type is not).
204 // We only want to check enough bits to cover the vector elements, because
205 // we care if the resultant vector is all ones, not whether the individual
206 // constants are.
207 SDValue NotZero = N->getOperand(Num: i);
208 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
209 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: NotZero)) {
210 if (CN->getAPIntValue().countr_one() < EltSize)
211 return false;
212 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val&: NotZero)) {
213 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
214 return false;
215 } else
216 return false;
217
218 // Okay, we have at least one ~0 value, check to see if the rest match or are
219 // undefs. Even with the above element type twiddling, this should be OK, as
220 // the same type legalization should have applied to all the elements.
221 for (++i; i != e; ++i)
222 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
223 return false;
224 return true;
225}
226
227bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228 // Look through a bit convert.
229 while (N->getOpcode() == ISD::BITCAST)
230 N = N->getOperand(Num: 0).getNode();
231
232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233 APInt SplatVal;
234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235 }
236
237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238
239 bool IsAllUndef = true;
240 for (const SDValue &Op : N->op_values()) {
241 if (Op.isUndef())
242 continue;
243 IsAllUndef = false;
244 // Do not accept build_vectors that aren't all constants or which have non-0
245 // elements. We have to be a bit careful here, as the type of the constant
246 // may not be the same as the type of the vector elements due to type
247 // legalization (the elements are promoted to a legal type for the target
248 // and a vector of a type may be legal when the base element type is not).
249 // We only want to check enough bits to cover the vector elements, because
250 // we care if the resultant vector is all zeros, not whether the individual
251 // constants are.
252 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
253 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: Op)) {
254 if (CN->getAPIntValue().countr_zero() < EltSize)
255 return false;
256 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val: Op)) {
257 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
258 return false;
259 } else
260 return false;
261 }
262
263 // Do not accept an all-undef vector.
264 if (IsAllUndef)
265 return false;
266 return true;
267}
268
269bool ISD::isBuildVectorAllOnes(const SDNode *N) {
270 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
271}
272
273bool ISD::isBuildVectorAllZeros(const SDNode *N) {
274 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
275}
276
277bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
278 if (N->getOpcode() != ISD::BUILD_VECTOR)
279 return false;
280
281 for (const SDValue &Op : N->op_values()) {
282 if (Op.isUndef())
283 continue;
284 if (!isa<ConstantSDNode>(Val: Op))
285 return false;
286 }
287 return true;
288}
289
290bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
291 if (N->getOpcode() != ISD::BUILD_VECTOR)
292 return false;
293
294 for (const SDValue &Op : N->op_values()) {
295 if (Op.isUndef())
296 continue;
297 if (!isa<ConstantFPSDNode>(Val: Op))
298 return false;
299 }
300 return true;
301}
302
303bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
304 bool Signed) {
305 assert(N->getValueType(0).isVector() && "Expected a vector!");
306
307 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
308 if (EltSize <= NewEltSize)
309 return false;
310
311 if (N->getOpcode() == ISD::ZERO_EXTEND) {
312 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
313 NewEltSize) &&
314 !Signed;
315 }
316 if (N->getOpcode() == ISD::SIGN_EXTEND) {
317 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
318 NewEltSize) &&
319 Signed;
320 }
321 if (N->getOpcode() != ISD::BUILD_VECTOR)
322 return false;
323
324 for (const SDValue &Op : N->op_values()) {
325 if (Op.isUndef())
326 continue;
327 if (!isa<ConstantSDNode>(Val: Op))
328 return false;
329
330 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
331 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
332 return false;
333 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
334 return false;
335 }
336
337 return true;
338}
339
340bool ISD::allOperandsUndef(const SDNode *N) {
341 // Return false if the node has no operands.
342 // This is "logically inconsistent" with the definition of "all" but
343 // is probably the desired behavior.
344 if (N->getNumOperands() == 0)
345 return false;
346 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
347}
348
349bool ISD::isFreezeUndef(const SDNode *N) {
350 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
351}
352
353template <typename ConstNodeType>
354bool ISD::matchUnaryPredicateImpl(SDValue Op,
355 std::function<bool(ConstNodeType *)> Match,
356 bool AllowUndefs) {
357 // FIXME: Add support for scalar UNDEF cases?
358 if (auto *C = dyn_cast<ConstNodeType>(Op))
359 return Match(C);
360
361 // FIXME: Add support for vector UNDEF cases?
362 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
363 ISD::SPLAT_VECTOR != Op.getOpcode())
364 return false;
365
366 EVT SVT = Op.getValueType().getScalarType();
367 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
368 if (AllowUndefs && Op.getOperand(i).isUndef()) {
369 if (!Match(nullptr))
370 return false;
371 continue;
372 }
373
374 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
375 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
376 return false;
377 }
378 return true;
379}
380// Build used template types.
381template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
382 SDValue, std::function<bool(ConstantSDNode *)>, bool);
383template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
384 SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
385
386bool ISD::matchBinaryPredicate(
387 SDValue LHS, SDValue RHS,
388 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
389 bool AllowUndefs, bool AllowTypeMismatch) {
390 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
391 return false;
392
393 // TODO: Add support for scalar UNDEF cases?
394 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
395 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
396 return Match(LHSCst, RHSCst);
397
398 // TODO: Add support for vector UNDEF cases?
399 if (LHS.getOpcode() != RHS.getOpcode() ||
400 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
401 LHS.getOpcode() != ISD::SPLAT_VECTOR))
402 return false;
403
404 EVT SVT = LHS.getValueType().getScalarType();
405 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
406 SDValue LHSOp = LHS.getOperand(i);
407 SDValue RHSOp = RHS.getOperand(i);
408 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
409 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
410 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
411 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
412 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
413 return false;
414 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
415 LHSOp.getValueType() != RHSOp.getValueType()))
416 return false;
417 if (!Match(LHSCst, RHSCst))
418 return false;
419 }
420 return true;
421}
422
423ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
424 switch (VecReduceOpcode) {
425 default:
426 llvm_unreachable("Expected VECREDUCE opcode");
427 case ISD::VECREDUCE_FADD:
428 case ISD::VECREDUCE_SEQ_FADD:
429 case ISD::VP_REDUCE_FADD:
430 case ISD::VP_REDUCE_SEQ_FADD:
431 return ISD::FADD;
432 case ISD::VECREDUCE_FMUL:
433 case ISD::VECREDUCE_SEQ_FMUL:
434 case ISD::VP_REDUCE_FMUL:
435 case ISD::VP_REDUCE_SEQ_FMUL:
436 return ISD::FMUL;
437 case ISD::VECREDUCE_ADD:
438 case ISD::VP_REDUCE_ADD:
439 return ISD::ADD;
440 case ISD::VECREDUCE_MUL:
441 case ISD::VP_REDUCE_MUL:
442 return ISD::MUL;
443 case ISD::VECREDUCE_AND:
444 case ISD::VP_REDUCE_AND:
445 return ISD::AND;
446 case ISD::VECREDUCE_OR:
447 case ISD::VP_REDUCE_OR:
448 return ISD::OR;
449 case ISD::VECREDUCE_XOR:
450 case ISD::VP_REDUCE_XOR:
451 return ISD::XOR;
452 case ISD::VECREDUCE_SMAX:
453 case ISD::VP_REDUCE_SMAX:
454 return ISD::SMAX;
455 case ISD::VECREDUCE_SMIN:
456 case ISD::VP_REDUCE_SMIN:
457 return ISD::SMIN;
458 case ISD::VECREDUCE_UMAX:
459 case ISD::VP_REDUCE_UMAX:
460 return ISD::UMAX;
461 case ISD::VECREDUCE_UMIN:
462 case ISD::VP_REDUCE_UMIN:
463 return ISD::UMIN;
464 case ISD::VECREDUCE_FMAX:
465 case ISD::VP_REDUCE_FMAX:
466 return ISD::FMAXNUM;
467 case ISD::VECREDUCE_FMIN:
468 case ISD::VP_REDUCE_FMIN:
469 return ISD::FMINNUM;
470 case ISD::VECREDUCE_FMAXIMUM:
471 return ISD::FMAXIMUM;
472 case ISD::VECREDUCE_FMINIMUM:
473 return ISD::FMINIMUM;
474 }
475}
476
477bool ISD::isVPOpcode(unsigned Opcode) {
478 switch (Opcode) {
479 default:
480 return false;
481#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
482 case ISD::VPSD: \
483 return true;
484#include "llvm/IR/VPIntrinsics.def"
485 }
486}
487
488bool ISD::isVPBinaryOp(unsigned Opcode) {
489 switch (Opcode) {
490 default:
491 break;
492#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
493#define VP_PROPERTY_BINARYOP return true;
494#define END_REGISTER_VP_SDNODE(VPSD) break;
495#include "llvm/IR/VPIntrinsics.def"
496 }
497 return false;
498}
499
500bool ISD::isVPReduction(unsigned Opcode) {
501 switch (Opcode) {
502 default:
503 break;
504#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
505#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
506#define END_REGISTER_VP_SDNODE(VPSD) break;
507#include "llvm/IR/VPIntrinsics.def"
508 }
509 return false;
510}
511
512/// The operand position of the vector mask.
513std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
514 switch (Opcode) {
515 default:
516 return std::nullopt;
517#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
518 case ISD::VPSD: \
519 return MASKPOS;
520#include "llvm/IR/VPIntrinsics.def"
521 }
522}
523
524/// The operand position of the explicit vector length parameter.
525std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
526 switch (Opcode) {
527 default:
528 return std::nullopt;
529#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
530 case ISD::VPSD: \
531 return EVLPOS;
532#include "llvm/IR/VPIntrinsics.def"
533 }
534}
535
536std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
537 bool hasFPExcept) {
538 // FIXME: Return strict opcodes in case of fp exceptions.
539 switch (VPOpcode) {
540 default:
541 return std::nullopt;
542#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
543#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
544#define END_REGISTER_VP_SDNODE(VPOPC) break;
545#include "llvm/IR/VPIntrinsics.def"
546 }
547 return std::nullopt;
548}
549
550unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
551 switch (Opcode) {
552 default:
553 llvm_unreachable("can not translate this Opcode to VP.");
554#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
555#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
556#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
557#include "llvm/IR/VPIntrinsics.def"
558 }
559}
560
561ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
562 switch (ExtType) {
563 case ISD::EXTLOAD:
564 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
565 case ISD::SEXTLOAD:
566 return ISD::SIGN_EXTEND;
567 case ISD::ZEXTLOAD:
568 return ISD::ZERO_EXTEND;
569 default:
570 break;
571 }
572
573 llvm_unreachable("Invalid LoadExtType");
574}
575
576ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
577 // To perform this operation, we just need to swap the L and G bits of the
578 // operation.
579 unsigned OldL = (Operation >> 2) & 1;
580 unsigned OldG = (Operation >> 1) & 1;
581 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
582 (OldL << 1) | // New G bit
583 (OldG << 2)); // New L bit.
584}
585
586static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
587 unsigned Operation = Op;
588 if (isIntegerLike)
589 Operation ^= 7; // Flip L, G, E bits, but not U.
590 else
591 Operation ^= 15; // Flip all of the condition bits.
592
593 if (Operation > ISD::SETTRUE2)
594 Operation &= ~8; // Don't let N and U bits get set.
595
596 return ISD::CondCode(Operation);
597}
598
599ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
600 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
601}
602
603ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
604 bool isIntegerLike) {
605 return getSetCCInverseImpl(Op, isIntegerLike);
606}
607
608/// For an integer comparison, return 1 if the comparison is a signed operation
609/// and 2 if the result is an unsigned comparison. Return zero if the operation
610/// does not depend on the sign of the input (setne and seteq).
611static int isSignedOp(ISD::CondCode Opcode) {
612 switch (Opcode) {
613 default: llvm_unreachable("Illegal integer setcc operation!");
614 case ISD::SETEQ:
615 case ISD::SETNE: return 0;
616 case ISD::SETLT:
617 case ISD::SETLE:
618 case ISD::SETGT:
619 case ISD::SETGE: return 1;
620 case ISD::SETULT:
621 case ISD::SETULE:
622 case ISD::SETUGT:
623 case ISD::SETUGE: return 2;
624 }
625}
626
627ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
628 EVT Type) {
629 bool IsInteger = Type.isInteger();
630 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
631 // Cannot fold a signed integer setcc with an unsigned integer setcc.
632 return ISD::SETCC_INVALID;
633
634 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
635
636 // If the N and U bits get set, then the resultant comparison DOES suddenly
637 // care about orderedness, and it is true when ordered.
638 if (Op > ISD::SETTRUE2)
639 Op &= ~16; // Clear the U bit if the N bit is set.
640
641 // Canonicalize illegal integer setcc's.
642 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
643 Op = ISD::SETNE;
644
645 return ISD::CondCode(Op);
646}
647
648ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
649 EVT Type) {
650 bool IsInteger = Type.isInteger();
651 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
652 // Cannot fold a signed setcc with an unsigned setcc.
653 return ISD::SETCC_INVALID;
654
655 // Combine all of the condition bits.
656 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
657
658 // Canonicalize illegal integer setcc's.
659 if (IsInteger) {
660 switch (Result) {
661 default: break;
662 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
663 case ISD::SETOEQ: // SETEQ & SETU[LG]E
664 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
665 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
666 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
667 }
668 }
669
670 return Result;
671}
672
673//===----------------------------------------------------------------------===//
674// SDNode Profile Support
675//===----------------------------------------------------------------------===//
676
677/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
678static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
679 ID.AddInteger(I: OpC);
680}
681
682/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
683/// solely with their pointer.
684static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
685 ID.AddPointer(Ptr: VTList.VTs);
686}
687
688/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
689static void AddNodeIDOperands(FoldingSetNodeID &ID,
690 ArrayRef<SDValue> Ops) {
691 for (const auto &Op : Ops) {
692 ID.AddPointer(Ptr: Op.getNode());
693 ID.AddInteger(I: Op.getResNo());
694 }
695}
696
697/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
698static void AddNodeIDOperands(FoldingSetNodeID &ID,
699 ArrayRef<SDUse> Ops) {
700 for (const auto &Op : Ops) {
701 ID.AddPointer(Ptr: Op.getNode());
702 ID.AddInteger(I: Op.getResNo());
703 }
704}
705
706static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
707 SDVTList VTList, ArrayRef<SDValue> OpList) {
708 AddNodeIDOpcode(ID, OpC);
709 AddNodeIDValueTypes(ID, VTList);
710 AddNodeIDOperands(ID, Ops: OpList);
711}
712
713/// If this is an SDNode with special info, add this info to the NodeID data.
714static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
715 switch (N->getOpcode()) {
716 case ISD::TargetExternalSymbol:
717 case ISD::ExternalSymbol:
718 case ISD::MCSymbol:
719 llvm_unreachable("Should only be used on nodes with operands");
720 default: break; // Normal nodes don't need extra info.
721 case ISD::TargetConstant:
722 case ISD::Constant: {
723 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
724 ID.AddPointer(Ptr: C->getConstantIntValue());
725 ID.AddBoolean(B: C->isOpaque());
726 break;
727 }
728 case ISD::TargetConstantFP:
729 case ISD::ConstantFP:
730 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
731 break;
732 case ISD::TargetGlobalAddress:
733 case ISD::GlobalAddress:
734 case ISD::TargetGlobalTLSAddress:
735 case ISD::GlobalTLSAddress: {
736 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
737 ID.AddPointer(Ptr: GA->getGlobal());
738 ID.AddInteger(I: GA->getOffset());
739 ID.AddInteger(I: GA->getTargetFlags());
740 break;
741 }
742 case ISD::BasicBlock:
743 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
744 break;
745 case ISD::Register:
746 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg());
747 break;
748 case ISD::RegisterMask:
749 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
750 break;
751 case ISD::SRCVALUE:
752 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
753 break;
754 case ISD::FrameIndex:
755 case ISD::TargetFrameIndex:
756 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
757 break;
758 case ISD::LIFETIME_START:
759 case ISD::LIFETIME_END:
760 if (cast<LifetimeSDNode>(Val: N)->hasOffset()) {
761 ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getSize());
762 ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getOffset());
763 }
764 break;
765 case ISD::PSEUDO_PROBE:
766 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
767 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
768 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
769 break;
770 case ISD::JumpTable:
771 case ISD::TargetJumpTable:
772 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
773 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
774 break;
775 case ISD::ConstantPool:
776 case ISD::TargetConstantPool: {
777 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
778 ID.AddInteger(I: CP->getAlign().value());
779 ID.AddInteger(I: CP->getOffset());
780 if (CP->isMachineConstantPoolEntry())
781 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
782 else
783 ID.AddPointer(Ptr: CP->getConstVal());
784 ID.AddInteger(I: CP->getTargetFlags());
785 break;
786 }
787 case ISD::TargetIndex: {
788 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
789 ID.AddInteger(I: TI->getIndex());
790 ID.AddInteger(I: TI->getOffset());
791 ID.AddInteger(I: TI->getTargetFlags());
792 break;
793 }
794 case ISD::LOAD: {
795 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
796 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
797 ID.AddInteger(I: LD->getRawSubclassData());
798 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
799 ID.AddInteger(I: LD->getMemOperand()->getFlags());
800 break;
801 }
802 case ISD::STORE: {
803 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
804 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
805 ID.AddInteger(I: ST->getRawSubclassData());
806 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
807 ID.AddInteger(I: ST->getMemOperand()->getFlags());
808 break;
809 }
810 case ISD::VP_LOAD: {
811 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
812 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
813 ID.AddInteger(I: ELD->getRawSubclassData());
814 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
815 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
816 break;
817 }
818 case ISD::VP_STORE: {
819 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
820 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
821 ID.AddInteger(I: EST->getRawSubclassData());
822 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
823 ID.AddInteger(I: EST->getMemOperand()->getFlags());
824 break;
825 }
826 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
827 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
828 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
829 ID.AddInteger(I: SLD->getRawSubclassData());
830 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
831 break;
832 }
833 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
834 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
835 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
836 ID.AddInteger(I: SST->getRawSubclassData());
837 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
838 break;
839 }
840 case ISD::VP_GATHER: {
841 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
842 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
843 ID.AddInteger(I: EG->getRawSubclassData());
844 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
845 ID.AddInteger(I: EG->getMemOperand()->getFlags());
846 break;
847 }
848 case ISD::VP_SCATTER: {
849 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
850 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
851 ID.AddInteger(I: ES->getRawSubclassData());
852 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
853 ID.AddInteger(I: ES->getMemOperand()->getFlags());
854 break;
855 }
856 case ISD::MLOAD: {
857 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
858 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
859 ID.AddInteger(I: MLD->getRawSubclassData());
860 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
861 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
862 break;
863 }
864 case ISD::MSTORE: {
865 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
866 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
867 ID.AddInteger(I: MST->getRawSubclassData());
868 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
869 ID.AddInteger(I: MST->getMemOperand()->getFlags());
870 break;
871 }
872 case ISD::MGATHER: {
873 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
874 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
875 ID.AddInteger(I: MG->getRawSubclassData());
876 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
877 ID.AddInteger(I: MG->getMemOperand()->getFlags());
878 break;
879 }
880 case ISD::MSCATTER: {
881 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
882 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
883 ID.AddInteger(I: MS->getRawSubclassData());
884 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
885 ID.AddInteger(I: MS->getMemOperand()->getFlags());
886 break;
887 }
888 case ISD::ATOMIC_CMP_SWAP:
889 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
890 case ISD::ATOMIC_SWAP:
891 case ISD::ATOMIC_LOAD_ADD:
892 case ISD::ATOMIC_LOAD_SUB:
893 case ISD::ATOMIC_LOAD_AND:
894 case ISD::ATOMIC_LOAD_CLR:
895 case ISD::ATOMIC_LOAD_OR:
896 case ISD::ATOMIC_LOAD_XOR:
897 case ISD::ATOMIC_LOAD_NAND:
898 case ISD::ATOMIC_LOAD_MIN:
899 case ISD::ATOMIC_LOAD_MAX:
900 case ISD::ATOMIC_LOAD_UMIN:
901 case ISD::ATOMIC_LOAD_UMAX:
902 case ISD::ATOMIC_LOAD:
903 case ISD::ATOMIC_STORE: {
904 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
905 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
906 ID.AddInteger(I: AT->getRawSubclassData());
907 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
908 ID.AddInteger(I: AT->getMemOperand()->getFlags());
909 break;
910 }
911 case ISD::VECTOR_SHUFFLE: {
912 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
913 for (int M : Mask)
914 ID.AddInteger(I: M);
915 break;
916 }
917 case ISD::TargetBlockAddress:
918 case ISD::BlockAddress: {
919 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
920 ID.AddPointer(Ptr: BA->getBlockAddress());
921 ID.AddInteger(I: BA->getOffset());
922 ID.AddInteger(I: BA->getTargetFlags());
923 break;
924 }
925 case ISD::AssertAlign:
926 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
927 break;
928 case ISD::PREFETCH:
929 case ISD::INTRINSIC_VOID:
930 case ISD::INTRINSIC_W_CHAIN:
931 // Handled by MemIntrinsicSDNode check after the switch.
932 break;
933 } // end switch (N->getOpcode())
934
935 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
936 // to check.
937 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
938 ID.AddInteger(I: MN->getRawSubclassData());
939 ID.AddInteger(I: MN->getPointerInfo().getAddrSpace());
940 ID.AddInteger(I: MN->getMemOperand()->getFlags());
941 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
942 }
943}
944
945/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
946/// data.
947static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
948 AddNodeIDOpcode(ID, OpC: N->getOpcode());
949 // Add the return value info.
950 AddNodeIDValueTypes(ID, VTList: N->getVTList());
951 // Add the operand info.
952 AddNodeIDOperands(ID, Ops: N->ops());
953
954 // Handle SDNode leafs with special info.
955 AddNodeIDCustom(ID, N);
956}
957
958//===----------------------------------------------------------------------===//
959// SelectionDAG Class
960//===----------------------------------------------------------------------===//
961
962/// doNotCSE - Return true if CSE should not be performed for this node.
963static bool doNotCSE(SDNode *N) {
964 if (N->getValueType(ResNo: 0) == MVT::Glue)
965 return true; // Never CSE anything that produces a glue result.
966
967 switch (N->getOpcode()) {
968 default: break;
969 case ISD::HANDLENODE:
970 case ISD::EH_LABEL:
971 return true; // Never CSE these nodes.
972 }
973
974 // Check that remaining values produced are not flags.
975 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
976 if (N->getValueType(ResNo: i) == MVT::Glue)
977 return true; // Never CSE anything that produces a glue result.
978
979 return false;
980}
981
982/// RemoveDeadNodes - This method deletes all unreachable nodes in the
983/// SelectionDAG.
984void SelectionDAG::RemoveDeadNodes() {
985 // Create a dummy node (which is not added to allnodes), that adds a reference
986 // to the root node, preventing it from being deleted.
987 HandleSDNode Dummy(getRoot());
988
989 SmallVector<SDNode*, 128> DeadNodes;
990
991 // Add all obviously-dead nodes to the DeadNodes worklist.
992 for (SDNode &Node : allnodes())
993 if (Node.use_empty())
994 DeadNodes.push_back(Elt: &Node);
995
996 RemoveDeadNodes(DeadNodes);
997
998 // If the root changed (e.g. it was a dead load, update the root).
999 setRoot(Dummy.getValue());
1000}
1001
1002/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1003/// given list, and any nodes that become unreachable as a result.
1004void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1005
1006 // Process the worklist, deleting the nodes and adding their uses to the
1007 // worklist.
1008 while (!DeadNodes.empty()) {
1009 SDNode *N = DeadNodes.pop_back_val();
1010 // Skip to next node if we've already managed to delete the node. This could
1011 // happen if replacing a node causes a node previously added to the node to
1012 // be deleted.
1013 if (N->getOpcode() == ISD::DELETED_NODE)
1014 continue;
1015
1016 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1017 DUL->NodeDeleted(N, nullptr);
1018
1019 // Take the node out of the appropriate CSE map.
1020 RemoveNodeFromCSEMaps(N);
1021
1022 // Next, brutally remove the operand list. This is safe to do, as there are
1023 // no cycles in the graph.
1024 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1025 SDUse &Use = *I++;
1026 SDNode *Operand = Use.getNode();
1027 Use.set(SDValue());
1028
1029 // Now that we removed this operand, see if there are no uses of it left.
1030 if (Operand->use_empty())
1031 DeadNodes.push_back(Elt: Operand);
1032 }
1033
1034 DeallocateNode(N);
1035 }
1036}
1037
1038void SelectionDAG::RemoveDeadNode(SDNode *N){
1039 SmallVector<SDNode*, 16> DeadNodes(1, N);
1040
1041 // Create a dummy node that adds a reference to the root node, preventing
1042 // it from being deleted. (This matters if the root is an operand of the
1043 // dead node.)
1044 HandleSDNode Dummy(getRoot());
1045
1046 RemoveDeadNodes(DeadNodes);
1047}
1048
1049void SelectionDAG::DeleteNode(SDNode *N) {
1050 // First take this out of the appropriate CSE map.
1051 RemoveNodeFromCSEMaps(N);
1052
1053 // Finally, remove uses due to operands of this node, remove from the
1054 // AllNodes list, and delete the node.
1055 DeleteNodeNotInCSEMaps(N);
1056}
1057
1058void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1059 assert(N->getIterator() != AllNodes.begin() &&
1060 "Cannot delete the entry node!");
1061 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1062
1063 // Drop all of the operands and decrement used node's use counts.
1064 N->DropOperands();
1065
1066 DeallocateNode(N);
1067}
1068
1069void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1070 assert(!(V->isVariadic() && isParameter));
1071 if (isParameter)
1072 ByvalParmDbgValues.push_back(Elt: V);
1073 else
1074 DbgValues.push_back(Elt: V);
1075 for (const SDNode *Node : V->getSDNodes())
1076 if (Node)
1077 DbgValMap[Node].push_back(Elt: V);
1078}
1079
1080void SDDbgInfo::erase(const SDNode *Node) {
1081 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1082 if (I == DbgValMap.end())
1083 return;
1084 for (auto &Val: I->second)
1085 Val->setIsInvalidated();
1086 DbgValMap.erase(I);
1087}
1088
1089void SelectionDAG::DeallocateNode(SDNode *N) {
1090 // If we have operands, deallocate them.
1091 removeOperands(Node: N);
1092
1093 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1094
1095 // Set the opcode to DELETED_NODE to help catch bugs when node
1096 // memory is reallocated.
1097 // FIXME: There are places in SDag that have grown a dependency on the opcode
1098 // value in the released node.
1099 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1100 N->NodeType = ISD::DELETED_NODE;
1101
1102 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1103 // them and forget about that node.
1104 DbgInfo->erase(Node: N);
1105
1106 // Invalidate extra info.
1107 SDEI.erase(Val: N);
1108}
1109
1110#ifndef NDEBUG
1111/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1112static void VerifySDNode(SDNode *N) {
1113 switch (N->getOpcode()) {
1114 default:
1115 break;
1116 case ISD::BUILD_PAIR: {
1117 EVT VT = N->getValueType(ResNo: 0);
1118 assert(N->getNumValues() == 1 && "Too many results!");
1119 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1120 "Wrong return type!");
1121 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1122 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1123 "Mismatched operand types!");
1124 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1125 "Wrong operand type!");
1126 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1127 "Wrong return type size");
1128 break;
1129 }
1130 case ISD::BUILD_VECTOR: {
1131 assert(N->getNumValues() == 1 && "Too many results!");
1132 assert(N->getValueType(0).isVector() && "Wrong return type!");
1133 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1134 "Wrong number of operands!");
1135 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
1136 for (const SDUse &Op : N->ops()) {
1137 assert((Op.getValueType() == EltVT ||
1138 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1139 EltVT.bitsLE(Op.getValueType()))) &&
1140 "Wrong operand type!");
1141 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1142 "Operands must all have the same type");
1143 }
1144 break;
1145 }
1146 }
1147}
1148#endif // NDEBUG
1149
1150/// Insert a newly allocated node into the DAG.
1151///
1152/// Handles insertion into the all nodes list and CSE map, as well as
1153/// verification and other common operations when a new node is allocated.
1154void SelectionDAG::InsertNode(SDNode *N) {
1155 AllNodes.push_back(val: N);
1156#ifndef NDEBUG
1157 N->PersistentId = NextPersistentId++;
1158 VerifySDNode(N);
1159#endif
1160 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1161 DUL->NodeInserted(N);
1162}
1163
1164/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1165/// correspond to it. This is useful when we're about to delete or repurpose
1166/// the node. We don't want future request for structurally identical nodes
1167/// to return N anymore.
1168bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1169 bool Erased = false;
1170 switch (N->getOpcode()) {
1171 case ISD::HANDLENODE: return false; // noop.
1172 case ISD::CONDCODE:
1173 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1174 "Cond code doesn't exist!");
1175 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1176 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1177 break;
1178 case ISD::ExternalSymbol:
1179 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1180 break;
1181 case ISD::TargetExternalSymbol: {
1182 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1183 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1184 ESN->getSymbol(), ESN->getTargetFlags()));
1185 break;
1186 }
1187 case ISD::MCSymbol: {
1188 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1189 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1190 break;
1191 }
1192 case ISD::VALUETYPE: {
1193 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1194 if (VT.isExtended()) {
1195 Erased = ExtendedValueTypeNodes.erase(x: VT);
1196 } else {
1197 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1198 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1199 }
1200 break;
1201 }
1202 default:
1203 // Remove it from the CSE Map.
1204 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1205 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1206 Erased = CSEMap.RemoveNode(N);
1207 break;
1208 }
1209#ifndef NDEBUG
1210 // Verify that the node was actually in one of the CSE maps, unless it has a
1211 // glue result (which cannot be CSE'd) or is one of the special cases that are
1212 // not subject to CSE.
1213 if (!Erased && N->getValueType(ResNo: N->getNumValues()-1) != MVT::Glue &&
1214 !N->isMachineOpcode() && !doNotCSE(N)) {
1215 N->dump(G: this);
1216 dbgs() << "\n";
1217 llvm_unreachable("Node is not in map!");
1218 }
1219#endif
1220 return Erased;
1221}
1222
1223/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1224/// maps and modified in place. Add it back to the CSE maps, unless an identical
1225/// node already exists, in which case transfer all its users to the existing
1226/// node. This transfer can potentially trigger recursive merging.
1227void
1228SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1229 // For node types that aren't CSE'd, just act as if no identical node
1230 // already exists.
1231 if (!doNotCSE(N)) {
1232 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1233 if (Existing != N) {
1234 // If there was already an existing matching node, use ReplaceAllUsesWith
1235 // to replace the dead one with the existing one. This can cause
1236 // recursive merging of other unrelated nodes down the line.
1237 ReplaceAllUsesWith(From: N, To: Existing);
1238
1239 // N is now dead. Inform the listeners and delete it.
1240 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1241 DUL->NodeDeleted(N, Existing);
1242 DeleteNodeNotInCSEMaps(N);
1243 return;
1244 }
1245 }
1246
1247 // If the node doesn't already exist, we updated it. Inform listeners.
1248 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1249 DUL->NodeUpdated(N);
1250}
1251
1252/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1253/// were replaced with those specified. If this node is never memoized,
1254/// return null, otherwise return a pointer to the slot it would take. If a
1255/// node already exists with these operands, the slot will be non-null.
1256SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1257 void *&InsertPos) {
1258 if (doNotCSE(N))
1259 return nullptr;
1260
1261 SDValue Ops[] = { Op };
1262 FoldingSetNodeID ID;
1263 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1264 AddNodeIDCustom(ID, N);
1265 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1266 if (Node)
1267 Node->intersectFlagsWith(Flags: N->getFlags());
1268 return Node;
1269}
1270
1271/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1272/// were replaced with those specified. If this node is never memoized,
1273/// return null, otherwise return a pointer to the slot it would take. If a
1274/// node already exists with these operands, the slot will be non-null.
1275SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1276 SDValue Op1, SDValue Op2,
1277 void *&InsertPos) {
1278 if (doNotCSE(N))
1279 return nullptr;
1280
1281 SDValue Ops[] = { Op1, Op2 };
1282 FoldingSetNodeID ID;
1283 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1284 AddNodeIDCustom(ID, N);
1285 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1286 if (Node)
1287 Node->intersectFlagsWith(Flags: N->getFlags());
1288 return Node;
1289}
1290
1291/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1292/// were replaced with those specified. If this node is never memoized,
1293/// return null, otherwise return a pointer to the slot it would take. If a
1294/// node already exists with these operands, the slot will be non-null.
1295SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1296 void *&InsertPos) {
1297 if (doNotCSE(N))
1298 return nullptr;
1299
1300 FoldingSetNodeID ID;
1301 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1302 AddNodeIDCustom(ID, N);
1303 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1304 if (Node)
1305 Node->intersectFlagsWith(Flags: N->getFlags());
1306 return Node;
1307}
1308
1309Align SelectionDAG::getEVTAlign(EVT VT) const {
1310 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1311 : VT.getTypeForEVT(Context&: *getContext());
1312
1313 return getDataLayout().getABITypeAlign(Ty);
1314}
1315
1316// EntryNode could meaningfully have debug info if we can find it...
1317SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1318 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1319 getVTList(MVT::Other, MVT::Glue)),
1320 Root(getEntryNode()) {
1321 InsertNode(N: &EntryNode);
1322 DbgInfo = new SDDbgInfo();
1323}
1324
1325void SelectionDAG::init(MachineFunction &NewMF,
1326 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1327 const TargetLibraryInfo *LibraryInfo,
1328 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1329 BlockFrequencyInfo *BFIin,
1330 FunctionVarLocs const *VarLocs) {
1331 MF = &NewMF;
1332 SDAGISelPass = PassPtr;
1333 ORE = &NewORE;
1334 TLI = getSubtarget().getTargetLowering();
1335 TSI = getSubtarget().getSelectionDAGInfo();
1336 LibInfo = LibraryInfo;
1337 Context = &MF->getFunction().getContext();
1338 UA = NewUA;
1339 PSI = PSIin;
1340 BFI = BFIin;
1341 FnVarLocs = VarLocs;
1342}
1343
1344SelectionDAG::~SelectionDAG() {
1345 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1346 allnodes_clear();
1347 OperandRecycler.clear(OperandAllocator);
1348 delete DbgInfo;
1349}
1350
1351bool SelectionDAG::shouldOptForSize() const {
1352 return MF->getFunction().hasOptSize() ||
1353 llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1354}
1355
1356void SelectionDAG::allnodes_clear() {
1357 assert(&*AllNodes.begin() == &EntryNode);
1358 AllNodes.remove(IT: AllNodes.begin());
1359 while (!AllNodes.empty())
1360 DeallocateNode(N: &AllNodes.front());
1361#ifndef NDEBUG
1362 NextPersistentId = 0;
1363#endif
1364}
1365
1366SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1367 void *&InsertPos) {
1368 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1369 if (N) {
1370 switch (N->getOpcode()) {
1371 default: break;
1372 case ISD::Constant:
1373 case ISD::ConstantFP:
1374 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1375 "debug location. Use another overload.");
1376 }
1377 }
1378 return N;
1379}
1380
1381SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1382 const SDLoc &DL, void *&InsertPos) {
1383 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1384 if (N) {
1385 switch (N->getOpcode()) {
1386 case ISD::Constant:
1387 case ISD::ConstantFP:
1388 // Erase debug location from the node if the node is used at several
1389 // different places. Do not propagate one location to all uses as it
1390 // will cause a worse single stepping debugging experience.
1391 if (N->getDebugLoc() != DL.getDebugLoc())
1392 N->setDebugLoc(DebugLoc());
1393 break;
1394 default:
1395 // When the node's point of use is located earlier in the instruction
1396 // sequence than its prior point of use, update its debug info to the
1397 // earlier location.
1398 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1399 N->setDebugLoc(DL.getDebugLoc());
1400 break;
1401 }
1402 }
1403 return N;
1404}
1405
1406void SelectionDAG::clear() {
1407 allnodes_clear();
1408 OperandRecycler.clear(OperandAllocator);
1409 OperandAllocator.Reset();
1410 CSEMap.clear();
1411
1412 ExtendedValueTypeNodes.clear();
1413 ExternalSymbols.clear();
1414 TargetExternalSymbols.clear();
1415 MCSymbols.clear();
1416 SDEI.clear();
1417 std::fill(first: CondCodeNodes.begin(), last: CondCodeNodes.end(),
1418 value: static_cast<CondCodeSDNode*>(nullptr));
1419 std::fill(first: ValueTypeNodes.begin(), last: ValueTypeNodes.end(),
1420 value: static_cast<SDNode*>(nullptr));
1421
1422 EntryNode.UseList = nullptr;
1423 InsertNode(N: &EntryNode);
1424 Root = getEntryNode();
1425 DbgInfo->clear();
1426}
1427
1428SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1429 return VT.bitsGT(VT: Op.getValueType())
1430 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1431 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1432 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1433}
1434
1435std::pair<SDValue, SDValue>
1436SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1437 const SDLoc &DL, EVT VT) {
1438 assert(!VT.bitsEq(Op.getValueType()) &&
1439 "Strict no-op FP extend/round not allowed.");
1440 SDValue Res =
1441 VT.bitsGT(Op.getValueType())
1442 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1443 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1444 {Chain, Op, getIntPtrConstant(0, DL)});
1445
1446 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1447}
1448
1449SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1450 return VT.bitsGT(VT: Op.getValueType()) ?
1451 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1452 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1453}
1454
1455SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1456 return VT.bitsGT(VT: Op.getValueType()) ?
1457 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1458 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1459}
1460
1461SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1462 return VT.bitsGT(VT: Op.getValueType()) ?
1463 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1464 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1465}
1466
1467SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1468 EVT VT) {
1469 assert(!VT.isVector());
1470 auto Type = Op.getValueType();
1471 SDValue DestOp;
1472 if (Type == VT)
1473 return Op;
1474 auto Size = Op.getValueSizeInBits();
1475 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1476 if (DestOp.getValueType() == VT)
1477 return DestOp;
1478
1479 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1480}
1481
1482SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1483 EVT VT) {
1484 assert(!VT.isVector());
1485 auto Type = Op.getValueType();
1486 SDValue DestOp;
1487 if (Type == VT)
1488 return Op;
1489 auto Size = Op.getValueSizeInBits();
1490 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1491 if (DestOp.getValueType() == VT)
1492 return DestOp;
1493
1494 return getSExtOrTrunc(Op: DestOp, DL, VT);
1495}
1496
1497SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1498 EVT VT) {
1499 assert(!VT.isVector());
1500 auto Type = Op.getValueType();
1501 SDValue DestOp;
1502 if (Type == VT)
1503 return Op;
1504 auto Size = Op.getValueSizeInBits();
1505 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1506 if (DestOp.getValueType() == VT)
1507 return DestOp;
1508
1509 return getZExtOrTrunc(Op: DestOp, DL, VT);
1510}
1511
1512SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1513 EVT OpVT) {
1514 if (VT.bitsLE(VT: Op.getValueType()))
1515 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1516
1517 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1518 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1519}
1520
1521SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1522 EVT OpVT = Op.getValueType();
1523 assert(VT.isInteger() && OpVT.isInteger() &&
1524 "Cannot getZeroExtendInReg FP types");
1525 assert(VT.isVector() == OpVT.isVector() &&
1526 "getZeroExtendInReg type should be vector iff the operand "
1527 "type is vector!");
1528 assert((!VT.isVector() ||
1529 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1530 "Vector element counts must match in getZeroExtendInReg");
1531 assert(VT.bitsLE(OpVT) && "Not extending!");
1532 if (OpVT == VT)
1533 return Op;
1534 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1535 loBitsSet: VT.getScalarSizeInBits());
1536 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1537}
1538
1539SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1540 // Only unsigned pointer semantics are supported right now. In the future this
1541 // might delegate to TLI to check pointer signedness.
1542 return getZExtOrTrunc(Op, DL, VT);
1543}
1544
1545SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1546 // Only unsigned pointer semantics are supported right now. In the future this
1547 // might delegate to TLI to check pointer signedness.
1548 return getZeroExtendInReg(Op, DL, VT);
1549}
1550
1551SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1552 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1553}
1554
1555/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1556SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1557 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1558}
1559
1560SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1561 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1562 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1563}
1564
1565SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1566 SDValue Mask, SDValue EVL, EVT VT) {
1567 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1568 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1569}
1570
1571SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1572 SDValue Mask, SDValue EVL) {
1573 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1574}
1575
1576SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1577 SDValue Mask, SDValue EVL) {
1578 if (VT.bitsGT(VT: Op.getValueType()))
1579 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1580 if (VT.bitsLT(VT: Op.getValueType()))
1581 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1582 return Op;
1583}
1584
1585SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1586 EVT OpVT) {
1587 if (!V)
1588 return getConstant(Val: 0, DL, VT);
1589
1590 switch (TLI->getBooleanContents(Type: OpVT)) {
1591 case TargetLowering::ZeroOrOneBooleanContent:
1592 case TargetLowering::UndefinedBooleanContent:
1593 return getConstant(Val: 1, DL, VT);
1594 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1595 return getAllOnesConstant(DL, VT);
1596 }
1597 llvm_unreachable("Unexpected boolean content enum!");
1598}
1599
1600SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1601 bool isT, bool isO) {
1602 EVT EltVT = VT.getScalarType();
1603 assert((EltVT.getSizeInBits() >= 64 ||
1604 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1605 "getConstant with a uint64_t value that doesn't fit in the type!");
1606 return getConstant(Val: APInt(EltVT.getSizeInBits(), Val), DL, VT, isTarget: isT, isOpaque: isO);
1607}
1608
1609SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1610 bool isT, bool isO) {
1611 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1612}
1613
1614SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1615 EVT VT, bool isT, bool isO) {
1616 assert(VT.isInteger() && "Cannot create FP integer constant!");
1617
1618 EVT EltVT = VT.getScalarType();
1619 const ConstantInt *Elt = &Val;
1620
1621 // In some cases the vector type is legal but the element type is illegal and
1622 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1623 // inserted value (the type does not need to match the vector element type).
1624 // Any extra bits introduced will be truncated away.
1625 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1626 TargetLowering::TypePromoteInteger) {
1627 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1628 APInt NewVal;
1629 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1630 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1631 else
1632 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1633 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1634 }
1635 // In other cases the element type is illegal and needs to be expanded, for
1636 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1637 // the value into n parts and use a vector type with n-times the elements.
1638 // Then bitcast to the type requested.
1639 // Legalizing constants too early makes the DAGCombiner's job harder so we
1640 // only legalize if the DAG tells us we must produce legal types.
1641 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1642 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1643 TargetLowering::TypeExpandInteger) {
1644 const APInt &NewVal = Elt->getValue();
1645 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1646 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1647
1648 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1649 if (VT.isScalableVector() ||
1650 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1651 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1652 "Can only handle an even split!");
1653 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1654
1655 SmallVector<SDValue, 2> ScalarParts;
1656 for (unsigned i = 0; i != Parts; ++i)
1657 ScalarParts.push_back(Elt: getConstant(
1658 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1659 VT: ViaEltVT, isT, isO));
1660
1661 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1662 }
1663
1664 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1665 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1666
1667 // Check the temporary vector is the correct size. If this fails then
1668 // getTypeToTransformTo() probably returned a type whose size (in bits)
1669 // isn't a power-of-2 factor of the requested type size.
1670 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1671
1672 SmallVector<SDValue, 2> EltParts;
1673 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1674 EltParts.push_back(Elt: getConstant(
1675 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1676 VT: ViaEltVT, isT, isO));
1677
1678 // EltParts is currently in little endian order. If we actually want
1679 // big-endian order then reverse it now.
1680 if (getDataLayout().isBigEndian())
1681 std::reverse(first: EltParts.begin(), last: EltParts.end());
1682
1683 // The elements must be reversed when the element order is different
1684 // to the endianness of the elements (because the BITCAST is itself a
1685 // vector shuffle in this situation). However, we do not need any code to
1686 // perform this reversal because getConstant() is producing a vector
1687 // splat.
1688 // This situation occurs in MIPS MSA.
1689
1690 SmallVector<SDValue, 8> Ops;
1691 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1692 llvm::append_range(C&: Ops, R&: EltParts);
1693
1694 SDValue V =
1695 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1696 return V;
1697 }
1698
1699 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1700 "APInt size does not match type size!");
1701 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1702 FoldingSetNodeID ID;
1703 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt);
1704 ID.AddPointer(Ptr: Elt);
1705 ID.AddBoolean(B: isO);
1706 void *IP = nullptr;
1707 SDNode *N = nullptr;
1708 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1709 if (!VT.isVector())
1710 return SDValue(N, 0);
1711
1712 if (!N) {
1713 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: EltVT);
1714 CSEMap.InsertNode(N, InsertPos: IP);
1715 InsertNode(N);
1716 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1717 }
1718
1719 SDValue Result(N, 0);
1720 if (VT.isVector())
1721 Result = getSplat(VT, DL, Op: Result);
1722 return Result;
1723}
1724
1725SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1726 bool isTarget) {
1727 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1728}
1729
1730SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1731 const SDLoc &DL, bool LegalTypes) {
1732 assert(VT.isInteger() && "Shift amount is not an integer type!");
1733 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout(), LegalTypes);
1734 return getConstant(Val, DL, VT: ShiftVT);
1735}
1736
1737SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1738 const SDLoc &DL, bool LegalTypes) {
1739 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1740 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL, LegalTypes);
1741}
1742
1743SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1744 bool isTarget) {
1745 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1746}
1747
1748SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1749 bool isTarget) {
1750 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1751}
1752
1753SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1754 EVT VT, bool isTarget) {
1755 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1756
1757 EVT EltVT = VT.getScalarType();
1758
1759 // Do the map lookup using the actual bit pattern for the floating point
1760 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1761 // we don't have issues with SNANs.
1762 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1763 FoldingSetNodeID ID;
1764 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt);
1765 ID.AddPointer(Ptr: &V);
1766 void *IP = nullptr;
1767 SDNode *N = nullptr;
1768 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1769 if (!VT.isVector())
1770 return SDValue(N, 0);
1771
1772 if (!N) {
1773 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args: &V, Args&: EltVT);
1774 CSEMap.InsertNode(N, InsertPos: IP);
1775 InsertNode(N);
1776 }
1777
1778 SDValue Result(N, 0);
1779 if (VT.isVector())
1780 Result = getSplat(VT, DL, Op: Result);
1781 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1782 return Result;
1783}
1784
1785SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1786 bool isTarget) {
1787 EVT EltVT = VT.getScalarType();
1788 if (EltVT == MVT::f32)
1789 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1790 if (EltVT == MVT::f64)
1791 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1792 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1793 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1794 bool Ignored;
1795 APFloat APF = APFloat(Val);
1796 APF.convert(ToSemantics: EVTToAPFloatSemantics(VT: EltVT), RM: APFloat::rmNearestTiesToEven,
1797 losesInfo: &Ignored);
1798 return getConstantFP(V: APF, DL, VT, isTarget);
1799 }
1800 llvm_unreachable("Unsupported type in getConstantFP");
1801}
1802
1803SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1804 EVT VT, int64_t Offset, bool isTargetGA,
1805 unsigned TargetFlags) {
1806 assert((TargetFlags == 0 || isTargetGA) &&
1807 "Cannot set target flags on target-independent globals");
1808
1809 // Truncate (with sign-extension) the offset value to the pointer size.
1810 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1811 if (BitWidth < 64)
1812 Offset = SignExtend64(X: Offset, B: BitWidth);
1813
1814 unsigned Opc;
1815 if (GV->isThreadLocal())
1816 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1817 else
1818 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1819
1820 FoldingSetNodeID ID;
1821 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1822 ID.AddPointer(Ptr: GV);
1823 ID.AddInteger(I: Offset);
1824 ID.AddInteger(I: TargetFlags);
1825 void *IP = nullptr;
1826 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1827 return SDValue(E, 0);
1828
1829 auto *N = newSDNode<GlobalAddressSDNode>(
1830 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VT, Args&: Offset, Args&: TargetFlags);
1831 CSEMap.InsertNode(N, InsertPos: IP);
1832 InsertNode(N);
1833 return SDValue(N, 0);
1834}
1835
1836SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1837 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1838 FoldingSetNodeID ID;
1839 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1840 ID.AddInteger(I: FI);
1841 void *IP = nullptr;
1842 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1843 return SDValue(E, 0);
1844
1845 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VT, Args&: isTarget);
1846 CSEMap.InsertNode(N, InsertPos: IP);
1847 InsertNode(N);
1848 return SDValue(N, 0);
1849}
1850
1851SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1852 unsigned TargetFlags) {
1853 assert((TargetFlags == 0 || isTarget) &&
1854 "Cannot set target flags on target-independent jump tables");
1855 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1856 FoldingSetNodeID ID;
1857 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1858 ID.AddInteger(I: JTI);
1859 ID.AddInteger(I: TargetFlags);
1860 void *IP = nullptr;
1861 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1862 return SDValue(E, 0);
1863
1864 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VT, Args&: isTarget, Args&: TargetFlags);
1865 CSEMap.InsertNode(N, InsertPos: IP);
1866 InsertNode(N);
1867 return SDValue(N, 0);
1868}
1869
1870SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1871 const SDLoc &DL) {
1872 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1873 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1874 getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1875}
1876
1877SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1878 MaybeAlign Alignment, int Offset,
1879 bool isTarget, unsigned TargetFlags) {
1880 assert((TargetFlags == 0 || isTarget) &&
1881 "Cannot set target flags on target-independent globals");
1882 if (!Alignment)
1883 Alignment = shouldOptForSize()
1884 ? getDataLayout().getABITypeAlign(Ty: C->getType())
1885 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
1886 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1887 FoldingSetNodeID ID;
1888 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1889 ID.AddInteger(I: Alignment->value());
1890 ID.AddInteger(I: Offset);
1891 ID.AddPointer(Ptr: C);
1892 ID.AddInteger(I: TargetFlags);
1893 void *IP = nullptr;
1894 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1895 return SDValue(E, 0);
1896
1897 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment,
1898 Args&: TargetFlags);
1899 CSEMap.InsertNode(N, InsertPos: IP);
1900 InsertNode(N);
1901 SDValue V = SDValue(N, 0);
1902 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
1903 return V;
1904}
1905
1906SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1907 MaybeAlign Alignment, int Offset,
1908 bool isTarget, unsigned TargetFlags) {
1909 assert((TargetFlags == 0 || isTarget) &&
1910 "Cannot set target flags on target-independent globals");
1911 if (!Alignment)
1912 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
1913 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1914 FoldingSetNodeID ID;
1915 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
1916 ID.AddInteger(I: Alignment->value());
1917 ID.AddInteger(I: Offset);
1918 C->addSelectionDAGCSEId(ID);
1919 ID.AddInteger(I: TargetFlags);
1920 void *IP = nullptr;
1921 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1922 return SDValue(E, 0);
1923
1924 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment,
1925 Args&: TargetFlags);
1926 CSEMap.InsertNode(N, InsertPos: IP);
1927 InsertNode(N);
1928 return SDValue(N, 0);
1929}
1930
1931SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1932 FoldingSetNodeID ID;
1933 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1934 ID.AddPointer(Ptr: MBB);
1935 void *IP = nullptr;
1936 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1937 return SDValue(E, 0);
1938
1939 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
1940 CSEMap.InsertNode(N, InsertPos: IP);
1941 InsertNode(N);
1942 return SDValue(N, 0);
1943}
1944
1945SDValue SelectionDAG::getValueType(EVT VT) {
1946 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1947 ValueTypeNodes.size())
1948 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
1949
1950 SDNode *&N = VT.isExtended() ?
1951 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1952
1953 if (N) return SDValue(N, 0);
1954 N = newSDNode<VTSDNode>(Args&: VT);
1955 InsertNode(N);
1956 return SDValue(N, 0);
1957}
1958
1959SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1960 SDNode *&N = ExternalSymbols[Sym];
1961 if (N) return SDValue(N, 0);
1962 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args&: VT);
1963 InsertNode(N);
1964 return SDValue(N, 0);
1965}
1966
1967SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1968 SDNode *&N = MCSymbols[Sym];
1969 if (N)
1970 return SDValue(N, 0);
1971 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args&: VT);
1972 InsertNode(N);
1973 return SDValue(N, 0);
1974}
1975
1976SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1977 unsigned TargetFlags) {
1978 SDNode *&N =
1979 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1980 if (N) return SDValue(N, 0);
1981 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args&: VT);
1982 InsertNode(N);
1983 return SDValue(N, 0);
1984}
1985
1986SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1987 if ((unsigned)Cond >= CondCodeNodes.size())
1988 CondCodeNodes.resize(new_size: Cond+1);
1989
1990 if (!CondCodeNodes[Cond]) {
1991 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
1992 CondCodeNodes[Cond] = N;
1993 InsertNode(N);
1994 }
1995
1996 return SDValue(CondCodeNodes[Cond], 0);
1997}
1998
1999SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
2000 bool ConstantFold) {
2001 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2002 "APInt size does not match type size!");
2003
2004 if (MulImm == 0)
2005 return getConstant(Val: 0, DL, VT);
2006
2007 if (ConstantFold) {
2008 const MachineFunction &MF = getMachineFunction();
2009 const Function &F = MF.getFunction();
2010 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2011 if (const APInt *C = CR.getSingleElement())
2012 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2013 }
2014
2015 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2016}
2017
2018SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
2019 bool ConstantFold) {
2020 if (EC.isScalable())
2021 return getVScale(DL, VT,
2022 MulImm: APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2023
2024 return getConstant(Val: EC.getKnownMinValue(), DL, VT);
2025}
2026
2027SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2028 APInt One(ResVT.getScalarSizeInBits(), 1);
2029 return getStepVector(DL, ResVT, StepVal: One);
2030}
2031
2032SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
2033 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2034 if (ResVT.isScalableVector())
2035 return getNode(
2036 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2037 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2038
2039 SmallVector<SDValue, 16> OpsStepConstants;
2040 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2041 OpsStepConstants.push_back(
2042 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2043 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2044}
2045
2046/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2047/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2048static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2049 std::swap(a&: N1, b&: N2);
2050 ShuffleVectorSDNode::commuteMask(Mask: M);
2051}
2052
2053SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2054 SDValue N2, ArrayRef<int> Mask) {
2055 assert(VT.getVectorNumElements() == Mask.size() &&
2056 "Must have the same number of vector elements as mask elements!");
2057 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2058 "Invalid VECTOR_SHUFFLE");
2059
2060 // Canonicalize shuffle undef, undef -> undef
2061 if (N1.isUndef() && N2.isUndef())
2062 return getUNDEF(VT);
2063
2064 // Validate that all indices in Mask are within the range of the elements
2065 // input to the shuffle.
2066 int NElts = Mask.size();
2067 assert(llvm::all_of(Mask,
2068 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2069 "Index out of range");
2070
2071 // Copy the mask so we can do any needed cleanup.
2072 SmallVector<int, 8> MaskVec(Mask);
2073
2074 // Canonicalize shuffle v, v -> v, undef
2075 if (N1 == N2) {
2076 N2 = getUNDEF(VT);
2077 for (int i = 0; i != NElts; ++i)
2078 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2079 }
2080
2081 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2082 if (N1.isUndef())
2083 commuteShuffle(N1, N2, M: MaskVec);
2084
2085 if (TLI->hasVectorBlend()) {
2086 // If shuffling a splat, try to blend the splat instead. We do this here so
2087 // that even when this arises during lowering we don't have to re-handle it.
2088 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2089 BitVector UndefElements;
2090 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2091 if (!Splat)
2092 return;
2093
2094 for (int i = 0; i < NElts; ++i) {
2095 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2096 continue;
2097
2098 // If this input comes from undef, mark it as such.
2099 if (UndefElements[MaskVec[i] - Offset]) {
2100 MaskVec[i] = -1;
2101 continue;
2102 }
2103
2104 // If we can blend a non-undef lane, use that instead.
2105 if (!UndefElements[i])
2106 MaskVec[i] = i + Offset;
2107 }
2108 };
2109 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2110 BlendSplat(N1BV, 0);
2111 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2112 BlendSplat(N2BV, NElts);
2113 }
2114
2115 // Canonicalize all index into lhs, -> shuffle lhs, undef
2116 // Canonicalize all index into rhs, -> shuffle rhs, undef
2117 bool AllLHS = true, AllRHS = true;
2118 bool N2Undef = N2.isUndef();
2119 for (int i = 0; i != NElts; ++i) {
2120 if (MaskVec[i] >= NElts) {
2121 if (N2Undef)
2122 MaskVec[i] = -1;
2123 else
2124 AllLHS = false;
2125 } else if (MaskVec[i] >= 0) {
2126 AllRHS = false;
2127 }
2128 }
2129 if (AllLHS && AllRHS)
2130 return getUNDEF(VT);
2131 if (AllLHS && !N2Undef)
2132 N2 = getUNDEF(VT);
2133 if (AllRHS) {
2134 N1 = getUNDEF(VT);
2135 commuteShuffle(N1, N2, M: MaskVec);
2136 }
2137 // Reset our undef status after accounting for the mask.
2138 N2Undef = N2.isUndef();
2139 // Re-check whether both sides ended up undef.
2140 if (N1.isUndef() && N2Undef)
2141 return getUNDEF(VT);
2142
2143 // If Identity shuffle return that node.
2144 bool Identity = true, AllSame = true;
2145 for (int i = 0; i != NElts; ++i) {
2146 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2147 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2148 }
2149 if (Identity && NElts)
2150 return N1;
2151
2152 // Shuffling a constant splat doesn't change the result.
2153 if (N2Undef) {
2154 SDValue V = N1;
2155
2156 // Look through any bitcasts. We check that these don't change the number
2157 // (and size) of elements and just changes their types.
2158 while (V.getOpcode() == ISD::BITCAST)
2159 V = V->getOperand(Num: 0);
2160
2161 // A splat should always show up as a build vector node.
2162 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2163 BitVector UndefElements;
2164 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2165 // If this is a splat of an undef, shuffling it is also undef.
2166 if (Splat && Splat.isUndef())
2167 return getUNDEF(VT);
2168
2169 bool SameNumElts =
2170 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2171
2172 // We only have a splat which can skip shuffles if there is a splatted
2173 // value and no undef lanes rearranged by the shuffle.
2174 if (Splat && UndefElements.none()) {
2175 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2176 // number of elements match or the value splatted is a zero constant.
2177 if (SameNumElts || isNullConstant(V: Splat))
2178 return N1;
2179 }
2180
2181 // If the shuffle itself creates a splat, build the vector directly.
2182 if (AllSame && SameNumElts) {
2183 EVT BuildVT = BV->getValueType(ResNo: 0);
2184 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2185 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2186
2187 // We may have jumped through bitcasts, so the type of the
2188 // BUILD_VECTOR may not match the type of the shuffle.
2189 if (BuildVT != VT)
2190 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2191 return NewBV;
2192 }
2193 }
2194 }
2195
2196 FoldingSetNodeID ID;
2197 SDValue Ops[2] = { N1, N2 };
2198 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: getVTList(VT), OpList: Ops);
2199 for (int i = 0; i != NElts; ++i)
2200 ID.AddInteger(I: MaskVec[i]);
2201
2202 void* IP = nullptr;
2203 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2204 return SDValue(E, 0);
2205
2206 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2207 // SDNode doesn't have access to it. This memory will be "leaked" when
2208 // the node is deallocated, but recovered when the NodeAllocator is released.
2209 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2210 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2211
2212 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VT, Args: dl.getIROrder(),
2213 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2214 createOperands(Node: N, Vals: Ops);
2215
2216 CSEMap.InsertNode(N, InsertPos: IP);
2217 InsertNode(N);
2218 SDValue V = SDValue(N, 0);
2219 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2220 return V;
2221}
2222
2223SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2224 EVT VT = SV.getValueType(ResNo: 0);
2225 SmallVector<int, 8> MaskVec(SV.getMask());
2226 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2227
2228 SDValue Op0 = SV.getOperand(Num: 0);
2229 SDValue Op1 = SV.getOperand(Num: 1);
2230 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2231}
2232
2233SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
2234 FoldingSetNodeID ID;
2235 AddNodeIDNode(ID, OpC: ISD::Register, VTList: getVTList(VT), OpList: std::nullopt);
2236 ID.AddInteger(I: RegNo);
2237 void *IP = nullptr;
2238 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2239 return SDValue(E, 0);
2240
2241 auto *N = newSDNode<RegisterSDNode>(Args&: RegNo, Args&: VT);
2242 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2243 CSEMap.InsertNode(N, InsertPos: IP);
2244 InsertNode(N);
2245 return SDValue(N, 0);
2246}
2247
2248SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2249 FoldingSetNodeID ID;
2250 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2251 ID.AddPointer(Ptr: RegMask);
2252 void *IP = nullptr;
2253 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2254 return SDValue(E, 0);
2255
2256 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2257 CSEMap.InsertNode(N, InsertPos: IP);
2258 InsertNode(N);
2259 return SDValue(N, 0);
2260}
2261
2262SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2263 MCSymbol *Label) {
2264 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2265}
2266
2267SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2268 SDValue Root, MCSymbol *Label) {
2269 FoldingSetNodeID ID;
2270 SDValue Ops[] = { Root };
2271 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2272 ID.AddPointer(Ptr: Label);
2273 void *IP = nullptr;
2274 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2275 return SDValue(E, 0);
2276
2277 auto *N =
2278 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2279 createOperands(Node: N, Vals: Ops);
2280
2281 CSEMap.InsertNode(N, InsertPos: IP);
2282 InsertNode(N);
2283 return SDValue(N, 0);
2284}
2285
2286SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2287 int64_t Offset, bool isTarget,
2288 unsigned TargetFlags) {
2289 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2290
2291 FoldingSetNodeID ID;
2292 AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt);
2293 ID.AddPointer(Ptr: BA);
2294 ID.AddInteger(I: Offset);
2295 ID.AddInteger(I: TargetFlags);
2296 void *IP = nullptr;
2297 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2298 return SDValue(E, 0);
2299
2300 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VT, Args&: BA, Args&: Offset, Args&: TargetFlags);
2301 CSEMap.InsertNode(N, InsertPos: IP);
2302 InsertNode(N);
2303 return SDValue(N, 0);
2304}
2305
2306SDValue SelectionDAG::getSrcValue(const Value *V) {
2307 FoldingSetNodeID ID;
2308 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2309 ID.AddPointer(Ptr: V);
2310
2311 void *IP = nullptr;
2312 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2313 return SDValue(E, 0);
2314
2315 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2316 CSEMap.InsertNode(N, InsertPos: IP);
2317 InsertNode(N);
2318 return SDValue(N, 0);
2319}
2320
2321SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2322 FoldingSetNodeID ID;
2323 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2324 ID.AddPointer(Ptr: MD);
2325
2326 void *IP = nullptr;
2327 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2328 return SDValue(E, 0);
2329
2330 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2331 CSEMap.InsertNode(N, InsertPos: IP);
2332 InsertNode(N);
2333 return SDValue(N, 0);
2334}
2335
2336SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2337 if (VT == V.getValueType())
2338 return V;
2339
2340 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2341}
2342
2343SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2344 unsigned SrcAS, unsigned DestAS) {
2345 SDValue Ops[] = {Ptr};
2346 FoldingSetNodeID ID;
2347 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: getVTList(VT), OpList: Ops);
2348 ID.AddInteger(I: SrcAS);
2349 ID.AddInteger(I: DestAS);
2350
2351 void *IP = nullptr;
2352 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2353 return SDValue(E, 0);
2354
2355 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2356 Args&: VT, Args&: SrcAS, Args&: DestAS);
2357 createOperands(Node: N, Vals: Ops);
2358
2359 CSEMap.InsertNode(N, InsertPos: IP);
2360 InsertNode(N);
2361 return SDValue(N, 0);
2362}
2363
2364SDValue SelectionDAG::getFreeze(SDValue V) {
2365 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2366}
2367
2368/// getShiftAmountOperand - Return the specified value casted to
2369/// the target's desired shift amount type.
2370SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2371 EVT OpTy = Op.getValueType();
2372 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2373 if (OpTy == ShTy || OpTy.isVector()) return Op;
2374
2375 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2376}
2377
2378SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2379 SDLoc dl(Node);
2380 const TargetLowering &TLI = getTargetLoweringInfo();
2381 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2382 EVT VT = Node->getValueType(ResNo: 0);
2383 SDValue Tmp1 = Node->getOperand(Num: 0);
2384 SDValue Tmp2 = Node->getOperand(Num: 1);
2385 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2386
2387 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2388 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2389 SDValue VAList = VAListLoad;
2390
2391 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2392 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2393 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2394
2395 VAList =
2396 getNode(Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2397 N2: getConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2398 }
2399
2400 // Increment the pointer, VAList, to the next vaarg
2401 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2402 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2403 Ty: VT.getTypeForEVT(Context&: *getContext())),
2404 DL: dl, VT: VAList.getValueType()));
2405 // Store the incremented VAList to the legalized pointer
2406 Tmp1 =
2407 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2408 // Load the actual argument out of the pointer VAList
2409 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2410}
2411
2412SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2413 SDLoc dl(Node);
2414 const TargetLowering &TLI = getTargetLoweringInfo();
2415 // This defaults to loading a pointer from the input and storing it to the
2416 // output, returning the chain.
2417 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2418 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2419 SDValue Tmp1 =
2420 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2421 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2422 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2423 PtrInfo: MachinePointerInfo(VD));
2424}
2425
2426Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2427 const DataLayout &DL = getDataLayout();
2428 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2429 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2430
2431 if (TLI->isTypeLegal(VT) || !VT.isVector())
2432 return RedAlign;
2433
2434 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2435 const Align StackAlign = TFI->getStackAlign();
2436
2437 // See if we can choose a smaller ABI alignment in cases where it's an
2438 // illegal vector type that will get broken down.
2439 if (RedAlign > StackAlign) {
2440 EVT IntermediateVT;
2441 MVT RegisterVT;
2442 unsigned NumIntermediates;
2443 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2444 NumIntermediates, RegisterVT);
2445 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2446 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2447 if (RedAlign2 < RedAlign)
2448 RedAlign = RedAlign2;
2449 }
2450
2451 return RedAlign;
2452}
2453
2454SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2455 MachineFrameInfo &MFI = MF->getFrameInfo();
2456 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2457 int StackID = 0;
2458 if (Bytes.isScalable())
2459 StackID = TFI->getStackIDForScalableVectors();
2460 // The stack id gives an indication of whether the object is scalable or
2461 // not, so it's safe to pass in the minimum size here.
2462 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2463 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2464 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2465}
2466
2467SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2468 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2469 Align StackAlign =
2470 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2471 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2472}
2473
2474SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2475 TypeSize VT1Size = VT1.getStoreSize();
2476 TypeSize VT2Size = VT2.getStoreSize();
2477 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2478 "Don't know how to choose the maximum size when creating a stack "
2479 "temporary");
2480 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2481 ? VT1Size
2482 : VT2Size;
2483
2484 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2485 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2486 const DataLayout &DL = getDataLayout();
2487 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2488 return CreateStackTemporary(Bytes, Alignment: Align);
2489}
2490
2491SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2492 ISD::CondCode Cond, const SDLoc &dl) {
2493 EVT OpVT = N1.getValueType();
2494
2495 auto GetUndefBooleanConstant = [&]() {
2496 if (VT.getScalarType() == MVT::i1 ||
2497 TLI->getBooleanContents(Type: OpVT) ==
2498 TargetLowering::UndefinedBooleanContent)
2499 return getUNDEF(VT);
2500 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2501 // so we cannot use getUNDEF(). Return zero instead.
2502 return getConstant(Val: 0, DL: dl, VT);
2503 };
2504
2505 // These setcc operations always fold.
2506 switch (Cond) {
2507 default: break;
2508 case ISD::SETFALSE:
2509 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2510 case ISD::SETTRUE:
2511 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2512
2513 case ISD::SETOEQ:
2514 case ISD::SETOGT:
2515 case ISD::SETOGE:
2516 case ISD::SETOLT:
2517 case ISD::SETOLE:
2518 case ISD::SETONE:
2519 case ISD::SETO:
2520 case ISD::SETUO:
2521 case ISD::SETUEQ:
2522 case ISD::SETUNE:
2523 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2524 break;
2525 }
2526
2527 if (OpVT.isInteger()) {
2528 // For EQ and NE, we can always pick a value for the undef to make the
2529 // predicate pass or fail, so we can return undef.
2530 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2531 // icmp eq/ne X, undef -> undef.
2532 if ((N1.isUndef() || N2.isUndef()) &&
2533 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2534 return GetUndefBooleanConstant();
2535
2536 // If both operands are undef, we can return undef for int comparison.
2537 // icmp undef, undef -> undef.
2538 if (N1.isUndef() && N2.isUndef())
2539 return GetUndefBooleanConstant();
2540
2541 // icmp X, X -> true/false
2542 // icmp X, undef -> true/false because undef could be X.
2543 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2544 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2545 }
2546
2547 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2548 const APInt &C2 = N2C->getAPIntValue();
2549 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2550 const APInt &C1 = N1C->getAPIntValue();
2551
2552 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2553 DL: dl, VT, OpVT);
2554 }
2555 }
2556
2557 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2558 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2559
2560 if (N1CFP && N2CFP) {
2561 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2562 switch (Cond) {
2563 default: break;
2564 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2565 return GetUndefBooleanConstant();
2566 [[fallthrough]];
2567 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2568 OpVT);
2569 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2570 return GetUndefBooleanConstant();
2571 [[fallthrough]];
2572 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2573 R==APFloat::cmpLessThan, DL: dl, VT,
2574 OpVT);
2575 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2576 return GetUndefBooleanConstant();
2577 [[fallthrough]];
2578 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2579 OpVT);
2580 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2581 return GetUndefBooleanConstant();
2582 [[fallthrough]];
2583 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2584 VT, OpVT);
2585 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2586 return GetUndefBooleanConstant();
2587 [[fallthrough]];
2588 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2589 R==APFloat::cmpEqual, DL: dl, VT,
2590 OpVT);
2591 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2592 return GetUndefBooleanConstant();
2593 [[fallthrough]];
2594 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2595 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2596 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2597 OpVT);
2598 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2599 OpVT);
2600 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2601 R==APFloat::cmpEqual, DL: dl, VT,
2602 OpVT);
2603 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2604 OpVT);
2605 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2606 R==APFloat::cmpLessThan, DL: dl, VT,
2607 OpVT);
2608 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2609 R==APFloat::cmpUnordered, DL: dl, VT,
2610 OpVT);
2611 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2612 VT, OpVT);
2613 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2614 OpVT);
2615 }
2616 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2617 // Ensure that the constant occurs on the RHS.
2618 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2619 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2620 return SDValue();
2621 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond);
2622 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2623 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2624 // If an operand is known to be a nan (or undef that could be a nan), we can
2625 // fold it.
2626 // Choosing NaN for the undef will always make unordered comparison succeed
2627 // and ordered comparison fails.
2628 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2629 switch (ISD::getUnorderedFlavor(Cond)) {
2630 default:
2631 llvm_unreachable("Unknown flavor!");
2632 case 0: // Known false.
2633 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2634 case 1: // Known true.
2635 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2636 case 2: // Undefined.
2637 return GetUndefBooleanConstant();
2638 }
2639 }
2640
2641 // Could not fold it.
2642 return SDValue();
2643}
2644
2645/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2646/// use this predicate to simplify operations downstream.
2647bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2648 unsigned BitWidth = Op.getScalarValueSizeInBits();
2649 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2650}
2651
2652/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2653/// this predicate to simplify operations downstream. Mask is known to be zero
2654/// for bits that V cannot have.
2655bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2656 unsigned Depth) const {
2657 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2658}
2659
2660/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2661/// DemandedElts. We use this predicate to simplify operations downstream.
2662/// Mask is known to be zero for bits that V cannot have.
2663bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2664 const APInt &DemandedElts,
2665 unsigned Depth) const {
2666 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2667}
2668
2669/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2670/// DemandedElts. We use this predicate to simplify operations downstream.
2671bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2672 unsigned Depth /* = 0 */) const {
2673 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2674}
2675
2676/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2677bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2678 unsigned Depth) const {
2679 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2680}
2681
2682APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2683 const APInt &DemandedElts,
2684 unsigned Depth) const {
2685 EVT VT = Op.getValueType();
2686 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2687
2688 unsigned NumElts = VT.getVectorNumElements();
2689 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2690
2691 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2692 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2693 if (!DemandedElts[EltIdx])
2694 continue; // Don't query elements that are not demanded.
2695 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2696 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2697 KnownZeroElements.setBit(EltIdx);
2698 }
2699 return KnownZeroElements;
2700}
2701
2702/// isSplatValue - Return true if the vector V has the same value
2703/// across all DemandedElts. For scalable vectors, we don't know the
2704/// number of lanes at compile time. Instead, we use a 1 bit APInt
2705/// to represent a conservative value for all lanes; that is, that
2706/// one bit value is implicitly splatted across all lanes.
2707bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2708 APInt &UndefElts, unsigned Depth) const {
2709 unsigned Opcode = V.getOpcode();
2710 EVT VT = V.getValueType();
2711 assert(VT.isVector() && "Vector type expected");
2712 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2713 "scalable demanded bits are ignored");
2714
2715 if (!DemandedElts)
2716 return false; // No demanded elts, better to assume we don't know anything.
2717
2718 if (Depth >= MaxRecursionDepth)
2719 return false; // Limit search depth.
2720
2721 // Deal with some common cases here that work for both fixed and scalable
2722 // vector types.
2723 switch (Opcode) {
2724 case ISD::SPLAT_VECTOR:
2725 UndefElts = V.getOperand(i: 0).isUndef()
2726 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2727 : APInt(DemandedElts.getBitWidth(), 0);
2728 return true;
2729 case ISD::ADD:
2730 case ISD::SUB:
2731 case ISD::AND:
2732 case ISD::XOR:
2733 case ISD::OR: {
2734 APInt UndefLHS, UndefRHS;
2735 SDValue LHS = V.getOperand(i: 0);
2736 SDValue RHS = V.getOperand(i: 1);
2737 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2738 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1)) {
2739 UndefElts = UndefLHS | UndefRHS;
2740 return true;
2741 }
2742 return false;
2743 }
2744 case ISD::ABS:
2745 case ISD::TRUNCATE:
2746 case ISD::SIGN_EXTEND:
2747 case ISD::ZERO_EXTEND:
2748 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2749 default:
2750 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2751 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2752 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
2753 Depth);
2754 break;
2755}
2756
2757 // We don't support other cases than those above for scalable vectors at
2758 // the moment.
2759 if (VT.isScalableVector())
2760 return false;
2761
2762 unsigned NumElts = VT.getVectorNumElements();
2763 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2764 UndefElts = APInt::getZero(numBits: NumElts);
2765
2766 switch (Opcode) {
2767 case ISD::BUILD_VECTOR: {
2768 SDValue Scl;
2769 for (unsigned i = 0; i != NumElts; ++i) {
2770 SDValue Op = V.getOperand(i);
2771 if (Op.isUndef()) {
2772 UndefElts.setBit(i);
2773 continue;
2774 }
2775 if (!DemandedElts[i])
2776 continue;
2777 if (Scl && Scl != Op)
2778 return false;
2779 Scl = Op;
2780 }
2781 return true;
2782 }
2783 case ISD::VECTOR_SHUFFLE: {
2784 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
2785 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
2786 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
2787 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
2788 for (int i = 0; i != (int)NumElts; ++i) {
2789 int M = Mask[i];
2790 if (M < 0) {
2791 UndefElts.setBit(i);
2792 continue;
2793 }
2794 if (!DemandedElts[i])
2795 continue;
2796 if (M < (int)NumElts)
2797 DemandedLHS.setBit(M);
2798 else
2799 DemandedRHS.setBit(M - NumElts);
2800 }
2801
2802 // If we aren't demanding either op, assume there's no splat.
2803 // If we are demanding both ops, assume there's no splat.
2804 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2805 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2806 return false;
2807
2808 // See if the demanded elts of the source op is a splat or we only demand
2809 // one element, which should always be a splat.
2810 // TODO: Handle source ops splats with undefs.
2811 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2812 APInt SrcUndefs;
2813 return (SrcElts.popcount() == 1) ||
2814 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
2815 (SrcElts & SrcUndefs).isZero());
2816 };
2817 if (!DemandedLHS.isZero())
2818 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
2819 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
2820 }
2821 case ISD::EXTRACT_SUBVECTOR: {
2822 // Offset the demanded elts by the subvector index.
2823 SDValue Src = V.getOperand(i: 0);
2824 // We don't support scalable vectors at the moment.
2825 if (Src.getValueType().isScalableVector())
2826 return false;
2827 uint64_t Idx = V.getConstantOperandVal(i: 1);
2828 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2829 APInt UndefSrcElts;
2830 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
2831 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
2832 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
2833 return true;
2834 }
2835 break;
2836 }
2837 case ISD::ANY_EXTEND_VECTOR_INREG:
2838 case ISD::SIGN_EXTEND_VECTOR_INREG:
2839 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2840 // Widen the demanded elts by the src element count.
2841 SDValue Src = V.getOperand(i: 0);
2842 // We don't support scalable vectors at the moment.
2843 if (Src.getValueType().isScalableVector())
2844 return false;
2845 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2846 APInt UndefSrcElts;
2847 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
2848 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
2849 UndefElts = UndefSrcElts.trunc(width: NumElts);
2850 return true;
2851 }
2852 break;
2853 }
2854 case ISD::BITCAST: {
2855 SDValue Src = V.getOperand(i: 0);
2856 EVT SrcVT = Src.getValueType();
2857 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2858 unsigned BitWidth = VT.getScalarSizeInBits();
2859
2860 // Ignore bitcasts from unsupported types.
2861 // TODO: Add fp support?
2862 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2863 break;
2864
2865 // Bitcast 'small element' vector to 'large element' vector.
2866 if ((BitWidth % SrcBitWidth) == 0) {
2867 // See if each sub element is a splat.
2868 unsigned Scale = BitWidth / SrcBitWidth;
2869 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2870 APInt ScaledDemandedElts =
2871 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
2872 for (unsigned I = 0; I != Scale; ++I) {
2873 APInt SubUndefElts;
2874 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
2875 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
2876 SubDemandedElts &= ScaledDemandedElts;
2877 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
2878 return false;
2879 // TODO: Add support for merging sub undef elements.
2880 if (!SubUndefElts.isZero())
2881 return false;
2882 }
2883 return true;
2884 }
2885 break;
2886 }
2887 }
2888
2889 return false;
2890}
2891
2892/// Helper wrapper to main isSplatValue function.
2893bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2894 EVT VT = V.getValueType();
2895 assert(VT.isVector() && "Vector type expected");
2896
2897 APInt UndefElts;
2898 // Since the number of lanes in a scalable vector is unknown at compile time,
2899 // we track one bit which is implicitly broadcast to all lanes. This means
2900 // that all lanes in a scalable vector are considered demanded.
2901 APInt DemandedElts
2902 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2903 return isSplatValue(V, DemandedElts, UndefElts) &&
2904 (AllowUndefs || !UndefElts);
2905}
2906
2907SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2908 V = peekThroughExtractSubvectors(V);
2909
2910 EVT VT = V.getValueType();
2911 unsigned Opcode = V.getOpcode();
2912 switch (Opcode) {
2913 default: {
2914 APInt UndefElts;
2915 // Since the number of lanes in a scalable vector is unknown at compile time,
2916 // we track one bit which is implicitly broadcast to all lanes. This means
2917 // that all lanes in a scalable vector are considered demanded.
2918 APInt DemandedElts
2919 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2920
2921 if (isSplatValue(V, DemandedElts, UndefElts)) {
2922 if (VT.isScalableVector()) {
2923 // DemandedElts and UndefElts are ignored for scalable vectors, since
2924 // the only supported cases are SPLAT_VECTOR nodes.
2925 SplatIdx = 0;
2926 } else {
2927 // Handle case where all demanded elements are UNDEF.
2928 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
2929 SplatIdx = 0;
2930 return getUNDEF(VT);
2931 }
2932 SplatIdx = (UndefElts & DemandedElts).countr_one();
2933 }
2934 return V;
2935 }
2936 break;
2937 }
2938 case ISD::SPLAT_VECTOR:
2939 SplatIdx = 0;
2940 return V;
2941 case ISD::VECTOR_SHUFFLE: {
2942 assert(!VT.isScalableVector());
2943 // Check if this is a shuffle node doing a splat.
2944 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2945 // getTargetVShiftNode currently struggles without the splat source.
2946 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
2947 if (!SVN->isSplat())
2948 break;
2949 int Idx = SVN->getSplatIndex();
2950 int NumElts = V.getValueType().getVectorNumElements();
2951 SplatIdx = Idx % NumElts;
2952 return V.getOperand(i: Idx / NumElts);
2953 }
2954 }
2955
2956 return SDValue();
2957}
2958
2959SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2960 int SplatIdx;
2961 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2962 EVT SVT = SrcVector.getValueType().getScalarType();
2963 EVT LegalSVT = SVT;
2964 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
2965 if (!SVT.isInteger())
2966 return SDValue();
2967 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
2968 if (LegalSVT.bitsLT(VT: SVT))
2969 return SDValue();
2970 }
2971 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(V), VT: LegalSVT, N1: SrcVector,
2972 N2: getVectorIdxConstant(Val: SplatIdx, DL: SDLoc(V)));
2973 }
2974 return SDValue();
2975}
2976
2977const APInt *
2978SelectionDAG::getValidShiftAmountConstant(SDValue V,
2979 const APInt &DemandedElts) const {
2980 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2981 V.getOpcode() == ISD::SRA) &&
2982 "Unknown shift node");
2983 unsigned BitWidth = V.getScalarValueSizeInBits();
2984 if (ConstantSDNode *SA = isConstOrConstSplat(N: V.getOperand(i: 1), DemandedElts)) {
2985 // Shifting more than the bitwidth is not valid.
2986 const APInt &ShAmt = SA->getAPIntValue();
2987 if (ShAmt.ult(RHS: BitWidth))
2988 return &ShAmt;
2989 }
2990 return nullptr;
2991}
2992
2993const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2994 SDValue V, const APInt &DemandedElts) const {
2995 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2996 V.getOpcode() == ISD::SRA) &&
2997 "Unknown shift node");
2998 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2999 return ValidAmt;
3000 unsigned BitWidth = V.getScalarValueSizeInBits();
3001 auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1));
3002 if (!BV)
3003 return nullptr;
3004 const APInt *MinShAmt = nullptr;
3005 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3006 if (!DemandedElts[i])
3007 continue;
3008 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3009 if (!SA)
3010 return nullptr;
3011 // Shifting more than the bitwidth is not valid.
3012 const APInt &ShAmt = SA->getAPIntValue();
3013 if (ShAmt.uge(RHS: BitWidth))
3014 return nullptr;
3015 if (MinShAmt && MinShAmt->ule(RHS: ShAmt))
3016 continue;
3017 MinShAmt = &ShAmt;
3018 }
3019 return MinShAmt;
3020}
3021
3022const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
3023 SDValue V, const APInt &DemandedElts) const {
3024 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3025 V.getOpcode() == ISD::SRA) &&
3026 "Unknown shift node");
3027 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3028 return ValidAmt;
3029 unsigned BitWidth = V.getScalarValueSizeInBits();
3030 auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1));
3031 if (!BV)
3032 return nullptr;
3033 const APInt *MaxShAmt = nullptr;
3034 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3035 if (!DemandedElts[i])
3036 continue;
3037 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3038 if (!SA)
3039 return nullptr;
3040 // Shifting more than the bitwidth is not valid.
3041 const APInt &ShAmt = SA->getAPIntValue();
3042 if (ShAmt.uge(RHS: BitWidth))
3043 return nullptr;
3044 if (MaxShAmt && MaxShAmt->uge(RHS: ShAmt))
3045 continue;
3046 MaxShAmt = &ShAmt;
3047 }
3048 return MaxShAmt;
3049}
3050
3051/// Determine which bits of Op are known to be either zero or one and return
3052/// them in Known. For vectors, the known bits are those that are shared by
3053/// every vector element.
3054KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3055 EVT VT = Op.getValueType();
3056
3057 // Since the number of lanes in a scalable vector is unknown at compile time,
3058 // we track one bit which is implicitly broadcast to all lanes. This means
3059 // that all lanes in a scalable vector are considered demanded.
3060 APInt DemandedElts = VT.isFixedLengthVector()
3061 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3062 : APInt(1, 1);
3063 return computeKnownBits(Op, DemandedElts, Depth);
3064}
3065
3066/// Determine which bits of Op are known to be either zero or one and return
3067/// them in Known. The DemandedElts argument allows us to only collect the known
3068/// bits that are shared by the requested vector elements.
3069KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3070 unsigned Depth) const {
3071 unsigned BitWidth = Op.getScalarValueSizeInBits();
3072
3073 KnownBits Known(BitWidth); // Don't know anything.
3074
3075 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
3076 // We know all of the bits for a constant!
3077 return KnownBits::makeConstant(C: C->getAPIntValue());
3078 }
3079 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
3080 // We know all of the bits for a constant fp!
3081 return KnownBits::makeConstant(C: C->getValueAPF().bitcastToAPInt());
3082 }
3083
3084 if (Depth >= MaxRecursionDepth)
3085 return Known; // Limit search depth.
3086
3087 KnownBits Known2;
3088 unsigned NumElts = DemandedElts.getBitWidth();
3089 assert((!Op.getValueType().isFixedLengthVector() ||
3090 NumElts == Op.getValueType().getVectorNumElements()) &&
3091 "Unexpected vector size");
3092
3093 if (!DemandedElts)
3094 return Known; // No demanded elts, better to assume we don't know anything.
3095
3096 unsigned Opcode = Op.getOpcode();
3097 switch (Opcode) {
3098 case ISD::MERGE_VALUES:
3099 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3100 Depth: Depth + 1);
3101 case ISD::SPLAT_VECTOR: {
3102 SDValue SrcOp = Op.getOperand(i: 0);
3103 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3104 "Expected SPLAT_VECTOR implicit truncation");
3105 // Implicitly truncate the bits to match the official semantics of
3106 // SPLAT_VECTOR.
3107 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3108 break;
3109 }
3110 case ISD::SPLAT_VECTOR_PARTS: {
3111 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3112 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3113 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3114 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3115 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3116 }
3117 break;
3118 }
3119 case ISD::STEP_VECTOR: {
3120 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3121
3122 if (Step.isPowerOf2())
3123 Known.Zero.setLowBits(Step.logBase2());
3124
3125 const Function &F = getMachineFunction().getFunction();
3126
3127 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3128 break;
3129 const APInt MinNumElts =
3130 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3131
3132 bool Overflow;
3133 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3134 .getUnsignedMax()
3135 .umul_ov(RHS: MinNumElts, Overflow);
3136 if (Overflow)
3137 break;
3138
3139 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3140 if (Overflow)
3141 break;
3142
3143 Known.Zero.setHighBits(MaxValue.countl_zero());
3144 break;
3145 }
3146 case ISD::BUILD_VECTOR:
3147 assert(!Op.getValueType().isScalableVector());
3148 // Collect the known bits that are shared by every demanded vector element.
3149 Known.Zero.setAllBits(); Known.One.setAllBits();
3150 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3151 if (!DemandedElts[i])
3152 continue;
3153
3154 SDValue SrcOp = Op.getOperand(i);
3155 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3156
3157 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3158 if (SrcOp.getValueSizeInBits() != BitWidth) {
3159 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3160 "Expected BUILD_VECTOR implicit truncation");
3161 Known2 = Known2.trunc(BitWidth);
3162 }
3163
3164 // Known bits are the values that are shared by every demanded element.
3165 Known = Known.intersectWith(RHS: Known2);
3166
3167 // If we don't know any bits, early out.
3168 if (Known.isUnknown())
3169 break;
3170 }
3171 break;
3172 case ISD::VECTOR_SHUFFLE: {
3173 assert(!Op.getValueType().isScalableVector());
3174 // Collect the known bits that are shared by every vector element referenced
3175 // by the shuffle.
3176 APInt DemandedLHS, DemandedRHS;
3177 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3178 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3179 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3180 DemandedLHS, DemandedRHS))
3181 break;
3182
3183 // Known bits are the values that are shared by every demanded element.
3184 Known.Zero.setAllBits(); Known.One.setAllBits();
3185 if (!!DemandedLHS) {
3186 SDValue LHS = Op.getOperand(i: 0);
3187 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3188 Known = Known.intersectWith(RHS: Known2);
3189 }
3190 // If we don't know any bits, early out.
3191 if (Known.isUnknown())
3192 break;
3193 if (!!DemandedRHS) {
3194 SDValue RHS = Op.getOperand(i: 1);
3195 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3196 Known = Known.intersectWith(RHS: Known2);
3197 }
3198 break;
3199 }
3200 case ISD::VSCALE: {
3201 const Function &F = getMachineFunction().getFunction();
3202 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3203 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3204 break;
3205 }
3206 case ISD::CONCAT_VECTORS: {
3207 if (Op.getValueType().isScalableVector())
3208 break;
3209 // Split DemandedElts and test each of the demanded subvectors.
3210 Known.Zero.setAllBits(); Known.One.setAllBits();
3211 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3212 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3213 unsigned NumSubVectors = Op.getNumOperands();
3214 for (unsigned i = 0; i != NumSubVectors; ++i) {
3215 APInt DemandedSub =
3216 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3217 if (!!DemandedSub) {
3218 SDValue Sub = Op.getOperand(i);
3219 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3220 Known = Known.intersectWith(RHS: Known2);
3221 }
3222 // If we don't know any bits, early out.
3223 if (Known.isUnknown())
3224 break;
3225 }
3226 break;
3227 }
3228 case ISD::INSERT_SUBVECTOR: {
3229 if (Op.getValueType().isScalableVector())
3230 break;
3231 // Demand any elements from the subvector and the remainder from the src its
3232 // inserted into.
3233 SDValue Src = Op.getOperand(i: 0);
3234 SDValue Sub = Op.getOperand(i: 1);
3235 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3236 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3237 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3238 APInt DemandedSrcElts = DemandedElts;
3239 DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx);
3240
3241 Known.One.setAllBits();
3242 Known.Zero.setAllBits();
3243 if (!!DemandedSubElts) {
3244 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3245 if (Known.isUnknown())
3246 break; // early-out.
3247 }
3248 if (!!DemandedSrcElts) {
3249 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3250 Known = Known.intersectWith(RHS: Known2);
3251 }
3252 break;
3253 }
3254 case ISD::EXTRACT_SUBVECTOR: {
3255 // Offset the demanded elts by the subvector index.
3256 SDValue Src = Op.getOperand(i: 0);
3257 // Bail until we can represent demanded elements for scalable vectors.
3258 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3259 break;
3260 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3261 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3262 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3263 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3264 break;
3265 }
3266 case ISD::SCALAR_TO_VECTOR: {
3267 if (Op.getValueType().isScalableVector())
3268 break;
3269 // We know about scalar_to_vector as much as we know about it source,
3270 // which becomes the first element of otherwise unknown vector.
3271 if (DemandedElts != 1)
3272 break;
3273
3274 SDValue N0 = Op.getOperand(i: 0);
3275 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3276 if (N0.getValueSizeInBits() != BitWidth)
3277 Known = Known.trunc(BitWidth);
3278
3279 break;
3280 }
3281 case ISD::BITCAST: {
3282 if (Op.getValueType().isScalableVector())
3283 break;
3284
3285 SDValue N0 = Op.getOperand(i: 0);
3286 EVT SubVT = N0.getValueType();
3287 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3288
3289 // Ignore bitcasts from unsupported types.
3290 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3291 break;
3292
3293 // Fast handling of 'identity' bitcasts.
3294 if (BitWidth == SubBitWidth) {
3295 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3296 break;
3297 }
3298
3299 bool IsLE = getDataLayout().isLittleEndian();
3300
3301 // Bitcast 'small element' vector to 'large element' scalar/vector.
3302 if ((BitWidth % SubBitWidth) == 0) {
3303 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3304
3305 // Collect known bits for the (larger) output by collecting the known
3306 // bits from each set of sub elements and shift these into place.
3307 // We need to separately call computeKnownBits for each set of
3308 // sub elements as the knownbits for each is likely to be different.
3309 unsigned SubScale = BitWidth / SubBitWidth;
3310 APInt SubDemandedElts(NumElts * SubScale, 0);
3311 for (unsigned i = 0; i != NumElts; ++i)
3312 if (DemandedElts[i])
3313 SubDemandedElts.setBit(i * SubScale);
3314
3315 for (unsigned i = 0; i != SubScale; ++i) {
3316 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3317 Depth: Depth + 1);
3318 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3319 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3320 }
3321 }
3322
3323 // Bitcast 'large element' scalar/vector to 'small element' vector.
3324 if ((SubBitWidth % BitWidth) == 0) {
3325 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3326
3327 // Collect known bits for the (smaller) output by collecting the known
3328 // bits from the overlapping larger input elements and extracting the
3329 // sub sections we actually care about.
3330 unsigned SubScale = SubBitWidth / BitWidth;
3331 APInt SubDemandedElts =
3332 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3333 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3334
3335 Known.Zero.setAllBits(); Known.One.setAllBits();
3336 for (unsigned i = 0; i != NumElts; ++i)
3337 if (DemandedElts[i]) {
3338 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3339 unsigned Offset = (Shifts % SubScale) * BitWidth;
3340 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3341 // If we don't know any bits, early out.
3342 if (Known.isUnknown())
3343 break;
3344 }
3345 }
3346 break;
3347 }
3348 case ISD::AND:
3349 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3350 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3351
3352 Known &= Known2;
3353 break;
3354 case ISD::OR:
3355 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3356 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3357
3358 Known |= Known2;
3359 break;
3360 case ISD::XOR:
3361 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3362 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3363
3364 Known ^= Known2;
3365 break;
3366 case ISD::MUL: {
3367 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3368 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3369 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3370 // TODO: SelfMultiply can be poison, but not undef.
3371 if (SelfMultiply)
3372 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3373 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3374 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3375
3376 // If the multiplication is known not to overflow, the product of a number
3377 // with itself is non-negative. Only do this if we didn't already computed
3378 // the opposite value for the sign bit.
3379 if (Op->getFlags().hasNoSignedWrap() &&
3380 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3381 !Known.isNegative())
3382 Known.makeNonNegative();
3383 break;
3384 }
3385 case ISD::MULHU: {
3386 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3387 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3388 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3389 break;
3390 }
3391 case ISD::MULHS: {
3392 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3393 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3394 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3395 break;
3396 }
3397 case ISD::UMUL_LOHI: {
3398 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3399 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3400 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3401 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3402 if (Op.getResNo() == 0)
3403 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3404 else
3405 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3406 break;
3407 }
3408 case ISD::SMUL_LOHI: {
3409 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3410 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3411 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3412 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3413 if (Op.getResNo() == 0)
3414 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3415 else
3416 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3417 break;
3418 }
3419 case ISD::AVGCEILU: {
3420 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3421 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3422 Known = Known.zext(BitWidth: BitWidth + 1);
3423 Known2 = Known2.zext(BitWidth: BitWidth + 1);
3424 KnownBits One = KnownBits::makeConstant(C: APInt(1, 1));
3425 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry: One);
3426 Known = Known.extractBits(NumBits: BitWidth, BitPosition: 1);
3427 break;
3428 }
3429 case ISD::SELECT:
3430 case ISD::VSELECT:
3431 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3432 // If we don't know any bits, early out.
3433 if (Known.isUnknown())
3434 break;
3435 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3436
3437 // Only known if known in both the LHS and RHS.
3438 Known = Known.intersectWith(RHS: Known2);
3439 break;
3440 case ISD::SELECT_CC:
3441 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3442 // If we don't know any bits, early out.
3443 if (Known.isUnknown())
3444 break;
3445 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3446
3447 // Only known if known in both the LHS and RHS.
3448 Known = Known.intersectWith(RHS: Known2);
3449 break;
3450 case ISD::SMULO:
3451 case ISD::UMULO:
3452 if (Op.getResNo() != 1)
3453 break;
3454 // The boolean result conforms to getBooleanContents.
3455 // If we know the result of a setcc has the top bits zero, use this info.
3456 // We know that we have an integer-based boolean since these operations
3457 // are only available for integer.
3458 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3459 TargetLowering::ZeroOrOneBooleanContent &&
3460 BitWidth > 1)
3461 Known.Zero.setBitsFrom(1);
3462 break;
3463 case ISD::SETCC:
3464 case ISD::SETCCCARRY:
3465 case ISD::STRICT_FSETCC:
3466 case ISD::STRICT_FSETCCS: {
3467 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3468 // If we know the result of a setcc has the top bits zero, use this info.
3469 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3470 TargetLowering::ZeroOrOneBooleanContent &&
3471 BitWidth > 1)
3472 Known.Zero.setBitsFrom(1);
3473 break;
3474 }
3475 case ISD::SHL:
3476 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3477 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3478 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3479
3480 // Minimum shift low bits are known zero.
3481 if (const APInt *ShMinAmt =
3482 getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
3483 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3484 break;
3485 case ISD::SRL:
3486 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3487 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3488 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3489
3490 // Minimum shift high bits are known zero.
3491 if (const APInt *ShMinAmt =
3492 getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
3493 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3494 break;
3495 case ISD::SRA:
3496 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3497 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3498 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3499 break;
3500 case ISD::FSHL:
3501 case ISD::FSHR:
3502 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3503 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3504
3505 // For fshl, 0-shift returns the 1st arg.
3506 // For fshr, 0-shift returns the 2nd arg.
3507 if (Amt == 0) {
3508 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3509 DemandedElts, Depth: Depth + 1);
3510 break;
3511 }
3512
3513 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3514 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3515 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3516 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3517 if (Opcode == ISD::FSHL) {
3518 Known.One <<= Amt;
3519 Known.Zero <<= Amt;
3520 Known2.One.lshrInPlace(ShiftAmt: BitWidth - Amt);
3521 Known2.Zero.lshrInPlace(ShiftAmt: BitWidth - Amt);
3522 } else {
3523 Known.One <<= BitWidth - Amt;
3524 Known.Zero <<= BitWidth - Amt;
3525 Known2.One.lshrInPlace(ShiftAmt: Amt);
3526 Known2.Zero.lshrInPlace(ShiftAmt: Amt);
3527 }
3528 Known = Known.unionWith(RHS: Known2);
3529 }
3530 break;
3531 case ISD::SHL_PARTS:
3532 case ISD::SRA_PARTS:
3533 case ISD::SRL_PARTS: {
3534 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3535
3536 // Collect lo/hi source values and concatenate.
3537 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3538 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3539 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3540 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3541 Known = Known2.concat(Lo: Known);
3542
3543 // Collect shift amount.
3544 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3545
3546 if (Opcode == ISD::SHL_PARTS)
3547 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3548 else if (Opcode == ISD::SRA_PARTS)
3549 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3550 else // if (Opcode == ISD::SRL_PARTS)
3551 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3552
3553 // TODO: Minimum shift low/high bits are known zero.
3554
3555 if (Op.getResNo() == 0)
3556 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3557 else
3558 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3559 break;
3560 }
3561 case ISD::SIGN_EXTEND_INREG: {
3562 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3563 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3564 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3565 break;
3566 }
3567 case ISD::CTTZ:
3568 case ISD::CTTZ_ZERO_UNDEF: {
3569 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3570 // If we have a known 1, its position is our upper bound.
3571 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3572 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3573 Known.Zero.setBitsFrom(LowBits);
3574 break;
3575 }
3576 case ISD::CTLZ:
3577 case ISD::CTLZ_ZERO_UNDEF: {
3578 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3579 // If we have a known 1, its position is our upper bound.
3580 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3581 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3582 Known.Zero.setBitsFrom(LowBits);
3583 break;
3584 }
3585 case ISD::CTPOP: {
3586 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3587 // If we know some of the bits are zero, they can't be one.
3588 unsigned PossibleOnes = Known2.countMaxPopulation();
3589 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3590 break;
3591 }
3592 case ISD::PARITY: {
3593 // Parity returns 0 everywhere but the LSB.
3594 Known.Zero.setBitsFrom(1);
3595 break;
3596 }
3597 case ISD::LOAD: {
3598 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3599 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3600 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3601 // Determine any common known bits from the loaded constant pool value.
3602 Type *CstTy = Cst->getType();
3603 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3604 !Op.getValueType().isScalableVector()) {
3605 // If its a vector splat, then we can (quickly) reuse the scalar path.
3606 // NOTE: We assume all elements match and none are UNDEF.
3607 if (CstTy->isVectorTy()) {
3608 if (const Constant *Splat = Cst->getSplatValue()) {
3609 Cst = Splat;
3610 CstTy = Cst->getType();
3611 }
3612 }
3613 // TODO - do we need to handle different bitwidths?
3614 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3615 // Iterate across all vector elements finding common known bits.
3616 Known.One.setAllBits();
3617 Known.Zero.setAllBits();
3618 for (unsigned i = 0; i != NumElts; ++i) {
3619 if (!DemandedElts[i])
3620 continue;
3621 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3622 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3623 const APInt &Value = CInt->getValue();
3624 Known.One &= Value;
3625 Known.Zero &= ~Value;
3626 continue;
3627 }
3628 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3629 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3630 Known.One &= Value;
3631 Known.Zero &= ~Value;
3632 continue;
3633 }
3634 }
3635 Known.One.clearAllBits();
3636 Known.Zero.clearAllBits();
3637 break;
3638 }
3639 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3640 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
3641 Known = KnownBits::makeConstant(C: CInt->getValue());
3642 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
3643 Known =
3644 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
3645 }
3646 }
3647 }
3648 } else if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
3649 // If this is a ZEXTLoad and we are looking at the loaded value.
3650 EVT VT = LD->getMemoryVT();
3651 unsigned MemBits = VT.getScalarSizeInBits();
3652 Known.Zero.setBitsFrom(MemBits);
3653 } else if (const MDNode *Ranges = LD->getRanges()) {
3654 EVT VT = LD->getValueType(ResNo: 0);
3655
3656 // TODO: Handle for extending loads
3657 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
3658 if (VT.isVector()) {
3659 // Handle truncation to the first demanded element.
3660 // TODO: Figure out which demanded elements are covered
3661 if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3662 break;
3663
3664 // Handle the case where a load has a vector type, but scalar memory
3665 // with an attached range.
3666 EVT MemVT = LD->getMemoryVT();
3667 KnownBits KnownFull(MemVT.getSizeInBits());
3668
3669 computeKnownBitsFromRangeMetadata(Ranges: *Ranges, Known&: KnownFull);
3670 Known = KnownFull.trunc(BitWidth);
3671 } else
3672 computeKnownBitsFromRangeMetadata(Ranges: *Ranges, Known);
3673 }
3674 }
3675 break;
3676 }
3677 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3678 if (Op.getValueType().isScalableVector())
3679 break;
3680 EVT InVT = Op.getOperand(i: 0).getValueType();
3681 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3682 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3683 Known = Known.zext(BitWidth);
3684 break;
3685 }
3686 case ISD::ZERO_EXTEND: {
3687 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3688 Known = Known.zext(BitWidth);
3689 break;
3690 }
3691 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3692 if (Op.getValueType().isScalableVector())
3693 break;
3694 EVT InVT = Op.getOperand(i: 0).getValueType();
3695 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3696 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3697 // If the sign bit is known to be zero or one, then sext will extend
3698 // it to the top bits, else it will just zext.
3699 Known = Known.sext(BitWidth);
3700 break;
3701 }
3702 case ISD::SIGN_EXTEND: {
3703 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3704 // If the sign bit is known to be zero or one, then sext will extend
3705 // it to the top bits, else it will just zext.
3706 Known = Known.sext(BitWidth);
3707 break;
3708 }
3709 case ISD::ANY_EXTEND_VECTOR_INREG: {
3710 if (Op.getValueType().isScalableVector())
3711 break;
3712 EVT InVT = Op.getOperand(i: 0).getValueType();
3713 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
3714 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
3715 Known = Known.anyext(BitWidth);
3716 break;
3717 }
3718 case ISD::ANY_EXTEND: {
3719 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3720 Known = Known.anyext(BitWidth);
3721 break;
3722 }
3723 case ISD::TRUNCATE: {
3724 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3725 Known = Known.trunc(BitWidth);
3726 break;
3727 }
3728 case ISD::AssertZext: {
3729 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3730 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
3731 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3732 Known.Zero |= (~InMask);
3733 Known.One &= (~Known.Zero);
3734 break;
3735 }
3736 case ISD::AssertAlign: {
3737 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
3738 assert(LogOfAlign != 0);
3739
3740 // TODO: Should use maximum with source
3741 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3742 // well as clearing one bits.
3743 Known.Zero.setLowBits(LogOfAlign);
3744 Known.One.clearLowBits(loBits: LogOfAlign);
3745 break;
3746 }
3747 case ISD::FGETSIGN:
3748 // All bits are zero except the low bit.
3749 Known.Zero.setBitsFrom(1);
3750 break;
3751 case ISD::ADD:
3752 case ISD::SUB: {
3753 SDNodeFlags Flags = Op.getNode()->getFlags();
3754 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3755 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3756 Known = KnownBits::computeForAddSub(Add: Op.getOpcode() == ISD::ADD,
3757 NSW: Flags.hasNoSignedWrap(), LHS: Known, RHS: Known2);
3758 break;
3759 }
3760 case ISD::USUBO:
3761 case ISD::SSUBO:
3762 case ISD::USUBO_CARRY:
3763 case ISD::SSUBO_CARRY:
3764 if (Op.getResNo() == 1) {
3765 // If we know the result of a setcc has the top bits zero, use this info.
3766 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
3767 TargetLowering::ZeroOrOneBooleanContent &&
3768 BitWidth > 1)
3769 Known.Zero.setBitsFrom(1);
3770 break;
3771 }
3772 [[fallthrough]];
3773 case ISD::SUBC: {
3774 assert(Op.getResNo() == 0 &&
3775 "We only compute knownbits for the difference here.");
3776
3777 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3778 KnownBits Borrow(1);
3779 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3780 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3781 // Borrow has bit width 1
3782 Borrow = Borrow.trunc(BitWidth: 1);
3783 } else {
3784 Borrow.setAllZero();
3785 }
3786
3787 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3788 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3789 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
3790 break;
3791 }
3792 case ISD::UADDO:
3793 case ISD::SADDO:
3794 case ISD::UADDO_CARRY:
3795 case ISD::SADDO_CARRY:
3796 if (Op.getResNo() == 1) {
3797 // If we know the result of a setcc has the top bits zero, use this info.
3798 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
3799 TargetLowering::ZeroOrOneBooleanContent &&
3800 BitWidth > 1)
3801 Known.Zero.setBitsFrom(1);
3802 break;
3803 }
3804 [[fallthrough]];
3805 case ISD::ADDC:
3806 case ISD::ADDE: {
3807 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3808
3809 // With ADDE and UADDO_CARRY, a carry bit may be added in.
3810 KnownBits Carry(1);
3811 if (Opcode == ISD::ADDE)
3812 // Can't track carry from glue, set carry to unknown.
3813 Carry.resetAll();
3814 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3815 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3816 // Carry has bit width 1
3817 Carry = Carry.trunc(BitWidth: 1);
3818 } else {
3819 Carry.setAllZero();
3820 }
3821
3822 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3823 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3824 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
3825 break;
3826 }
3827 case ISD::UDIV: {
3828 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3829 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3830 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
3831 break;
3832 }
3833 case ISD::SDIV: {
3834 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3835 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3836 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
3837 break;
3838 }
3839 case ISD::SREM: {
3840 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3841 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3842 Known = KnownBits::srem(LHS: Known, RHS: Known2);
3843 break;
3844 }
3845 case ISD::UREM: {
3846 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3847 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3848 Known = KnownBits::urem(LHS: Known, RHS: Known2);
3849 break;
3850 }
3851 case ISD::EXTRACT_ELEMENT: {
3852 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
3853 const unsigned Index = Op.getConstantOperandVal(i: 1);
3854 const unsigned EltBitWidth = Op.getValueSizeInBits();
3855
3856 // Remove low part of known bits mask
3857 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
3858 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
3859
3860 // Remove high part of known bit mask
3861 Known = Known.trunc(BitWidth: EltBitWidth);
3862 break;
3863 }
3864 case ISD::EXTRACT_VECTOR_ELT: {
3865 SDValue InVec = Op.getOperand(i: 0);
3866 SDValue EltNo = Op.getOperand(i: 1);
3867 EVT VecVT = InVec.getValueType();
3868 // computeKnownBits not yet implemented for scalable vectors.
3869 if (VecVT.isScalableVector())
3870 break;
3871 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3872 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3873
3874 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3875 // anything about the extended bits.
3876 if (BitWidth > EltBitWidth)
3877 Known = Known.trunc(BitWidth: EltBitWidth);
3878
3879 // If we know the element index, just demand that vector element, else for
3880 // an unknown element index, ignore DemandedElts and demand them all.
3881 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
3882 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
3883 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
3884 DemandedSrcElts =
3885 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
3886
3887 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3888 if (BitWidth > EltBitWidth)
3889 Known = Known.anyext(BitWidth);
3890 break;
3891 }
3892 case ISD::INSERT_VECTOR_ELT: {
3893 if (Op.getValueType().isScalableVector())
3894 break;
3895
3896 // If we know the element index, split the demand between the
3897 // source vector and the inserted element, otherwise assume we need
3898 // the original demanded vector elements and the value.
3899 SDValue InVec = Op.getOperand(i: 0);
3900 SDValue InVal = Op.getOperand(i: 1);
3901 SDValue EltNo = Op.getOperand(i: 2);
3902 bool DemandedVal = true;
3903 APInt DemandedVecElts = DemandedElts;
3904 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
3905 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
3906 unsigned EltIdx = CEltNo->getZExtValue();
3907 DemandedVal = !!DemandedElts[EltIdx];
3908 DemandedVecElts.clearBit(BitPosition: EltIdx);
3909 }
3910 Known.One.setAllBits();
3911 Known.Zero.setAllBits();
3912 if (DemandedVal) {
3913 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
3914 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
3915 }
3916 if (!!DemandedVecElts) {
3917 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
3918 Known = Known.intersectWith(RHS: Known2);
3919 }
3920 break;
3921 }
3922 case ISD::BITREVERSE: {
3923 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3924 Known = Known2.reverseBits();
3925 break;
3926 }
3927 case ISD::BSWAP: {
3928 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3929 Known = Known2.byteSwap();
3930 break;
3931 }
3932 case ISD::ABS: {
3933 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3934 Known = Known2.abs();
3935 break;
3936 }
3937 case ISD::USUBSAT: {
3938 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3939 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3940 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
3941 break;
3942 }
3943 case ISD::UMIN: {
3944 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3945 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3946 Known = KnownBits::umin(LHS: Known, RHS: Known2);
3947 break;
3948 }
3949 case ISD::UMAX: {
3950 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3951 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3952 Known = KnownBits::umax(LHS: Known, RHS: Known2);
3953 break;
3954 }
3955 case ISD::SMIN:
3956 case ISD::SMAX: {
3957 // If we have a clamp pattern, we know that the number of sign bits will be
3958 // the minimum of the clamp min/max range.
3959 bool IsMax = (Opcode == ISD::SMAX);
3960 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3961 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
3962 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3963 CstHigh =
3964 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
3965 if (CstLow && CstHigh) {
3966 if (!IsMax)
3967 std::swap(a&: CstLow, b&: CstHigh);
3968
3969 const APInt &ValueLow = CstLow->getAPIntValue();
3970 const APInt &ValueHigh = CstHigh->getAPIntValue();
3971 if (ValueLow.sle(RHS: ValueHigh)) {
3972 unsigned LowSignBits = ValueLow.getNumSignBits();
3973 unsigned HighSignBits = ValueHigh.getNumSignBits();
3974 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
3975 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3976 Known.One.setHighBits(MinSignBits);
3977 break;
3978 }
3979 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3980 Known.Zero.setHighBits(MinSignBits);
3981 break;
3982 }
3983 }
3984 }
3985
3986 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3987 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3988 if (IsMax)
3989 Known = KnownBits::smax(LHS: Known, RHS: Known2);
3990 else
3991 Known = KnownBits::smin(LHS: Known, RHS: Known2);
3992
3993 // For SMAX, if CstLow is non-negative we know the result will be
3994 // non-negative and thus all sign bits are 0.
3995 // TODO: There's an equivalent of this for smin with negative constant for
3996 // known ones.
3997 if (IsMax && CstLow) {
3998 const APInt &ValueLow = CstLow->getAPIntValue();
3999 if (ValueLow.isNonNegative()) {
4000 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4001 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4002 }
4003 }
4004
4005 break;
4006 }
4007 case ISD::FP_TO_UINT_SAT: {
4008 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4009 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4010 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4011 break;
4012 }
4013 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4014 if (Op.getResNo() == 1) {
4015 // The boolean result conforms to getBooleanContents.
4016 // If we know the result of a setcc has the top bits zero, use this info.
4017 // We know that we have an integer-based boolean since these operations
4018 // are only available for integer.
4019 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4020 TargetLowering::ZeroOrOneBooleanContent &&
4021 BitWidth > 1)
4022 Known.Zero.setBitsFrom(1);
4023 break;
4024 }
4025 [[fallthrough]];
4026 case ISD::ATOMIC_CMP_SWAP:
4027 case ISD::ATOMIC_SWAP:
4028 case ISD::ATOMIC_LOAD_ADD:
4029 case ISD::ATOMIC_LOAD_SUB:
4030 case ISD::ATOMIC_LOAD_AND:
4031 case ISD::ATOMIC_LOAD_CLR:
4032 case ISD::ATOMIC_LOAD_OR:
4033 case ISD::ATOMIC_LOAD_XOR:
4034 case ISD::ATOMIC_LOAD_NAND:
4035 case ISD::ATOMIC_LOAD_MIN:
4036 case ISD::ATOMIC_LOAD_MAX:
4037 case ISD::ATOMIC_LOAD_UMIN:
4038 case ISD::ATOMIC_LOAD_UMAX:
4039 case ISD::ATOMIC_LOAD: {
4040 unsigned MemBits =
4041 cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits();
4042 // If we are looking at the loaded value.
4043 if (Op.getResNo() == 0) {
4044 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4045 Known.Zero.setBitsFrom(MemBits);
4046 }
4047 break;
4048 }
4049 case ISD::FrameIndex:
4050 case ISD::TargetFrameIndex:
4051 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4052 Known, MF: getMachineFunction());
4053 break;
4054
4055 default:
4056 if (Opcode < ISD::BUILTIN_OP_END)
4057 break;
4058 [[fallthrough]];
4059 case ISD::INTRINSIC_WO_CHAIN:
4060 case ISD::INTRINSIC_W_CHAIN:
4061 case ISD::INTRINSIC_VOID:
4062 // TODO: Probably okay to remove after audit; here to reduce change size
4063 // in initial enablement patch for scalable vectors
4064 if (Op.getValueType().isScalableVector())
4065 break;
4066
4067 // Allow the target to implement this method for its nodes.
4068 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4069 break;
4070 }
4071
4072 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4073 return Known;
4074}
4075
4076/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4077static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4078 switch (OR) {
4079 case ConstantRange::OverflowResult::MayOverflow:
4080 return SelectionDAG::OFK_Sometime;
4081 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4082 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4083 return SelectionDAG::OFK_Always;
4084 case ConstantRange::OverflowResult::NeverOverflows:
4085 return SelectionDAG::OFK_Never;
4086 }
4087 llvm_unreachable("Unknown OverflowResult");
4088}
4089
4090SelectionDAG::OverflowKind
4091SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4092 // X + 0 never overflow
4093 if (isNullConstant(V: N1))
4094 return OFK_Never;
4095
4096 // If both operands each have at least two sign bits, the addition
4097 // cannot overflow.
4098 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4099 return OFK_Never;
4100
4101 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4102 return OFK_Sometime;
4103}
4104
4105SelectionDAG::OverflowKind
4106SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4107 // X + 0 never overflow
4108 if (isNullConstant(V: N1))
4109 return OFK_Never;
4110
4111 // mulhi + 1 never overflow
4112 KnownBits N1Known = computeKnownBits(Op: N1);
4113 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4114 N1Known.getMaxValue().ult(RHS: 2))
4115 return OFK_Never;
4116
4117 KnownBits N0Known = computeKnownBits(Op: N0);
4118 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4119 N0Known.getMaxValue().ult(RHS: 2))
4120 return OFK_Never;
4121
4122 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4123 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4124 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4125 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4126}
4127
4128SelectionDAG::OverflowKind
4129SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4130 // X - 0 never overflow
4131 if (isNullConstant(V: N1))
4132 return OFK_Never;
4133
4134 // If both operands each have at least two sign bits, the subtraction
4135 // cannot overflow.
4136 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4137 return OFK_Never;
4138
4139 KnownBits N0Known = computeKnownBits(Op: N0);
4140 KnownBits N1Known = computeKnownBits(Op: N1);
4141 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4142 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4143 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4144}
4145
4146SelectionDAG::OverflowKind
4147SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4148 // X - 0 never overflow
4149 if (isNullConstant(V: N1))
4150 return OFK_Never;
4151
4152 KnownBits N0Known = computeKnownBits(Op: N0);
4153 KnownBits N1Known = computeKnownBits(Op: N1);
4154 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4155 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4156 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4157}
4158
4159SelectionDAG::OverflowKind
4160SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4161 // X * 0 and X * 1 never overflow.
4162 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4163 return OFK_Never;
4164
4165 KnownBits N0Known = computeKnownBits(Op: N0);
4166 KnownBits N1Known = computeKnownBits(Op: N1);
4167 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4168 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4169 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4170}
4171
4172SelectionDAG::OverflowKind
4173SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4174 // X * 0 and X * 1 never overflow.
4175 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4176 return OFK_Never;
4177
4178 // Get the size of the result.
4179 unsigned BitWidth = N0.getScalarValueSizeInBits();
4180
4181 // Sum of the sign bits.
4182 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4183
4184 // If we have enough sign bits, then there's no overflow.
4185 if (SignBits > BitWidth + 1)
4186 return OFK_Never;
4187
4188 if (SignBits == BitWidth + 1) {
4189 // The overflow occurs when the true multiplication of the
4190 // the operands is the minimum negative number.
4191 KnownBits N0Known = computeKnownBits(Op: N0);
4192 KnownBits N1Known = computeKnownBits(Op: N1);
4193 // If one of the operands is non-negative, then there's no
4194 // overflow.
4195 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4196 return OFK_Never;
4197 }
4198
4199 return OFK_Sometime;
4200}
4201
4202bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4203 if (Depth >= MaxRecursionDepth)
4204 return false; // Limit search depth.
4205
4206 EVT OpVT = Val.getValueType();
4207 unsigned BitWidth = OpVT.getScalarSizeInBits();
4208
4209 // Is the constant a known power of 2?
4210 if (ISD::matchUnaryPredicate(Op: Val, Match: [BitWidth](ConstantSDNode *C) {
4211 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4212 }))
4213 return true;
4214
4215 // A left-shift of a constant one will have exactly one bit set because
4216 // shifting the bit off the end is undefined.
4217 if (Val.getOpcode() == ISD::SHL) {
4218 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4219 if (C && C->getAPIntValue() == 1)
4220 return true;
4221 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4222 isKnownNeverZero(Op: Val, Depth);
4223 }
4224
4225 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4226 // one bit set.
4227 if (Val.getOpcode() == ISD::SRL) {
4228 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4229 if (C && C->getAPIntValue().isSignMask())
4230 return true;
4231 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4232 isKnownNeverZero(Op: Val, Depth);
4233 }
4234
4235 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4236 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4237
4238 // Are all operands of a build vector constant powers of two?
4239 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4240 if (llvm::all_of(Range: Val->ops(), P: [BitWidth](SDValue E) {
4241 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: E))
4242 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4243 return false;
4244 }))
4245 return true;
4246
4247 // Is the operand of a splat vector a constant power of two?
4248 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4249 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4250 if (C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2())
4251 return true;
4252
4253 // vscale(power-of-two) is a power-of-two for some targets
4254 if (Val.getOpcode() == ISD::VSCALE &&
4255 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4256 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1))
4257 return true;
4258
4259 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4260 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4261 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1) &&
4262 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4263
4264 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4265 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), Depth: Depth + 1) &&
4266 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1);
4267
4268 if (Val.getOpcode() == ISD::AND) {
4269 // Looking for `x & -x` pattern:
4270 // If x == 0:
4271 // x & -x -> 0
4272 // If x != 0:
4273 // x & -x -> non-zero pow2
4274 // so if we find the pattern return whether we know `x` is non-zero.
4275 for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) {
4276 SDValue NegOp = Val.getOperand(i: OpIdx);
4277 if (NegOp.getOpcode() == ISD::SUB &&
4278 NegOp.getOperand(i: 1) == Val.getOperand(i: 1 - OpIdx) &&
4279 isNullOrNullSplat(V: NegOp.getOperand(i: 0)))
4280 return isKnownNeverZero(Op: Val.getOperand(i: 1 - OpIdx), Depth);
4281 }
4282 }
4283
4284 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4285 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4286
4287 // More could be done here, though the above checks are enough
4288 // to handle some common cases.
4289 return false;
4290}
4291
4292unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4293 EVT VT = Op.getValueType();
4294
4295 // Since the number of lanes in a scalable vector is unknown at compile time,
4296 // we track one bit which is implicitly broadcast to all lanes. This means
4297 // that all lanes in a scalable vector are considered demanded.
4298 APInt DemandedElts = VT.isFixedLengthVector()
4299 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4300 : APInt(1, 1);
4301 return ComputeNumSignBits(Op, DemandedElts, Depth);
4302}
4303
4304unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4305 unsigned Depth) const {
4306 EVT VT = Op.getValueType();
4307 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4308 unsigned VTBits = VT.getScalarSizeInBits();
4309 unsigned NumElts = DemandedElts.getBitWidth();
4310 unsigned Tmp, Tmp2;
4311 unsigned FirstAnswer = 1;
4312
4313 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4314 const APInt &Val = C->getAPIntValue();
4315 return Val.getNumSignBits();
4316 }
4317
4318 if (Depth >= MaxRecursionDepth)
4319 return 1; // Limit search depth.
4320
4321 if (!DemandedElts)
4322 return 1; // No demanded elts, better to assume we don't know anything.
4323
4324 unsigned Opcode = Op.getOpcode();
4325 switch (Opcode) {
4326 default: break;
4327 case ISD::AssertSext:
4328 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4329 return VTBits-Tmp+1;
4330 case ISD::AssertZext:
4331 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4332 return VTBits-Tmp;
4333 case ISD::MERGE_VALUES:
4334 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4335 Depth: Depth + 1);
4336 case ISD::SPLAT_VECTOR: {
4337 // Check if the sign bits of source go down as far as the truncated value.
4338 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4339 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4340 if (NumSrcSignBits > (NumSrcBits - VTBits))
4341 return NumSrcSignBits - (NumSrcBits - VTBits);
4342 break;
4343 }
4344 case ISD::BUILD_VECTOR:
4345 assert(!VT.isScalableVector());
4346 Tmp = VTBits;
4347 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4348 if (!DemandedElts[i])
4349 continue;
4350
4351 SDValue SrcOp = Op.getOperand(i);
4352 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4353 // for constant nodes to ensure we only look at the sign bits.
4354 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4355 APInt T = C->getAPIntValue().trunc(width: VTBits);
4356 Tmp2 = T.getNumSignBits();
4357 } else {
4358 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4359
4360 if (SrcOp.getValueSizeInBits() != VTBits) {
4361 assert(SrcOp.getValueSizeInBits() > VTBits &&
4362 "Expected BUILD_VECTOR implicit truncation");
4363 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4364 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4365 }
4366 }
4367 Tmp = std::min(a: Tmp, b: Tmp2);
4368 }
4369 return Tmp;
4370
4371 case ISD::VECTOR_SHUFFLE: {
4372 // Collect the minimum number of sign bits that are shared by every vector
4373 // element referenced by the shuffle.
4374 APInt DemandedLHS, DemandedRHS;
4375 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4376 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4377 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4378 DemandedLHS, DemandedRHS))
4379 return 1;
4380
4381 Tmp = std::numeric_limits<unsigned>::max();
4382 if (!!DemandedLHS)
4383 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4384 if (!!DemandedRHS) {
4385 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4386 Tmp = std::min(a: Tmp, b: Tmp2);
4387 }
4388 // If we don't know anything, early out and try computeKnownBits fall-back.
4389 if (Tmp == 1)
4390 break;
4391 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4392 return Tmp;
4393 }
4394
4395 case ISD::BITCAST: {
4396 if (VT.isScalableVector())
4397 break;
4398 SDValue N0 = Op.getOperand(i: 0);
4399 EVT SrcVT = N0.getValueType();
4400 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4401
4402 // Ignore bitcasts from unsupported types..
4403 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4404 break;
4405
4406 // Fast handling of 'identity' bitcasts.
4407 if (VTBits == SrcBits)
4408 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4409
4410 bool IsLE = getDataLayout().isLittleEndian();
4411
4412 // Bitcast 'large element' scalar/vector to 'small element' vector.
4413 if ((SrcBits % VTBits) == 0) {
4414 assert(VT.isVector() && "Expected bitcast to vector");
4415
4416 unsigned Scale = SrcBits / VTBits;
4417 APInt SrcDemandedElts =
4418 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4419
4420 // Fast case - sign splat can be simply split across the small elements.
4421 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4422 if (Tmp == SrcBits)
4423 return VTBits;
4424
4425 // Slow case - determine how far the sign extends into each sub-element.
4426 Tmp2 = VTBits;
4427 for (unsigned i = 0; i != NumElts; ++i)
4428 if (DemandedElts[i]) {
4429 unsigned SubOffset = i % Scale;
4430 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4431 SubOffset = SubOffset * VTBits;
4432 if (Tmp <= SubOffset)
4433 return 1;
4434 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4435 }
4436 return Tmp2;
4437 }
4438 break;
4439 }
4440
4441 case ISD::FP_TO_SINT_SAT:
4442 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4443 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4444 return VTBits - Tmp + 1;
4445 case ISD::SIGN_EXTEND:
4446 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
4447 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
4448 case ISD::SIGN_EXTEND_INREG:
4449 // Max of the input and what this extends.
4450 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4451 Tmp = VTBits-Tmp+1;
4452 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4453 return std::max(a: Tmp, b: Tmp2);
4454 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4455 if (VT.isScalableVector())
4456 break;
4457 SDValue Src = Op.getOperand(i: 0);
4458 EVT SrcVT = Src.getValueType();
4459 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
4460 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4461 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
4462 }
4463 case ISD::SRA:
4464 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4465 // SRA X, C -> adds C sign bits.
4466 if (const APInt *ShAmt =
4467 getValidMinimumShiftAmountConstant(V: Op, DemandedElts))
4468 Tmp = std::min<uint64_t>(a: Tmp + ShAmt->getZExtValue(), b: VTBits);
4469 return Tmp;
4470 case ISD::SHL:
4471 if (const APInt *ShAmt =
4472 getValidMaximumShiftAmountConstant(V: Op, DemandedElts)) {
4473 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4474 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4475 if (ShAmt->ult(RHS: Tmp))
4476 return Tmp - ShAmt->getZExtValue();
4477 }
4478 break;
4479 case ISD::AND:
4480 case ISD::OR:
4481 case ISD::XOR: // NOT is handled here.
4482 // Logical binary ops preserve the number of sign bits at the worst.
4483 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4484 if (Tmp != 1) {
4485 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4486 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4487 // We computed what we know about the sign bits as our first
4488 // answer. Now proceed to the generic code that uses
4489 // computeKnownBits, and pick whichever answer is better.
4490 }
4491 break;
4492
4493 case ISD::SELECT:
4494 case ISD::VSELECT:
4495 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4496 if (Tmp == 1) return 1; // Early out.
4497 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4498 return std::min(a: Tmp, b: Tmp2);
4499 case ISD::SELECT_CC:
4500 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4501 if (Tmp == 1) return 1; // Early out.
4502 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
4503 return std::min(a: Tmp, b: Tmp2);
4504
4505 case ISD::SMIN:
4506 case ISD::SMAX: {
4507 // If we have a clamp pattern, we know that the number of sign bits will be
4508 // the minimum of the clamp min/max range.
4509 bool IsMax = (Opcode == ISD::SMAX);
4510 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4511 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4512 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4513 CstHigh =
4514 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4515 if (CstLow && CstHigh) {
4516 if (!IsMax)
4517 std::swap(a&: CstLow, b&: CstHigh);
4518 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
4519 Tmp = CstLow->getAPIntValue().getNumSignBits();
4520 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4521 return std::min(a: Tmp, b: Tmp2);
4522 }
4523 }
4524
4525 // Fallback - just get the minimum number of sign bits of the operands.
4526 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4527 if (Tmp == 1)
4528 return 1; // Early out.
4529 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4530 return std::min(a: Tmp, b: Tmp2);
4531 }
4532 case ISD::UMIN:
4533 case ISD::UMAX:
4534 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4535 if (Tmp == 1)
4536 return 1; // Early out.
4537 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4538 return std::min(a: Tmp, b: Tmp2);
4539 case ISD::SADDO:
4540 case ISD::UADDO:
4541 case ISD::SADDO_CARRY:
4542 case ISD::UADDO_CARRY:
4543 case ISD::SSUBO:
4544 case ISD::USUBO:
4545 case ISD::SSUBO_CARRY:
4546 case ISD::USUBO_CARRY:
4547 case ISD::SMULO:
4548 case ISD::UMULO:
4549 if (Op.getResNo() != 1)
4550 break;
4551 // The boolean result conforms to getBooleanContents. Fall through.
4552 // If setcc returns 0/-1, all bits are sign bits.
4553 // We know that we have an integer-based boolean since these operations
4554 // are only available for integer.
4555 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
4556 TargetLowering::ZeroOrNegativeOneBooleanContent)
4557 return VTBits;
4558 break;
4559 case ISD::SETCC:
4560 case ISD::SETCCCARRY:
4561 case ISD::STRICT_FSETCC:
4562 case ISD::STRICT_FSETCCS: {
4563 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4564 // If setcc returns 0/-1, all bits are sign bits.
4565 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
4566 TargetLowering::ZeroOrNegativeOneBooleanContent)
4567 return VTBits;
4568 break;
4569 }
4570 case ISD::ROTL:
4571 case ISD::ROTR:
4572 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4573
4574 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4575 if (Tmp == VTBits)
4576 return VTBits;
4577
4578 if (ConstantSDNode *C =
4579 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
4580 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
4581
4582 // Handle rotate right by N like a rotate left by 32-N.
4583 if (Opcode == ISD::ROTR)
4584 RotAmt = (VTBits - RotAmt) % VTBits;
4585
4586 // If we aren't rotating out all of the known-in sign bits, return the
4587 // number that are left. This handles rotl(sext(x), 1) for example.
4588 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4589 }
4590 break;
4591 case ISD::ADD:
4592 case ISD::ADDC:
4593 // Add can have at most one carry bit. Thus we know that the output
4594 // is, at worst, one more bit than the inputs.
4595 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4596 if (Tmp == 1) return 1; // Early out.
4597
4598 // Special case decrementing a value (ADD X, -1):
4599 if (ConstantSDNode *CRHS =
4600 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
4601 if (CRHS->isAllOnes()) {
4602 KnownBits Known =
4603 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4604
4605 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4606 // sign bits set.
4607 if ((Known.Zero | 1).isAllOnes())
4608 return VTBits;
4609
4610 // If we are subtracting one from a positive number, there is no carry
4611 // out of the result.
4612 if (Known.isNonNegative())
4613 return Tmp;
4614 }
4615
4616 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4617 if (Tmp2 == 1) return 1; // Early out.
4618 return std::min(a: Tmp, b: Tmp2) - 1;
4619 case ISD::SUB:
4620 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4621 if (Tmp2 == 1) return 1; // Early out.
4622
4623 // Handle NEG.
4624 if (ConstantSDNode *CLHS =
4625 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
4626 if (CLHS->isZero()) {
4627 KnownBits Known =
4628 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4629 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4630 // sign bits set.
4631 if ((Known.Zero | 1).isAllOnes())
4632 return VTBits;
4633
4634 // If the input is known to be positive (the sign bit is known clear),
4635 // the output of the NEG has the same number of sign bits as the input.
4636 if (Known.isNonNegative())
4637 return Tmp2;
4638
4639 // Otherwise, we treat this like a SUB.
4640 }
4641
4642 // Sub can have at most one carry bit. Thus we know that the output
4643 // is, at worst, one more bit than the inputs.
4644 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4645 if (Tmp == 1) return 1; // Early out.
4646 return std::min(a: Tmp, b: Tmp2) - 1;
4647 case ISD::MUL: {
4648 // The output of the Mul can be at most twice the valid bits in the inputs.
4649 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4650 if (SignBitsOp0 == 1)
4651 break;
4652 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
4653 if (SignBitsOp1 == 1)
4654 break;
4655 unsigned OutValidBits =
4656 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4657 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4658 }
4659 case ISD::SREM:
4660 // The sign bit is the LHS's sign bit, except when the result of the
4661 // remainder is zero. The magnitude of the result should be less than or
4662 // equal to the magnitude of the LHS. Therefore, the result should have
4663 // at least as many sign bits as the left hand side.
4664 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4665 case ISD::TRUNCATE: {
4666 // Check if the sign bits of source go down as far as the truncated value.
4667 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
4668 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4669 if (NumSrcSignBits > (NumSrcBits - VTBits))
4670 return NumSrcSignBits - (NumSrcBits - VTBits);
4671 break;
4672 }
4673 case ISD::EXTRACT_ELEMENT: {
4674 if (VT.isScalableVector())
4675 break;
4676 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4677 const int BitWidth = Op.getValueSizeInBits();
4678 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
4679
4680 // Get reverse index (starting from 1), Op1 value indexes elements from
4681 // little end. Sign starts at big end.
4682 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
4683
4684 // If the sign portion ends in our element the subtraction gives correct
4685 // result. Otherwise it gives either negative or > bitwidth result
4686 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 0, hi: BitWidth);
4687 }
4688 case ISD::INSERT_VECTOR_ELT: {
4689 if (VT.isScalableVector())
4690 break;
4691 // If we know the element index, split the demand between the
4692 // source vector and the inserted element, otherwise assume we need
4693 // the original demanded vector elements and the value.
4694 SDValue InVec = Op.getOperand(i: 0);
4695 SDValue InVal = Op.getOperand(i: 1);
4696 SDValue EltNo = Op.getOperand(i: 2);
4697 bool DemandedVal = true;
4698 APInt DemandedVecElts = DemandedElts;
4699 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4700 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4701 unsigned EltIdx = CEltNo->getZExtValue();
4702 DemandedVal = !!DemandedElts[EltIdx];
4703 DemandedVecElts.clearBit(BitPosition: EltIdx);
4704 }
4705 Tmp = std::numeric_limits<unsigned>::max();
4706 if (DemandedVal) {
4707 // TODO - handle implicit truncation of inserted elements.
4708 if (InVal.getScalarValueSizeInBits() != VTBits)
4709 break;
4710 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
4711 Tmp = std::min(a: Tmp, b: Tmp2);
4712 }
4713 if (!!DemandedVecElts) {
4714 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4715 Tmp = std::min(a: Tmp, b: Tmp2);
4716 }
4717 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4718 return Tmp;
4719 }
4720 case ISD::EXTRACT_VECTOR_ELT: {
4721 assert(!VT.isScalableVector());
4722 SDValue InVec = Op.getOperand(i: 0);
4723 SDValue EltNo = Op.getOperand(i: 1);
4724 EVT VecVT = InVec.getValueType();
4725 // ComputeNumSignBits not yet implemented for scalable vectors.
4726 if (VecVT.isScalableVector())
4727 break;
4728 const unsigned BitWidth = Op.getValueSizeInBits();
4729 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
4730 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4731
4732 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4733 // anything about sign bits. But if the sizes match we can derive knowledge
4734 // about sign bits from the vector operand.
4735 if (BitWidth != EltBitWidth)
4736 break;
4737
4738 // If we know the element index, just demand that vector element, else for
4739 // an unknown element index, ignore DemandedElts and demand them all.
4740 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4741 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4742 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4743 DemandedSrcElts =
4744 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4745
4746 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4747 }
4748 case ISD::EXTRACT_SUBVECTOR: {
4749 // Offset the demanded elts by the subvector index.
4750 SDValue Src = Op.getOperand(i: 0);
4751 // Bail until we can represent demanded elements for scalable vectors.
4752 if (Src.getValueType().isScalableVector())
4753 break;
4754 uint64_t Idx = Op.getConstantOperandVal(i: 1);
4755 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4756 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
4757 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4758 }
4759 case ISD::CONCAT_VECTORS: {
4760 if (VT.isScalableVector())
4761 break;
4762 // Determine the minimum number of sign bits across all demanded
4763 // elts of the input vectors. Early out if the result is already 1.
4764 Tmp = std::numeric_limits<unsigned>::max();
4765 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
4766 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4767 unsigned NumSubVectors = Op.getNumOperands();
4768 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4769 APInt DemandedSub =
4770 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
4771 if (!DemandedSub)
4772 continue;
4773 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
4774 Tmp = std::min(a: Tmp, b: Tmp2);
4775 }
4776 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4777 return Tmp;
4778 }
4779 case ISD::INSERT_SUBVECTOR: {
4780 if (VT.isScalableVector())
4781 break;
4782 // Demand any elements from the subvector and the remainder from the src its
4783 // inserted into.
4784 SDValue Src = Op.getOperand(i: 0);
4785 SDValue Sub = Op.getOperand(i: 1);
4786 uint64_t Idx = Op.getConstantOperandVal(i: 2);
4787 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4788 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
4789 APInt DemandedSrcElts = DemandedElts;
4790 DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx);
4791
4792 Tmp = std::numeric_limits<unsigned>::max();
4793 if (!!DemandedSubElts) {
4794 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
4795 if (Tmp == 1)
4796 return 1; // early-out
4797 }
4798 if (!!DemandedSrcElts) {
4799 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4800 Tmp = std::min(a: Tmp, b: Tmp2);
4801 }
4802 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4803 return Tmp;
4804 }
4805 case ISD::LOAD: {
4806 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4807 if (const MDNode *Ranges = LD->getRanges()) {
4808 if (DemandedElts != 1)
4809 break;
4810
4811 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
4812 if (VTBits > CR.getBitWidth()) {
4813 switch (LD->getExtensionType()) {
4814 case ISD::SEXTLOAD:
4815 CR = CR.signExtend(BitWidth: VTBits);
4816 break;
4817 case ISD::ZEXTLOAD:
4818 CR = CR.zeroExtend(BitWidth: VTBits);
4819 break;
4820 default:
4821 break;
4822 }
4823 }
4824
4825 if (VTBits != CR.getBitWidth())
4826 break;
4827 return std::min(a: CR.getSignedMin().getNumSignBits(),
4828 b: CR.getSignedMax().getNumSignBits());
4829 }
4830
4831 break;
4832 }
4833 case ISD::ATOMIC_CMP_SWAP:
4834 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4835 case ISD::ATOMIC_SWAP:
4836 case ISD::ATOMIC_LOAD_ADD:
4837 case ISD::ATOMIC_LOAD_SUB:
4838 case ISD::ATOMIC_LOAD_AND:
4839 case ISD::ATOMIC_LOAD_CLR:
4840 case ISD::ATOMIC_LOAD_OR:
4841 case ISD::ATOMIC_LOAD_XOR:
4842 case ISD::ATOMIC_LOAD_NAND:
4843 case ISD::ATOMIC_LOAD_MIN:
4844 case ISD::ATOMIC_LOAD_MAX:
4845 case ISD::ATOMIC_LOAD_UMIN:
4846 case ISD::ATOMIC_LOAD_UMAX:
4847 case ISD::ATOMIC_LOAD: {
4848 Tmp = cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits();
4849 // If we are looking at the loaded value.
4850 if (Op.getResNo() == 0) {
4851 if (Tmp == VTBits)
4852 return 1; // early-out
4853 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4854 return VTBits - Tmp + 1;
4855 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4856 return VTBits - Tmp;
4857 }
4858 break;
4859 }
4860 }
4861
4862 // If we are looking at the loaded value of the SDNode.
4863 if (Op.getResNo() == 0) {
4864 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4865 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op)) {
4866 unsigned ExtType = LD->getExtensionType();
4867 switch (ExtType) {
4868 default: break;
4869 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4870 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4871 return VTBits - Tmp + 1;
4872 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4873 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4874 return VTBits - Tmp;
4875 case ISD::NON_EXTLOAD:
4876 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4877 // We only need to handle vectors - computeKnownBits should handle
4878 // scalar cases.
4879 Type *CstTy = Cst->getType();
4880 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4881 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4882 VTBits == CstTy->getScalarSizeInBits()) {
4883 Tmp = VTBits;
4884 for (unsigned i = 0; i != NumElts; ++i) {
4885 if (!DemandedElts[i])
4886 continue;
4887 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4888 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4889 const APInt &Value = CInt->getValue();
4890 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
4891 continue;
4892 }
4893 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4894 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4895 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
4896 continue;
4897 }
4898 }
4899 // Unknown type. Conservatively assume no bits match sign bit.
4900 return 1;
4901 }
4902 return Tmp;
4903 }
4904 }
4905 break;
4906 }
4907 }
4908 }
4909
4910 // Allow the target to implement this method for its nodes.
4911 if (Opcode >= ISD::BUILTIN_OP_END ||
4912 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4913 Opcode == ISD::INTRINSIC_W_CHAIN ||
4914 Opcode == ISD::INTRINSIC_VOID) {
4915 // TODO: This can probably be removed once target code is audited. This
4916 // is here purely to reduce patch size and review complexity.
4917 if (!VT.isScalableVector()) {
4918 unsigned NumBits =
4919 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
4920 if (NumBits > 1)
4921 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
4922 }
4923 }
4924
4925 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4926 // use this information.
4927 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4928 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4929}
4930
4931unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4932 unsigned Depth) const {
4933 unsigned SignBits = ComputeNumSignBits(Op, Depth);
4934 return Op.getScalarValueSizeInBits() - SignBits + 1;
4935}
4936
4937unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4938 const APInt &DemandedElts,
4939 unsigned Depth) const {
4940 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
4941 return Op.getScalarValueSizeInBits() - SignBits + 1;
4942}
4943
4944bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4945 unsigned Depth) const {
4946 // Early out for FREEZE.
4947 if (Op.getOpcode() == ISD::FREEZE)
4948 return true;
4949
4950 // TODO: Assume we don't know anything for now.
4951 EVT VT = Op.getValueType();
4952 if (VT.isScalableVector())
4953 return false;
4954
4955 APInt DemandedElts = VT.isVector()
4956 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4957 : APInt(1, 1);
4958 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4959}
4960
4961bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4962 const APInt &DemandedElts,
4963 bool PoisonOnly,
4964 unsigned Depth) const {
4965 unsigned Opcode = Op.getOpcode();
4966
4967 // Early out for FREEZE.
4968 if (Opcode == ISD::FREEZE)
4969 return true;
4970
4971 if (Depth >= MaxRecursionDepth)
4972 return false; // Limit search depth.
4973
4974 if (isIntOrFPConstant(V: Op))
4975 return true;
4976
4977 switch (Opcode) {
4978 case ISD::VALUETYPE:
4979 case ISD::FrameIndex:
4980 case ISD::TargetFrameIndex:
4981 return true;
4982
4983 case ISD::UNDEF:
4984 return PoisonOnly;
4985
4986 case ISD::BUILD_VECTOR:
4987 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
4988 // this shouldn't affect the result.
4989 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
4990 if (!DemandedElts[i])
4991 continue;
4992 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
4993 Depth: Depth + 1))
4994 return false;
4995 }
4996 return true;
4997
4998 // TODO: Search for noundef attributes from library functions.
4999
5000 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5001
5002 default:
5003 // Allow the target to implement this method for its nodes.
5004 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5005 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5006 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5007 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5008 break;
5009 }
5010
5011 // If Op can't create undef/poison and none of its operands are undef/poison
5012 // then Op is never undef/poison.
5013 // NOTE: TargetNodes should handle this in themselves in
5014 // isGuaranteedNotToBeUndefOrPoisonForTargetNode.
5015 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5016 Depth) &&
5017 all_of(Range: Op->ops(), P: [&](SDValue V) {
5018 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5019 });
5020}
5021
5022bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5023 bool ConsiderFlags,
5024 unsigned Depth) const {
5025 // TODO: Assume we don't know anything for now.
5026 EVT VT = Op.getValueType();
5027 if (VT.isScalableVector())
5028 return true;
5029
5030 APInt DemandedElts = VT.isVector()
5031 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5032 : APInt(1, 1);
5033 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5034 Depth);
5035}
5036
5037bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5038 bool PoisonOnly, bool ConsiderFlags,
5039 unsigned Depth) const {
5040 // TODO: Assume we don't know anything for now.
5041 EVT VT = Op.getValueType();
5042 if (VT.isScalableVector())
5043 return true;
5044
5045 unsigned Opcode = Op.getOpcode();
5046 switch (Opcode) {
5047 case ISD::FREEZE:
5048 case ISD::CONCAT_VECTORS:
5049 case ISD::INSERT_SUBVECTOR:
5050 case ISD::AND:
5051 case ISD::XOR:
5052 case ISD::ROTL:
5053 case ISD::ROTR:
5054 case ISD::FSHL:
5055 case ISD::FSHR:
5056 case ISD::BSWAP:
5057 case ISD::CTPOP:
5058 case ISD::BITREVERSE:
5059 case ISD::PARITY:
5060 case ISD::SIGN_EXTEND:
5061 case ISD::TRUNCATE:
5062 case ISD::SIGN_EXTEND_INREG:
5063 case ISD::SIGN_EXTEND_VECTOR_INREG:
5064 case ISD::ZERO_EXTEND_VECTOR_INREG:
5065 case ISD::BITCAST:
5066 case ISD::BUILD_VECTOR:
5067 case ISD::BUILD_PAIR:
5068 return false;
5069
5070 // Matches hasPoisonGeneratingFlags().
5071 case ISD::ZERO_EXTEND:
5072 return ConsiderFlags && Op->getFlags().hasNonNeg();
5073
5074 case ISD::ADD:
5075 case ISD::SUB:
5076 case ISD::MUL:
5077 // Matches hasPoisonGeneratingFlags().
5078 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5079 Op->getFlags().hasNoUnsignedWrap());
5080
5081 case ISD::SHL:
5082 // If the max shift amount isn't in range, then the shift can create poison.
5083 if (!getValidMaximumShiftAmountConstant(V: Op, DemandedElts))
5084 return true;
5085
5086 // Matches hasPoisonGeneratingFlags().
5087 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5088 Op->getFlags().hasNoUnsignedWrap());
5089
5090 // Matches hasPoisonGeneratingFlags().
5091 case ISD::OR:
5092 return ConsiderFlags && Op->getFlags().hasDisjoint();
5093
5094 case ISD::INSERT_VECTOR_ELT:{
5095 // Ensure that the element index is in bounds.
5096 EVT VecVT = Op.getOperand(i: 0).getValueType();
5097 KnownBits KnownIdx = computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
5098 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5099 }
5100
5101 default:
5102 // Allow the target to implement this method for its nodes.
5103 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5104 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5105 return TLI->canCreateUndefOrPoisonForTargetNode(
5106 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5107 break;
5108 }
5109
5110 // Be conservative and return true.
5111 return true;
5112}
5113
5114bool SelectionDAG::isADDLike(SDValue Op) const {
5115 unsigned Opcode = Op.getOpcode();
5116 if (Opcode == ISD::OR)
5117 return Op->getFlags().hasDisjoint() ||
5118 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5119 if (Opcode == ISD::XOR)
5120 return isMinSignedConstant(V: Op.getOperand(i: 1));
5121 return false;
5122}
5123
5124bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5125 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
5126 !isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
5127 return false;
5128
5129 if (Op.getOpcode() == ISD::OR &&
5130 !MaskedValueIsZero(V: Op.getOperand(i: 0), Mask: Op.getConstantOperandAPInt(i: 1)))
5131 return false;
5132
5133 return true;
5134}
5135
5136bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5137 // If we're told that NaNs won't happen, assume they won't.
5138 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5139 return true;
5140
5141 if (Depth >= MaxRecursionDepth)
5142 return false; // Limit search depth.
5143
5144 // If the value is a constant, we can obviously see if it is a NaN or not.
5145 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
5146 return !C->getValueAPF().isNaN() ||
5147 (SNaN && !C->getValueAPF().isSignaling());
5148 }
5149
5150 unsigned Opcode = Op.getOpcode();
5151 switch (Opcode) {
5152 case ISD::FADD:
5153 case ISD::FSUB:
5154 case ISD::FMUL:
5155 case ISD::FDIV:
5156 case ISD::FREM:
5157 case ISD::FSIN:
5158 case ISD::FCOS:
5159 case ISD::FMA:
5160 case ISD::FMAD: {
5161 if (SNaN)
5162 return true;
5163 // TODO: Need isKnownNeverInfinity
5164 return false;
5165 }
5166 case ISD::FCANONICALIZE:
5167 case ISD::FEXP:
5168 case ISD::FEXP2:
5169 case ISD::FEXP10:
5170 case ISD::FTRUNC:
5171 case ISD::FFLOOR:
5172 case ISD::FCEIL:
5173 case ISD::FROUND:
5174 case ISD::FROUNDEVEN:
5175 case ISD::FRINT:
5176 case ISD::LRINT:
5177 case ISD::LLRINT:
5178 case ISD::FNEARBYINT:
5179 case ISD::FLDEXP: {
5180 if (SNaN)
5181 return true;
5182 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5183 }
5184 case ISD::FABS:
5185 case ISD::FNEG:
5186 case ISD::FCOPYSIGN: {
5187 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5188 }
5189 case ISD::SELECT:
5190 return isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
5191 isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
5192 case ISD::FP_EXTEND:
5193 case ISD::FP_ROUND: {
5194 if (SNaN)
5195 return true;
5196 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5197 }
5198 case ISD::SINT_TO_FP:
5199 case ISD::UINT_TO_FP:
5200 return true;
5201 case ISD::FSQRT: // Need is known positive
5202 case ISD::FLOG:
5203 case ISD::FLOG2:
5204 case ISD::FLOG10:
5205 case ISD::FPOWI:
5206 case ISD::FPOW: {
5207 if (SNaN)
5208 return true;
5209 // TODO: Refine on operand
5210 return false;
5211 }
5212 case ISD::FMINNUM:
5213 case ISD::FMAXNUM: {
5214 // Only one needs to be known not-nan, since it will be returned if the
5215 // other ends up being one.
5216 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) ||
5217 isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
5218 }
5219 case ISD::FMINNUM_IEEE:
5220 case ISD::FMAXNUM_IEEE: {
5221 if (SNaN)
5222 return true;
5223 // This can return a NaN if either operand is an sNaN, or if both operands
5224 // are NaN.
5225 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN: false, Depth: Depth + 1) &&
5226 isKnownNeverSNaN(Op: Op.getOperand(i: 1), Depth: Depth + 1)) ||
5227 (isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN: false, Depth: Depth + 1) &&
5228 isKnownNeverSNaN(Op: Op.getOperand(i: 0), Depth: Depth + 1));
5229 }
5230 case ISD::FMINIMUM:
5231 case ISD::FMAXIMUM: {
5232 // TODO: Does this quiet or return the origina NaN as-is?
5233 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
5234 isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
5235 }
5236 case ISD::EXTRACT_VECTOR_ELT: {
5237 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
5238 }
5239 case ISD::BUILD_VECTOR: {
5240 for (const SDValue &Opnd : Op->ops())
5241 if (!isKnownNeverNaN(Op: Opnd, SNaN, Depth: Depth + 1))
5242 return false;
5243 return true;
5244 }
5245 default:
5246 if (Opcode >= ISD::BUILTIN_OP_END ||
5247 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5248 Opcode == ISD::INTRINSIC_W_CHAIN ||
5249 Opcode == ISD::INTRINSIC_VOID) {
5250 return TLI->isKnownNeverNaNForTargetNode(Op, DAG: *this, SNaN, Depth);
5251 }
5252
5253 return false;
5254 }
5255}
5256
5257bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
5258 assert(Op.getValueType().isFloatingPoint() &&
5259 "Floating point type expected");
5260
5261 // If the value is a constant, we can obviously see if it is a zero or not.
5262 return ISD::matchUnaryFpPredicate(
5263 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
5264}
5265
5266bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
5267 if (Depth >= MaxRecursionDepth)
5268 return false; // Limit search depth.
5269
5270 assert(!Op.getValueType().isFloatingPoint() &&
5271 "Floating point types unsupported - use isKnownNeverZeroFloat");
5272
5273 // If the value is a constant, we can obviously see if it is a zero or not.
5274 if (ISD::matchUnaryPredicate(Op,
5275 Match: [](ConstantSDNode *C) { return !C->isZero(); }))
5276 return true;
5277
5278 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5279 // some degree.
5280 switch (Op.getOpcode()) {
5281 default:
5282 break;
5283
5284 case ISD::OR:
5285 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5286 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5287
5288 case ISD::VSELECT:
5289 case ISD::SELECT:
5290 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5291 isKnownNeverZero(Op: Op.getOperand(i: 2), Depth: Depth + 1);
5292
5293 case ISD::SHL: {
5294 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5295 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5296 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5297 // 1 << X is never zero.
5298 if (ValKnown.One[0])
5299 return true;
5300 // If max shift cnt of known ones is non-zero, result is non-zero.
5301 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5302 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
5303 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
5304 return true;
5305 break;
5306 }
5307 case ISD::UADDSAT:
5308 case ISD::UMAX:
5309 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5310 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5311
5312 // TODO for smin/smax: If either operand is known negative/positive
5313 // respectively we don't need the other to be known at all.
5314 case ISD::SMAX:
5315 case ISD::SMIN:
5316 case ISD::UMIN:
5317 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5318 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5319
5320 case ISD::ROTL:
5321 case ISD::ROTR:
5322 case ISD::BITREVERSE:
5323 case ISD::BSWAP:
5324 case ISD::CTPOP:
5325 case ISD::ABS:
5326 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5327
5328 case ISD::SRA:
5329 case ISD::SRL: {
5330 if (Op->getFlags().hasExact())
5331 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5332 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5333 if (ValKnown.isNegative())
5334 return true;
5335 // If max shift cnt of known ones is non-zero, result is non-zero.
5336 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5337 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
5338 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
5339 return true;
5340 break;
5341 }
5342 case ISD::UDIV:
5343 case ISD::SDIV:
5344 // div exact can only produce a zero if the dividend is zero.
5345 // TODO: For udiv this is also true if Op1 u<= Op0
5346 if (Op->getFlags().hasExact())
5347 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5348 break;
5349
5350 case ISD::ADD:
5351 if (Op->getFlags().hasNoUnsignedWrap())
5352 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5353 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
5354 return true;
5355 // TODO: There are a lot more cases we can prove for add.
5356 break;
5357
5358 case ISD::SUB: {
5359 if (isNullConstant(V: Op.getOperand(i: 0)))
5360 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5361
5362 std::optional<bool> ne =
5363 KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1),
5364 RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1));
5365 return ne && *ne;
5366 }
5367
5368 case ISD::MUL:
5369 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5370 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5371 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
5372 return true;
5373 break;
5374
5375 case ISD::ZERO_EXTEND:
5376 case ISD::SIGN_EXTEND:
5377 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5378 }
5379
5380 return computeKnownBits(Op, Depth).isNonZero();
5381}
5382
5383bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
5384 // Check the obvious case.
5385 if (A == B) return true;
5386
5387 // For negative and positive zero.
5388 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
5389 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
5390 if (CA->isZero() && CB->isZero()) return true;
5391
5392 // Otherwise they may not be equal.
5393 return false;
5394}
5395
5396// Only bits set in Mask must be negated, other bits may be arbitrary.
5397SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
5398 if (isBitwiseNot(V, AllowUndefs))
5399 return V.getOperand(i: 0);
5400
5401 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5402 // bits in the non-extended part.
5403 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
5404 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5405 return SDValue();
5406 SDValue ExtArg = V.getOperand(i: 0);
5407 if (ExtArg.getScalarValueSizeInBits() >=
5408 MaskC->getAPIntValue().getActiveBits() &&
5409 isBitwiseNot(V: ExtArg, AllowUndefs) &&
5410 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
5411 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
5412 return ExtArg.getOperand(i: 0).getOperand(i: 0);
5413 return SDValue();
5414}
5415
5416static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
5417 // Match masked merge pattern (X & ~M) op (Y & M)
5418 // Including degenerate case (X & ~M) op M
5419 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5420 SDValue Other) {
5421 if (SDValue NotOperand =
5422 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
5423 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5424 NotOperand->getOpcode() == ISD::TRUNCATE)
5425 NotOperand = NotOperand->getOperand(Num: 0);
5426
5427 if (Other == NotOperand)
5428 return true;
5429 if (Other->getOpcode() == ISD::AND)
5430 return NotOperand == Other->getOperand(Num: 0) ||
5431 NotOperand == Other->getOperand(Num: 1);
5432 }
5433 return false;
5434 };
5435
5436 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5437 A = A->getOperand(Num: 0);
5438
5439 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5440 B = B->getOperand(Num: 0);
5441
5442 if (A->getOpcode() == ISD::AND)
5443 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
5444 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
5445 return false;
5446}
5447
5448// FIXME: unify with llvm::haveNoCommonBitsSet.
5449bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
5450 assert(A.getValueType() == B.getValueType() &&
5451 "Values must have the same type");
5452 if (haveNoCommonBitsSetCommutative(A, B) ||
5453 haveNoCommonBitsSetCommutative(A: B, B: A))
5454 return true;
5455 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
5456 RHS: computeKnownBits(Op: B));
5457}
5458
5459static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5460 SelectionDAG &DAG) {
5461 if (cast<ConstantSDNode>(Val&: Step)->isZero())
5462 return DAG.getConstant(Val: 0, DL, VT);
5463
5464 return SDValue();
5465}
5466
5467static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
5468 ArrayRef<SDValue> Ops,
5469 SelectionDAG &DAG) {
5470 int NumOps = Ops.size();
5471 assert(NumOps != 0 && "Can't build an empty vector!");
5472 assert(!VT.isScalableVector() &&
5473 "BUILD_VECTOR cannot be used with scalable types");
5474 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5475 "Incorrect element count in BUILD_VECTOR!");
5476
5477 // BUILD_VECTOR of UNDEFs is UNDEF.
5478 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
5479 return DAG.getUNDEF(VT);
5480
5481 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5482 SDValue IdentitySrc;
5483 bool IsIdentity = true;
5484 for (int i = 0; i != NumOps; ++i) {
5485 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5486 Ops[i].getOperand(i: 0).getValueType() != VT ||
5487 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
5488 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
5489 Ops[i].getConstantOperandAPInt(i: 1) != i) {
5490 IsIdentity = false;
5491 break;
5492 }
5493 IdentitySrc = Ops[i].getOperand(i: 0);
5494 }
5495 if (IsIdentity)
5496 return IdentitySrc;
5497
5498 return SDValue();
5499}
5500
5501/// Try to simplify vector concatenation to an input value, undef, or build
5502/// vector.
5503static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
5504 ArrayRef<SDValue> Ops,
5505 SelectionDAG &DAG) {
5506 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5507 assert(llvm::all_of(Ops,
5508 [Ops](SDValue Op) {
5509 return Ops[0].getValueType() == Op.getValueType();
5510 }) &&
5511 "Concatenation of vectors with inconsistent value types!");
5512 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5513 VT.getVectorElementCount() &&
5514 "Incorrect element count in vector concatenation!");
5515
5516 if (Ops.size() == 1)
5517 return Ops[0];
5518
5519 // Concat of UNDEFs is UNDEF.
5520 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
5521 return DAG.getUNDEF(VT);
5522
5523 // Scan the operands and look for extract operations from a single source
5524 // that correspond to insertion at the same location via this concatenation:
5525 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5526 SDValue IdentitySrc;
5527 bool IsIdentity = true;
5528 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5529 SDValue Op = Ops[i];
5530 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5531 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5532 Op.getOperand(i: 0).getValueType() != VT ||
5533 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
5534 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
5535 IsIdentity = false;
5536 break;
5537 }
5538 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5539 "Unexpected identity source vector for concat of extracts");
5540 IdentitySrc = Op.getOperand(i: 0);
5541 }
5542 if (IsIdentity) {
5543 assert(IdentitySrc && "Failed to set source vector of extracts");
5544 return IdentitySrc;
5545 }
5546
5547 // The code below this point is only designed to work for fixed width
5548 // vectors, so we bail out for now.
5549 if (VT.isScalableVector())
5550 return SDValue();
5551
5552 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5553 // simplified to one big BUILD_VECTOR.
5554 // FIXME: Add support for SCALAR_TO_VECTOR as well.
5555 EVT SVT = VT.getScalarType();
5556 SmallVector<SDValue, 16> Elts;
5557 for (SDValue Op : Ops) {
5558 EVT OpVT = Op.getValueType();
5559 if (Op.isUndef())
5560 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
5561 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5562 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
5563 else
5564 return SDValue();
5565 }
5566
5567 // BUILD_VECTOR requires all inputs to be of the same type, find the
5568 // maximum type and extend them all.
5569 for (SDValue Op : Elts)
5570 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
5571
5572 if (SVT.bitsGT(VT: VT.getScalarType())) {
5573 for (SDValue &Op : Elts) {
5574 if (Op.isUndef())
5575 Op = DAG.getUNDEF(VT: SVT);
5576 else
5577 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
5578 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
5579 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
5580 }
5581 }
5582
5583 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
5584 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
5585 return V;
5586}
5587
5588/// Gets or creates the specified node.
5589SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5590 FoldingSetNodeID ID;
5591 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT), OpList: std::nullopt);
5592 void *IP = nullptr;
5593 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
5594 return SDValue(E, 0);
5595
5596 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(),
5597 Args: getVTList(VT));
5598 CSEMap.InsertNode(N, InsertPos: IP);
5599
5600 InsertNode(N);
5601 SDValue V = SDValue(N, 0);
5602 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
5603 return V;
5604}
5605
5606SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5607 SDValue N1) {
5608 SDNodeFlags Flags;
5609 if (Inserter)
5610 Flags = Inserter->getFlags();
5611 return getNode(Opcode, DL, VT, Operand: N1, Flags);
5612}
5613
5614SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5615 SDValue N1, const SDNodeFlags Flags) {
5616 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5617
5618 // Constant fold unary operations with a vector integer or float operand.
5619 switch (Opcode) {
5620 default:
5621 // FIXME: Entirely reasonable to perform folding of other unary
5622 // operations here as the need arises.
5623 break;
5624 case ISD::FNEG:
5625 case ISD::FABS:
5626 case ISD::FCEIL:
5627 case ISD::FTRUNC:
5628 case ISD::FFLOOR:
5629 case ISD::FP_EXTEND:
5630 case ISD::FP_TO_SINT:
5631 case ISD::FP_TO_UINT:
5632 case ISD::FP_TO_FP16:
5633 case ISD::FP_TO_BF16:
5634 case ISD::TRUNCATE:
5635 case ISD::ANY_EXTEND:
5636 case ISD::ZERO_EXTEND:
5637 case ISD::SIGN_EXTEND:
5638 case ISD::UINT_TO_FP:
5639 case ISD::SINT_TO_FP:
5640 case ISD::FP16_TO_FP:
5641 case ISD::BF16_TO_FP:
5642 case ISD::BITCAST:
5643 case ISD::ABS:
5644 case ISD::BITREVERSE:
5645 case ISD::BSWAP:
5646 case ISD::CTLZ:
5647 case ISD::CTLZ_ZERO_UNDEF:
5648 case ISD::CTTZ:
5649 case ISD::CTTZ_ZERO_UNDEF:
5650 case ISD::CTPOP:
5651 case ISD::STEP_VECTOR: {
5652 SDValue Ops = {N1};
5653 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5654 return Fold;
5655 }
5656 }
5657
5658 unsigned OpOpcode = N1.getNode()->getOpcode();
5659 switch (Opcode) {
5660 case ISD::STEP_VECTOR:
5661 assert(VT.isScalableVector() &&
5662 "STEP_VECTOR can only be used with scalable types");
5663 assert(OpOpcode == ISD::TargetConstant &&
5664 VT.getVectorElementType() == N1.getValueType() &&
5665 "Unexpected step operand");
5666 break;
5667 case ISD::FREEZE:
5668 assert(VT == N1.getValueType() && "Unexpected VT!");
5669 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly*/ false,
5670 /*Depth*/ 1))
5671 return N1;
5672 break;
5673 case ISD::TokenFactor:
5674 case ISD::MERGE_VALUES:
5675 case ISD::CONCAT_VECTORS:
5676 return N1; // Factor, merge or concat of one node? No need.
5677 case ISD::BUILD_VECTOR: {
5678 // Attempt to simplify BUILD_VECTOR.
5679 SDValue Ops[] = {N1};
5680 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
5681 return V;
5682 break;
5683 }
5684 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5685 case ISD::FP_EXTEND:
5686 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
5687 "Invalid FP cast!");
5688 if (N1.getValueType() == VT) return N1; // noop conversion.
5689 assert((!VT.isVector() || VT.getVectorElementCount() ==
5690 N1.getValueType().getVectorElementCount()) &&
5691 "Vector element count mismatch!");
5692 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5693 if (N1.isUndef())
5694 return getUNDEF(VT);
5695 break;
5696 case ISD::FP_TO_SINT:
5697 case ISD::FP_TO_UINT:
5698 if (N1.isUndef())
5699 return getUNDEF(VT);
5700 break;
5701 case ISD::SINT_TO_FP:
5702 case ISD::UINT_TO_FP:
5703 // [us]itofp(undef) = 0, because the result value is bounded.
5704 if (N1.isUndef())
5705 return getConstantFP(Val: 0.0, DL, VT);
5706 break;
5707 case ISD::SIGN_EXTEND:
5708 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5709 "Invalid SIGN_EXTEND!");
5710 assert(VT.isVector() == N1.getValueType().isVector() &&
5711 "SIGN_EXTEND result type type should be vector iff the operand "
5712 "type is vector!");
5713 if (N1.getValueType() == VT) return N1; // noop extension
5714 assert((!VT.isVector() || VT.getVectorElementCount() ==
5715 N1.getValueType().getVectorElementCount()) &&
5716 "Vector element count mismatch!");
5717 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5718 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
5719 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0));
5720 if (OpOpcode == ISD::UNDEF)
5721 // sext(undef) = 0, because the top bits will all be the same.
5722 return getConstant(Val: 0, DL, VT);
5723 break;
5724 case ISD::ZERO_EXTEND:
5725 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5726 "Invalid ZERO_EXTEND!");
5727 assert(VT.isVector() == N1.getValueType().isVector() &&
5728 "ZERO_EXTEND result type type should be vector iff the operand "
5729 "type is vector!");
5730 if (N1.getValueType() == VT) return N1; // noop extension
5731 assert((!VT.isVector() || VT.getVectorElementCount() ==
5732 N1.getValueType().getVectorElementCount()) &&
5733 "Vector element count mismatch!");
5734 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5735 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
5736 return getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0));
5737 if (OpOpcode == ISD::UNDEF)
5738 // zext(undef) = 0, because the top bits will be zero.
5739 return getConstant(Val: 0, DL, VT);
5740
5741 // Skip unnecessary zext_inreg pattern:
5742 // (zext (trunc x)) -> x iff the upper bits are known zero.
5743 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
5744 // use to recognise zext_inreg patterns.
5745 if (OpOpcode == ISD::TRUNCATE) {
5746 SDValue OpOp = N1.getOperand(i: 0);
5747 if (OpOp.getValueType() == VT) {
5748 if (OpOp.getOpcode() != ISD::AND) {
5749 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
5750 loBit: N1.getScalarValueSizeInBits());
5751 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
5752 transferDbgValues(From: N1, To: OpOp);
5753 return OpOp;
5754 }
5755 }
5756 }
5757 }
5758 break;
5759 case ISD::ANY_EXTEND:
5760 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5761 "Invalid ANY_EXTEND!");
5762 assert(VT.isVector() == N1.getValueType().isVector() &&
5763 "ANY_EXTEND result type type should be vector iff the operand "
5764 "type is vector!");
5765 if (N1.getValueType() == VT) return N1; // noop extension
5766 assert((!VT.isVector() || VT.getVectorElementCount() ==
5767 N1.getValueType().getVectorElementCount()) &&
5768 "Vector element count mismatch!");
5769 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5770
5771 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5772 OpOpcode == ISD::ANY_EXTEND)
5773 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
5774 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0));
5775 if (OpOpcode == ISD::UNDEF)
5776 return getUNDEF(VT);
5777
5778 // (ext (trunc x)) -> x
5779 if (OpOpcode == ISD::TRUNCATE) {
5780 SDValue OpOp = N1.getOperand(i: 0);
5781 if (OpOp.getValueType() == VT) {
5782 transferDbgValues(From: N1, To: OpOp);
5783 return OpOp;
5784 }
5785 }
5786 break;
5787 case ISD::TRUNCATE:
5788 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5789 "Invalid TRUNCATE!");
5790 assert(VT.isVector() == N1.getValueType().isVector() &&
5791 "TRUNCATE result type type should be vector iff the operand "
5792 "type is vector!");
5793 if (N1.getValueType() == VT) return N1; // noop truncate
5794 assert((!VT.isVector() || VT.getVectorElementCount() ==
5795 N1.getValueType().getVectorElementCount()) &&
5796 "Vector element count mismatch!");
5797 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5798 if (OpOpcode == ISD::TRUNCATE)
5799 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
5800 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5801 OpOpcode == ISD::ANY_EXTEND) {
5802 // If the source is smaller than the dest, we still need an extend.
5803 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
5804 VT: VT.getScalarType()))
5805 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0));
5806 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
5807 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
5808 return N1.getOperand(i: 0);
5809 }
5810 if (OpOpcode == ISD::UNDEF)
5811 return getUNDEF(VT);
5812 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5813 return getVScale(DL, VT,
5814 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
5815 break;
5816 case ISD::ANY_EXTEND_VECTOR_INREG:
5817 case ISD::ZERO_EXTEND_VECTOR_INREG:
5818 case ISD::SIGN_EXTEND_VECTOR_INREG:
5819 assert(VT.isVector() && "This DAG node is restricted to vector types.");
5820 assert(N1.getValueType().bitsLE(VT) &&
5821 "The input must be the same size or smaller than the result.");
5822 assert(VT.getVectorMinNumElements() <
5823 N1.getValueType().getVectorMinNumElements() &&
5824 "The destination vector type must have fewer lanes than the input.");
5825 break;
5826 case ISD::ABS:
5827 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5828 if (OpOpcode == ISD::UNDEF)
5829 return getConstant(Val: 0, DL, VT);
5830 break;
5831 case ISD::BSWAP:
5832 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5833 assert((VT.getScalarSizeInBits() % 16 == 0) &&
5834 "BSWAP types must be a multiple of 16 bits!");
5835 if (OpOpcode == ISD::UNDEF)
5836 return getUNDEF(VT);
5837 // bswap(bswap(X)) -> X.
5838 if (OpOpcode == ISD::BSWAP)
5839 return N1.getOperand(i: 0);
5840 break;
5841 case ISD::BITREVERSE:
5842 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5843 if (OpOpcode == ISD::UNDEF)
5844 return getUNDEF(VT);
5845 break;
5846 case ISD::BITCAST:
5847 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
5848 "Cannot BITCAST between types of different sizes!");
5849 if (VT == N1.getValueType()) return N1; // noop conversion.
5850 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5851 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
5852 if (OpOpcode == ISD::UNDEF)
5853 return getUNDEF(VT);
5854 break;
5855 case ISD::SCALAR_TO_VECTOR:
5856 assert(VT.isVector() && !N1.getValueType().isVector() &&
5857 (VT.getVectorElementType() == N1.getValueType() ||
5858 (VT.getVectorElementType().isInteger() &&
5859 N1.getValueType().isInteger() &&
5860 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
5861 "Illegal SCALAR_TO_VECTOR node!");
5862 if (OpOpcode == ISD::UNDEF)
5863 return getUNDEF(VT);
5864 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5865 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5866 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
5867 N1.getConstantOperandVal(i: 1) == 0 &&
5868 N1.getOperand(i: 0).getValueType() == VT)
5869 return N1.getOperand(i: 0);
5870 break;
5871 case ISD::FNEG:
5872 // Negation of an unknown bag of bits is still completely undefined.
5873 if (OpOpcode == ISD::UNDEF)
5874 return getUNDEF(VT);
5875
5876 if (OpOpcode == ISD::FNEG) // --X -> X
5877 return N1.getOperand(i: 0);
5878 break;
5879 case ISD::FABS:
5880 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5881 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
5882 break;
5883 case ISD::VSCALE:
5884 assert(VT == N1.getValueType() && "Unexpected VT!");
5885 break;
5886 case ISD::CTPOP:
5887 if (N1.getValueType().getScalarType() == MVT::i1)
5888 return N1;
5889 break;
5890 case ISD::CTLZ:
5891 case ISD::CTTZ:
5892 if (N1.getValueType().getScalarType() == MVT::i1)
5893 return getNOT(DL, Val: N1, VT: N1.getValueType());
5894 break;
5895 case ISD::VECREDUCE_ADD:
5896 if (N1.getValueType().getScalarType() == MVT::i1)
5897 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
5898 break;
5899 case ISD::VECREDUCE_SMIN:
5900 case ISD::VECREDUCE_UMAX:
5901 if (N1.getValueType().getScalarType() == MVT::i1)
5902 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
5903 break;
5904 case ISD::VECREDUCE_SMAX:
5905 case ISD::VECREDUCE_UMIN:
5906 if (N1.getValueType().getScalarType() == MVT::i1)
5907 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
5908 break;
5909 }
5910
5911 SDNode *N;
5912 SDVTList VTs = getVTList(VT);
5913 SDValue Ops[] = {N1};
5914 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
5915 FoldingSetNodeID ID;
5916 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
5917 void *IP = nullptr;
5918 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
5919 E->intersectFlagsWith(Flags);
5920 return SDValue(E, 0);
5921 }
5922
5923 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
5924 N->setFlags(Flags);
5925 createOperands(Node: N, Vals: Ops);
5926 CSEMap.InsertNode(N, InsertPos: IP);
5927 } else {
5928 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
5929 createOperands(Node: N, Vals: Ops);
5930 }
5931
5932 InsertNode(N);
5933 SDValue V = SDValue(N, 0);
5934 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
5935 return V;
5936}
5937
5938static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5939 const APInt &C2) {
5940 switch (Opcode) {
5941 case ISD::ADD: return C1 + C2;
5942 case ISD::SUB: return C1 - C2;
5943 case ISD::MUL: return C1 * C2;
5944 case ISD::AND: return C1 & C2;
5945 case ISD::OR: return C1 | C2;
5946 case ISD::XOR: return C1 ^ C2;
5947 case ISD::SHL: return C1 << C2;
5948 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
5949 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
5950 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
5951 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
5952 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
5953 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
5954 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
5955 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
5956 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
5957 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
5958 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
5959 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
5960 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
5961 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
5962 case ISD::UDIV:
5963 if (!C2.getBoolValue())
5964 break;
5965 return C1.udiv(RHS: C2);
5966 case ISD::UREM:
5967 if (!C2.getBoolValue())
5968 break;
5969 return C1.urem(RHS: C2);
5970 case ISD::SDIV:
5971 if (!C2.getBoolValue())
5972 break;
5973 return C1.sdiv(RHS: C2);
5974 case ISD::SREM:
5975 if (!C2.getBoolValue())
5976 break;
5977 return C1.srem(RHS: C2);
5978 case ISD::MULHS: {
5979 unsigned FullWidth = C1.getBitWidth() * 2;
5980 APInt C1Ext = C1.sext(width: FullWidth);
5981 APInt C2Ext = C2.sext(width: FullWidth);
5982 return (C1Ext * C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: C1.getBitWidth());
5983 }
5984 case ISD::MULHU: {
5985 unsigned FullWidth = C1.getBitWidth() * 2;
5986 APInt C1Ext = C1.zext(width: FullWidth);
5987 APInt C2Ext = C2.zext(width: FullWidth);
5988 return (C1Ext * C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: C1.getBitWidth());
5989 }
5990 case ISD::AVGFLOORS: {
5991 unsigned FullWidth = C1.getBitWidth() + 1;
5992 APInt C1Ext = C1.sext(width: FullWidth);
5993 APInt C2Ext = C2.sext(width: FullWidth);
5994 return (C1Ext + C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: 1);
5995 }
5996 case ISD::AVGFLOORU: {
5997 unsigned FullWidth = C1.getBitWidth() + 1;
5998 APInt C1Ext = C1.zext(width: FullWidth);
5999 APInt C2Ext = C2.zext(width: FullWidth);
6000 return (C1Ext + C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: 1);
6001 }
6002 case ISD::AVGCEILS: {
6003 unsigned FullWidth = C1.getBitWidth() + 1;
6004 APInt C1Ext = C1.sext(width: FullWidth);
6005 APInt C2Ext = C2.sext(width: FullWidth);
6006 return (C1Ext + C2Ext + 1).extractBits(numBits: C1.getBitWidth(), bitPosition: 1);
6007 }
6008 case ISD::AVGCEILU: {
6009 unsigned FullWidth = C1.getBitWidth() + 1;
6010 APInt C1Ext = C1.zext(width: FullWidth);
6011 APInt C2Ext = C2.zext(width: FullWidth);
6012 return (C1Ext + C2Ext + 1).extractBits(numBits: C1.getBitWidth(), bitPosition: 1);
6013 }
6014 case ISD::ABDS:
6015 return APIntOps::smax(A: C1, B: C2) - APIntOps::smin(A: C1, B: C2);
6016 case ISD::ABDU:
6017 return APIntOps::umax(A: C1, B: C2) - APIntOps::umin(A: C1, B: C2);
6018 }
6019 return std::nullopt;
6020}
6021
6022// Handle constant folding with UNDEF.
6023// TODO: Handle more cases.
6024static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6025 bool IsUndef1, const APInt &C2,
6026 bool IsUndef2) {
6027 if (!(IsUndef1 || IsUndef2))
6028 return FoldValue(Opcode, C1, C2);
6029
6030 // Fold and(x, undef) -> 0
6031 // Fold mul(x, undef) -> 0
6032 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6033 return APInt::getZero(numBits: C1.getBitWidth());
6034
6035 return std::nullopt;
6036}
6037
6038SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
6039 const GlobalAddressSDNode *GA,
6040 const SDNode *N2) {
6041 if (GA->getOpcode() != ISD::GlobalAddress)
6042 return SDValue();
6043 if (!TLI->isOffsetFoldingLegal(GA))
6044 return SDValue();
6045 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
6046 if (!C2)
6047 return SDValue();
6048 int64_t Offset = C2->getSExtValue();
6049 switch (Opcode) {
6050 case ISD::ADD: break;
6051 case ISD::SUB: Offset = -uint64_t(Offset); break;
6052 default: return SDValue();
6053 }
6054 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
6055 Offset: GA->getOffset() + uint64_t(Offset));
6056}
6057
6058bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6059 switch (Opcode) {
6060 case ISD::SDIV:
6061 case ISD::UDIV:
6062 case ISD::SREM:
6063 case ISD::UREM: {
6064 // If a divisor is zero/undef or any element of a divisor vector is
6065 // zero/undef, the whole op is undef.
6066 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6067 SDValue Divisor = Ops[1];
6068 if (Divisor.isUndef() || isNullConstant(V: Divisor))
6069 return true;
6070
6071 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
6072 llvm::any_of(Range: Divisor->op_values(),
6073 P: [](SDValue V) { return V.isUndef() ||
6074 isNullConstant(V); });
6075 // TODO: Handle signed overflow.
6076 }
6077 // TODO: Handle oversized shifts.
6078 default:
6079 return false;
6080 }
6081}
6082
6083SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
6084 EVT VT, ArrayRef<SDValue> Ops) {
6085 // If the opcode is a target-specific ISD node, there's nothing we can
6086 // do here and the operand rules may not line up with the below, so
6087 // bail early.
6088 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6089 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6090 // foldCONCAT_VECTORS in getNode before this is called.
6091 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6092 return SDValue();
6093
6094 unsigned NumOps = Ops.size();
6095 if (NumOps == 0)
6096 return SDValue();
6097
6098 if (isUndef(Opcode, Ops))
6099 return getUNDEF(VT);
6100
6101 // Handle unary special cases.
6102 if (NumOps == 1) {
6103 SDValue N1 = Ops[0];
6104
6105 // Constant fold unary operations with an integer constant operand. Even
6106 // opaque constant will be folded, because the folding of unary operations
6107 // doesn't create new constants with different values. Nevertheless, the
6108 // opaque flag is preserved during folding to prevent future folding with
6109 // other constants.
6110 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
6111 const APInt &Val = C->getAPIntValue();
6112 switch (Opcode) {
6113 case ISD::SIGN_EXTEND:
6114 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6115 isT: C->isTargetOpcode(), isO: C->isOpaque());
6116 case ISD::TRUNCATE:
6117 if (C->isOpaque())
6118 break;
6119 [[fallthrough]];
6120 case ISD::ZERO_EXTEND:
6121 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6122 isT: C->isTargetOpcode(), isO: C->isOpaque());
6123 case ISD::ANY_EXTEND:
6124 // Some targets like RISCV prefer to sign extend some types.
6125 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
6126 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6127 isT: C->isTargetOpcode(), isO: C->isOpaque());
6128 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6129 isT: C->isTargetOpcode(), isO: C->isOpaque());
6130 case ISD::ABS:
6131 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
6132 isO: C->isOpaque());
6133 case ISD::BITREVERSE:
6134 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
6135 isO: C->isOpaque());
6136 case ISD::BSWAP:
6137 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
6138 isO: C->isOpaque());
6139 case ISD::CTPOP:
6140 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
6141 isO: C->isOpaque());
6142 case ISD::CTLZ:
6143 case ISD::CTLZ_ZERO_UNDEF:
6144 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
6145 isO: C->isOpaque());
6146 case ISD::CTTZ:
6147 case ISD::CTTZ_ZERO_UNDEF:
6148 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
6149 isO: C->isOpaque());
6150 case ISD::UINT_TO_FP:
6151 case ISD::SINT_TO_FP: {
6152 APFloat apf(EVTToAPFloatSemantics(VT),
6153 APInt::getZero(numBits: VT.getSizeInBits()));
6154 (void)apf.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
6155 RM: APFloat::rmNearestTiesToEven);
6156 return getConstantFP(V: apf, DL, VT);
6157 }
6158 case ISD::FP16_TO_FP:
6159 case ISD::BF16_TO_FP: {
6160 bool Ignored;
6161 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6162 : APFloat::BFloat(),
6163 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
6164
6165 // This can return overflow, underflow, or inexact; we don't care.
6166 // FIXME need to be more flexible about rounding mode.
6167 (void)FPV.convert(ToSemantics: EVTToAPFloatSemantics(VT),
6168 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
6169 return getConstantFP(V: FPV, DL, VT);
6170 }
6171 case ISD::STEP_VECTOR:
6172 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
6173 return V;
6174 break;
6175 case ISD::BITCAST:
6176 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
6177 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6178 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
6179 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6180 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
6181 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6182 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
6183 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
6184 break;
6185 }
6186 }
6187
6188 // Constant fold unary operations with a floating point constant operand.
6189 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
6190 APFloat V = C->getValueAPF(); // make copy
6191 switch (Opcode) {
6192 case ISD::FNEG:
6193 V.changeSign();
6194 return getConstantFP(V, DL, VT);
6195 case ISD::FABS:
6196 V.clearSign();
6197 return getConstantFP(V, DL, VT);
6198 case ISD::FCEIL: {
6199 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
6200 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6201 return getConstantFP(V, DL, VT);
6202 return SDValue();
6203 }
6204 case ISD::FTRUNC: {
6205 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
6206 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6207 return getConstantFP(V, DL, VT);
6208 return SDValue();
6209 }
6210 case ISD::FFLOOR: {
6211 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
6212 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6213 return getConstantFP(V, DL, VT);
6214 return SDValue();
6215 }
6216 case ISD::FP_EXTEND: {
6217 bool ignored;
6218 // This can return overflow, underflow, or inexact; we don't care.
6219 // FIXME need to be more flexible about rounding mode.
6220 (void)V.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven,
6221 losesInfo: &ignored);
6222 return getConstantFP(V, DL, VT);
6223 }
6224 case ISD::FP_TO_SINT:
6225 case ISD::FP_TO_UINT: {
6226 bool ignored;
6227 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6228 // FIXME need to be more flexible about rounding mode.
6229 APFloat::opStatus s =
6230 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
6231 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6232 break;
6233 return getConstant(Val: IntVal, DL, VT);
6234 }
6235 case ISD::FP_TO_FP16:
6236 case ISD::FP_TO_BF16: {
6237 bool Ignored;
6238 // This can return overflow, underflow, or inexact; we don't care.
6239 // FIXME need to be more flexible about rounding mode.
6240 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6241 : APFloat::BFloat(),
6242 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
6243 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6244 }
6245 case ISD::BITCAST:
6246 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
6247 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6248 VT);
6249 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
6250 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6251 VT);
6252 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
6253 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6254 VT);
6255 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
6256 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6257 break;
6258 }
6259 }
6260
6261 // Early-out if we failed to constant fold a bitcast.
6262 if (Opcode == ISD::BITCAST)
6263 return SDValue();
6264 }
6265
6266 // Handle binops special cases.
6267 if (NumOps == 2) {
6268 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6269 return CFP;
6270
6271 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
6272 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
6273 if (C1->isOpaque() || C2->isOpaque())
6274 return SDValue();
6275
6276 std::optional<APInt> FoldAttempt =
6277 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
6278 if (!FoldAttempt)
6279 return SDValue();
6280
6281 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
6282 assert((!Folded || !VT.isVector()) &&
6283 "Can't fold vectors ops with scalar operands");
6284 return Folded;
6285 }
6286 }
6287
6288 // fold (add Sym, c) -> Sym+c
6289 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
6290 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
6291 if (TLI->isCommutativeBinOp(Opcode))
6292 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
6293 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
6294 }
6295
6296 // This is for vector folding only from here on.
6297 if (!VT.isVector())
6298 return SDValue();
6299
6300 ElementCount NumElts = VT.getVectorElementCount();
6301
6302 // See if we can fold through bitcasted integer ops.
6303 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6304 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6305 Ops[0].getOpcode() == ISD::BITCAST &&
6306 Ops[1].getOpcode() == ISD::BITCAST) {
6307 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
6308 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
6309 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
6310 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
6311 EVT BVVT = N1.getValueType();
6312 if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6313 bool IsLE = getDataLayout().isLittleEndian();
6314 unsigned EltBits = VT.getScalarSizeInBits();
6315 SmallVector<APInt> RawBits1, RawBits2;
6316 BitVector UndefElts1, UndefElts2;
6317 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
6318 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
6319 SmallVector<APInt> RawBits;
6320 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6321 std::optional<APInt> Fold = FoldValueWithUndef(
6322 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
6323 if (!Fold)
6324 break;
6325 RawBits.push_back(Elt: *Fold);
6326 }
6327 if (RawBits.size() == NumElts.getFixedValue()) {
6328 // We have constant folded, but we need to cast this again back to
6329 // the original (possibly legalized) type.
6330 SmallVector<APInt> DstBits;
6331 BitVector DstUndefs;
6332 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
6333 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
6334 SrcUndefElements: BitVector(RawBits.size(), false));
6335 EVT BVEltVT = BV1->getOperand(Num: 0).getValueType();
6336 unsigned BVEltBits = BVEltVT.getSizeInBits();
6337 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
6338 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6339 if (DstUndefs[I])
6340 continue;
6341 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
6342 }
6343 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
6344 }
6345 }
6346 }
6347 }
6348
6349 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6350 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6351 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6352 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6353 APInt RHSVal;
6354 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
6355 APInt NewStep = Opcode == ISD::MUL
6356 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
6357 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
6358 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
6359 }
6360 }
6361
6362 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6363 return !Op.getValueType().isVector() ||
6364 Op.getValueType().getVectorElementCount() == NumElts;
6365 };
6366
6367 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6368 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6369 Op.getOpcode() == ISD::BUILD_VECTOR ||
6370 Op.getOpcode() == ISD::SPLAT_VECTOR;
6371 };
6372
6373 // All operands must be vector types with the same number of elements as
6374 // the result type and must be either UNDEF or a build/splat vector
6375 // or UNDEF scalars.
6376 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
6377 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
6378 return SDValue();
6379
6380 // If we are comparing vectors, then the result needs to be a i1 boolean that
6381 // is then extended back to the legal result type depending on how booleans
6382 // are represented.
6383 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6384 ISD::NodeType ExtendCode =
6385 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
6386 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
6387 : ISD::SIGN_EXTEND;
6388
6389 // Find legal integer scalar type for constant promotion and
6390 // ensure that its scalar size is at least as large as source.
6391 EVT LegalSVT = VT.getScalarType();
6392 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6393 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
6394 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
6395 return SDValue();
6396 }
6397
6398 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6399 // only have one operand to check. For fixed-length vector types we may have
6400 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
6401 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6402
6403 // Constant fold each scalar lane separately.
6404 SmallVector<SDValue, 4> ScalarResults;
6405 for (unsigned I = 0; I != NumVectorElts; I++) {
6406 SmallVector<SDValue, 4> ScalarOps;
6407 for (SDValue Op : Ops) {
6408 EVT InSVT = Op.getValueType().getScalarType();
6409 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6410 Op.getOpcode() != ISD::SPLAT_VECTOR) {
6411 if (Op.isUndef())
6412 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
6413 else
6414 ScalarOps.push_back(Elt: Op);
6415 continue;
6416 }
6417
6418 SDValue ScalarOp =
6419 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6420 EVT ScalarVT = ScalarOp.getValueType();
6421
6422 // Build vector (integer) scalar operands may need implicit
6423 // truncation - do this before constant folding.
6424 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
6425 // Don't create illegally-typed nodes unless they're constants or undef
6426 // - if we fail to constant fold we can't guarantee the (dead) nodes
6427 // we're creating will be cleaned up before being visited for
6428 // legalization.
6429 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6430 !isa<ConstantSDNode>(Val: ScalarOp) &&
6431 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
6432 TargetLowering::TypeLegal)
6433 return SDValue();
6434 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
6435 }
6436
6437 ScalarOps.push_back(Elt: ScalarOp);
6438 }
6439
6440 // Constant fold the scalar operands.
6441 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps);
6442
6443 // Legalize the (integer) scalar constant if necessary.
6444 if (LegalSVT != SVT)
6445 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
6446
6447 // Scalar folding only succeeded if the result is a constant or UNDEF.
6448 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6449 ScalarResult.getOpcode() != ISD::ConstantFP)
6450 return SDValue();
6451 ScalarResults.push_back(Elt: ScalarResult);
6452 }
6453
6454 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
6455 : getBuildVector(VT, DL, Ops: ScalarResults);
6456 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
6457 return V;
6458}
6459
6460SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
6461 EVT VT, ArrayRef<SDValue> Ops) {
6462 // TODO: Add support for unary/ternary fp opcodes.
6463 if (Ops.size() != 2)
6464 return SDValue();
6465
6466 // TODO: We don't do any constant folding for strict FP opcodes here, but we
6467 // should. That will require dealing with a potentially non-default
6468 // rounding mode, checking the "opStatus" return value from the APFloat
6469 // math calculations, and possibly other variations.
6470 SDValue N1 = Ops[0];
6471 SDValue N2 = Ops[1];
6472 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
6473 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
6474 if (N1CFP && N2CFP) {
6475 APFloat C1 = N1CFP->getValueAPF(); // make copy
6476 const APFloat &C2 = N2CFP->getValueAPF();
6477 switch (Opcode) {
6478 case ISD::FADD:
6479 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6480 return getConstantFP(V: C1, DL, VT);
6481 case ISD::FSUB:
6482 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6483 return getConstantFP(V: C1, DL, VT);
6484 case ISD::FMUL:
6485 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6486 return getConstantFP(V: C1, DL, VT);
6487 case ISD::FDIV:
6488 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
6489 return getConstantFP(V: C1, DL, VT);
6490 case ISD::FREM:
6491 C1.mod(RHS: C2);
6492 return getConstantFP(V: C1, DL, VT);
6493 case ISD::FCOPYSIGN:
6494 C1.copySign(RHS: C2);
6495 return getConstantFP(V: C1, DL, VT);
6496 case ISD::FMINNUM:
6497 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
6498 case ISD::FMAXNUM:
6499 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
6500 case ISD::FMINIMUM:
6501 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
6502 case ISD::FMAXIMUM:
6503 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
6504 default: break;
6505 }
6506 }
6507 if (N1CFP && Opcode == ISD::FP_ROUND) {
6508 APFloat C1 = N1CFP->getValueAPF(); // make copy
6509 bool Unused;
6510 // This can return overflow, underflow, or inexact; we don't care.
6511 // FIXME need to be more flexible about rounding mode.
6512 (void) C1.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven,
6513 losesInfo: &Unused);
6514 return getConstantFP(V: C1, DL, VT);
6515 }
6516
6517 switch (Opcode) {
6518 case ISD::FSUB:
6519 // -0.0 - undef --> undef (consistent with "fneg undef")
6520 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
6521 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6522 return getUNDEF(VT);
6523 [[fallthrough]];
6524
6525 case ISD::FADD:
6526 case ISD::FMUL:
6527 case ISD::FDIV:
6528 case ISD::FREM:
6529 // If both operands are undef, the result is undef. If 1 operand is undef,
6530 // the result is NaN. This should match the behavior of the IR optimizer.
6531 if (N1.isUndef() && N2.isUndef())
6532 return getUNDEF(VT);
6533 if (N1.isUndef() || N2.isUndef())
6534 return getConstantFP(V: APFloat::getNaN(Sem: EVTToAPFloatSemantics(VT)), DL, VT);
6535 }
6536 return SDValue();
6537}
6538
6539SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
6540 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6541
6542 // There's no need to assert on a byte-aligned pointer. All pointers are at
6543 // least byte aligned.
6544 if (A == Align(1))
6545 return Val;
6546
6547 FoldingSetNodeID ID;
6548 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: getVTList(VT: Val.getValueType()), OpList: {Val});
6549 ID.AddInteger(I: A.value());
6550
6551 void *IP = nullptr;
6552 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6553 return SDValue(E, 0);
6554
6555 auto *N = newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
6556 Args: Val.getValueType(), Args&: A);
6557 createOperands(Node: N, Vals: {Val});
6558
6559 CSEMap.InsertNode(N, InsertPos: IP);
6560 InsertNode(N);
6561
6562 SDValue V(N, 0);
6563 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6564 return V;
6565}
6566
6567SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6568 SDValue N1, SDValue N2) {
6569 SDNodeFlags Flags;
6570 if (Inserter)
6571 Flags = Inserter->getFlags();
6572 return getNode(Opcode, DL, VT, N1, N2, Flags);
6573}
6574
6575void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
6576 SDValue &N2) const {
6577 if (!TLI->isCommutativeBinOp(Opcode))
6578 return;
6579
6580 // Canonicalize:
6581 // binop(const, nonconst) -> binop(nonconst, const)
6582 SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
6583 SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
6584 SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
6585 SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
6586 if ((N1C && !N2C) || (N1CFP && !N2CFP))
6587 std::swap(a&: N1, b&: N2);
6588
6589 // Canonicalize:
6590 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6591 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6592 N2.getOpcode() == ISD::STEP_VECTOR)
6593 std::swap(a&: N1, b&: N2);
6594}
6595
6596SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6597 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6598 assert(N1.getOpcode() != ISD::DELETED_NODE &&
6599 N2.getOpcode() != ISD::DELETED_NODE &&
6600 "Operand is DELETED_NODE!");
6601
6602 canonicalizeCommutativeBinop(Opcode, N1, N2);
6603
6604 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
6605 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
6606
6607 // Don't allow undefs in vector splats - we might be returning N2 when folding
6608 // to zero etc.
6609 ConstantSDNode *N2CV =
6610 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6611
6612 switch (Opcode) {
6613 default: break;
6614 case ISD::TokenFactor:
6615 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6616 N2.getValueType() == MVT::Other && "Invalid token factor!");
6617 // Fold trivial token factors.
6618 if (N1.getOpcode() == ISD::EntryToken) return N2;
6619 if (N2.getOpcode() == ISD::EntryToken) return N1;
6620 if (N1 == N2) return N1;
6621 break;
6622 case ISD::BUILD_VECTOR: {
6623 // Attempt to simplify BUILD_VECTOR.
6624 SDValue Ops[] = {N1, N2};
6625 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6626 return V;
6627 break;
6628 }
6629 case ISD::CONCAT_VECTORS: {
6630 SDValue Ops[] = {N1, N2};
6631 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
6632 return V;
6633 break;
6634 }
6635 case ISD::AND:
6636 assert(VT.isInteger() && "This operator does not apply to FP types!");
6637 assert(N1.getValueType() == N2.getValueType() &&
6638 N1.getValueType() == VT && "Binary operator types must match!");
6639 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
6640 // worth handling here.
6641 if (N2CV && N2CV->isZero())
6642 return N2;
6643 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6644 return N1;
6645 break;
6646 case ISD::OR:
6647 case ISD::XOR:
6648 case ISD::ADD:
6649 case ISD::SUB:
6650 assert(VT.isInteger() && "This operator does not apply to FP types!");
6651 assert(N1.getValueType() == N2.getValueType() &&
6652 N1.getValueType() == VT && "Binary operator types must match!");
6653 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
6654 // it's worth handling here.
6655 if (N2CV && N2CV->isZero())
6656 return N1;
6657 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6658 VT.getVectorElementType() == MVT::i1)
6659 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
6660 break;
6661 case ISD::MUL:
6662 assert(VT.isInteger() && "This operator does not apply to FP types!");
6663 assert(N1.getValueType() == N2.getValueType() &&
6664 N1.getValueType() == VT && "Binary operator types must match!");
6665 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6666 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
6667 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6668 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
6669 const APInt &N2CImm = N2C->getAPIntValue();
6670 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
6671 }
6672 break;
6673 case ISD::UDIV:
6674 case ISD::UREM:
6675 case ISD::MULHU:
6676 case ISD::MULHS:
6677 case ISD::SDIV:
6678 case ISD::SREM:
6679 case ISD::SADDSAT:
6680 case ISD::SSUBSAT:
6681 case ISD::UADDSAT:
6682 case ISD::USUBSAT:
6683 assert(VT.isInteger() && "This operator does not apply to FP types!");
6684 assert(N1.getValueType() == N2.getValueType() &&
6685 N1.getValueType() == VT && "Binary operator types must match!");
6686 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6687 // fold (add_sat x, y) -> (or x, y) for bool types.
6688 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6689 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
6690 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
6691 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6692 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
6693 }
6694 break;
6695 case ISD::ABDS:
6696 case ISD::ABDU:
6697 assert(VT.isInteger() && "This operator does not apply to FP types!");
6698 assert(N1.getValueType() == N2.getValueType() &&
6699 N1.getValueType() == VT && "Binary operator types must match!");
6700 break;
6701 case ISD::SMIN:
6702 case ISD::UMAX:
6703 assert(VT.isInteger() && "This operator does not apply to FP types!");
6704 assert(N1.getValueType() == N2.getValueType() &&
6705 N1.getValueType() == VT && "Binary operator types must match!");
6706 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6707 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
6708 break;
6709 case ISD::SMAX:
6710 case ISD::UMIN:
6711 assert(VT.isInteger() && "This operator does not apply to FP types!");
6712 assert(N1.getValueType() == N2.getValueType() &&
6713 N1.getValueType() == VT && "Binary operator types must match!");
6714 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6715 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
6716 break;
6717 case ISD::FADD:
6718 case ISD::FSUB:
6719 case ISD::FMUL:
6720 case ISD::FDIV:
6721 case ISD::FREM:
6722 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6723 assert(N1.getValueType() == N2.getValueType() &&
6724 N1.getValueType() == VT && "Binary operator types must match!");
6725 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
6726 return V;
6727 break;
6728 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
6729 assert(N1.getValueType() == VT &&
6730 N1.getValueType().isFloatingPoint() &&
6731 N2.getValueType().isFloatingPoint() &&
6732 "Invalid FCOPYSIGN!");
6733 break;
6734 case ISD::SHL:
6735 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6736 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
6737 const APInt &ShiftImm = N2C->getAPIntValue();
6738 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
6739 }
6740 [[fallthrough]];
6741 case ISD::SRA:
6742 case ISD::SRL:
6743 if (SDValue V = simplifyShift(X: N1, Y: N2))
6744 return V;
6745 [[fallthrough]];
6746 case ISD::ROTL:
6747 case ISD::ROTR:
6748 assert(VT == N1.getValueType() &&
6749 "Shift operators return type must be the same as their first arg");
6750 assert(VT.isInteger() && N2.getValueType().isInteger() &&
6751 "Shifts only work on integers");
6752 assert((!VT.isVector() || VT == N2.getValueType()) &&
6753 "Vector shift amounts must be in the same as their first arg");
6754 // Verify that the shift amount VT is big enough to hold valid shift
6755 // amounts. This catches things like trying to shift an i1024 value by an
6756 // i8, which is easy to fall into in generic code that uses
6757 // TLI.getShiftAmount().
6758 assert(N2.getValueType().getScalarSizeInBits() >=
6759 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
6760 "Invalid use of small shift amount with oversized value!");
6761
6762 // Always fold shifts of i1 values so the code generator doesn't need to
6763 // handle them. Since we know the size of the shift has to be less than the
6764 // size of the value, the shift/rotate count is guaranteed to be zero.
6765 if (VT == MVT::i1)
6766 return N1;
6767 if (N2CV && N2CV->isZero())
6768 return N1;
6769 break;
6770 case ISD::FP_ROUND:
6771 assert(VT.isFloatingPoint() &&
6772 N1.getValueType().isFloatingPoint() &&
6773 VT.bitsLE(N1.getValueType()) &&
6774 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6775 "Invalid FP_ROUND!");
6776 if (N1.getValueType() == VT) return N1; // noop conversion.
6777 break;
6778 case ISD::AssertSext:
6779 case ISD::AssertZext: {
6780 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
6781 assert(VT == N1.getValueType() && "Not an inreg extend!");
6782 assert(VT.isInteger() && EVT.isInteger() &&
6783 "Cannot *_EXTEND_INREG FP types");
6784 assert(!EVT.isVector() &&
6785 "AssertSExt/AssertZExt type should be the vector element type "
6786 "rather than the vector type!");
6787 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6788 if (VT.getScalarType() == EVT) return N1; // noop assertion.
6789 break;
6790 }
6791 case ISD::SIGN_EXTEND_INREG: {
6792 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
6793 assert(VT == N1.getValueType() && "Not an inreg extend!");
6794 assert(VT.isInteger() && EVT.isInteger() &&
6795 "Cannot *_EXTEND_INREG FP types");
6796 assert(EVT.isVector() == VT.isVector() &&
6797 "SIGN_EXTEND_INREG type should be vector iff the operand "
6798 "type is vector!");
6799 assert((!EVT.isVector() ||
6800 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
6801 "Vector element counts must match in SIGN_EXTEND_INREG");
6802 assert(EVT.bitsLE(VT) && "Not extending!");
6803 if (EVT == VT) return N1; // Not actually extending
6804
6805 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6806 unsigned FromBits = EVT.getScalarSizeInBits();
6807 Val <<= Val.getBitWidth() - FromBits;
6808 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
6809 return getConstant(Val, DL, VT: ConstantVT);
6810 };
6811
6812 if (N1C) {
6813 const APInt &Val = N1C->getAPIntValue();
6814 return SignExtendInReg(Val, VT);
6815 }
6816
6817 if (ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
6818 SmallVector<SDValue, 8> Ops;
6819 llvm::EVT OpVT = N1.getOperand(i: 0).getValueType();
6820 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6821 SDValue Op = N1.getOperand(i);
6822 if (Op.isUndef()) {
6823 Ops.push_back(Elt: getUNDEF(VT: OpVT));
6824 continue;
6825 }
6826 ConstantSDNode *C = cast<ConstantSDNode>(Val&: Op);
6827 APInt Val = C->getAPIntValue();
6828 Ops.push_back(Elt: SignExtendInReg(Val, OpVT));
6829 }
6830 return getBuildVector(VT, DL, Ops);
6831 }
6832
6833 if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6834 isa<ConstantSDNode>(Val: N1.getOperand(i: 0)))
6835 return getNode(
6836 Opcode: ISD::SPLAT_VECTOR, DL, VT,
6837 N1: SignExtendInReg(N1.getConstantOperandAPInt(i: 0),
6838 N1.getOperand(i: 0).getValueType()));
6839 break;
6840 }
6841 case ISD::FP_TO_SINT_SAT:
6842 case ISD::FP_TO_UINT_SAT: {
6843 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6844 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6845 assert(N1.getValueType().isVector() == VT.isVector() &&
6846 "FP_TO_*INT_SAT type should be vector iff the operand type is "
6847 "vector!");
6848 assert((!VT.isVector() || VT.getVectorElementCount() ==
6849 N1.getValueType().getVectorElementCount()) &&
6850 "Vector element counts must match in FP_TO_*INT_SAT");
6851 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6852 "Type to saturate to must be a scalar.");
6853 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6854 "Not extending!");
6855 break;
6856 }
6857 case ISD::EXTRACT_VECTOR_ELT:
6858 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
6859 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6860 element type of the vector.");
6861
6862 // Extract from an undefined value or using an undefined index is undefined.
6863 if (N1.isUndef() || N2.isUndef())
6864 return getUNDEF(VT);
6865
6866 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6867 // vectors. For scalable vectors we will provide appropriate support for
6868 // dealing with arbitrary indices.
6869 if (N2C && N1.getValueType().isFixedLengthVector() &&
6870 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
6871 return getUNDEF(VT);
6872
6873 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6874 // expanding copies of large vectors from registers. This only works for
6875 // fixed length vectors, since we need to know the exact number of
6876 // elements.
6877 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
6878 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
6879 unsigned Factor =
6880 N1.getOperand(i: 0).getValueType().getVectorNumElements();
6881 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT,
6882 N1: N1.getOperand(i: N2C->getZExtValue() / Factor),
6883 N2: getVectorIdxConstant(Val: N2C->getZExtValue() % Factor, DL));
6884 }
6885
6886 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
6887 // lowering is expanding large vector constants.
6888 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
6889 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
6890 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
6891 N1.getValueType().isFixedLengthVector()) &&
6892 "BUILD_VECTOR used for scalable vectors");
6893 unsigned Index =
6894 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
6895 SDValue Elt = N1.getOperand(i: Index);
6896
6897 if (VT != Elt.getValueType())
6898 // If the vector element type is not legal, the BUILD_VECTOR operands
6899 // are promoted and implicitly truncated, and the result implicitly
6900 // extended. Make that explicit here.
6901 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
6902
6903 return Elt;
6904 }
6905
6906 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
6907 // operations are lowered to scalars.
6908 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
6909 // If the indices are the same, return the inserted element else
6910 // if the indices are known different, extract the element from
6911 // the original vector.
6912 SDValue N1Op2 = N1.getOperand(i: 2);
6913 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
6914
6915 if (N1Op2C && N2C) {
6916 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
6917 if (VT == N1.getOperand(i: 1).getValueType())
6918 return N1.getOperand(i: 1);
6919 if (VT.isFloatingPoint()) {
6920 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
6921 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
6922 }
6923 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
6924 }
6925 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
6926 }
6927 }
6928
6929 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
6930 // when vector types are scalarized and v1iX is legal.
6931 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
6932 // Here we are completely ignoring the extract element index (N2),
6933 // which is fine for fixed width vectors, since any index other than 0
6934 // is undefined anyway. However, this cannot be ignored for scalable
6935 // vectors - in theory we could support this, but we don't want to do this
6936 // without a profitability check.
6937 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6938 N1.getValueType().isFixedLengthVector() &&
6939 N1.getValueType().getVectorNumElements() == 1) {
6940 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
6941 N2: N1.getOperand(i: 1));
6942 }
6943 break;
6944 case ISD::EXTRACT_ELEMENT:
6945 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
6946 assert(!N1.getValueType().isVector() && !VT.isVector() &&
6947 (N1.getValueType().isInteger() == VT.isInteger()) &&
6948 N1.getValueType() != VT &&
6949 "Wrong types for EXTRACT_ELEMENT!");
6950
6951 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
6952 // 64-bit integers into 32-bit parts. Instead of building the extract of
6953 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
6954 if (N1.getOpcode() == ISD::BUILD_PAIR)
6955 return N1.getOperand(i: N2C->getZExtValue());
6956
6957 // EXTRACT_ELEMENT of a constant int is also very common.
6958 if (N1C) {
6959 unsigned ElementSize = VT.getSizeInBits();
6960 unsigned Shift = ElementSize * N2C->getZExtValue();
6961 const APInt &Val = N1C->getAPIntValue();
6962 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
6963 }
6964 break;
6965 case ISD::EXTRACT_SUBVECTOR: {
6966 EVT N1VT = N1.getValueType();
6967 assert(VT.isVector() && N1VT.isVector() &&
6968 "Extract subvector VTs must be vectors!");
6969 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
6970 "Extract subvector VTs must have the same element type!");
6971 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
6972 "Cannot extract a scalable vector from a fixed length vector!");
6973 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6974 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
6975 "Extract subvector must be from larger vector to smaller vector!");
6976 assert(N2C && "Extract subvector index must be a constant");
6977 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6978 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
6979 N1VT.getVectorMinNumElements()) &&
6980 "Extract subvector overflow!");
6981 assert(N2C->getAPIntValue().getBitWidth() ==
6982 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
6983 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
6984
6985 // Trivial extraction.
6986 if (VT == N1VT)
6987 return N1;
6988
6989 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
6990 if (N1.isUndef())
6991 return getUNDEF(VT);
6992
6993 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
6994 // the concat have the same type as the extract.
6995 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
6996 VT == N1.getOperand(i: 0).getValueType()) {
6997 unsigned Factor = VT.getVectorMinNumElements();
6998 return N1.getOperand(i: N2C->getZExtValue() / Factor);
6999 }
7000
7001 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7002 // during shuffle legalization.
7003 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
7004 VT == N1.getOperand(i: 1).getValueType())
7005 return N1.getOperand(i: 1);
7006 break;
7007 }
7008 }
7009
7010 // Perform trivial constant folding.
7011 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}))
7012 return SV;
7013
7014 // Canonicalize an UNDEF to the RHS, even over a constant.
7015 if (N1.isUndef()) {
7016 if (TLI->isCommutativeBinOp(Opcode)) {
7017 std::swap(a&: N1, b&: N2);
7018 } else {
7019 switch (Opcode) {
7020 case ISD::SUB:
7021 return getUNDEF(VT); // fold op(undef, arg2) -> undef
7022 case ISD::SIGN_EXTEND_INREG:
7023 case ISD::UDIV:
7024 case ISD::SDIV:
7025 case ISD::UREM:
7026 case ISD::SREM:
7027 case ISD::SSUBSAT:
7028 case ISD::USUBSAT:
7029 return getConstant(Val: 0, DL, VT); // fold op(undef, arg2) -> 0
7030 }
7031 }
7032 }
7033
7034 // Fold a bunch of operators when the RHS is undef.
7035 if (N2.isUndef()) {
7036 switch (Opcode) {
7037 case ISD::XOR:
7038 if (N1.isUndef())
7039 // Handle undef ^ undef -> 0 special case. This is a common
7040 // idiom (misuse).
7041 return getConstant(Val: 0, DL, VT);
7042 [[fallthrough]];
7043 case ISD::ADD:
7044 case ISD::SUB:
7045 case ISD::UDIV:
7046 case ISD::SDIV:
7047 case ISD::UREM:
7048 case ISD::SREM:
7049 return getUNDEF(VT); // fold op(arg1, undef) -> undef
7050 case ISD::MUL:
7051 case ISD::AND:
7052 case ISD::SSUBSAT:
7053 case ISD::USUBSAT:
7054 return getConstant(Val: 0, DL, VT); // fold op(arg1, undef) -> 0
7055 case ISD::OR:
7056 case ISD::SADDSAT:
7057 case ISD::UADDSAT:
7058 return getAllOnesConstant(DL, VT);
7059 }
7060 }
7061
7062 // Memoize this node if possible.
7063 SDNode *N;
7064 SDVTList VTs = getVTList(VT);
7065 SDValue Ops[] = {N1, N2};
7066 if (VT != MVT::Glue) {
7067 FoldingSetNodeID ID;
7068 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7069 void *IP = nullptr;
7070 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7071 E->intersectFlagsWith(Flags);
7072 return SDValue(E, 0);
7073 }
7074
7075 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7076 N->setFlags(Flags);
7077 createOperands(Node: N, Vals: Ops);
7078 CSEMap.InsertNode(N, InsertPos: IP);
7079 } else {
7080 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7081 createOperands(Node: N, Vals: Ops);
7082 }
7083
7084 InsertNode(N);
7085 SDValue V = SDValue(N, 0);
7086 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7087 return V;
7088}
7089
7090SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7091 SDValue N1, SDValue N2, SDValue N3) {
7092 SDNodeFlags Flags;
7093 if (Inserter)
7094 Flags = Inserter->getFlags();
7095 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7096}
7097
7098SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7099 SDValue N1, SDValue N2, SDValue N3,
7100 const SDNodeFlags Flags) {
7101 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7102 N2.getOpcode() != ISD::DELETED_NODE &&
7103 N3.getOpcode() != ISD::DELETED_NODE &&
7104 "Operand is DELETED_NODE!");
7105 // Perform various simplifications.
7106 switch (Opcode) {
7107 case ISD::FMA:
7108 case ISD::FMAD: {
7109 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7110 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7111 N3.getValueType() == VT && "FMA types must match!");
7112 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
7113 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
7114 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(Val&: N3);
7115 if (N1CFP && N2CFP && N3CFP) {
7116 APFloat V1 = N1CFP->getValueAPF();
7117 const APFloat &V2 = N2CFP->getValueAPF();
7118 const APFloat &V3 = N3CFP->getValueAPF();
7119 if (Opcode == ISD::FMAD) {
7120 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7121 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7122 } else
7123 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7124 return getConstantFP(V: V1, DL, VT);
7125 }
7126 break;
7127 }
7128 case ISD::BUILD_VECTOR: {
7129 // Attempt to simplify BUILD_VECTOR.
7130 SDValue Ops[] = {N1, N2, N3};
7131 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7132 return V;
7133 break;
7134 }
7135 case ISD::CONCAT_VECTORS: {
7136 SDValue Ops[] = {N1, N2, N3};
7137 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7138 return V;
7139 break;
7140 }
7141 case ISD::SETCC: {
7142 assert(VT.isInteger() && "SETCC result type must be an integer!");
7143 assert(N1.getValueType() == N2.getValueType() &&
7144 "SETCC operands must have the same type!");
7145 assert(VT.isVector() == N1.getValueType().isVector() &&
7146 "SETCC type should be vector iff the operand type is vector!");
7147 assert((!VT.isVector() || VT.getVectorElementCount() ==
7148 N1.getValueType().getVectorElementCount()) &&
7149 "SETCC vector element counts must match!");
7150 // Use FoldSetCC to simplify SETCC's.
7151 if (SDValue V = FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL))
7152 return V;
7153 // Vector constant folding.
7154 SDValue Ops[] = {N1, N2, N3};
7155 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7156 NewSDValueDbgMsg(V, Msg: "New node vector constant folding: ", G: this);
7157 return V;
7158 }
7159 break;
7160 }
7161 case ISD::SELECT:
7162 case ISD::VSELECT:
7163 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
7164 return V;
7165 break;
7166 case ISD::VECTOR_SHUFFLE:
7167 llvm_unreachable("should use getVectorShuffle constructor!");
7168 case ISD::VECTOR_SPLICE: {
7169 if (cast<ConstantSDNode>(Val&: N3)->isZero())
7170 return N1;
7171 break;
7172 }
7173 case ISD::INSERT_VECTOR_ELT: {
7174 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
7175 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7176 // for scalable vectors where we will generate appropriate code to
7177 // deal with out-of-bounds cases correctly.
7178 if (N3C && N1.getValueType().isFixedLengthVector() &&
7179 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
7180 return getUNDEF(VT);
7181
7182 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7183 if (N3.isUndef())
7184 return getUNDEF(VT);
7185
7186 // If the inserted element is an UNDEF, just use the input vector.
7187 if (N2.isUndef())
7188 return N1;
7189
7190 break;
7191 }
7192 case ISD::INSERT_SUBVECTOR: {
7193 // Inserting undef into undef is still undef.
7194 if (N1.isUndef() && N2.isUndef())
7195 return getUNDEF(VT);
7196
7197 EVT N2VT = N2.getValueType();
7198 assert(VT == N1.getValueType() &&
7199 "Dest and insert subvector source types must match!");
7200 assert(VT.isVector() && N2VT.isVector() &&
7201 "Insert subvector VTs must be vectors!");
7202 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
7203 "Insert subvector VTs must have the same element type!");
7204 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7205 "Cannot insert a scalable vector into a fixed length vector!");
7206 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7207 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
7208 "Insert subvector must be from smaller vector to larger vector!");
7209 assert(isa<ConstantSDNode>(N3) &&
7210 "Insert subvector index must be constant");
7211 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7212 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7213 VT.getVectorMinNumElements()) &&
7214 "Insert subvector overflow!");
7215 assert(N3->getAsAPIntVal().getBitWidth() ==
7216 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7217 "Constant index for INSERT_SUBVECTOR has an invalid size");
7218
7219 // Trivial insertion.
7220 if (VT == N2VT)
7221 return N2;
7222
7223 // If this is an insert of an extracted vector into an undef vector, we
7224 // can just use the input to the extract.
7225 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7226 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT)
7227 return N2.getOperand(i: 0);
7228 break;
7229 }
7230 case ISD::BITCAST:
7231 // Fold bit_convert nodes from a type to themselves.
7232 if (N1.getValueType() == VT)
7233 return N1;
7234 break;
7235 case ISD::VP_TRUNCATE:
7236 case ISD::VP_SIGN_EXTEND:
7237 case ISD::VP_ZERO_EXTEND:
7238 // Don't create noop casts.
7239 if (N1.getValueType() == VT)
7240 return N1;
7241 break;
7242 }
7243
7244 // Memoize node if it doesn't produce a glue result.
7245 SDNode *N;
7246 SDVTList VTs = getVTList(VT);
7247 SDValue Ops[] = {N1, N2, N3};
7248 if (VT != MVT::Glue) {
7249 FoldingSetNodeID ID;
7250 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7251 void *IP = nullptr;
7252 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7253 E->intersectFlagsWith(Flags);
7254 return SDValue(E, 0);
7255 }
7256
7257 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7258 N->setFlags(Flags);
7259 createOperands(Node: N, Vals: Ops);
7260 CSEMap.InsertNode(N, InsertPos: IP);
7261 } else {
7262 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7263 createOperands(Node: N, Vals: Ops);
7264 }
7265
7266 InsertNode(N);
7267 SDValue V = SDValue(N, 0);
7268 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7269 return V;
7270}
7271
7272SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7273 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7274 SDValue Ops[] = { N1, N2, N3, N4 };
7275 return getNode(Opcode, DL, VT, Ops);
7276}
7277
7278SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7279 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7280 SDValue N5) {
7281 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7282 return getNode(Opcode, DL, VT, Ops);
7283}
7284
7285/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7286/// the incoming stack arguments to be loaded from the stack.
7287SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
7288 SmallVector<SDValue, 8> ArgChains;
7289
7290 // Include the original chain at the beginning of the list. When this is
7291 // used by target LowerCall hooks, this helps legalize find the
7292 // CALLSEQ_BEGIN node.
7293 ArgChains.push_back(Elt: Chain);
7294
7295 // Add a chain value for each stack argument.
7296 for (SDNode *U : getEntryNode().getNode()->uses())
7297 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
7298 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
7299 if (FI->getIndex() < 0)
7300 ArgChains.push_back(Elt: SDValue(L, 1));
7301
7302 // Build a tokenfactor for all the chains.
7303 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7304}
7305
7306/// getMemsetValue - Vectorized representation of the memset value
7307/// operand.
7308static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
7309 const SDLoc &dl) {
7310 assert(!Value.isUndef());
7311
7312 unsigned NumBits = VT.getScalarSizeInBits();
7313 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
7314 assert(C->getAPIntValue().getBitWidth() == 8);
7315 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
7316 if (VT.isInteger()) {
7317 bool IsOpaque = VT.getSizeInBits() > 64 ||
7318 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
7319 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
7320 }
7321 return DAG.getConstantFP(V: APFloat(DAG.EVTToAPFloatSemantics(VT), Val), DL: dl,
7322 VT);
7323 }
7324
7325 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7326 EVT IntVT = VT.getScalarType();
7327 if (!IntVT.isInteger())
7328 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
7329
7330 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
7331 if (NumBits > 8) {
7332 // Use a multiplication with 0x010101... to extend the input to the
7333 // required length.
7334 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
7335 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
7336 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
7337 }
7338
7339 if (VT != Value.getValueType() && !VT.isInteger())
7340 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
7341 if (VT != Value.getValueType())
7342 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
7343
7344 return Value;
7345}
7346
7347/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7348/// used when a memcpy is turned into a memset when the source is a constant
7349/// string ptr.
7350static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
7351 const TargetLowering &TLI,
7352 const ConstantDataArraySlice &Slice) {
7353 // Handle vector with all elements zero.
7354 if (Slice.Array == nullptr) {
7355 if (VT.isInteger())
7356 return DAG.getConstant(Val: 0, DL: dl, VT);
7357 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7358 return DAG.getConstantFP(Val: 0.0, DL: dl, VT);
7359 if (VT.isVector()) {
7360 unsigned NumElts = VT.getVectorNumElements();
7361 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7362 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
7363 N1: DAG.getConstant(Val: 0, DL: dl,
7364 VT: EVT::getVectorVT(Context&: *DAG.getContext(),
7365 VT: EltVT, NumElements: NumElts)));
7366 }
7367 llvm_unreachable("Expected type!");
7368 }
7369
7370 assert(!VT.isVector() && "Can't handle vector type here!");
7371 unsigned NumVTBits = VT.getSizeInBits();
7372 unsigned NumVTBytes = NumVTBits / 8;
7373 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
7374
7375 APInt Val(NumVTBits, 0);
7376 if (DAG.getDataLayout().isLittleEndian()) {
7377 for (unsigned i = 0; i != NumBytes; ++i)
7378 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7379 } else {
7380 for (unsigned i = 0; i != NumBytes; ++i)
7381 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7382 }
7383
7384 // If the "cost" of materializing the integer immediate is less than the cost
7385 // of a load, then it is cost effective to turn the load into the immediate.
7386 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
7387 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
7388 return DAG.getConstant(Val, DL: dl, VT);
7389 return SDValue();
7390}
7391
7392SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
7393 const SDLoc &DL,
7394 const SDNodeFlags Flags) {
7395 EVT VT = Base.getValueType();
7396 SDValue Index;
7397
7398 if (Offset.isScalable())
7399 Index = getVScale(DL, VT: Base.getValueType(),
7400 MulImm: APInt(Base.getValueSizeInBits().getFixedValue(),
7401 Offset.getKnownMinValue()));
7402 else
7403 Index = getConstant(Val: Offset.getFixedValue(), DL, VT);
7404
7405 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
7406}
7407
7408SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
7409 const SDLoc &DL,
7410 const SDNodeFlags Flags) {
7411 assert(Offset.getValueType().isInteger());
7412 EVT BasePtrVT = Ptr.getValueType();
7413 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
7414}
7415
7416/// Returns true if memcpy source is constant data.
7417static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
7418 uint64_t SrcDelta = 0;
7419 GlobalAddressSDNode *G = nullptr;
7420 if (Src.getOpcode() == ISD::GlobalAddress)
7421 G = cast<GlobalAddressSDNode>(Val&: Src);
7422 else if (Src.getOpcode() == ISD::ADD &&
7423 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
7424 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
7425 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
7426 SrcDelta = Src.getConstantOperandVal(i: 1);
7427 }
7428 if (!G)
7429 return false;
7430
7431 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
7432 Offset: SrcDelta + G->getOffset());
7433}
7434
7435static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
7436 SelectionDAG &DAG) {
7437 // On Darwin, -Os means optimize for size without hurting performance, so
7438 // only really optimize for size when -Oz (MinSize) is used.
7439 if (MF.getTarget().getTargetTriple().isOSDarwin())
7440 return MF.getFunction().hasMinSize();
7441 return DAG.shouldOptForSize();
7442}
7443
7444static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
7445 SmallVector<SDValue, 32> &OutChains, unsigned From,
7446 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7447 SmallVector<SDValue, 16> &OutStoreChains) {
7448 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7449 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7450 SmallVector<SDValue, 16> GluedLoadChains;
7451 for (unsigned i = From; i < To; ++i) {
7452 OutChains.push_back(Elt: OutLoadChains[i]);
7453 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
7454 }
7455
7456 // Chain for all loads.
7457 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7458 GluedLoadChains);
7459
7460 for (unsigned i = From; i < To; ++i) {
7461 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
7462 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
7463 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
7464 MMO: ST->getMemOperand());
7465 OutChains.push_back(Elt: NewStore);
7466 }
7467}
7468
7469static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7470 SDValue Chain, SDValue Dst, SDValue Src,
7471 uint64_t Size, Align Alignment,
7472 bool isVol, bool AlwaysInline,
7473 MachinePointerInfo DstPtrInfo,
7474 MachinePointerInfo SrcPtrInfo,
7475 const AAMDNodes &AAInfo, AAResults *AA) {
7476 // Turn a memcpy of undef to nop.
7477 // FIXME: We need to honor volatile even is Src is undef.
7478 if (Src.isUndef())
7479 return Chain;
7480
7481 // Expand memcpy to a series of load and store ops if the size operand falls
7482 // below a certain threshold.
7483 // TODO: In the AlwaysInline case, if the size is big then generate a loop
7484 // rather than maybe a humongous number of loads and stores.
7485 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7486 const DataLayout &DL = DAG.getDataLayout();
7487 LLVMContext &C = *DAG.getContext();
7488 std::vector<EVT> MemOps;
7489 bool DstAlignCanChange = false;
7490 MachineFunction &MF = DAG.getMachineFunction();
7491 MachineFrameInfo &MFI = MF.getFrameInfo();
7492 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7493 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7494 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7495 DstAlignCanChange = true;
7496 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
7497 if (!SrcAlign || Alignment > *SrcAlign)
7498 SrcAlign = Alignment;
7499 assert(SrcAlign && "SrcAlign must be set");
7500 ConstantDataArraySlice Slice;
7501 // If marked as volatile, perform a copy even when marked as constant.
7502 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7503 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7504 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7505 const MemOp Op = isZeroConstant
7506 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
7507 /*IsZeroMemset*/ true, IsVolatile: isVol)
7508 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
7509 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
7510 if (!TLI.findOptimalMemOpLowering(
7511 MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
7512 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes()))
7513 return SDValue();
7514
7515 if (DstAlignCanChange) {
7516 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
7517 Align NewAlign = DL.getABITypeAlign(Ty);
7518
7519 // Don't promote to an alignment that would require dynamic stack
7520 // realignment which may conflict with optimizations such as tail call
7521 // optimization.
7522 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7523 if (!TRI->hasStackRealignment(MF))
7524 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7525 NewAlign = NewAlign.previous();
7526
7527 if (NewAlign > Alignment) {
7528 // Give the stack frame object a larger alignment if needed.
7529 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7530 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7531 Alignment = NewAlign;
7532 }
7533 }
7534
7535 // Prepare AAInfo for loads/stores after lowering this memcpy.
7536 AAMDNodes NewAAInfo = AAInfo;
7537 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7538
7539 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
7540 bool isConstant =
7541 AA && SrcVal &&
7542 AA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
7543
7544 MachineMemOperand::Flags MMOFlags =
7545 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7546 SmallVector<SDValue, 16> OutLoadChains;
7547 SmallVector<SDValue, 16> OutStoreChains;
7548 SmallVector<SDValue, 32> OutChains;
7549 unsigned NumMemOps = MemOps.size();
7550 uint64_t SrcOff = 0, DstOff = 0;
7551 for (unsigned i = 0; i != NumMemOps; ++i) {
7552 EVT VT = MemOps[i];
7553 unsigned VTSize = VT.getSizeInBits() / 8;
7554 SDValue Value, Store;
7555
7556 if (VTSize > Size) {
7557 // Issuing an unaligned load / store pair that overlaps with the previous
7558 // pair. Adjust the offset accordingly.
7559 assert(i == NumMemOps-1 && i != 0);
7560 SrcOff -= VTSize - Size;
7561 DstOff -= VTSize - Size;
7562 }
7563
7564 if (CopyFromConstant &&
7565 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7566 // It's unlikely a store of a vector immediate can be done in a single
7567 // instruction. It would require a load from a constantpool first.
7568 // We only handle zero vectors here.
7569 // FIXME: Handle other cases where store of vector immediate is done in
7570 // a single instruction.
7571 ConstantDataArraySlice SubSlice;
7572 if (SrcOff < Slice.Length) {
7573 SubSlice = Slice;
7574 SubSlice.move(Delta: SrcOff);
7575 } else {
7576 // This is an out-of-bounds access and hence UB. Pretend we read zero.
7577 SubSlice.Array = nullptr;
7578 SubSlice.Offset = 0;
7579 SubSlice.Length = VTSize;
7580 }
7581 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
7582 if (Value.getNode()) {
7583 Store = DAG.getStore(
7584 Chain, dl, Val: Value,
7585 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7586 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
7587 OutChains.push_back(Elt: Store);
7588 }
7589 }
7590
7591 if (!Store.getNode()) {
7592 // The type might not be legal for the target. This should only happen
7593 // if the type is smaller than a legal type, as on PPC, so the right
7594 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
7595 // to Load/Store if NVT==VT.
7596 // FIXME does the case above also need this?
7597 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
7598 assert(NVT.bitsGE(VT));
7599
7600 bool isDereferenceable =
7601 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
7602 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7603 if (isDereferenceable)
7604 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7605 if (isConstant)
7606 SrcMMOFlags |= MachineMemOperand::MOInvariant;
7607
7608 Value = DAG.getExtLoad(
7609 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
7610 Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
7611 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
7612 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
7613 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
7614
7615 Store = DAG.getTruncStore(
7616 Chain, dl, Val: Value,
7617 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7618 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
7619 OutStoreChains.push_back(Elt: Store);
7620 }
7621 SrcOff += VTSize;
7622 DstOff += VTSize;
7623 Size -= VTSize;
7624 }
7625
7626 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7627 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
7628 unsigned NumLdStInMemcpy = OutStoreChains.size();
7629
7630 if (NumLdStInMemcpy) {
7631 // It may be that memcpy might be converted to memset if it's memcpy
7632 // of constants. In such a case, we won't have loads and stores, but
7633 // just stores. In the absence of loads, there is nothing to gang up.
7634 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7635 // If target does not care, just leave as it.
7636 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7637 OutChains.push_back(Elt: OutLoadChains[i]);
7638 OutChains.push_back(Elt: OutStoreChains[i]);
7639 }
7640 } else {
7641 // Ld/St less than/equal limit set by target.
7642 if (NumLdStInMemcpy <= GluedLdStLimit) {
7643 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
7644 To: NumLdStInMemcpy, OutLoadChains,
7645 OutStoreChains);
7646 } else {
7647 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7648 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7649 unsigned GlueIter = 0;
7650
7651 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7652 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7653 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7654
7655 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
7656 OutLoadChains, OutStoreChains);
7657 GlueIter += GluedLdStLimit;
7658 }
7659
7660 // Residual ld/st.
7661 if (RemainingLdStInMemcpy) {
7662 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
7663 To: RemainingLdStInMemcpy, OutLoadChains,
7664 OutStoreChains);
7665 }
7666 }
7667 }
7668 }
7669 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7670}
7671
7672static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7673 SDValue Chain, SDValue Dst, SDValue Src,
7674 uint64_t Size, Align Alignment,
7675 bool isVol, bool AlwaysInline,
7676 MachinePointerInfo DstPtrInfo,
7677 MachinePointerInfo SrcPtrInfo,
7678 const AAMDNodes &AAInfo) {
7679 // Turn a memmove of undef to nop.
7680 // FIXME: We need to honor volatile even is Src is undef.
7681 if (Src.isUndef())
7682 return Chain;
7683
7684 // Expand memmove to a series of load and store ops if the size operand falls
7685 // below a certain threshold.
7686 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7687 const DataLayout &DL = DAG.getDataLayout();
7688 LLVMContext &C = *DAG.getContext();
7689 std::vector<EVT> MemOps;
7690 bool DstAlignCanChange = false;
7691 MachineFunction &MF = DAG.getMachineFunction();
7692 MachineFrameInfo &MFI = MF.getFrameInfo();
7693 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7694 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7695 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7696 DstAlignCanChange = true;
7697 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
7698 if (!SrcAlign || Alignment > *SrcAlign)
7699 SrcAlign = Alignment;
7700 assert(SrcAlign && "SrcAlign must be set");
7701 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7702 if (!TLI.findOptimalMemOpLowering(
7703 MemOps, Limit,
7704 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
7705 /*IsVolatile*/ true),
7706 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
7707 FuncAttributes: MF.getFunction().getAttributes()))
7708 return SDValue();
7709
7710 if (DstAlignCanChange) {
7711 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
7712 Align NewAlign = DL.getABITypeAlign(Ty);
7713
7714 // Don't promote to an alignment that would require dynamic stack
7715 // realignment which may conflict with optimizations such as tail call
7716 // optimization.
7717 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7718 if (!TRI->hasStackRealignment(MF))
7719 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7720 NewAlign = NewAlign.previous();
7721
7722 if (NewAlign > Alignment) {
7723 // Give the stack frame object a larger alignment if needed.
7724 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7725 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7726 Alignment = NewAlign;
7727 }
7728 }
7729
7730 // Prepare AAInfo for loads/stores after lowering this memmove.
7731 AAMDNodes NewAAInfo = AAInfo;
7732 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7733
7734 MachineMemOperand::Flags MMOFlags =
7735 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7736 uint64_t SrcOff = 0, DstOff = 0;
7737 SmallVector<SDValue, 8> LoadValues;
7738 SmallVector<SDValue, 8> LoadChains;
7739 SmallVector<SDValue, 8> OutChains;
7740 unsigned NumMemOps = MemOps.size();
7741 for (unsigned i = 0; i < NumMemOps; i++) {
7742 EVT VT = MemOps[i];
7743 unsigned VTSize = VT.getSizeInBits() / 8;
7744 SDValue Value;
7745
7746 bool isDereferenceable =
7747 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
7748 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7749 if (isDereferenceable)
7750 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7751
7752 Value = DAG.getLoad(
7753 VT, dl, Chain,
7754 Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
7755 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
7756 LoadValues.push_back(Elt: Value);
7757 LoadChains.push_back(Elt: Value.getValue(R: 1));
7758 SrcOff += VTSize;
7759 }
7760 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7761 OutChains.clear();
7762 for (unsigned i = 0; i < NumMemOps; i++) {
7763 EVT VT = MemOps[i];
7764 unsigned VTSize = VT.getSizeInBits() / 8;
7765 SDValue Store;
7766
7767 Store = DAG.getStore(
7768 Chain, dl, Val: LoadValues[i],
7769 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7770 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
7771 OutChains.push_back(Elt: Store);
7772 DstOff += VTSize;
7773 }
7774
7775 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7776}
7777
7778/// Lower the call to 'memset' intrinsic function into a series of store
7779/// operations.
7780///
7781/// \param DAG Selection DAG where lowered code is placed.
7782/// \param dl Link to corresponding IR location.
7783/// \param Chain Control flow dependency.
7784/// \param Dst Pointer to destination memory location.
7785/// \param Src Value of byte to write into the memory.
7786/// \param Size Number of bytes to write.
7787/// \param Alignment Alignment of the destination in bytes.
7788/// \param isVol True if destination is volatile.
7789/// \param AlwaysInline Makes sure no function call is generated.
7790/// \param DstPtrInfo IR information on the memory pointer.
7791/// \returns New head in the control flow, if lowering was successful, empty
7792/// SDValue otherwise.
7793///
7794/// The function tries to replace 'llvm.memset' intrinsic with several store
7795/// operations and value calculation code. This is usually profitable for small
7796/// memory size or when the semantic requires inlining.
7797static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
7798 SDValue Chain, SDValue Dst, SDValue Src,
7799 uint64_t Size, Align Alignment, bool isVol,
7800 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7801 const AAMDNodes &AAInfo) {
7802 // Turn a memset of undef to nop.
7803 // FIXME: We need to honor volatile even is Src is undef.
7804 if (Src.isUndef())
7805 return Chain;
7806
7807 // Expand memset to a series of load/store ops if the size operand
7808 // falls below a certain threshold.
7809 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7810 std::vector<EVT> MemOps;
7811 bool DstAlignCanChange = false;
7812 MachineFunction &MF = DAG.getMachineFunction();
7813 MachineFrameInfo &MFI = MF.getFrameInfo();
7814 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7815 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
7816 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
7817 DstAlignCanChange = true;
7818 bool IsZeroVal = isNullConstant(V: Src);
7819 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7820
7821 if (!TLI.findOptimalMemOpLowering(
7822 MemOps, Limit,
7823 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
7824 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes()))
7825 return SDValue();
7826
7827 if (DstAlignCanChange) {
7828 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
7829 const DataLayout &DL = DAG.getDataLayout();
7830 Align NewAlign = DL.getABITypeAlign(Ty);
7831
7832 // Don't promote to an alignment that would require dynamic stack
7833 // realignment which may conflict with optimizations such as tail call
7834 // optimization.
7835 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7836 if (!TRI->hasStackRealignment(MF))
7837 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign))
7838 NewAlign = NewAlign.previous();
7839
7840 if (NewAlign > Alignment) {
7841 // Give the stack frame object a larger alignment if needed.
7842 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
7843 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
7844 Alignment = NewAlign;
7845 }
7846 }
7847
7848 SmallVector<SDValue, 8> OutChains;
7849 uint64_t DstOff = 0;
7850 unsigned NumMemOps = MemOps.size();
7851
7852 // Find the largest store and generate the bit pattern for it.
7853 EVT LargestVT = MemOps[0];
7854 for (unsigned i = 1; i < NumMemOps; i++)
7855 if (MemOps[i].bitsGT(VT: LargestVT))
7856 LargestVT = MemOps[i];
7857 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
7858
7859 // Prepare AAInfo for loads/stores after lowering this memset.
7860 AAMDNodes NewAAInfo = AAInfo;
7861 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7862
7863 for (unsigned i = 0; i < NumMemOps; i++) {
7864 EVT VT = MemOps[i];
7865 unsigned VTSize = VT.getSizeInBits() / 8;
7866 if (VTSize > Size) {
7867 // Issuing an unaligned load / store pair that overlaps with the previous
7868 // pair. Adjust the offset accordingly.
7869 assert(i == NumMemOps-1 && i != 0);
7870 DstOff -= VTSize - Size;
7871 }
7872
7873 // If this store is smaller than the largest store see whether we can get
7874 // the smaller value for free with a truncate or extract vector element and
7875 // then store.
7876 SDValue Value = MemSetValue;
7877 if (VT.bitsLT(VT: LargestVT)) {
7878 unsigned Index;
7879 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
7880 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
7881 if (!LargestVT.isVector() && !VT.isVector() &&
7882 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
7883 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
7884 else if (LargestVT.isVector() && !VT.isVector() &&
7885 TLI.shallExtractConstSplatVectorElementToStore(
7886 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
7887 ElemSizeInBits: VT.getSizeInBits(), Index) &&
7888 TLI.isTypeLegal(VT: SVT) &&
7889 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
7890 // Target which can combine store(extractelement VectorTy, Idx) can get
7891 // the smaller value for free.
7892 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
7893 Value = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: TailValue,
7894 N2: DAG.getVectorIdxConstant(Val: Index, DL: dl));
7895 } else
7896 Value = getMemsetValue(Value: Src, VT, DAG, dl);
7897 }
7898 assert(Value.getValueType() == VT && "Value with wrong type.");
7899 SDValue Store = DAG.getStore(
7900 Chain, dl, Val: Value,
7901 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
7902 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
7903 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
7904 AAInfo: NewAAInfo);
7905 OutChains.push_back(Elt: Store);
7906 DstOff += VT.getSizeInBits() / 8;
7907 Size -= VTSize;
7908 }
7909
7910 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7911}
7912
7913static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
7914 unsigned AS) {
7915 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
7916 // pointer operands can be losslessly bitcasted to pointers of address space 0
7917 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
7918 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
7919 Twine(AS));
7920 }
7921}
7922
7923SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
7924 SDValue Src, SDValue Size, Align Alignment,
7925 bool isVol, bool AlwaysInline, bool isTailCall,
7926 MachinePointerInfo DstPtrInfo,
7927 MachinePointerInfo SrcPtrInfo,
7928 const AAMDNodes &AAInfo, AAResults *AA) {
7929 // Check to see if we should lower the memcpy to loads and stores first.
7930 // For cases within the target-specified limits, this is the best choice.
7931 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
7932 if (ConstantSize) {
7933 // Memcpy with size zero? Just return the original chain.
7934 if (ConstantSize->isZero())
7935 return Chain;
7936
7937 SDValue Result = getMemcpyLoadsAndStores(
7938 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
7939 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7940 if (Result.getNode())
7941 return Result;
7942 }
7943
7944 // Then check to see if we should lower the memcpy with target-specific
7945 // code. If the target chooses to do this, this is the next best.
7946 if (TSI) {
7947 SDValue Result = TSI->EmitTargetCodeForMemcpy(
7948 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
7949 DstPtrInfo, SrcPtrInfo);
7950 if (Result.getNode())
7951 return Result;
7952 }
7953
7954 // If we really need inline code and the target declined to provide it,
7955 // use a (potentially long) sequence of loads and stores.
7956 if (AlwaysInline) {
7957 assert(ConstantSize && "AlwaysInline requires a constant size!");
7958 return getMemcpyLoadsAndStores(
7959 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
7960 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7961 }
7962
7963 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
7964 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
7965
7966 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
7967 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
7968 // respect volatile, so they may do things like read or write memory
7969 // beyond the given memory regions. But fixing this isn't easy, and most
7970 // people don't care.
7971
7972 // Emit a library call.
7973 TargetLowering::ArgListTy Args;
7974 TargetLowering::ArgListEntry Entry;
7975 Entry.Ty = PointerType::getUnqual(C&: *getContext());
7976 Entry.Node = Dst; Args.push_back(x: Entry);
7977 Entry.Node = Src; Args.push_back(x: Entry);
7978
7979 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
7980 Entry.Node = Size; Args.push_back(x: Entry);
7981 // FIXME: pass in SDLoc
7982 TargetLowering::CallLoweringInfo CLI(*this);
7983 CLI.setDebugLoc(dl)
7984 .setChain(Chain)
7985 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMCPY),
7986 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
7987 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMCPY),
7988 VT: TLI->getPointerTy(DL: getDataLayout())),
7989 ArgsList: std::move(Args))
7990 .setDiscardResult()
7991 .setTailCall(isTailCall);
7992
7993 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
7994 return CallResult.second;
7995}
7996
7997SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
7998 SDValue Dst, SDValue Src, SDValue Size,
7999 Type *SizeTy, unsigned ElemSz,
8000 bool isTailCall,
8001 MachinePointerInfo DstPtrInfo,
8002 MachinePointerInfo SrcPtrInfo) {
8003 // Emit a library call.
8004 TargetLowering::ArgListTy Args;
8005 TargetLowering::ArgListEntry Entry;
8006 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8007 Entry.Node = Dst;
8008 Args.push_back(x: Entry);
8009
8010 Entry.Node = Src;
8011 Args.push_back(x: Entry);
8012
8013 Entry.Ty = SizeTy;
8014 Entry.Node = Size;
8015 Args.push_back(x: Entry);
8016
8017 RTLIB::Libcall LibraryCall =
8018 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8019 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8020 report_fatal_error(reason: "Unsupported element size");
8021
8022 TargetLowering::CallLoweringInfo CLI(*this);
8023 CLI.setDebugLoc(dl)
8024 .setChain(Chain)
8025 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8026 ResultType: Type::getVoidTy(C&: *getContext()),
8027 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8028 VT: TLI->getPointerTy(DL: getDataLayout())),
8029 ArgsList: std::move(Args))
8030 .setDiscardResult()
8031 .setTailCall(isTailCall);
8032
8033 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8034 return CallResult.second;
8035}
8036
8037SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
8038 SDValue Src, SDValue Size, Align Alignment,
8039 bool isVol, bool isTailCall,
8040 MachinePointerInfo DstPtrInfo,
8041 MachinePointerInfo SrcPtrInfo,
8042 const AAMDNodes &AAInfo, AAResults *AA) {
8043 // Check to see if we should lower the memmove to loads and stores first.
8044 // For cases within the target-specified limits, this is the best choice.
8045 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8046 if (ConstantSize) {
8047 // Memmove with size zero? Just return the original chain.
8048 if (ConstantSize->isZero())
8049 return Chain;
8050
8051 SDValue Result = getMemmoveLoadsAndStores(
8052 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8053 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
8054 if (Result.getNode())
8055 return Result;
8056 }
8057
8058 // Then check to see if we should lower the memmove with target-specific
8059 // code. If the target chooses to do this, this is the next best.
8060 if (TSI) {
8061 SDValue Result =
8062 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
8063 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
8064 if (Result.getNode())
8065 return Result;
8066 }
8067
8068 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8069 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
8070
8071 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8072 // not be safe. See memcpy above for more details.
8073
8074 // Emit a library call.
8075 TargetLowering::ArgListTy Args;
8076 TargetLowering::ArgListEntry Entry;
8077 Entry.Ty = PointerType::getUnqual(C&: *getContext());
8078 Entry.Node = Dst; Args.push_back(x: Entry);
8079 Entry.Node = Src; Args.push_back(x: Entry);
8080
8081 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8082 Entry.Node = Size; Args.push_back(x: Entry);
8083 // FIXME: pass in SDLoc
8084 TargetLowering::CallLoweringInfo CLI(*this);
8085 CLI.setDebugLoc(dl)
8086 .setChain(Chain)
8087 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMMOVE),
8088 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
8089 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMMOVE),
8090 VT: TLI->getPointerTy(DL: getDataLayout())),
8091 ArgsList: std::move(Args))
8092 .setDiscardResult()
8093 .setTailCall(isTailCall);
8094
8095 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8096 return CallResult.second;
8097}
8098
8099SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
8100 SDValue Dst, SDValue Src, SDValue Size,
8101 Type *SizeTy, unsigned ElemSz,
8102 bool isTailCall,
8103 MachinePointerInfo DstPtrInfo,
8104 MachinePointerInfo SrcPtrInfo) {
8105 // Emit a library call.
8106 TargetLowering::ArgListTy Args;
8107 TargetLowering::ArgListEntry Entry;
8108 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8109 Entry.Node = Dst;
8110 Args.push_back(x: Entry);
8111
8112 Entry.Node = Src;
8113 Args.push_back(x: Entry);
8114
8115 Entry.Ty = SizeTy;
8116 Entry.Node = Size;
8117 Args.push_back(x: Entry);
8118
8119 RTLIB::Libcall LibraryCall =
8120 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8121 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8122 report_fatal_error(reason: "Unsupported element size");
8123
8124 TargetLowering::CallLoweringInfo CLI(*this);
8125 CLI.setDebugLoc(dl)
8126 .setChain(Chain)
8127 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8128 ResultType: Type::getVoidTy(C&: *getContext()),
8129 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8130 VT: TLI->getPointerTy(DL: getDataLayout())),
8131 ArgsList: std::move(Args))
8132 .setDiscardResult()
8133 .setTailCall(isTailCall);
8134
8135 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8136 return CallResult.second;
8137}
8138
8139SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
8140 SDValue Src, SDValue Size, Align Alignment,
8141 bool isVol, bool AlwaysInline, bool isTailCall,
8142 MachinePointerInfo DstPtrInfo,
8143 const AAMDNodes &AAInfo) {
8144 // Check to see if we should lower the memset to stores first.
8145 // For cases within the target-specified limits, this is the best choice.
8146 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8147 if (ConstantSize) {
8148 // Memset with size zero? Just return the original chain.
8149 if (ConstantSize->isZero())
8150 return Chain;
8151
8152 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
8153 Size: ConstantSize->getZExtValue(), Alignment,
8154 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
8155
8156 if (Result.getNode())
8157 return Result;
8158 }
8159
8160 // Then check to see if we should lower the memset with target-specific
8161 // code. If the target chooses to do this, this is the next best.
8162 if (TSI) {
8163 SDValue Result = TSI->EmitTargetCodeForMemset(
8164 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
8165 if (Result.getNode())
8166 return Result;
8167 }
8168
8169 // If we really need inline code and the target declined to provide it,
8170 // use a (potentially long) sequence of loads and stores.
8171 if (AlwaysInline) {
8172 assert(ConstantSize && "AlwaysInline requires a constant size!");
8173 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
8174 Size: ConstantSize->getZExtValue(), Alignment,
8175 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
8176 assert(Result &&
8177 "getMemsetStores must return a valid sequence when AlwaysInline");
8178 return Result;
8179 }
8180
8181 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8182
8183 // Emit a library call.
8184 auto &Ctx = *getContext();
8185 const auto& DL = getDataLayout();
8186
8187 TargetLowering::CallLoweringInfo CLI(*this);
8188 // FIXME: pass in SDLoc
8189 CLI.setDebugLoc(dl).setChain(Chain);
8190
8191 const char *BzeroName = getTargetLoweringInfo().getLibcallName(Call: RTLIB::BZERO);
8192
8193 // Helper function to create an Entry from Node and Type.
8194 const auto CreateEntry = [](SDValue Node, Type *Ty) {
8195 TargetLowering::ArgListEntry Entry;
8196 Entry.Node = Node;
8197 Entry.Ty = Ty;
8198 return Entry;
8199 };
8200
8201 // If zeroing out and bzero is present, use it.
8202 if (isNullConstant(V: Src) && BzeroName) {
8203 TargetLowering::ArgListTy Args;
8204 Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
8205 Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
8206 CLI.setLibCallee(
8207 CC: TLI->getLibcallCallingConv(Call: RTLIB::BZERO), ResultType: Type::getVoidTy(C&: Ctx),
8208 Target: getExternalSymbol(Sym: BzeroName, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
8209 } else {
8210 TargetLowering::ArgListTy Args;
8211 Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
8212 Args.push_back(x: CreateEntry(Src, Src.getValueType().getTypeForEVT(Context&: Ctx)));
8213 Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
8214 CLI.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMSET),
8215 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
8216 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMSET),
8217 VT: TLI->getPointerTy(DL)),
8218 ArgsList: std::move(Args));
8219 }
8220
8221 CLI.setDiscardResult().setTailCall(isTailCall);
8222
8223 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8224 return CallResult.second;
8225}
8226
8227SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
8228 SDValue Dst, SDValue Value, SDValue Size,
8229 Type *SizeTy, unsigned ElemSz,
8230 bool isTailCall,
8231 MachinePointerInfo DstPtrInfo) {
8232 // Emit a library call.
8233 TargetLowering::ArgListTy Args;
8234 TargetLowering::ArgListEntry Entry;
8235 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8236 Entry.Node = Dst;
8237 Args.push_back(x: Entry);
8238
8239 Entry.Ty = Type::getInt8Ty(C&: *getContext());
8240 Entry.Node = Value;
8241 Args.push_back(x: Entry);
8242
8243 Entry.Ty = SizeTy;
8244 Entry.Node = Size;
8245 Args.push_back(x: Entry);
8246
8247 RTLIB::Libcall LibraryCall =
8248 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8249 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8250 report_fatal_error(reason: "Unsupported element size");
8251
8252 TargetLowering::CallLoweringInfo CLI(*this);
8253 CLI.setDebugLoc(dl)
8254 .setChain(Chain)
8255 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8256 ResultType: Type::getVoidTy(C&: *getContext()),
8257 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8258 VT: TLI->getPointerTy(DL: getDataLayout())),
8259 ArgsList: std::move(Args))
8260 .setDiscardResult()
8261 .setTailCall(isTailCall);
8262
8263 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8264 return CallResult.second;
8265}
8266
8267SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8268 SDVTList VTList, ArrayRef<SDValue> Ops,
8269 MachineMemOperand *MMO) {
8270 FoldingSetNodeID ID;
8271 ID.AddInteger(I: MemVT.getRawBits());
8272 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
8273 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8274 ID.AddInteger(I: MMO->getFlags());
8275 void* IP = nullptr;
8276 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8277 cast<AtomicSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8278 return SDValue(E, 0);
8279 }
8280
8281 auto *N = newSDNode<AtomicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8282 Args&: VTList, Args&: MemVT, Args&: MMO);
8283 createOperands(Node: N, Vals: Ops);
8284
8285 CSEMap.InsertNode(N, InsertPos: IP);
8286 InsertNode(N);
8287 return SDValue(N, 0);
8288}
8289
8290SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
8291 EVT MemVT, SDVTList VTs, SDValue Chain,
8292 SDValue Ptr, SDValue Cmp, SDValue Swp,
8293 MachineMemOperand *MMO) {
8294 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8295 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
8296 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8297
8298 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8299 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8300}
8301
8302SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8303 SDValue Chain, SDValue Ptr, SDValue Val,
8304 MachineMemOperand *MMO) {
8305 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8306 Opcode == ISD::ATOMIC_LOAD_SUB ||
8307 Opcode == ISD::ATOMIC_LOAD_AND ||
8308 Opcode == ISD::ATOMIC_LOAD_CLR ||
8309 Opcode == ISD::ATOMIC_LOAD_OR ||
8310 Opcode == ISD::ATOMIC_LOAD_XOR ||
8311 Opcode == ISD::ATOMIC_LOAD_NAND ||
8312 Opcode == ISD::ATOMIC_LOAD_MIN ||
8313 Opcode == ISD::ATOMIC_LOAD_MAX ||
8314 Opcode == ISD::ATOMIC_LOAD_UMIN ||
8315 Opcode == ISD::ATOMIC_LOAD_UMAX ||
8316 Opcode == ISD::ATOMIC_LOAD_FADD ||
8317 Opcode == ISD::ATOMIC_LOAD_FSUB ||
8318 Opcode == ISD::ATOMIC_LOAD_FMAX ||
8319 Opcode == ISD::ATOMIC_LOAD_FMIN ||
8320 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8321 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8322 Opcode == ISD::ATOMIC_SWAP ||
8323 Opcode == ISD::ATOMIC_STORE) &&
8324 "Invalid Atomic Op");
8325
8326 EVT VT = Val.getValueType();
8327
8328 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8329 getVTList(VT, MVT::Other);
8330 SDValue Ops[] = {Chain, Ptr, Val};
8331 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8332}
8333
8334SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8335 EVT VT, SDValue Chain, SDValue Ptr,
8336 MachineMemOperand *MMO) {
8337 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8338
8339 SDVTList VTs = getVTList(VT, MVT::Other);
8340 SDValue Ops[] = {Chain, Ptr};
8341 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
8342}
8343
8344/// getMergeValues - Create a MERGE_VALUES node from the given operands.
8345SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
8346 if (Ops.size() == 1)
8347 return Ops[0];
8348
8349 SmallVector<EVT, 4> VTs;
8350 VTs.reserve(N: Ops.size());
8351 for (const SDValue &Op : Ops)
8352 VTs.push_back(Elt: Op.getValueType());
8353 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
8354}
8355
8356SDValue SelectionDAG::getMemIntrinsicNode(
8357 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8358 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8359 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
8360 if (!Size && MemVT.isScalableVector())
8361 Size = MemoryLocation::UnknownSize;
8362 else if (!Size)
8363 Size = MemVT.getStoreSize();
8364
8365 MachineFunction &MF = getMachineFunction();
8366 MachineMemOperand *MMO =
8367 MF.getMachineMemOperand(PtrInfo, f: Flags, s: Size, base_alignment: Alignment, AAInfo);
8368
8369 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8370}
8371
8372SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
8373 SDVTList VTList,
8374 ArrayRef<SDValue> Ops, EVT MemVT,
8375 MachineMemOperand *MMO) {
8376 assert((Opcode == ISD::INTRINSIC_VOID ||
8377 Opcode == ISD::INTRINSIC_W_CHAIN ||
8378 Opcode == ISD::PREFETCH ||
8379 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8380 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8381 "Opcode is not a memory-accessing opcode!");
8382
8383 // Memoize the node unless it returns a glue result.
8384 MemIntrinsicSDNode *N;
8385 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8386 FoldingSetNodeID ID;
8387 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
8388 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8389 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MMO));
8390 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8391 ID.AddInteger(I: MMO->getFlags());
8392 ID.AddInteger(I: MemVT.getRawBits());
8393 void *IP = nullptr;
8394 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8395 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8396 return SDValue(E, 0);
8397 }
8398
8399 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8400 Args&: VTList, Args&: MemVT, Args&: MMO);
8401 createOperands(Node: N, Vals: Ops);
8402
8403 CSEMap.InsertNode(N, InsertPos: IP);
8404 } else {
8405 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
8406 Args&: VTList, Args&: MemVT, Args&: MMO);
8407 createOperands(Node: N, Vals: Ops);
8408 }
8409 InsertNode(N);
8410 SDValue V(N, 0);
8411 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8412 return V;
8413}
8414
8415SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
8416 SDValue Chain, int FrameIndex,
8417 int64_t Size, int64_t Offset) {
8418 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8419 const auto VTs = getVTList(MVT::Other);
8420 SDValue Ops[2] = {
8421 Chain,
8422 getFrameIndex(FI: FrameIndex,
8423 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
8424 isTarget: true)};
8425
8426 FoldingSetNodeID ID;
8427 AddNodeIDNode(ID, Opcode, VTs, Ops);
8428 ID.AddInteger(I: FrameIndex);
8429 ID.AddInteger(I: Size);
8430 ID.AddInteger(I: Offset);
8431 void *IP = nullptr;
8432 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
8433 return SDValue(E, 0);
8434
8435 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8436 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8437 createOperands(Node: N, Vals: Ops);
8438 CSEMap.InsertNode(N, InsertPos: IP);
8439 InsertNode(N);
8440 SDValue V(N, 0);
8441 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8442 return V;
8443}
8444
8445SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
8446 uint64_t Guid, uint64_t Index,
8447 uint32_t Attr) {
8448 const unsigned Opcode = ISD::PSEUDO_PROBE;
8449 const auto VTs = getVTList(MVT::Other);
8450 SDValue Ops[] = {Chain};
8451 FoldingSetNodeID ID;
8452 AddNodeIDNode(ID, Opcode, VTs, Ops);
8453 ID.AddInteger(I: Guid);
8454 ID.AddInteger(I: Index);
8455 void *IP = nullptr;
8456 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
8457 return SDValue(E, 0);
8458
8459 auto *N = newSDNode<PseudoProbeSDNode>(
8460 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8461 createOperands(Node: N, Vals: Ops);
8462 CSEMap.InsertNode(N, IP);
8463 InsertNode(N: N);
8464 SDValue V(N, 0);
8465 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8466 return V;
8467}
8468
8469/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8470/// MachinePointerInfo record from it. This is particularly useful because the
8471/// code generator has many cases where it doesn't bother passing in a
8472/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8473static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8474 SelectionDAG &DAG, SDValue Ptr,
8475 int64_t Offset = 0) {
8476 // If this is FI+Offset, we can model it.
8477 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
8478 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
8479 FI: FI->getIndex(), Offset);
8480
8481 // If this is (FI+Offset1)+Offset2, we can model it.
8482 if (Ptr.getOpcode() != ISD::ADD ||
8483 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
8484 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
8485 return Info;
8486
8487 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
8488 return MachinePointerInfo::getFixedStack(
8489 MF&: DAG.getMachineFunction(), FI,
8490 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
8491}
8492
8493/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8494/// MachinePointerInfo record from it. This is particularly useful because the
8495/// code generator has many cases where it doesn't bother passing in a
8496/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8497static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8498 SelectionDAG &DAG, SDValue Ptr,
8499 SDValue OffsetOp) {
8500 // If the 'Offset' value isn't a constant, we can't handle this.
8501 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
8502 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
8503 if (OffsetOp.isUndef())
8504 return InferPointerInfo(Info, DAG, Ptr);
8505 return Info;
8506}
8507
8508SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8509 EVT VT, const SDLoc &dl, SDValue Chain,
8510 SDValue Ptr, SDValue Offset,
8511 MachinePointerInfo PtrInfo, EVT MemVT,
8512 Align Alignment,
8513 MachineMemOperand::Flags MMOFlags,
8514 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8515 assert(Chain.getValueType() == MVT::Other &&
8516 "Invalid chain type");
8517
8518 MMOFlags |= MachineMemOperand::MOLoad;
8519 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8520 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8521 // clients.
8522 if (PtrInfo.V.isNull())
8523 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
8524
8525 uint64_t Size = MemoryLocation::getSizeOrUnknown(T: MemVT.getStoreSize());
8526 MachineFunction &MF = getMachineFunction();
8527 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size,
8528 base_alignment: Alignment, AAInfo, Ranges);
8529 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8530}
8531
8532SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8533 EVT VT, const SDLoc &dl, SDValue Chain,
8534 SDValue Ptr, SDValue Offset, EVT MemVT,
8535 MachineMemOperand *MMO) {
8536 if (VT == MemVT) {
8537 ExtType = ISD::NON_EXTLOAD;
8538 } else if (ExtType == ISD::NON_EXTLOAD) {
8539 assert(VT == MemVT && "Non-extending load from different memory type!");
8540 } else {
8541 // Extending load.
8542 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8543 "Should only be an extending load, not truncating!");
8544 assert(VT.isInteger() == MemVT.isInteger() &&
8545 "Cannot convert from FP to Int or Int -> FP!");
8546 assert(VT.isVector() == MemVT.isVector() &&
8547 "Cannot use an ext load to convert to or from a vector!");
8548 assert((!VT.isVector() ||
8549 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
8550 "Cannot use an ext load to change the number of vector elements!");
8551 }
8552
8553 bool Indexed = AM != ISD::UNINDEXED;
8554 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8555
8556 SDVTList VTs = Indexed ?
8557 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8558 SDValue Ops[] = { Chain, Ptr, Offset };
8559 FoldingSetNodeID ID;
8560 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
8561 ID.AddInteger(I: MemVT.getRawBits());
8562 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
8563 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
8564 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8565 ID.AddInteger(I: MMO->getFlags());
8566 void *IP = nullptr;
8567 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8568 cast<LoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8569 return SDValue(E, 0);
8570 }
8571 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8572 Args&: ExtType, Args&: MemVT, Args&: MMO);
8573 createOperands(Node: N, Vals: Ops);
8574
8575 CSEMap.InsertNode(N, InsertPos: IP);
8576 InsertNode(N);
8577 SDValue V(N, 0);
8578 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8579 return V;
8580}
8581
8582SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8583 SDValue Ptr, MachinePointerInfo PtrInfo,
8584 MaybeAlign Alignment,
8585 MachineMemOperand::Flags MMOFlags,
8586 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8587 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8588 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8589 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
8590}
8591
8592SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8593 SDValue Ptr, MachineMemOperand *MMO) {
8594 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8595 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8596 MemVT: VT, MMO);
8597}
8598
8599SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8600 EVT VT, SDValue Chain, SDValue Ptr,
8601 MachinePointerInfo PtrInfo, EVT MemVT,
8602 MaybeAlign Alignment,
8603 MachineMemOperand::Flags MMOFlags,
8604 const AAMDNodes &AAInfo) {
8605 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8606 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
8607 MemVT, Alignment, MMOFlags, AAInfo);
8608}
8609
8610SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8611 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8612 MachineMemOperand *MMO) {
8613 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8614 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
8615 MemVT, MMO);
8616}
8617
8618SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
8619 SDValue Base, SDValue Offset,
8620 ISD::MemIndexedMode AM) {
8621 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
8622 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8623 // Don't propagate the invariant or dereferenceable flags.
8624 auto MMOFlags =
8625 LD->getMemOperand()->getFlags() &
8626 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8627 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
8628 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
8629 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
8630}
8631
8632SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8633 SDValue Ptr, MachinePointerInfo PtrInfo,
8634 Align Alignment,
8635 MachineMemOperand::Flags MMOFlags,
8636 const AAMDNodes &AAInfo) {
8637 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8638
8639 MMOFlags |= MachineMemOperand::MOStore;
8640 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8641
8642 if (PtrInfo.V.isNull())
8643 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
8644
8645 MachineFunction &MF = getMachineFunction();
8646 uint64_t Size =
8647 MemoryLocation::getSizeOrUnknown(T: Val.getValueType().getStoreSize());
8648 MachineMemOperand *MMO =
8649 MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size, base_alignment: Alignment, AAInfo);
8650 return getStore(Chain, dl, Val, Ptr, MMO);
8651}
8652
8653SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8654 SDValue Ptr, MachineMemOperand *MMO) {
8655 assert(Chain.getValueType() == MVT::Other &&
8656 "Invalid chain type");
8657 EVT VT = Val.getValueType();
8658 SDVTList VTs = getVTList(MVT::Other);
8659 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8660 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8661 FoldingSetNodeID ID;
8662 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8663 ID.AddInteger(I: VT.getRawBits());
8664 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
8665 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO));
8666 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8667 ID.AddInteger(I: MMO->getFlags());
8668 void *IP = nullptr;
8669 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8670 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8671 return SDValue(E, 0);
8672 }
8673 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
8674 Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO);
8675 createOperands(Node: N, Vals: Ops);
8676
8677 CSEMap.InsertNode(N, InsertPos: IP);
8678 InsertNode(N);
8679 SDValue V(N, 0);
8680 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8681 return V;
8682}
8683
8684SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8685 SDValue Ptr, MachinePointerInfo PtrInfo,
8686 EVT SVT, Align Alignment,
8687 MachineMemOperand::Flags MMOFlags,
8688 const AAMDNodes &AAInfo) {
8689 assert(Chain.getValueType() == MVT::Other &&
8690 "Invalid chain type");
8691
8692 MMOFlags |= MachineMemOperand::MOStore;
8693 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8694
8695 if (PtrInfo.V.isNull())
8696 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
8697
8698 MachineFunction &MF = getMachineFunction();
8699 MachineMemOperand *MMO = MF.getMachineMemOperand(
8700 PtrInfo, f: MMOFlags, s: MemoryLocation::getSizeOrUnknown(T: SVT.getStoreSize()),
8701 base_alignment: Alignment, AAInfo);
8702 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8703}
8704
8705SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8706 SDValue Ptr, EVT SVT,
8707 MachineMemOperand *MMO) {
8708 EVT VT = Val.getValueType();
8709
8710 assert(Chain.getValueType() == MVT::Other &&
8711 "Invalid chain type");
8712 if (VT == SVT)
8713 return getStore(Chain, dl, Val, Ptr, MMO);
8714
8715 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8716 "Should only be a truncating store, not extending!");
8717 assert(VT.isInteger() == SVT.isInteger() &&
8718 "Can't do FP-INT conversion!");
8719 assert(VT.isVector() == SVT.isVector() &&
8720 "Cannot use trunc store to convert to or from a vector!");
8721 assert((!VT.isVector() ||
8722 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8723 "Cannot use trunc store to change the number of vector elements!");
8724
8725 SDVTList VTs = getVTList(MVT::Other);
8726 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8727 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8728 FoldingSetNodeID ID;
8729 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8730 ID.AddInteger(I: SVT.getRawBits());
8731 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
8732 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO));
8733 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8734 ID.AddInteger(I: MMO->getFlags());
8735 void *IP = nullptr;
8736 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8737 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8738 return SDValue(E, 0);
8739 }
8740 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
8741 Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO);
8742 createOperands(Node: N, Vals: Ops);
8743
8744 CSEMap.InsertNode(N, InsertPos: IP);
8745 InsertNode(N);
8746 SDValue V(N, 0);
8747 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8748 return V;
8749}
8750
8751SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
8752 SDValue Base, SDValue Offset,
8753 ISD::MemIndexedMode AM) {
8754 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
8755 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8756 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8757 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8758 FoldingSetNodeID ID;
8759 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
8760 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
8761 ID.AddInteger(I: ST->getRawSubclassData());
8762 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
8763 ID.AddInteger(I: ST->getMemOperand()->getFlags());
8764 void *IP = nullptr;
8765 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
8766 return SDValue(E, 0);
8767
8768 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8769 Args: ST->isTruncatingStore(), Args: ST->getMemoryVT(),
8770 Args: ST->getMemOperand());
8771 createOperands(Node: N, Vals: Ops);
8772
8773 CSEMap.InsertNode(N, InsertPos: IP);
8774 InsertNode(N);
8775 SDValue V(N, 0);
8776 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8777 return V;
8778}
8779
8780SDValue SelectionDAG::getLoadVP(
8781 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8782 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8783 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8784 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8785 const MDNode *Ranges, bool IsExpanding) {
8786 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8787
8788 MMOFlags |= MachineMemOperand::MOLoad;
8789 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8790 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8791 // clients.
8792 if (PtrInfo.V.isNull())
8793 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
8794
8795 uint64_t Size = MemoryLocation::getSizeOrUnknown(T: MemVT.getStoreSize());
8796 MachineFunction &MF = getMachineFunction();
8797 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size,
8798 base_alignment: Alignment, AAInfo, Ranges);
8799 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8800 MMO, IsExpanding);
8801}
8802
8803SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
8804 ISD::LoadExtType ExtType, EVT VT,
8805 const SDLoc &dl, SDValue Chain, SDValue Ptr,
8806 SDValue Offset, SDValue Mask, SDValue EVL,
8807 EVT MemVT, MachineMemOperand *MMO,
8808 bool IsExpanding) {
8809 bool Indexed = AM != ISD::UNINDEXED;
8810 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8811
8812 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8813 : getVTList(VT, MVT::Other);
8814 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8815 FoldingSetNodeID ID;
8816 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
8817 ID.AddInteger(I: MemVT.getRawBits());
8818 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
8819 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
8820 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8821 ID.AddInteger(I: MMO->getFlags());
8822 void *IP = nullptr;
8823 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8824 cast<VPLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8825 return SDValue(E, 0);
8826 }
8827 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8828 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
8829 createOperands(Node: N, Vals: Ops);
8830
8831 CSEMap.InsertNode(N, InsertPos: IP);
8832 InsertNode(N);
8833 SDValue V(N, 0);
8834 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8835 return V;
8836}
8837
8838SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8839 SDValue Ptr, SDValue Mask, SDValue EVL,
8840 MachinePointerInfo PtrInfo,
8841 MaybeAlign Alignment,
8842 MachineMemOperand::Flags MMOFlags,
8843 const AAMDNodes &AAInfo, const MDNode *Ranges,
8844 bool IsExpanding) {
8845 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8846 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8847 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
8848 IsExpanding);
8849}
8850
8851SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8852 SDValue Ptr, SDValue Mask, SDValue EVL,
8853 MachineMemOperand *MMO, bool IsExpanding) {
8854 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8855 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
8856 Mask, EVL, MemVT: VT, MMO, IsExpanding);
8857}
8858
8859SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8860 EVT VT, SDValue Chain, SDValue Ptr,
8861 SDValue Mask, SDValue EVL,
8862 MachinePointerInfo PtrInfo, EVT MemVT,
8863 MaybeAlign Alignment,
8864 MachineMemOperand::Flags MMOFlags,
8865 const AAMDNodes &AAInfo, bool IsExpanding) {
8866 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8867 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
8868 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
8869 IsExpanding);
8870}
8871
8872SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8873 EVT VT, SDValue Chain, SDValue Ptr,
8874 SDValue Mask, SDValue EVL, EVT MemVT,
8875 MachineMemOperand *MMO, bool IsExpanding) {
8876 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8877 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
8878 EVL, MemVT, MMO, IsExpanding);
8879}
8880
8881SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
8882 SDValue Base, SDValue Offset,
8883 ISD::MemIndexedMode AM) {
8884 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
8885 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8886 // Don't propagate the invariant or dereferenceable flags.
8887 auto MMOFlags =
8888 LD->getMemOperand()->getFlags() &
8889 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8890 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
8891 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
8892 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
8893 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
8894 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
8895}
8896
8897SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
8898 SDValue Ptr, SDValue Offset, SDValue Mask,
8899 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
8900 ISD::MemIndexedMode AM, bool IsTruncating,
8901 bool IsCompressing) {
8902 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8903 bool Indexed = AM != ISD::UNINDEXED;
8904 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
8905 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
8906 : getVTList(MVT::Other);
8907 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
8908 FoldingSetNodeID ID;
8909 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
8910 ID.AddInteger(I: MemVT.getRawBits());
8911 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
8912 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
8913 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8914 ID.AddInteger(I: MMO->getFlags());
8915 void *IP = nullptr;
8916 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8917 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8918 return SDValue(E, 0);
8919 }
8920 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
8921 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
8922 createOperands(Node: N, Vals: Ops);
8923
8924 CSEMap.InsertNode(N, InsertPos: IP);
8925 InsertNode(N);
8926 SDValue V(N, 0);
8927 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8928 return V;
8929}
8930
8931SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8932 SDValue Val, SDValue Ptr, SDValue Mask,
8933 SDValue EVL, MachinePointerInfo PtrInfo,
8934 EVT SVT, Align Alignment,
8935 MachineMemOperand::Flags MMOFlags,
8936 const AAMDNodes &AAInfo,
8937 bool IsCompressing) {
8938 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8939
8940 MMOFlags |= MachineMemOperand::MOStore;
8941 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8942
8943 if (PtrInfo.V.isNull())
8944 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
8945
8946 MachineFunction &MF = getMachineFunction();
8947 MachineMemOperand *MMO = MF.getMachineMemOperand(
8948 PtrInfo, f: MMOFlags, s: MemoryLocation::getSizeOrUnknown(T: SVT.getStoreSize()),
8949 base_alignment: Alignment, AAInfo);
8950 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
8951 IsCompressing);
8952}
8953
8954SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8955 SDValue Val, SDValue Ptr, SDValue Mask,
8956 SDValue EVL, EVT SVT,
8957 MachineMemOperand *MMO,
8958 bool IsCompressing) {
8959 EVT VT = Val.getValueType();
8960
8961 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8962 if (VT == SVT)
8963 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
8964 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
8965 /*IsTruncating*/ false, IsCompressing);
8966
8967 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8968 "Should only be a truncating store, not extending!");
8969 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
8970 assert(VT.isVector() == SVT.isVector() &&
8971 "Cannot use trunc store to convert to or from a vector!");
8972 assert((!VT.isVector() ||
8973 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8974 "Cannot use trunc store to change the number of vector elements!");
8975
8976 SDVTList VTs = getVTList(MVT::Other);
8977 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
8978 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
8979 FoldingSetNodeID ID;
8980 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
8981 ID.AddInteger(I: SVT.getRawBits());
8982 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
8983 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
8984 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
8985 ID.AddInteger(I: MMO->getFlags());
8986 void *IP = nullptr;
8987 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
8988 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
8989 return SDValue(E, 0);
8990 }
8991 auto *N =
8992 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
8993 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
8994 createOperands(Node: N, Vals: Ops);
8995
8996 CSEMap.InsertNode(N, InsertPos: IP);
8997 InsertNode(N);
8998 SDValue V(N, 0);
8999 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9000 return V;
9001}
9002
9003SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
9004 SDValue Base, SDValue Offset,
9005 ISD::MemIndexedMode AM) {
9006 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
9007 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9008 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9009 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9010 Offset, ST->getMask(), ST->getVectorLength()};
9011 FoldingSetNodeID ID;
9012 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9013 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
9014 ID.AddInteger(I: ST->getRawSubclassData());
9015 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
9016 ID.AddInteger(I: ST->getMemOperand()->getFlags());
9017 void *IP = nullptr;
9018 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9019 return SDValue(E, 0);
9020
9021 auto *N = newSDNode<VPStoreSDNode>(
9022 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
9023 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
9024 createOperands(Node: N, Vals: Ops);
9025
9026 CSEMap.InsertNode(N, InsertPos: IP);
9027 InsertNode(N);
9028 SDValue V(N, 0);
9029 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9030 return V;
9031}
9032
9033SDValue SelectionDAG::getStridedLoadVP(
9034 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9035 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9036 SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
9037 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9038 const MDNode *Ranges, bool IsExpanding) {
9039 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9040
9041 MMOFlags |= MachineMemOperand::MOLoad;
9042 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9043 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9044 // clients.
9045 if (PtrInfo.V.isNull())
9046 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
9047
9048 uint64_t Size = MemoryLocation::UnknownSize;
9049 MachineFunction &MF = getMachineFunction();
9050 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size,
9051 base_alignment: Alignment, AAInfo, Ranges);
9052 return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask,
9053 EVL, MemVT, MMO, IsExpanding);
9054}
9055
9056SDValue SelectionDAG::getStridedLoadVP(
9057 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9058 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9059 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9060 bool Indexed = AM != ISD::UNINDEXED;
9061 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9062
9063 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9064 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9065 : getVTList(VT, MVT::Other);
9066 FoldingSetNodeID ID;
9067 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
9068 ID.AddInteger(I: VT.getRawBits());
9069 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9070 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
9071 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9072
9073 void *IP = nullptr;
9074 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9075 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9076 return SDValue(E, 0);
9077 }
9078
9079 auto *N =
9080 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
9081 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
9082 createOperands(Node: N, Vals: Ops);
9083 CSEMap.InsertNode(N, InsertPos: IP);
9084 InsertNode(N);
9085 SDValue V(N, 0);
9086 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9087 return V;
9088}
9089
9090SDValue SelectionDAG::getStridedLoadVP(
9091 EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride,
9092 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment,
9093 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9094 const MDNode *Ranges, bool IsExpanding) {
9095 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9096 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9097 Offset: Undef, Stride, Mask, EVL, PtrInfo, MemVT: VT, Alignment,
9098 MMOFlags, AAInfo, Ranges, IsExpanding);
9099}
9100
9101SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
9102 SDValue Ptr, SDValue Stride,
9103 SDValue Mask, SDValue EVL,
9104 MachineMemOperand *MMO,
9105 bool IsExpanding) {
9106 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9107 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9108 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
9109}
9110
9111SDValue SelectionDAG::getExtStridedLoadVP(
9112 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9113 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL,
9114 MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment,
9115 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9116 bool IsExpanding) {
9117 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9118 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
9119 Stride, Mask, EVL, PtrInfo, MemVT, Alignment,
9120 MMOFlags, AAInfo, Ranges: nullptr, IsExpanding);
9121}
9122
9123SDValue SelectionDAG::getExtStridedLoadVP(
9124 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9125 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9126 MachineMemOperand *MMO, bool IsExpanding) {
9127 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9128 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
9129 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9130}
9131
9132SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL,
9133 SDValue Base, SDValue Offset,
9134 ISD::MemIndexedMode AM) {
9135 auto *SLD = cast<VPStridedLoadSDNode>(Val&: OrigLoad);
9136 assert(SLD->getOffset().isUndef() &&
9137 "Strided load is already a indexed load!");
9138 // Don't propagate the invariant or dereferenceable flags.
9139 auto MMOFlags =
9140 SLD->getMemOperand()->getFlags() &
9141 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9142 return getStridedLoadVP(
9143 AM, ExtType: SLD->getExtensionType(), VT: OrigLoad.getValueType(), DL, Chain: SLD->getChain(),
9144 Ptr: Base, Offset, Stride: SLD->getStride(), Mask: SLD->getMask(), EVL: SLD->getVectorLength(),
9145 PtrInfo: SLD->getPointerInfo(), MemVT: SLD->getMemoryVT(), Alignment: SLD->getAlign(), MMOFlags,
9146 AAInfo: SLD->getAAInfo(), Ranges: nullptr, IsExpanding: SLD->isExpandingLoad());
9147}
9148
9149SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
9150 SDValue Val, SDValue Ptr,
9151 SDValue Offset, SDValue Stride,
9152 SDValue Mask, SDValue EVL, EVT MemVT,
9153 MachineMemOperand *MMO,
9154 ISD::MemIndexedMode AM,
9155 bool IsTruncating, bool IsCompressing) {
9156 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9157 bool Indexed = AM != ISD::UNINDEXED;
9158 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9159 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9160 : getVTList(MVT::Other);
9161 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9162 FoldingSetNodeID ID;
9163 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9164 ID.AddInteger(I: MemVT.getRawBits());
9165 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9166 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9167 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9168 void *IP = nullptr;
9169 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9170 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9171 return SDValue(E, 0);
9172 }
9173 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9174 Args&: VTs, Args&: AM, Args&: IsTruncating,
9175 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9176 createOperands(Node: N, Vals: Ops);
9177
9178 CSEMap.InsertNode(N, InsertPos: IP);
9179 InsertNode(N);
9180 SDValue V(N, 0);
9181 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9182 return V;
9183}
9184
9185SDValue SelectionDAG::getTruncStridedStoreVP(
9186 SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride,
9187 SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT,
9188 Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9189 bool IsCompressing) {
9190 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9191
9192 MMOFlags |= MachineMemOperand::MOStore;
9193 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9194
9195 if (PtrInfo.V.isNull())
9196 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
9197
9198 MachineFunction &MF = getMachineFunction();
9199 MachineMemOperand *MMO = MF.getMachineMemOperand(
9200 PtrInfo, f: MMOFlags, s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo);
9201 return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT,
9202 MMO, IsCompressing);
9203}
9204
9205SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
9206 SDValue Val, SDValue Ptr,
9207 SDValue Stride, SDValue Mask,
9208 SDValue EVL, EVT SVT,
9209 MachineMemOperand *MMO,
9210 bool IsCompressing) {
9211 EVT VT = Val.getValueType();
9212
9213 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9214 if (VT == SVT)
9215 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
9216 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
9217 /*IsTruncating*/ false, IsCompressing);
9218
9219 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9220 "Should only be a truncating store, not extending!");
9221 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9222 assert(VT.isVector() == SVT.isVector() &&
9223 "Cannot use trunc store to convert to or from a vector!");
9224 assert((!VT.isVector() ||
9225 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9226 "Cannot use trunc store to change the number of vector elements!");
9227
9228 SDVTList VTs = getVTList(MVT::Other);
9229 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9230 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9231 FoldingSetNodeID ID;
9232 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9233 ID.AddInteger(I: SVT.getRawBits());
9234 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9235 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
9236 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9237 void *IP = nullptr;
9238 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9239 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9240 return SDValue(E, 0);
9241 }
9242 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9243 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
9244 Args&: IsCompressing, Args&: SVT, Args&: MMO);
9245 createOperands(Node: N, Vals: Ops);
9246
9247 CSEMap.InsertNode(N, InsertPos: IP);
9248 InsertNode(N);
9249 SDValue V(N, 0);
9250 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9251 return V;
9252}
9253
9254SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore,
9255 const SDLoc &DL, SDValue Base,
9256 SDValue Offset,
9257 ISD::MemIndexedMode AM) {
9258 auto *SST = cast<VPStridedStoreSDNode>(Val&: OrigStore);
9259 assert(SST->getOffset().isUndef() &&
9260 "Strided store is already an indexed store!");
9261 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9262 SDValue Ops[] = {
9263 SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(),
9264 SST->getMask(), SST->getVectorLength()};
9265 FoldingSetNodeID ID;
9266 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9267 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
9268 ID.AddInteger(I: SST->getRawSubclassData());
9269 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
9270 void *IP = nullptr;
9271 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
9272 return SDValue(E, 0);
9273
9274 auto *N = newSDNode<VPStridedStoreSDNode>(
9275 Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM, Args: SST->isTruncatingStore(),
9276 Args: SST->isCompressingStore(), Args: SST->getMemoryVT(), Args: SST->getMemOperand());
9277 createOperands(Node: N, Vals: Ops);
9278
9279 CSEMap.InsertNode(N, InsertPos: IP);
9280 InsertNode(N);
9281 SDValue V(N, 0);
9282 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9283 return V;
9284}
9285
9286SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9287 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
9288 ISD::MemIndexType IndexType) {
9289 assert(Ops.size() == 6 && "Incompatible number of operands");
9290
9291 FoldingSetNodeID ID;
9292 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
9293 ID.AddInteger(I: VT.getRawBits());
9294 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
9295 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
9296 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9297 ID.AddInteger(I: MMO->getFlags());
9298 void *IP = nullptr;
9299 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9300 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9301 return SDValue(E, 0);
9302 }
9303
9304 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9305 Args&: VT, Args&: MMO, Args&: IndexType);
9306 createOperands(Node: N, Vals: Ops);
9307
9308 assert(N->getMask().getValueType().getVectorElementCount() ==
9309 N->getValueType(0).getVectorElementCount() &&
9310 "Vector width mismatch between mask and data");
9311 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9312 N->getValueType(0).getVectorElementCount().isScalable() &&
9313 "Scalable flags of index and data do not match");
9314 assert(ElementCount::isKnownGE(
9315 N->getIndex().getValueType().getVectorElementCount(),
9316 N->getValueType(0).getVectorElementCount()) &&
9317 "Vector width mismatch between index and data");
9318 assert(isa<ConstantSDNode>(N->getScale()) &&
9319 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9320 "Scale should be a constant power of 2");
9321
9322 CSEMap.InsertNode(N, InsertPos: IP);
9323 InsertNode(N);
9324 SDValue V(N, 0);
9325 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9326 return V;
9327}
9328
9329SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9330 ArrayRef<SDValue> Ops,
9331 MachineMemOperand *MMO,
9332 ISD::MemIndexType IndexType) {
9333 assert(Ops.size() == 7 && "Incompatible number of operands");
9334
9335 FoldingSetNodeID ID;
9336 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
9337 ID.AddInteger(I: VT.getRawBits());
9338 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
9339 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
9340 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9341 ID.AddInteger(I: MMO->getFlags());
9342 void *IP = nullptr;
9343 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9344 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9345 return SDValue(E, 0);
9346 }
9347 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9348 Args&: VT, Args&: MMO, Args&: IndexType);
9349 createOperands(Node: N, Vals: Ops);
9350
9351 assert(N->getMask().getValueType().getVectorElementCount() ==
9352 N->getValue().getValueType().getVectorElementCount() &&
9353 "Vector width mismatch between mask and data");
9354 assert(
9355 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9356 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9357 "Scalable flags of index and data do not match");
9358 assert(ElementCount::isKnownGE(
9359 N->getIndex().getValueType().getVectorElementCount(),
9360 N->getValue().getValueType().getVectorElementCount()) &&
9361 "Vector width mismatch between index and data");
9362 assert(isa<ConstantSDNode>(N->getScale()) &&
9363 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9364 "Scale should be a constant power of 2");
9365
9366 CSEMap.InsertNode(N, InsertPos: IP);
9367 InsertNode(N);
9368 SDValue V(N, 0);
9369 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9370 return V;
9371}
9372
9373SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9374 SDValue Base, SDValue Offset, SDValue Mask,
9375 SDValue PassThru, EVT MemVT,
9376 MachineMemOperand *MMO,
9377 ISD::MemIndexedMode AM,
9378 ISD::LoadExtType ExtTy, bool isExpanding) {
9379 bool Indexed = AM != ISD::UNINDEXED;
9380 assert((Indexed || Offset.isUndef()) &&
9381 "Unindexed masked load with an offset!");
9382 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9383 : getVTList(VT, MVT::Other);
9384 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9385 FoldingSetNodeID ID;
9386 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
9387 ID.AddInteger(I: MemVT.getRawBits());
9388 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9389 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
9390 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9391 ID.AddInteger(I: MMO->getFlags());
9392 void *IP = nullptr;
9393 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9394 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9395 return SDValue(E, 0);
9396 }
9397 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9398 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
9399 createOperands(Node: N, Vals: Ops);
9400
9401 CSEMap.InsertNode(N, InsertPos: IP);
9402 InsertNode(N);
9403 SDValue V(N, 0);
9404 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9405 return V;
9406}
9407
9408SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
9409 SDValue Base, SDValue Offset,
9410 ISD::MemIndexedMode AM) {
9411 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
9412 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9413 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
9414 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
9415 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
9416 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
9417}
9418
9419SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
9420 SDValue Val, SDValue Base, SDValue Offset,
9421 SDValue Mask, EVT MemVT,
9422 MachineMemOperand *MMO,
9423 ISD::MemIndexedMode AM, bool IsTruncating,
9424 bool IsCompressing) {
9425 assert(Chain.getValueType() == MVT::Other &&
9426 "Invalid chain type");
9427 bool Indexed = AM != ISD::UNINDEXED;
9428 assert((Indexed || Offset.isUndef()) &&
9429 "Unindexed masked store with an offset!");
9430 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9431 : getVTList(MVT::Other);
9432 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9433 FoldingSetNodeID ID;
9434 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
9435 ID.AddInteger(I: MemVT.getRawBits());
9436 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9437 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9438 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9439 ID.AddInteger(I: MMO->getFlags());
9440 void *IP = nullptr;
9441 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9442 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9443 return SDValue(E, 0);
9444 }
9445 auto *N =
9446 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9447 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9448 createOperands(Node: N, Vals: Ops);
9449
9450 CSEMap.InsertNode(N, InsertPos: IP);
9451 InsertNode(N);
9452 SDValue V(N, 0);
9453 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9454 return V;
9455}
9456
9457SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
9458 SDValue Base, SDValue Offset,
9459 ISD::MemIndexedMode AM) {
9460 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
9461 assert(ST->getOffset().isUndef() &&
9462 "Masked store is already a indexed store!");
9463 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
9464 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
9465 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
9466}
9467
9468SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9469 ArrayRef<SDValue> Ops,
9470 MachineMemOperand *MMO,
9471 ISD::MemIndexType IndexType,
9472 ISD::LoadExtType ExtTy) {
9473 assert(Ops.size() == 6 && "Incompatible number of operands");
9474
9475 FoldingSetNodeID ID;
9476 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
9477 ID.AddInteger(I: MemVT.getRawBits());
9478 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9479 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
9480 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9481 ID.AddInteger(I: MMO->getFlags());
9482 void *IP = nullptr;
9483 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9484 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9485 return SDValue(E, 0);
9486 }
9487
9488 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9489 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
9490 createOperands(Node: N, Vals: Ops);
9491
9492 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9493 "Incompatible type of the PassThru value in MaskedGatherSDNode");
9494 assert(N->getMask().getValueType().getVectorElementCount() ==
9495 N->getValueType(0).getVectorElementCount() &&
9496 "Vector width mismatch between mask and data");
9497 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9498 N->getValueType(0).getVectorElementCount().isScalable() &&
9499 "Scalable flags of index and data do not match");
9500 assert(ElementCount::isKnownGE(
9501 N->getIndex().getValueType().getVectorElementCount(),
9502 N->getValueType(0).getVectorElementCount()) &&
9503 "Vector width mismatch between index and data");
9504 assert(isa<ConstantSDNode>(N->getScale()) &&
9505 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9506 "Scale should be a constant power of 2");
9507
9508 CSEMap.InsertNode(N, InsertPos: IP);
9509 InsertNode(N);
9510 SDValue V(N, 0);
9511 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9512 return V;
9513}
9514
9515SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9516 ArrayRef<SDValue> Ops,
9517 MachineMemOperand *MMO,
9518 ISD::MemIndexType IndexType,
9519 bool IsTrunc) {
9520 assert(Ops.size() == 6 && "Incompatible number of operands");
9521
9522 FoldingSetNodeID ID;
9523 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
9524 ID.AddInteger(I: MemVT.getRawBits());
9525 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9526 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
9527 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9528 ID.AddInteger(I: MMO->getFlags());
9529 void *IP = nullptr;
9530 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9531 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9532 return SDValue(E, 0);
9533 }
9534
9535 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9536 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
9537 createOperands(Node: N, Vals: Ops);
9538
9539 assert(N->getMask().getValueType().getVectorElementCount() ==
9540 N->getValue().getValueType().getVectorElementCount() &&
9541 "Vector width mismatch between mask and data");
9542 assert(
9543 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9544 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9545 "Scalable flags of index and data do not match");
9546 assert(ElementCount::isKnownGE(
9547 N->getIndex().getValueType().getVectorElementCount(),
9548 N->getValue().getValueType().getVectorElementCount()) &&
9549 "Vector width mismatch between index and data");
9550 assert(isa<ConstantSDNode>(N->getScale()) &&
9551 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9552 "Scale should be a constant power of 2");
9553
9554 CSEMap.InsertNode(N, InsertPos: IP);
9555 InsertNode(N);
9556 SDValue V(N, 0);
9557 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9558 return V;
9559}
9560
9561SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9562 EVT MemVT, MachineMemOperand *MMO) {
9563 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9564 SDVTList VTs = getVTList(MVT::Other);
9565 SDValue Ops[] = {Chain, Ptr};
9566 FoldingSetNodeID ID;
9567 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
9568 ID.AddInteger(I: MemVT.getRawBits());
9569 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9570 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
9571 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9572 ID.AddInteger(I: MMO->getFlags());
9573 void *IP = nullptr;
9574 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9575 return SDValue(E, 0);
9576
9577 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
9578 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
9579 createOperands(Node: N, Vals: Ops);
9580
9581 CSEMap.InsertNode(N, InsertPos: IP);
9582 InsertNode(N);
9583 SDValue V(N, 0);
9584 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9585 return V;
9586}
9587
9588SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9589 EVT MemVT, MachineMemOperand *MMO) {
9590 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9591 SDVTList VTs = getVTList(MVT::Other);
9592 SDValue Ops[] = {Chain, Ptr};
9593 FoldingSetNodeID ID;
9594 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
9595 ID.AddInteger(I: MemVT.getRawBits());
9596 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9597 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
9598 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9599 ID.AddInteger(I: MMO->getFlags());
9600 void *IP = nullptr;
9601 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9602 return SDValue(E, 0);
9603
9604 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
9605 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
9606 createOperands(Node: N, Vals: Ops);
9607
9608 CSEMap.InsertNode(N, InsertPos: IP);
9609 InsertNode(N);
9610 SDValue V(N, 0);
9611 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9612 return V;
9613}
9614
9615SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
9616 // select undef, T, F --> T (if T is a constant), otherwise F
9617 // select, ?, undef, F --> F
9618 // select, ?, T, undef --> T
9619 if (Cond.isUndef())
9620 return isConstantValueOfAnyType(N: T) ? T : F;
9621 if (T.isUndef())
9622 return F;
9623 if (F.isUndef())
9624 return T;
9625
9626 // select true, T, F --> T
9627 // select false, T, F --> F
9628 if (auto *CondC = dyn_cast<ConstantSDNode>(Val&: Cond))
9629 return CondC->isZero() ? F : T;
9630
9631 // TODO: This should simplify VSELECT with non-zero constant condition using
9632 // something like this (but check boolean contents to be complete?):
9633 if (ConstantSDNode *CondC = isConstOrConstSplat(N: Cond, /*AllowUndefs*/ false,
9634 /*AllowTruncation*/ true))
9635 if (CondC->isZero())
9636 return F;
9637
9638 // select ?, T, T --> T
9639 if (T == F)
9640 return T;
9641
9642 return SDValue();
9643}
9644
9645SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
9646 // shift undef, Y --> 0 (can always assume that the undef value is 0)
9647 if (X.isUndef())
9648 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
9649 // shift X, undef --> undef (because it may shift by the bitwidth)
9650 if (Y.isUndef())
9651 return getUNDEF(VT: X.getValueType());
9652
9653 // shift 0, Y --> 0
9654 // shift X, 0 --> X
9655 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
9656 return X;
9657
9658 // shift X, C >= bitwidth(X) --> undef
9659 // All vector elements must be too big (or undef) to avoid partial undefs.
9660 auto isShiftTooBig = [X](ConstantSDNode *Val) {
9661 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
9662 };
9663 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
9664 return getUNDEF(VT: X.getValueType());
9665
9666 return SDValue();
9667}
9668
9669SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
9670 SDNodeFlags Flags) {
9671 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9672 // (an undef operand can be chosen to be Nan/Inf), then the result of this
9673 // operation is poison. That result can be relaxed to undef.
9674 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
9675 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
9676 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9677 (YC && YC->getValueAPF().isNaN());
9678 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9679 (YC && YC->getValueAPF().isInfinity());
9680
9681 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9682 return getUNDEF(VT: X.getValueType());
9683
9684 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9685 return getUNDEF(VT: X.getValueType());
9686
9687 if (!YC)
9688 return SDValue();
9689
9690 // X + -0.0 --> X
9691 if (Opcode == ISD::FADD)
9692 if (YC->getValueAPF().isNegZero())
9693 return X;
9694
9695 // X - +0.0 --> X
9696 if (Opcode == ISD::FSUB)
9697 if (YC->getValueAPF().isPosZero())
9698 return X;
9699
9700 // X * 1.0 --> X
9701 // X / 1.0 --> X
9702 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9703 if (YC->getValueAPF().isExactlyValue(V: 1.0))
9704 return X;
9705
9706 // X * 0.0 --> 0.0
9707 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9708 if (YC->getValueAPF().isZero())
9709 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
9710
9711 return SDValue();
9712}
9713
9714SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
9715 SDValue Ptr, SDValue SV, unsigned Align) {
9716 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9717 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9718}
9719
9720SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9721 ArrayRef<SDUse> Ops) {
9722 switch (Ops.size()) {
9723 case 0: return getNode(Opcode, DL, VT);
9724 case 1: return getNode(Opcode, DL, VT, N1: static_cast<const SDValue>(Ops[0]));
9725 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
9726 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
9727 default: break;
9728 }
9729
9730 // Copy from an SDUse array into an SDValue array for use with
9731 // the regular getNode logic.
9732 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9733 return getNode(Opcode, DL, VT, Ops: NewOps);
9734}
9735
9736SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9737 ArrayRef<SDValue> Ops) {
9738 SDNodeFlags Flags;
9739 if (Inserter)
9740 Flags = Inserter->getFlags();
9741 return getNode(Opcode, DL, VT, Ops, Flags);
9742}
9743
9744SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9745 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9746 unsigned NumOps = Ops.size();
9747 switch (NumOps) {
9748 case 0: return getNode(Opcode, DL, VT);
9749 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
9750 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
9751 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
9752 default: break;
9753 }
9754
9755#ifndef NDEBUG
9756 for (const auto &Op : Ops)
9757 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9758 "Operand is DELETED_NODE!");
9759#endif
9760
9761 switch (Opcode) {
9762 default: break;
9763 case ISD::BUILD_VECTOR:
9764 // Attempt to simplify BUILD_VECTOR.
9765 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9766 return V;
9767 break;
9768 case ISD::CONCAT_VECTORS:
9769 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9770 return V;
9771 break;
9772 case ISD::SELECT_CC:
9773 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9774 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9775 "LHS and RHS of condition must have same type!");
9776 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9777 "True and False arms of SelectCC must have same type!");
9778 assert(Ops[2].getValueType() == VT &&
9779 "select_cc node must be of same type as true and false value!");
9780 assert((!Ops[0].getValueType().isVector() ||
9781 Ops[0].getValueType().getVectorElementCount() ==
9782 VT.getVectorElementCount()) &&
9783 "Expected select_cc with vector result to have the same sized "
9784 "comparison type!");
9785 break;
9786 case ISD::BR_CC:
9787 assert(NumOps == 5 && "BR_CC takes 5 operands!");
9788 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9789 "LHS/RHS of comparison should match types!");
9790 break;
9791 case ISD::VP_ADD:
9792 case ISD::VP_SUB:
9793 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9794 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9795 Opcode = ISD::VP_XOR;
9796 break;
9797 case ISD::VP_MUL:
9798 // If it is VP_MUL mask operation then turn it to VP_AND
9799 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9800 Opcode = ISD::VP_AND;
9801 break;
9802 case ISD::VP_REDUCE_MUL:
9803 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9804 if (VT == MVT::i1)
9805 Opcode = ISD::VP_REDUCE_AND;
9806 break;
9807 case ISD::VP_REDUCE_ADD:
9808 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9809 if (VT == MVT::i1)
9810 Opcode = ISD::VP_REDUCE_XOR;
9811 break;
9812 case ISD::VP_REDUCE_SMAX:
9813 case ISD::VP_REDUCE_UMIN:
9814 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9815 // VP_REDUCE_AND.
9816 if (VT == MVT::i1)
9817 Opcode = ISD::VP_REDUCE_AND;
9818 break;
9819 case ISD::VP_REDUCE_SMIN:
9820 case ISD::VP_REDUCE_UMAX:
9821 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9822 // VP_REDUCE_OR.
9823 if (VT == MVT::i1)
9824 Opcode = ISD::VP_REDUCE_OR;
9825 break;
9826 }
9827
9828 // Memoize nodes.
9829 SDNode *N;
9830 SDVTList VTs = getVTList(VT);
9831
9832 if (VT != MVT::Glue) {
9833 FoldingSetNodeID ID;
9834 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9835 void *IP = nullptr;
9836
9837 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
9838 return SDValue(E, 0);
9839
9840 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9841 createOperands(Node: N, Vals: Ops);
9842
9843 CSEMap.InsertNode(N, InsertPos: IP);
9844 } else {
9845 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9846 createOperands(Node: N, Vals: Ops);
9847 }
9848
9849 N->setFlags(Flags);
9850 InsertNode(N);
9851 SDValue V(N, 0);
9852 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9853 return V;
9854}
9855
9856SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9857 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9858 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops);
9859}
9860
9861SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9862 ArrayRef<SDValue> Ops) {
9863 SDNodeFlags Flags;
9864 if (Inserter)
9865 Flags = Inserter->getFlags();
9866 return getNode(Opcode, DL, VTList, Ops, Flags);
9867}
9868
9869SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9870 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9871 if (VTList.NumVTs == 1)
9872 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
9873
9874#ifndef NDEBUG
9875 for (const auto &Op : Ops)
9876 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9877 "Operand is DELETED_NODE!");
9878#endif
9879
9880 switch (Opcode) {
9881 case ISD::SADDO:
9882 case ISD::UADDO:
9883 case ISD::SSUBO:
9884 case ISD::USUBO: {
9885 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9886 "Invalid add/sub overflow op!");
9887 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9888 Ops[0].getValueType() == Ops[1].getValueType() &&
9889 Ops[0].getValueType() == VTList.VTs[0] &&
9890 "Binary operator types must match!");
9891 SDValue N1 = Ops[0], N2 = Ops[1];
9892 canonicalizeCommutativeBinop(Opcode, N1, N2);
9893
9894 // (X +- 0) -> X with zero-overflow.
9895 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
9896 /*AllowTruncation*/ true);
9897 if (N2CV && N2CV->isZero()) {
9898 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
9899 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
9900 }
9901
9902 if (VTList.VTs[0].isVector() &&
9903 VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9904 VTList.VTs[1].getVectorElementType() == MVT::i1) {
9905 SDValue F1 = getFreeze(V: N1);
9906 SDValue F2 = getFreeze(V: N2);
9907 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9908 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9909 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
9910 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
9911 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
9912 Flags);
9913 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9914 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9915 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
9916 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
9917 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
9918 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
9919 Flags);
9920 }
9921 }
9922 break;
9923 }
9924 case ISD::SMUL_LOHI:
9925 case ISD::UMUL_LOHI: {
9926 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9927 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9928 VTList.VTs[0] == Ops[0].getValueType() &&
9929 VTList.VTs[0] == Ops[1].getValueType() &&
9930 "Binary operator types must match!");
9931 // Constant fold.
9932 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
9933 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
9934 if (LHS && RHS) {
9935 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9936 unsigned OutWidth = Width * 2;
9937 APInt Val = LHS->getAPIntValue();
9938 APInt Mul = RHS->getAPIntValue();
9939 if (Opcode == ISD::SMUL_LOHI) {
9940 Val = Val.sext(width: OutWidth);
9941 Mul = Mul.sext(width: OutWidth);
9942 } else {
9943 Val = Val.zext(width: OutWidth);
9944 Mul = Mul.zext(width: OutWidth);
9945 }
9946 Val *= Mul;
9947
9948 SDValue Hi =
9949 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
9950 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
9951 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
9952 }
9953 break;
9954 }
9955 case ISD::FFREXP: {
9956 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9957 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9958 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9959
9960 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
9961 int FrexpExp;
9962 APFloat FrexpMant =
9963 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
9964 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
9965 SDValue Result1 =
9966 getConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0, DL, VT: VTList.VTs[1]);
9967 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
9968 }
9969
9970 break;
9971 }
9972 case ISD::STRICT_FP_EXTEND:
9973 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9974 "Invalid STRICT_FP_EXTEND!");
9975 assert(VTList.VTs[0].isFloatingPoint() &&
9976 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9977 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9978 "STRICT_FP_EXTEND result type should be vector iff the operand "
9979 "type is vector!");
9980 assert((!VTList.VTs[0].isVector() ||
9981 VTList.VTs[0].getVectorElementCount() ==
9982 Ops[1].getValueType().getVectorElementCount()) &&
9983 "Vector element count mismatch!");
9984 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
9985 "Invalid fpext node, dst <= src!");
9986 break;
9987 case ISD::STRICT_FP_ROUND:
9988 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
9989 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9990 "STRICT_FP_ROUND result type should be vector iff the operand "
9991 "type is vector!");
9992 assert((!VTList.VTs[0].isVector() ||
9993 VTList.VTs[0].getVectorElementCount() ==
9994 Ops[1].getValueType().getVectorElementCount()) &&
9995 "Vector element count mismatch!");
9996 assert(VTList.VTs[0].isFloatingPoint() &&
9997 Ops[1].getValueType().isFloatingPoint() &&
9998 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
9999 isa<ConstantSDNode>(Ops[2]) &&
10000 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10001 "Invalid STRICT_FP_ROUND!");
10002 break;
10003#if 0
10004 // FIXME: figure out how to safely handle things like
10005 // int foo(int x) { return 1 << (x & 255); }
10006 // int bar() { return foo(256); }
10007 case ISD::SRA_PARTS:
10008 case ISD::SRL_PARTS:
10009 case ISD::SHL_PARTS:
10010 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
10011 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10012 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10013 else if (N3.getOpcode() == ISD::AND)
10014 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10015 // If the and is only masking out bits that cannot effect the shift,
10016 // eliminate the and.
10017 unsigned NumBits = VT.getScalarSizeInBits()*2;
10018 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10019 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10020 }
10021 break;
10022#endif
10023 }
10024
10025 // Memoize the node unless it returns a glue result.
10026 SDNode *N;
10027 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10028 FoldingSetNodeID ID;
10029 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10030 void *IP = nullptr;
10031 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
10032 return SDValue(E, 0);
10033
10034 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10035 createOperands(Node: N, Vals: Ops);
10036 CSEMap.InsertNode(N, InsertPos: IP);
10037 } else {
10038 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10039 createOperands(Node: N, Vals: Ops);
10040 }
10041
10042 N->setFlags(Flags);
10043 InsertNode(N);
10044 SDValue V(N, 0);
10045 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10046 return V;
10047}
10048
10049SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10050 SDVTList VTList) {
10051 return getNode(Opcode, DL, VTList, Ops: std::nullopt);
10052}
10053
10054SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10055 SDValue N1) {
10056 SDValue Ops[] = { N1 };
10057 return getNode(Opcode, DL, VTList, Ops);
10058}
10059
10060SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10061 SDValue N1, SDValue N2) {
10062 SDValue Ops[] = { N1, N2 };
10063 return getNode(Opcode, DL, VTList, Ops);
10064}
10065
10066SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10067 SDValue N1, SDValue N2, SDValue N3) {
10068 SDValue Ops[] = { N1, N2, N3 };
10069 return getNode(Opcode, DL, VTList, Ops);
10070}
10071
10072SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10073 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10074 SDValue Ops[] = { N1, N2, N3, N4 };
10075 return getNode(Opcode, DL, VTList, Ops);
10076}
10077
10078SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10079 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10080 SDValue N5) {
10081 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10082 return getNode(Opcode, DL, VTList, Ops);
10083}
10084
10085SDVTList SelectionDAG::getVTList(EVT VT) {
10086 return makeVTList(VTs: SDNode::getValueTypeList(VT), NumVTs: 1);
10087}
10088
10089SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
10090 FoldingSetNodeID ID;
10091 ID.AddInteger(I: 2U);
10092 ID.AddInteger(I: VT1.getRawBits());
10093 ID.AddInteger(I: VT2.getRawBits());
10094
10095 void *IP = nullptr;
10096 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10097 if (!Result) {
10098 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
10099 Array[0] = VT1;
10100 Array[1] = VT2;
10101 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10102 VTListMap.InsertNode(N: Result, InsertPos: IP);
10103 }
10104 return Result->getSDVTList();
10105}
10106
10107SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
10108 FoldingSetNodeID ID;
10109 ID.AddInteger(I: 3U);
10110 ID.AddInteger(I: VT1.getRawBits());
10111 ID.AddInteger(I: VT2.getRawBits());
10112 ID.AddInteger(I: VT3.getRawBits());
10113
10114 void *IP = nullptr;
10115 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10116 if (!Result) {
10117 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
10118 Array[0] = VT1;
10119 Array[1] = VT2;
10120 Array[2] = VT3;
10121 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10122 VTListMap.InsertNode(N: Result, InsertPos: IP);
10123 }
10124 return Result->getSDVTList();
10125}
10126
10127SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
10128 FoldingSetNodeID ID;
10129 ID.AddInteger(I: 4U);
10130 ID.AddInteger(I: VT1.getRawBits());
10131 ID.AddInteger(I: VT2.getRawBits());
10132 ID.AddInteger(I: VT3.getRawBits());
10133 ID.AddInteger(I: VT4.getRawBits());
10134
10135 void *IP = nullptr;
10136 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10137 if (!Result) {
10138 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
10139 Array[0] = VT1;
10140 Array[1] = VT2;
10141 Array[2] = VT3;
10142 Array[3] = VT4;
10143 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10144 VTListMap.InsertNode(N: Result, InsertPos: IP);
10145 }
10146 return Result->getSDVTList();
10147}
10148
10149SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
10150 unsigned NumVTs = VTs.size();
10151 FoldingSetNodeID ID;
10152 ID.AddInteger(I: NumVTs);
10153 for (unsigned index = 0; index < NumVTs; index++) {
10154 ID.AddInteger(I: VTs[index].getRawBits());
10155 }
10156
10157 void *IP = nullptr;
10158 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10159 if (!Result) {
10160 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
10161 llvm::copy(Range&: VTs, Out: Array);
10162 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10163 VTListMap.InsertNode(N: Result, InsertPos: IP);
10164 }
10165 return Result->getSDVTList();
10166}
10167
10168
10169/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10170/// specified operands. If the resultant node already exists in the DAG,
10171/// this does not modify the specified node, instead it returns the node that
10172/// already exists. If the resultant node does not exist in the DAG, the
10173/// input node is returned. As a degenerate case, if you specify the same
10174/// input operands as the node already has, the input node is returned.
10175SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
10176 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10177
10178 // Check to see if there is no change.
10179 if (Op == N->getOperand(Num: 0)) return N;
10180
10181 // See if the modified node already exists.
10182 void *InsertPos = nullptr;
10183 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10184 return Existing;
10185
10186 // Nope it doesn't. Remove the node from its current place in the maps.
10187 if (InsertPos)
10188 if (!RemoveNodeFromCSEMaps(N))
10189 InsertPos = nullptr;
10190
10191 // Now we update the operands.
10192 N->OperandList[0].set(Op);
10193
10194 updateDivergence(N);
10195 // If this gets put into a CSE map, add it.
10196 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10197 return N;
10198}
10199
10200SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
10201 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10202
10203 // Check to see if there is no change.
10204 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
10205 return N; // No operands changed, just return the input node.
10206
10207 // See if the modified node already exists.
10208 void *InsertPos = nullptr;
10209 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10210 return Existing;
10211
10212 // Nope it doesn't. Remove the node from its current place in the maps.
10213 if (InsertPos)
10214 if (!RemoveNodeFromCSEMaps(N))
10215 InsertPos = nullptr;
10216
10217 // Now we update the operands.
10218 if (N->OperandList[0] != Op1)
10219 N->OperandList[0].set(Op1);
10220 if (N->OperandList[1] != Op2)
10221 N->OperandList[1].set(Op2);
10222
10223 updateDivergence(N);
10224 // If this gets put into a CSE map, add it.
10225 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10226 return N;
10227}
10228
10229SDNode *SelectionDAG::
10230UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
10231 SDValue Ops[] = { Op1, Op2, Op3 };
10232 return UpdateNodeOperands(N, Ops);
10233}
10234
10235SDNode *SelectionDAG::
10236UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10237 SDValue Op3, SDValue Op4) {
10238 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10239 return UpdateNodeOperands(N, Ops);
10240}
10241
10242SDNode *SelectionDAG::
10243UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10244 SDValue Op3, SDValue Op4, SDValue Op5) {
10245 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10246 return UpdateNodeOperands(N, Ops);
10247}
10248
10249SDNode *SelectionDAG::
10250UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
10251 unsigned NumOps = Ops.size();
10252 assert(N->getNumOperands() == NumOps &&
10253 "Update with wrong number of operands");
10254
10255 // If no operands changed just return the input node.
10256 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
10257 return N;
10258
10259 // See if the modified node already exists.
10260 void *InsertPos = nullptr;
10261 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10262 return Existing;
10263
10264 // Nope it doesn't. Remove the node from its current place in the maps.
10265 if (InsertPos)
10266 if (!RemoveNodeFromCSEMaps(N))
10267 InsertPos = nullptr;
10268
10269 // Now we update the operands.
10270 for (unsigned i = 0; i != NumOps; ++i)
10271 if (N->OperandList[i] != Ops[i])
10272 N->OperandList[i].set(Ops[i]);
10273
10274 updateDivergence(N);
10275 // If this gets put into a CSE map, add it.
10276 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10277 return N;
10278}
10279
10280/// DropOperands - Release the operands and set this node to have
10281/// zero operands.
10282void SDNode::DropOperands() {
10283 // Unlike the code in MorphNodeTo that does this, we don't need to
10284 // watch for dead nodes here.
10285 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10286 SDUse &Use = *I++;
10287 Use.set(SDValue());
10288 }
10289}
10290
10291void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
10292 ArrayRef<MachineMemOperand *> NewMemRefs) {
10293 if (NewMemRefs.empty()) {
10294 N->clearMemRefs();
10295 return;
10296 }
10297
10298 // Check if we can avoid allocating by storing a single reference directly.
10299 if (NewMemRefs.size() == 1) {
10300 N->MemRefs = NewMemRefs[0];
10301 N->NumMemRefs = 1;
10302 return;
10303 }
10304
10305 MachineMemOperand **MemRefsBuffer =
10306 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
10307 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
10308 N->MemRefs = MemRefsBuffer;
10309 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10310}
10311
10312/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10313/// machine opcode.
10314///
10315SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10316 EVT VT) {
10317 SDVTList VTs = getVTList(VT);
10318 return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt);
10319}
10320
10321SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10322 EVT VT, SDValue Op1) {
10323 SDVTList VTs = getVTList(VT);
10324 SDValue Ops[] = { Op1 };
10325 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10326}
10327
10328SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10329 EVT VT, SDValue Op1,
10330 SDValue Op2) {
10331 SDVTList VTs = getVTList(VT);
10332 SDValue Ops[] = { Op1, Op2 };
10333 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10334}
10335
10336SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10337 EVT VT, SDValue Op1,
10338 SDValue Op2, SDValue Op3) {
10339 SDVTList VTs = getVTList(VT);
10340 SDValue Ops[] = { Op1, Op2, Op3 };
10341 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10342}
10343
10344SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10345 EVT VT, ArrayRef<SDValue> Ops) {
10346 SDVTList VTs = getVTList(VT);
10347 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10348}
10349
10350SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10351 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10352 SDVTList VTs = getVTList(VT1, VT2);
10353 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10354}
10355
10356SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10357 EVT VT1, EVT VT2) {
10358 SDVTList VTs = getVTList(VT1, VT2);
10359 return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt);
10360}
10361
10362SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10363 EVT VT1, EVT VT2, EVT VT3,
10364 ArrayRef<SDValue> Ops) {
10365 SDVTList VTs = getVTList(VT1, VT2, VT3);
10366 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10367}
10368
10369SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10370 EVT VT1, EVT VT2,
10371 SDValue Op1, SDValue Op2) {
10372 SDVTList VTs = getVTList(VT1, VT2);
10373 SDValue Ops[] = { Op1, Op2 };
10374 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10375}
10376
10377SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10378 SDVTList VTs,ArrayRef<SDValue> Ops) {
10379 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
10380 // Reset the NodeID to -1.
10381 New->setNodeId(-1);
10382 if (New != N) {
10383 ReplaceAllUsesWith(From: N, To: New);
10384 RemoveDeadNode(N);
10385 }
10386 return New;
10387}
10388
10389/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10390/// the line number information on the merged node since it is not possible to
10391/// preserve the information that operation is associated with multiple lines.
10392/// This will make the debugger working better at -O0, were there is a higher
10393/// probability having other instructions associated with that line.
10394///
10395/// For IROrder, we keep the smaller of the two
10396SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10397 DebugLoc NLoc = N->getDebugLoc();
10398 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10399 N->setDebugLoc(DebugLoc());
10400 }
10401 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
10402 N->setIROrder(Order);
10403 return N;
10404}
10405
10406/// MorphNodeTo - This *mutates* the specified node to have the specified
10407/// return type, opcode, and operands.
10408///
10409/// Note that MorphNodeTo returns the resultant node. If there is already a
10410/// node of the specified opcode and operands, it returns that node instead of
10411/// the current one. Note that the SDLoc need not be the same.
10412///
10413/// Using MorphNodeTo is faster than creating a new node and swapping it in
10414/// with ReplaceAllUsesWith both because it often avoids allocating a new
10415/// node, and because it doesn't require CSE recalculation for any of
10416/// the node's users.
10417///
10418/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10419/// As a consequence it isn't appropriate to use from within the DAG combiner or
10420/// the legalizer which maintain worklists that would need to be updated when
10421/// deleting things.
10422SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
10423 SDVTList VTs, ArrayRef<SDValue> Ops) {
10424 // If an identical node already exists, use it.
10425 void *IP = nullptr;
10426 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10427 FoldingSetNodeID ID;
10428 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
10429 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
10430 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
10431 }
10432
10433 if (!RemoveNodeFromCSEMaps(N))
10434 IP = nullptr;
10435
10436 // Start the morphing.
10437 N->NodeType = Opc;
10438 N->ValueList = VTs.VTs;
10439 N->NumValues = VTs.NumVTs;
10440
10441 // Clear the operands list, updating used nodes to remove this from their
10442 // use list. Keep track of any operands that become dead as a result.
10443 SmallPtrSet<SDNode*, 16> DeadNodeSet;
10444 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10445 SDUse &Use = *I++;
10446 SDNode *Used = Use.getNode();
10447 Use.set(SDValue());
10448 if (Used->use_empty())
10449 DeadNodeSet.insert(Ptr: Used);
10450 }
10451
10452 // For MachineNode, initialize the memory references information.
10453 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
10454 MN->clearMemRefs();
10455
10456 // Swap for an appropriately sized array from the recycler.
10457 removeOperands(Node: N);
10458 createOperands(Node: N, Vals: Ops);
10459
10460 // Delete any nodes that are still dead after adding the uses for the
10461 // new operands.
10462 if (!DeadNodeSet.empty()) {
10463 SmallVector<SDNode *, 16> DeadNodes;
10464 for (SDNode *N : DeadNodeSet)
10465 if (N->use_empty())
10466 DeadNodes.push_back(Elt: N);
10467 RemoveDeadNodes(DeadNodes);
10468 }
10469
10470 if (IP)
10471 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
10472 return N;
10473}
10474
10475SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
10476 unsigned OrigOpc = Node->getOpcode();
10477 unsigned NewOpc;
10478 switch (OrigOpc) {
10479 default:
10480 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10481#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10482 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10483#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10484 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10485#include "llvm/IR/ConstrainedOps.def"
10486 }
10487
10488 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10489
10490 // We're taking this node out of the chain, so we need to re-link things.
10491 SDValue InputChain = Node->getOperand(Num: 0);
10492 SDValue OutputChain = SDValue(Node, 1);
10493 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
10494
10495 SmallVector<SDValue, 3> Ops;
10496 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10497 Ops.push_back(Elt: Node->getOperand(Num: i));
10498
10499 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
10500 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
10501
10502 // MorphNodeTo can operate in two ways: if an existing node with the
10503 // specified operands exists, it can just return it. Otherwise, it
10504 // updates the node in place to have the requested operands.
10505 if (Res == Node) {
10506 // If we updated the node in place, reset the node ID. To the isel,
10507 // this should be just like a newly allocated machine node.
10508 Res->setNodeId(-1);
10509 } else {
10510 ReplaceAllUsesWith(From: Node, To: Res);
10511 RemoveDeadNode(N: Node);
10512 }
10513
10514 return Res;
10515}
10516
10517/// getMachineNode - These are used for target selectors to create a new node
10518/// with specified return type(s), MachineInstr opcode, and operands.
10519///
10520/// Note that getMachineNode returns the resultant node. If there is already a
10521/// node of the specified opcode and operands, it returns that node instead of
10522/// the current one.
10523MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10524 EVT VT) {
10525 SDVTList VTs = getVTList(VT);
10526 return getMachineNode(Opcode, dl, VTs, Ops: std::nullopt);
10527}
10528
10529MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10530 EVT VT, SDValue Op1) {
10531 SDVTList VTs = getVTList(VT);
10532 SDValue Ops[] = { Op1 };
10533 return getMachineNode(Opcode, dl, VTs, Ops);
10534}
10535
10536MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10537 EVT VT, SDValue Op1, SDValue Op2) {
10538 SDVTList VTs = getVTList(VT);
10539 SDValue Ops[] = { Op1, Op2 };
10540 return getMachineNode(Opcode, dl, VTs, Ops);
10541}
10542
10543MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10544 EVT VT, SDValue Op1, SDValue Op2,
10545 SDValue Op3) {
10546 SDVTList VTs = getVTList(VT);
10547 SDValue Ops[] = { Op1, Op2, Op3 };
10548 return getMachineNode(Opcode, dl, VTs, Ops);
10549}
10550
10551MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10552 EVT VT, ArrayRef<SDValue> Ops) {
10553 SDVTList VTs = getVTList(VT);
10554 return getMachineNode(Opcode, dl, VTs, Ops);
10555}
10556
10557MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10558 EVT VT1, EVT VT2, SDValue Op1,
10559 SDValue Op2) {
10560 SDVTList VTs = getVTList(VT1, VT2);
10561 SDValue Ops[] = { Op1, Op2 };
10562 return getMachineNode(Opcode, dl, VTs, Ops);
10563}
10564
10565MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10566 EVT VT1, EVT VT2, SDValue Op1,
10567 SDValue Op2, SDValue Op3) {
10568 SDVTList VTs = getVTList(VT1, VT2);
10569 SDValue Ops[] = { Op1, Op2, Op3 };
10570 return getMachineNode(Opcode, dl, VTs, Ops);
10571}
10572
10573MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10574 EVT VT1, EVT VT2,
10575 ArrayRef<SDValue> Ops) {
10576 SDVTList VTs = getVTList(VT1, VT2);
10577 return getMachineNode(Opcode, dl, VTs, Ops);
10578}
10579
10580MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10581 EVT VT1, EVT VT2, EVT VT3,
10582 SDValue Op1, SDValue Op2) {
10583 SDVTList VTs = getVTList(VT1, VT2, VT3);
10584 SDValue Ops[] = { Op1, Op2 };
10585 return getMachineNode(Opcode, dl, VTs, Ops);
10586}
10587
10588MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10589 EVT VT1, EVT VT2, EVT VT3,
10590 SDValue Op1, SDValue Op2,
10591 SDValue Op3) {
10592 SDVTList VTs = getVTList(VT1, VT2, VT3);
10593 SDValue Ops[] = { Op1, Op2, Op3 };
10594 return getMachineNode(Opcode, dl, VTs, Ops);
10595}
10596
10597MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10598 EVT VT1, EVT VT2, EVT VT3,
10599 ArrayRef<SDValue> Ops) {
10600 SDVTList VTs = getVTList(VT1, VT2, VT3);
10601 return getMachineNode(Opcode, dl, VTs, Ops);
10602}
10603
10604MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10605 ArrayRef<EVT> ResultTys,
10606 ArrayRef<SDValue> Ops) {
10607 SDVTList VTs = getVTList(VTs: ResultTys);
10608 return getMachineNode(Opcode, dl, VTs, Ops);
10609}
10610
10611MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
10612 SDVTList VTs,
10613 ArrayRef<SDValue> Ops) {
10614 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10615 MachineSDNode *N;
10616 void *IP = nullptr;
10617
10618 if (DoCSE) {
10619 FoldingSetNodeID ID;
10620 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
10621 IP = nullptr;
10622 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10623 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
10624 }
10625 }
10626
10627 // Allocate a new MachineSDNode.
10628 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
10629 createOperands(Node: N, Vals: Ops);
10630
10631 if (DoCSE)
10632 CSEMap.InsertNode(N, InsertPos: IP);
10633
10634 InsertNode(N);
10635 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
10636 return N;
10637}
10638
10639/// getTargetExtractSubreg - A convenience function for creating
10640/// TargetOpcode::EXTRACT_SUBREG nodes.
10641SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10642 SDValue Operand) {
10643 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10644 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
10645 VT, Op1: Operand, Op2: SRIdxVal);
10646 return SDValue(Subreg, 0);
10647}
10648
10649/// getTargetInsertSubreg - A convenience function for creating
10650/// TargetOpcode::INSERT_SUBREG nodes.
10651SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10652 SDValue Operand, SDValue Subreg) {
10653 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10654 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
10655 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
10656 return SDValue(Result, 0);
10657}
10658
10659/// getNodeIfExists - Get the specified node if it's already available, or
10660/// else return NULL.
10661SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10662 ArrayRef<SDValue> Ops) {
10663 SDNodeFlags Flags;
10664 if (Inserter)
10665 Flags = Inserter->getFlags();
10666 return getNodeIfExists(Opcode, VTList, Ops, Flags);
10667}
10668
10669SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10670 ArrayRef<SDValue> Ops,
10671 const SDNodeFlags Flags) {
10672 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10673 FoldingSetNodeID ID;
10674 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10675 void *IP = nullptr;
10676 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP)) {
10677 E->intersectFlagsWith(Flags);
10678 return E;
10679 }
10680 }
10681 return nullptr;
10682}
10683
10684/// doesNodeExist - Check if a node exists without modifying its flags.
10685bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10686 ArrayRef<SDValue> Ops) {
10687 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10688 FoldingSetNodeID ID;
10689 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10690 void *IP = nullptr;
10691 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
10692 return true;
10693 }
10694 return false;
10695}
10696
10697/// getDbgValue - Creates a SDDbgValue node.
10698///
10699/// SDNode
10700SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
10701 SDNode *N, unsigned R, bool IsIndirect,
10702 const DebugLoc &DL, unsigned O) {
10703 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10704 "Expected inlined-at fields to agree");
10705 return new (DbgInfo->getAlloc())
10706 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
10707 {}, IsIndirect, DL, O,
10708 /*IsVariadic=*/false);
10709}
10710
10711/// Constant
10712SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
10713 DIExpression *Expr,
10714 const Value *C,
10715 const DebugLoc &DL, unsigned O) {
10716 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10717 "Expected inlined-at fields to agree");
10718 return new (DbgInfo->getAlloc())
10719 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
10720 /*IsIndirect=*/false, DL, O,
10721 /*IsVariadic=*/false);
10722}
10723
10724/// FrameIndex
10725SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10726 DIExpression *Expr, unsigned FI,
10727 bool IsIndirect,
10728 const DebugLoc &DL,
10729 unsigned O) {
10730 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10731 "Expected inlined-at fields to agree");
10732 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
10733}
10734
10735/// FrameIndex with dependencies
10736SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10737 DIExpression *Expr, unsigned FI,
10738 ArrayRef<SDNode *> Dependencies,
10739 bool IsIndirect,
10740 const DebugLoc &DL,
10741 unsigned O) {
10742 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10743 "Expected inlined-at fields to agree");
10744 return new (DbgInfo->getAlloc())
10745 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
10746 Dependencies, IsIndirect, DL, O,
10747 /*IsVariadic=*/false);
10748}
10749
10750/// VReg
10751SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
10752 unsigned VReg, bool IsIndirect,
10753 const DebugLoc &DL, unsigned O) {
10754 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10755 "Expected inlined-at fields to agree");
10756 return new (DbgInfo->getAlloc())
10757 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10758 {}, IsIndirect, DL, O,
10759 /*IsVariadic=*/false);
10760}
10761
10762SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
10763 ArrayRef<SDDbgOperand> Locs,
10764 ArrayRef<SDNode *> Dependencies,
10765 bool IsIndirect, const DebugLoc &DL,
10766 unsigned O, bool IsVariadic) {
10767 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10768 "Expected inlined-at fields to agree");
10769 return new (DbgInfo->getAlloc())
10770 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10771 DL, O, IsVariadic);
10772}
10773
10774void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
10775 unsigned OffsetInBits, unsigned SizeInBits,
10776 bool InvalidateDbg) {
10777 SDNode *FromNode = From.getNode();
10778 SDNode *ToNode = To.getNode();
10779 assert(FromNode && ToNode && "Can't modify dbg values");
10780
10781 // PR35338
10782 // TODO: assert(From != To && "Redundant dbg value transfer");
10783 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10784 if (From == To || FromNode == ToNode)
10785 return;
10786
10787 if (!FromNode->getHasDebugValue())
10788 return;
10789
10790 SDDbgOperand FromLocOp =
10791 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
10792 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
10793
10794 SmallVector<SDDbgValue *, 2> ClonedDVs;
10795 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
10796 if (Dbg->isInvalidated())
10797 continue;
10798
10799 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10800
10801 // Create a new location ops vector that is equal to the old vector, but
10802 // with each instance of FromLocOp replaced with ToLocOp.
10803 bool Changed = false;
10804 auto NewLocOps = Dbg->copyLocationOps();
10805 std::replace_if(
10806 first: NewLocOps.begin(), last: NewLocOps.end(),
10807 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
10808 bool Match = Op == FromLocOp;
10809 Changed |= Match;
10810 return Match;
10811 },
10812 new_value: ToLocOp);
10813 // Ignore this SDDbgValue if we didn't find a matching location.
10814 if (!Changed)
10815 continue;
10816
10817 DIVariable *Var = Dbg->getVariable();
10818 auto *Expr = Dbg->getExpression();
10819 // If a fragment is requested, update the expression.
10820 if (SizeInBits) {
10821 // When splitting a larger (e.g., sign-extended) value whose
10822 // lower bits are described with an SDDbgValue, do not attempt
10823 // to transfer the SDDbgValue to the upper bits.
10824 if (auto FI = Expr->getFragmentInfo())
10825 if (OffsetInBits + SizeInBits > FI->SizeInBits)
10826 continue;
10827 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10828 SizeInBits);
10829 if (!Fragment)
10830 continue;
10831 Expr = *Fragment;
10832 }
10833
10834 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10835 // Clone the SDDbgValue and move it to To.
10836 SDDbgValue *Clone = getDbgValueList(
10837 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
10838 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
10839 IsVariadic: Dbg->isVariadic());
10840 ClonedDVs.push_back(Elt: Clone);
10841
10842 if (InvalidateDbg) {
10843 // Invalidate value and indicate the SDDbgValue should not be emitted.
10844 Dbg->setIsInvalidated();
10845 Dbg->setIsEmitted();
10846 }
10847 }
10848
10849 for (SDDbgValue *Dbg : ClonedDVs) {
10850 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10851 "Transferred DbgValues should depend on the new SDNode");
10852 AddDbgValue(DB: Dbg, isParameter: false);
10853 }
10854}
10855
10856void SelectionDAG::salvageDebugInfo(SDNode &N) {
10857 if (!N.getHasDebugValue())
10858 return;
10859
10860 SmallVector<SDDbgValue *, 2> ClonedDVs;
10861 for (auto *DV : GetDbgValues(SD: &N)) {
10862 if (DV->isInvalidated())
10863 continue;
10864 switch (N.getOpcode()) {
10865 default:
10866 break;
10867 case ISD::ADD: {
10868 SDValue N0 = N.getOperand(Num: 0);
10869 SDValue N1 = N.getOperand(Num: 1);
10870 if (!isa<ConstantSDNode>(Val: N0)) {
10871 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
10872 uint64_t Offset;
10873 if (RHSConstant)
10874 Offset = N.getConstantOperandVal(Num: 1);
10875 // We are not allowed to turn indirect debug values variadic, so
10876 // don't salvage those.
10877 if (!RHSConstant && DV->isIndirect())
10878 continue;
10879
10880 // Rewrite an ADD constant node into a DIExpression. Since we are
10881 // performing arithmetic to compute the variable's *value* in the
10882 // DIExpression, we need to mark the expression with a
10883 // DW_OP_stack_value.
10884 auto *DIExpr = DV->getExpression();
10885 auto NewLocOps = DV->copyLocationOps();
10886 bool Changed = false;
10887 size_t OrigLocOpsSize = NewLocOps.size();
10888 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10889 // We're not given a ResNo to compare against because the whole
10890 // node is going away. We know that any ISD::ADD only has one
10891 // result, so we can assume any node match is using the result.
10892 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10893 NewLocOps[i].getSDNode() != &N)
10894 continue;
10895 NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo());
10896 if (RHSConstant) {
10897 SmallVector<uint64_t, 3> ExprOps;
10898 DIExpression::appendOffset(Ops&: ExprOps, Offset);
10899 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
10900 } else {
10901 // Convert to a variadic expression (if not already).
10902 // convertToVariadicExpression() returns a const pointer, so we use
10903 // a temporary const variable here.
10904 const auto *TmpDIExpr =
10905 DIExpression::convertToVariadicExpression(Expr: DIExpr);
10906 SmallVector<uint64_t, 3> ExprOps;
10907 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
10908 ExprOps.push_back(Elt: NewLocOps.size());
10909 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
10910 SDDbgOperand RHS =
10911 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
10912 NewLocOps.push_back(Elt: RHS);
10913 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
10914 }
10915 Changed = true;
10916 }
10917 (void)Changed;
10918 assert(Changed && "Salvage target doesn't use N");
10919
10920 bool IsVariadic =
10921 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10922
10923 auto AdditionalDependencies = DV->getAdditionalDependencies();
10924 SDDbgValue *Clone = getDbgValueList(
10925 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
10926 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
10927 ClonedDVs.push_back(Elt: Clone);
10928 DV->setIsInvalidated();
10929 DV->setIsEmitted();
10930 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10931 N0.getNode()->dumprFull(this);
10932 dbgs() << " into " << *DIExpr << '\n');
10933 }
10934 break;
10935 }
10936 case ISD::TRUNCATE: {
10937 SDValue N0 = N.getOperand(Num: 0);
10938 TypeSize FromSize = N0.getValueSizeInBits();
10939 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
10940
10941 DIExpression *DbgExpression = DV->getExpression();
10942 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
10943 auto NewLocOps = DV->copyLocationOps();
10944 bool Changed = false;
10945 for (size_t i = 0; i < NewLocOps.size(); ++i) {
10946 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10947 NewLocOps[i].getSDNode() != &N)
10948 continue;
10949
10950 NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo());
10951 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
10952 Changed = true;
10953 }
10954 assert(Changed && "Salvage target doesn't use N");
10955 (void)Changed;
10956
10957 SDDbgValue *Clone =
10958 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
10959 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
10960 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
10961
10962 ClonedDVs.push_back(Elt: Clone);
10963 DV->setIsInvalidated();
10964 DV->setIsEmitted();
10965 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10966 dbgs() << " into " << *DbgExpression << '\n');
10967 break;
10968 }
10969 }
10970 }
10971
10972 for (SDDbgValue *Dbg : ClonedDVs) {
10973 assert(!Dbg->getSDNodes().empty() &&
10974 "Salvaged DbgValue should depend on a new SDNode");
10975 AddDbgValue(DB: Dbg, isParameter: false);
10976 }
10977}
10978
10979/// Creates a SDDbgLabel node.
10980SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
10981 const DebugLoc &DL, unsigned O) {
10982 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
10983 "Expected inlined-at fields to agree");
10984 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
10985}
10986
10987namespace {
10988
10989/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
10990/// pointed to by a use iterator is deleted, increment the use iterator
10991/// so that it doesn't dangle.
10992///
10993class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
10994 SDNode::use_iterator &UI;
10995 SDNode::use_iterator &UE;
10996
10997 void NodeDeleted(SDNode *N, SDNode *E) override {
10998 // Increment the iterator as needed.
10999 while (UI != UE && N == *UI)
11000 ++UI;
11001 }
11002
11003public:
11004 RAUWUpdateListener(SelectionDAG &d,
11005 SDNode::use_iterator &ui,
11006 SDNode::use_iterator &ue)
11007 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11008};
11009
11010} // end anonymous namespace
11011
11012/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11013/// This can cause recursive merging of nodes in the DAG.
11014///
11015/// This version assumes From has a single result value.
11016///
11017void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
11018 SDNode *From = FromN.getNode();
11019 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
11020 "Cannot replace with this method!");
11021 assert(From != To.getNode() && "Cannot replace uses of with self");
11022
11023 // Preserve Debug Values
11024 transferDbgValues(From: FromN, To);
11025 // Preserve extra info.
11026 copyExtraInfo(From, To: To.getNode());
11027
11028 // Iterate over all the existing uses of From. New uses will be added
11029 // to the beginning of the use list, which we avoid visiting.
11030 // This specifically avoids visiting uses of From that arise while the
11031 // replacement is happening, because any such uses would be the result
11032 // of CSE: If an existing node looks like From after one of its operands
11033 // is replaced by To, we don't want to replace of all its users with To
11034 // too. See PR3018 for more info.
11035 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11036 RAUWUpdateListener Listener(*this, UI, UE);
11037 while (UI != UE) {
11038 SDNode *User = *UI;
11039
11040 // This node is about to morph, remove its old self from the CSE maps.
11041 RemoveNodeFromCSEMaps(N: User);
11042
11043 // A user can appear in a use list multiple times, and when this
11044 // happens the uses are usually next to each other in the list.
11045 // To help reduce the number of CSE recomputations, process all
11046 // the uses of this user that we can find this way.
11047 do {
11048 SDUse &Use = UI.getUse();
11049 ++UI;
11050 Use.set(To);
11051 if (To->isDivergent() != From->isDivergent())
11052 updateDivergence(N: User);
11053 } while (UI != UE && *UI == User);
11054 // Now that we have modified User, add it back to the CSE maps. If it
11055 // already exists there, recursively merge the results together.
11056 AddModifiedNodeToCSEMaps(N: User);
11057 }
11058
11059 // If we just RAUW'd the root, take note.
11060 if (FromN == getRoot())
11061 setRoot(To);
11062}
11063
11064/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11065/// This can cause recursive merging of nodes in the DAG.
11066///
11067/// This version assumes that for each value of From, there is a
11068/// corresponding value in To in the same position with the same type.
11069///
11070void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
11071#ifndef NDEBUG
11072 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11073 assert((!From->hasAnyUseOfValue(i) ||
11074 From->getValueType(i) == To->getValueType(i)) &&
11075 "Cannot use this version of ReplaceAllUsesWith!");
11076#endif
11077
11078 // Handle the trivial case.
11079 if (From == To)
11080 return;
11081
11082 // Preserve Debug Info. Only do this if there's a use.
11083 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11084 if (From->hasAnyUseOfValue(Value: i)) {
11085 assert((i < To->getNumValues()) && "Invalid To location");
11086 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
11087 }
11088 // Preserve extra info.
11089 copyExtraInfo(From, To);
11090
11091 // Iterate over just the existing users of From. See the comments in
11092 // the ReplaceAllUsesWith above.
11093 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11094 RAUWUpdateListener Listener(*this, UI, UE);
11095 while (UI != UE) {
11096 SDNode *User = *UI;
11097
11098 // This node is about to morph, remove its old self from the CSE maps.
11099 RemoveNodeFromCSEMaps(N: User);
11100
11101 // A user can appear in a use list multiple times, and when this
11102 // happens the uses are usually next to each other in the list.
11103 // To help reduce the number of CSE recomputations, process all
11104 // the uses of this user that we can find this way.
11105 do {
11106 SDUse &Use = UI.getUse();
11107 ++UI;
11108 Use.setNode(To);
11109 if (To->isDivergent() != From->isDivergent())
11110 updateDivergence(N: User);
11111 } while (UI != UE && *UI == User);
11112
11113 // Now that we have modified User, add it back to the CSE maps. If it
11114 // already exists there, recursively merge the results together.
11115 AddModifiedNodeToCSEMaps(N: User);
11116 }
11117
11118 // If we just RAUW'd the root, take note.
11119 if (From == getRoot().getNode())
11120 setRoot(SDValue(To, getRoot().getResNo()));
11121}
11122
11123/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11124/// This can cause recursive merging of nodes in the DAG.
11125///
11126/// This version can replace From with any result values. To must match the
11127/// number and types of values returned by From.
11128void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
11129 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11130 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
11131
11132 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11133 // Preserve Debug Info.
11134 transferDbgValues(From: SDValue(From, i), To: To[i]);
11135 // Preserve extra info.
11136 copyExtraInfo(From, To: To[i].getNode());
11137 }
11138
11139 // Iterate over just the existing users of From. See the comments in
11140 // the ReplaceAllUsesWith above.
11141 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11142 RAUWUpdateListener Listener(*this, UI, UE);
11143 while (UI != UE) {
11144 SDNode *User = *UI;
11145
11146 // This node is about to morph, remove its old self from the CSE maps.
11147 RemoveNodeFromCSEMaps(N: User);
11148
11149 // A user can appear in a use list multiple times, and when this happens the
11150 // uses are usually next to each other in the list. To help reduce the
11151 // number of CSE and divergence recomputations, process all the uses of this
11152 // user that we can find this way.
11153 bool To_IsDivergent = false;
11154 do {
11155 SDUse &Use = UI.getUse();
11156 const SDValue &ToOp = To[Use.getResNo()];
11157 ++UI;
11158 Use.set(ToOp);
11159 To_IsDivergent |= ToOp->isDivergent();
11160 } while (UI != UE && *UI == User);
11161
11162 if (To_IsDivergent != From->isDivergent())
11163 updateDivergence(N: User);
11164
11165 // Now that we have modified User, add it back to the CSE maps. If it
11166 // already exists there, recursively merge the results together.
11167 AddModifiedNodeToCSEMaps(N: User);
11168 }
11169
11170 // If we just RAUW'd the root, take note.
11171 if (From == getRoot().getNode())
11172 setRoot(SDValue(To[getRoot().getResNo()]));
11173}
11174
11175/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11176/// uses of other values produced by From.getNode() alone. The Deleted
11177/// vector is handled the same way as for ReplaceAllUsesWith.
11178void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
11179 // Handle the really simple, really trivial case efficiently.
11180 if (From == To) return;
11181
11182 // Handle the simple, trivial, case efficiently.
11183 if (From.getNode()->getNumValues() == 1) {
11184 ReplaceAllUsesWith(FromN: From, To);
11185 return;
11186 }
11187
11188 // Preserve Debug Info.
11189 transferDbgValues(From, To);
11190 copyExtraInfo(From: From.getNode(), To: To.getNode());
11191
11192 // Iterate over just the existing users of From. See the comments in
11193 // the ReplaceAllUsesWith above.
11194 SDNode::use_iterator UI = From.getNode()->use_begin(),
11195 UE = From.getNode()->use_end();
11196 RAUWUpdateListener Listener(*this, UI, UE);
11197 while (UI != UE) {
11198 SDNode *User = *UI;
11199 bool UserRemovedFromCSEMaps = false;
11200
11201 // A user can appear in a use list multiple times, and when this
11202 // happens the uses are usually next to each other in the list.
11203 // To help reduce the number of CSE recomputations, process all
11204 // the uses of this user that we can find this way.
11205 do {
11206 SDUse &Use = UI.getUse();
11207
11208 // Skip uses of different values from the same node.
11209 if (Use.getResNo() != From.getResNo()) {
11210 ++UI;
11211 continue;
11212 }
11213
11214 // If this node hasn't been modified yet, it's still in the CSE maps,
11215 // so remove its old self from the CSE maps.
11216 if (!UserRemovedFromCSEMaps) {
11217 RemoveNodeFromCSEMaps(N: User);
11218 UserRemovedFromCSEMaps = true;
11219 }
11220
11221 ++UI;
11222 Use.set(To);
11223 if (To->isDivergent() != From->isDivergent())
11224 updateDivergence(N: User);
11225 } while (UI != UE && *UI == User);
11226 // We are iterating over all uses of the From node, so if a use
11227 // doesn't use the specific value, no changes are made.
11228 if (!UserRemovedFromCSEMaps)
11229 continue;
11230
11231 // Now that we have modified User, add it back to the CSE maps. If it
11232 // already exists there, recursively merge the results together.
11233 AddModifiedNodeToCSEMaps(N: User);
11234 }
11235
11236 // If we just RAUW'd the root, take note.
11237 if (From == getRoot())
11238 setRoot(To);
11239}
11240
11241namespace {
11242
11243/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11244/// to record information about a use.
11245struct UseMemo {
11246 SDNode *User;
11247 unsigned Index;
11248 SDUse *Use;
11249};
11250
11251/// operator< - Sort Memos by User.
11252bool operator<(const UseMemo &L, const UseMemo &R) {
11253 return (intptr_t)L.User < (intptr_t)R.User;
11254}
11255
11256/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11257/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11258/// the node already has been taken care of recursively.
11259class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11260 SmallVector<UseMemo, 4> &Uses;
11261
11262 void NodeDeleted(SDNode *N, SDNode *E) override {
11263 for (UseMemo &Memo : Uses)
11264 if (Memo.User == N)
11265 Memo.User = nullptr;
11266 }
11267
11268public:
11269 RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11270 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11271};
11272
11273} // end anonymous namespace
11274
11275bool SelectionDAG::calculateDivergence(SDNode *N) {
11276 if (TLI->isSDNodeAlwaysUniform(N)) {
11277 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11278 "Conflicting divergence information!");
11279 return false;
11280 }
11281 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11282 return true;
11283 for (const auto &Op : N->ops()) {
11284 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11285 return true;
11286 }
11287 return false;
11288}
11289
11290void SelectionDAG::updateDivergence(SDNode *N) {
11291 SmallVector<SDNode *, 16> Worklist(1, N);
11292 do {
11293 N = Worklist.pop_back_val();
11294 bool IsDivergent = calculateDivergence(N);
11295 if (N->SDNodeBits.IsDivergent != IsDivergent) {
11296 N->SDNodeBits.IsDivergent = IsDivergent;
11297 llvm::append_range(C&: Worklist, R: N->uses());
11298 }
11299 } while (!Worklist.empty());
11300}
11301
11302void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11303 DenseMap<SDNode *, unsigned> Degree;
11304 Order.reserve(n: AllNodes.size());
11305 for (auto &N : allnodes()) {
11306 unsigned NOps = N.getNumOperands();
11307 Degree[&N] = NOps;
11308 if (0 == NOps)
11309 Order.push_back(x: &N);
11310 }
11311 for (size_t I = 0; I != Order.size(); ++I) {
11312 SDNode *N = Order[I];
11313 for (auto *U : N->uses()) {
11314 unsigned &UnsortedOps = Degree[U];
11315 if (0 == --UnsortedOps)
11316 Order.push_back(x: U);
11317 }
11318 }
11319}
11320
11321#ifndef NDEBUG
11322void SelectionDAG::VerifyDAGDivergence() {
11323 std::vector<SDNode *> TopoOrder;
11324 CreateTopologicalOrder(Order&: TopoOrder);
11325 for (auto *N : TopoOrder) {
11326 assert(calculateDivergence(N) == N->isDivergent() &&
11327 "Divergence bit inconsistency detected");
11328 }
11329}
11330#endif
11331
11332/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11333/// uses of other values produced by From.getNode() alone. The same value
11334/// may appear in both the From and To list. The Deleted vector is
11335/// handled the same way as for ReplaceAllUsesWith.
11336void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
11337 const SDValue *To,
11338 unsigned Num){
11339 // Handle the simple, trivial case efficiently.
11340 if (Num == 1)
11341 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
11342
11343 transferDbgValues(From: *From, To: *To);
11344 copyExtraInfo(From: From->getNode(), To: To->getNode());
11345
11346 // Read up all the uses and make records of them. This helps
11347 // processing new uses that are introduced during the
11348 // replacement process.
11349 SmallVector<UseMemo, 4> Uses;
11350 for (unsigned i = 0; i != Num; ++i) {
11351 unsigned FromResNo = From[i].getResNo();
11352 SDNode *FromNode = From[i].getNode();
11353 for (SDNode::use_iterator UI = FromNode->use_begin(),
11354 E = FromNode->use_end(); UI != E; ++UI) {
11355 SDUse &Use = UI.getUse();
11356 if (Use.getResNo() == FromResNo) {
11357 UseMemo Memo = { .User: *UI, .Index: i, .Use: &Use };
11358 Uses.push_back(Elt: Memo);
11359 }
11360 }
11361 }
11362
11363 // Sort the uses, so that all the uses from a given User are together.
11364 llvm::sort(C&: Uses);
11365 RAUOVWUpdateListener Listener(*this, Uses);
11366
11367 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11368 UseIndex != UseIndexEnd; ) {
11369 // We know that this user uses some value of From. If it is the right
11370 // value, update it.
11371 SDNode *User = Uses[UseIndex].User;
11372 // If the node has been deleted by recursive CSE updates when updating
11373 // another node, then just skip this entry.
11374 if (User == nullptr) {
11375 ++UseIndex;
11376 continue;
11377 }
11378
11379 // This node is about to morph, remove its old self from the CSE maps.
11380 RemoveNodeFromCSEMaps(N: User);
11381
11382 // The Uses array is sorted, so all the uses for a given User
11383 // are next to each other in the list.
11384 // To help reduce the number of CSE recomputations, process all
11385 // the uses of this user that we can find this way.
11386 do {
11387 unsigned i = Uses[UseIndex].Index;
11388 SDUse &Use = *Uses[UseIndex].Use;
11389 ++UseIndex;
11390
11391 Use.set(To[i]);
11392 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11393
11394 // Now that we have modified User, add it back to the CSE maps. If it
11395 // already exists there, recursively merge the results together.
11396 AddModifiedNodeToCSEMaps(N: User);
11397 }
11398}
11399
11400/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11401/// based on their topological order. It returns the maximum id and a vector
11402/// of the SDNodes* in assigned order by reference.
11403unsigned SelectionDAG::AssignTopologicalOrder() {
11404 unsigned DAGSize = 0;
11405
11406 // SortedPos tracks the progress of the algorithm. Nodes before it are
11407 // sorted, nodes after it are unsorted. When the algorithm completes
11408 // it is at the end of the list.
11409 allnodes_iterator SortedPos = allnodes_begin();
11410
11411 // Visit all the nodes. Move nodes with no operands to the front of
11412 // the list immediately. Annotate nodes that do have operands with their
11413 // operand count. Before we do this, the Node Id fields of the nodes
11414 // may contain arbitrary values. After, the Node Id fields for nodes
11415 // before SortedPos will contain the topological sort index, and the
11416 // Node Id fields for nodes At SortedPos and after will contain the
11417 // count of outstanding operands.
11418 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
11419 checkForCycles(N: &N, DAG: this);
11420 unsigned Degree = N.getNumOperands();
11421 if (Degree == 0) {
11422 // A node with no uses, add it to the result array immediately.
11423 N.setNodeId(DAGSize++);
11424 allnodes_iterator Q(&N);
11425 if (Q != SortedPos)
11426 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
11427 assert(SortedPos != AllNodes.end() && "Overran node list");
11428 ++SortedPos;
11429 } else {
11430 // Temporarily use the Node Id as scratch space for the degree count.
11431 N.setNodeId(Degree);
11432 }
11433 }
11434
11435 // Visit all the nodes. As we iterate, move nodes into sorted order,
11436 // such that by the time the end is reached all nodes will be sorted.
11437 for (SDNode &Node : allnodes()) {
11438 SDNode *N = &Node;
11439 checkForCycles(N, DAG: this);
11440 // N is in sorted position, so all its uses have one less operand
11441 // that needs to be sorted.
11442 for (SDNode *P : N->uses()) {
11443 unsigned Degree = P->getNodeId();
11444 assert(Degree != 0 && "Invalid node degree");
11445 --Degree;
11446 if (Degree == 0) {
11447 // All of P's operands are sorted, so P may sorted now.
11448 P->setNodeId(DAGSize++);
11449 if (P->getIterator() != SortedPos)
11450 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
11451 assert(SortedPos != AllNodes.end() && "Overran node list");
11452 ++SortedPos;
11453 } else {
11454 // Update P's outstanding operand count.
11455 P->setNodeId(Degree);
11456 }
11457 }
11458 if (Node.getIterator() == SortedPos) {
11459#ifndef NDEBUG
11460 allnodes_iterator I(N);
11461 SDNode *S = &*++I;
11462 dbgs() << "Overran sorted position:\n";
11463 S->dumprFull(G: this); dbgs() << "\n";
11464 dbgs() << "Checking if this is due to cycles\n";
11465 checkForCycles(DAG: this, force: true);
11466#endif
11467 llvm_unreachable(nullptr);
11468 }
11469 }
11470
11471 assert(SortedPos == AllNodes.end() &&
11472 "Topological sort incomplete!");
11473 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11474 "First node in topological sort is not the entry token!");
11475 assert(AllNodes.front().getNodeId() == 0 &&
11476 "First node in topological sort has non-zero id!");
11477 assert(AllNodes.front().getNumOperands() == 0 &&
11478 "First node in topological sort has operands!");
11479 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11480 "Last node in topologic sort has unexpected id!");
11481 assert(AllNodes.back().use_empty() &&
11482 "Last node in topologic sort has users!");
11483 assert(DAGSize == allnodes_size() && "Node count mismatch!");
11484 return DAGSize;
11485}
11486
11487/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11488/// value is produced by SD.
11489void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11490 for (SDNode *SD : DB->getSDNodes()) {
11491 if (!SD)
11492 continue;
11493 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11494 SD->setHasDebugValue(true);
11495 }
11496 DbgInfo->add(V: DB, isParameter);
11497}
11498
11499void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
11500
11501SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
11502 SDValue NewMemOpChain) {
11503 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11504 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11505 // The new memory operation must have the same position as the old load in
11506 // terms of memory dependency. Create a TokenFactor for the old load and new
11507 // memory operation and update uses of the old load's output chain to use that
11508 // TokenFactor.
11509 if (OldChain == NewMemOpChain || OldChain.use_empty())
11510 return NewMemOpChain;
11511
11512 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11513 OldChain, NewMemOpChain);
11514 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
11515 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
11516 return TokenFactor;
11517}
11518
11519SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
11520 SDValue NewMemOp) {
11521 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11522 SDValue OldChain = SDValue(OldLoad, 1);
11523 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
11524 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11525}
11526
11527SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
11528 Function **OutFunction) {
11529 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11530
11531 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
11532 auto *Module = MF->getFunction().getParent();
11533 auto *Function = Module->getFunction(Name: Symbol);
11534
11535 if (OutFunction != nullptr)
11536 *OutFunction = Function;
11537
11538 if (Function != nullptr) {
11539 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
11540 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
11541 }
11542
11543 std::string ErrorStr;
11544 raw_string_ostream ErrorFormatter(ErrorStr);
11545 ErrorFormatter << "Undefined external symbol ";
11546 ErrorFormatter << '"' << Symbol << '"';
11547 report_fatal_error(reason: Twine(ErrorFormatter.str()));
11548}
11549
11550//===----------------------------------------------------------------------===//
11551// SDNode Class
11552//===----------------------------------------------------------------------===//
11553
11554bool llvm::isNullConstant(SDValue V) {
11555 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11556 return Const != nullptr && Const->isZero();
11557}
11558
11559bool llvm::isNullFPConstant(SDValue V) {
11560 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
11561 return Const != nullptr && Const->isZero() && !Const->isNegative();
11562}
11563
11564bool llvm::isAllOnesConstant(SDValue V) {
11565 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11566 return Const != nullptr && Const->isAllOnes();
11567}
11568
11569bool llvm::isOneConstant(SDValue V) {
11570 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11571 return Const != nullptr && Const->isOne();
11572}
11573
11574bool llvm::isMinSignedConstant(SDValue V) {
11575 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
11576 return Const != nullptr && Const->isMinSignedValue();
11577}
11578
11579bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11580 unsigned OperandNo) {
11581 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11582 // TODO: Target-specific opcodes could be added.
11583 if (auto *Const = isConstOrConstSplat(N: V)) {
11584 switch (Opcode) {
11585 case ISD::ADD:
11586 case ISD::OR:
11587 case ISD::XOR:
11588 case ISD::UMAX:
11589 return Const->isZero();
11590 case ISD::MUL:
11591 return Const->isOne();
11592 case ISD::AND:
11593 case ISD::UMIN:
11594 return Const->isAllOnes();
11595 case ISD::SMAX:
11596 return Const->isMinSignedValue();
11597 case ISD::SMIN:
11598 return Const->isMaxSignedValue();
11599 case ISD::SUB:
11600 case ISD::SHL:
11601 case ISD::SRA:
11602 case ISD::SRL:
11603 return OperandNo == 1 && Const->isZero();
11604 case ISD::UDIV:
11605 case ISD::SDIV:
11606 return OperandNo == 1 && Const->isOne();
11607 }
11608 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
11609 switch (Opcode) {
11610 case ISD::FADD:
11611 return ConstFP->isZero() &&
11612 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11613 case ISD::FSUB:
11614 return OperandNo == 1 && ConstFP->isZero() &&
11615 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11616 case ISD::FMUL:
11617 return ConstFP->isExactlyValue(V: 1.0);
11618 case ISD::FDIV:
11619 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
11620 case ISD::FMINNUM:
11621 case ISD::FMAXNUM: {
11622 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11623 EVT VT = V.getValueType();
11624 const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT);
11625 APFloat NeutralAF = !Flags.hasNoNaNs()
11626 ? APFloat::getQNaN(Sem: Semantics)
11627 : !Flags.hasNoInfs()
11628 ? APFloat::getInf(Sem: Semantics)
11629 : APFloat::getLargest(Sem: Semantics);
11630 if (Opcode == ISD::FMAXNUM)
11631 NeutralAF.changeSign();
11632
11633 return ConstFP->isExactlyValue(V: NeutralAF);
11634 }
11635 }
11636 }
11637 return false;
11638}
11639
11640SDValue llvm::peekThroughBitcasts(SDValue V) {
11641 while (V.getOpcode() == ISD::BITCAST)
11642 V = V.getOperand(i: 0);
11643 return V;
11644}
11645
11646SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
11647 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
11648 V = V.getOperand(i: 0);
11649 return V;
11650}
11651
11652SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
11653 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11654 V = V.getOperand(i: 0);
11655 return V;
11656}
11657
11658SDValue llvm::peekThroughTruncates(SDValue V) {
11659 while (V.getOpcode() == ISD::TRUNCATE)
11660 V = V.getOperand(i: 0);
11661 return V;
11662}
11663
11664bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11665 if (V.getOpcode() != ISD::XOR)
11666 return false;
11667 V = peekThroughBitcasts(V: V.getOperand(i: 1));
11668 unsigned NumBits = V.getScalarValueSizeInBits();
11669 ConstantSDNode *C =
11670 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
11671 return C && (C->getAPIntValue().countr_one() >= NumBits);
11672}
11673
11674ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
11675 bool AllowTruncation) {
11676 EVT VT = N.getValueType();
11677 APInt DemandedElts = VT.isFixedLengthVector()
11678 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
11679 : APInt(1, 1);
11680 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11681}
11682
11683ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
11684 bool AllowUndefs,
11685 bool AllowTruncation) {
11686 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
11687 return CN;
11688
11689 // SplatVectors can truncate their operands. Ignore that case here unless
11690 // AllowTruncation is set.
11691 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11692 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
11693 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
11694 EVT CVT = CN->getValueType(ResNo: 0);
11695 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11696 if (AllowTruncation || CVT == VecEltVT)
11697 return CN;
11698 }
11699 }
11700
11701 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
11702 BitVector UndefElements;
11703 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
11704
11705 // BuildVectors can truncate their operands. Ignore that case here unless
11706 // AllowTruncation is set.
11707 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11708 if (CN && (UndefElements.none() || AllowUndefs)) {
11709 EVT CVT = CN->getValueType(ResNo: 0);
11710 EVT NSVT = N.getValueType().getScalarType();
11711 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11712 if (AllowTruncation || (CVT == NSVT))
11713 return CN;
11714 }
11715 }
11716
11717 return nullptr;
11718}
11719
11720ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
11721 EVT VT = N.getValueType();
11722 APInt DemandedElts = VT.isFixedLengthVector()
11723 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
11724 : APInt(1, 1);
11725 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11726}
11727
11728ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
11729 const APInt &DemandedElts,
11730 bool AllowUndefs) {
11731 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
11732 return CN;
11733
11734 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
11735 BitVector UndefElements;
11736 ConstantFPSDNode *CN =
11737 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
11738 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11739 if (CN && (UndefElements.none() || AllowUndefs))
11740 return CN;
11741 }
11742
11743 if (N.getOpcode() == ISD::SPLAT_VECTOR)
11744 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
11745 return CN;
11746
11747 return nullptr;
11748}
11749
11750bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11751 // TODO: may want to use peekThroughBitcast() here.
11752 ConstantSDNode *C =
11753 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11754 return C && C->isZero();
11755}
11756
11757bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11758 ConstantSDNode *C =
11759 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11760 return C && C->isOne();
11761}
11762
11763bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11764 N = peekThroughBitcasts(V: N);
11765 unsigned BitWidth = N.getScalarValueSizeInBits();
11766 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11767 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
11768}
11769
11770HandleSDNode::~HandleSDNode() {
11771 DropOperands();
11772}
11773
11774GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11775 const DebugLoc &DL,
11776 const GlobalValue *GA, EVT VT,
11777 int64_t o, unsigned TF)
11778 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11779 TheGlobal = GA;
11780}
11781
11782AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
11783 EVT VT, unsigned SrcAS,
11784 unsigned DestAS)
11785 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11786 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11787
11788MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11789 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11790 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11791 MemSDNodeBits.IsVolatile = MMO->isVolatile();
11792 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11793 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11794 MemSDNodeBits.IsInvariant = MMO->isInvariant();
11795
11796 // We check here that the size of the memory operand fits within the size of
11797 // the MMO. This is because the MMO might indicate only a possible address
11798 // range instead of specifying the affected memory addresses precisely.
11799 // TODO: Make MachineMemOperands aware of scalable vectors.
11800 assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
11801 "Size mismatch!");
11802}
11803
11804/// Profile - Gather unique data for the node.
11805///
11806void SDNode::Profile(FoldingSetNodeID &ID) const {
11807 AddNodeIDNode(ID, N: this);
11808}
11809
11810namespace {
11811
11812 struct EVTArray {
11813 std::vector<EVT> VTs;
11814
11815 EVTArray() {
11816 VTs.reserve(n: MVT::VALUETYPE_SIZE);
11817 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11818 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
11819 }
11820 };
11821
11822} // end anonymous namespace
11823
11824/// getValueTypeList - Return a pointer to the specified value type.
11825///
11826const EVT *SDNode::getValueTypeList(EVT VT) {
11827 static std::set<EVT, EVT::compareRawBits> EVTs;
11828 static EVTArray SimpleVTArray;
11829 static sys::SmartMutex<true> VTMutex;
11830
11831 if (VT.isExtended()) {
11832 sys::SmartScopedLock<true> Lock(VTMutex);
11833 return &(*EVTs.insert(x: VT).first);
11834 }
11835 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11836 return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11837}
11838
11839/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11840/// indicated value. This method ignores uses of other values defined by this
11841/// operation.
11842bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11843 assert(Value < getNumValues() && "Bad value!");
11844
11845 // TODO: Only iterate over uses of a given value of the node
11846 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11847 if (UI.getUse().getResNo() == Value) {
11848 if (NUses == 0)
11849 return false;
11850 --NUses;
11851 }
11852 }
11853
11854 // Found exactly the right number of uses?
11855 return NUses == 0;
11856}
11857
11858/// hasAnyUseOfValue - Return true if there are any use of the indicated
11859/// value. This method ignores uses of other values defined by this operation.
11860bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11861 assert(Value < getNumValues() && "Bad value!");
11862
11863 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11864 if (UI.getUse().getResNo() == Value)
11865 return true;
11866
11867 return false;
11868}
11869
11870/// isOnlyUserOf - Return true if this node is the only use of N.
11871bool SDNode::isOnlyUserOf(const SDNode *N) const {
11872 bool Seen = false;
11873 for (const SDNode *User : N->uses()) {
11874 if (User == this)
11875 Seen = true;
11876 else
11877 return false;
11878 }
11879
11880 return Seen;
11881}
11882
11883/// Return true if the only users of N are contained in Nodes.
11884bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
11885 bool Seen = false;
11886 for (const SDNode *User : N->uses()) {
11887 if (llvm::is_contained(Range&: Nodes, Element: User))
11888 Seen = true;
11889 else
11890 return false;
11891 }
11892
11893 return Seen;
11894}
11895
11896/// isOperand - Return true if this node is an operand of N.
11897bool SDValue::isOperandOf(const SDNode *N) const {
11898 return is_contained(Range: N->op_values(), Element: *this);
11899}
11900
11901bool SDNode::isOperandOf(const SDNode *N) const {
11902 return any_of(Range: N->op_values(),
11903 P: [this](SDValue Op) { return this == Op.getNode(); });
11904}
11905
11906/// reachesChainWithoutSideEffects - Return true if this operand (which must
11907/// be a chain) reaches the specified operand without crossing any
11908/// side-effecting instructions on any chain path. In practice, this looks
11909/// through token factors and non-volatile loads. In order to remain efficient,
11910/// this only looks a couple of nodes in, it does not do an exhaustive search.
11911///
11912/// Note that we only need to examine chains when we're searching for
11913/// side-effects; SelectionDAG requires that all side-effects are represented
11914/// by chains, even if another operand would force a specific ordering. This
11915/// constraint is necessary to allow transformations like splitting loads.
11916bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
11917 unsigned Depth) const {
11918 if (*this == Dest) return true;
11919
11920 // Don't search too deeply, we just want to be able to see through
11921 // TokenFactor's etc.
11922 if (Depth == 0) return false;
11923
11924 // If this is a token factor, all inputs to the TF happen in parallel.
11925 if (getOpcode() == ISD::TokenFactor) {
11926 // First, try a shallow search.
11927 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
11928 // We found the chain we want as an operand of this TokenFactor.
11929 // Essentially, we reach the chain without side-effects if we could
11930 // serialize the TokenFactor into a simple chain of operations with
11931 // Dest as the last operation. This is automatically true if the
11932 // chain has one use: there are no other ordering constraints.
11933 // If the chain has more than one use, we give up: some other
11934 // use of Dest might force a side-effect between Dest and the current
11935 // node.
11936 if (Dest.hasOneUse())
11937 return true;
11938 }
11939 // Next, try a deep search: check whether every operand of the TokenFactor
11940 // reaches Dest.
11941 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
11942 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
11943 });
11944 }
11945
11946 // Loads don't have side effects, look through them.
11947 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
11948 if (Ld->isUnordered())
11949 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
11950 }
11951 return false;
11952}
11953
11954bool SDNode::hasPredecessor(const SDNode *N) const {
11955 SmallPtrSet<const SDNode *, 32> Visited;
11956 SmallVector<const SDNode *, 16> Worklist;
11957 Worklist.push_back(Elt: this);
11958 return hasPredecessorHelper(N, Visited, Worklist);
11959}
11960
11961void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
11962 this->Flags.intersectWith(Flags);
11963}
11964
11965SDValue
11966SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
11967 ArrayRef<ISD::NodeType> CandidateBinOps,
11968 bool AllowPartials) {
11969 // The pattern must end in an extract from index 0.
11970 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11971 !isNullConstant(V: Extract->getOperand(Num: 1)))
11972 return SDValue();
11973
11974 // Match against one of the candidate binary ops.
11975 SDValue Op = Extract->getOperand(Num: 0);
11976 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
11977 return Op.getOpcode() == unsigned(BinOp);
11978 }))
11979 return SDValue();
11980
11981 // Floating-point reductions may require relaxed constraints on the final step
11982 // of the reduction because they may reorder intermediate operations.
11983 unsigned CandidateBinOp = Op.getOpcode();
11984 if (Op.getValueType().isFloatingPoint()) {
11985 SDNodeFlags Flags = Op->getFlags();
11986 switch (CandidateBinOp) {
11987 case ISD::FADD:
11988 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
11989 return SDValue();
11990 break;
11991 default:
11992 llvm_unreachable("Unhandled FP opcode for binop reduction");
11993 }
11994 }
11995
11996 // Matching failed - attempt to see if we did enough stages that a partial
11997 // reduction from a subvector is possible.
11998 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
11999 if (!AllowPartials || !Op)
12000 return SDValue();
12001 EVT OpVT = Op.getValueType();
12002 EVT OpSVT = OpVT.getScalarType();
12003 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
12004 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
12005 return SDValue();
12006 BinOp = (ISD::NodeType)CandidateBinOp;
12007 return getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Op), VT: SubVT, N1: Op,
12008 N2: getVectorIdxConstant(Val: 0, DL: SDLoc(Op)));
12009 };
12010
12011 // At each stage, we're looking for something that looks like:
12012 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
12013 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
12014 // i32 undef, i32 undef, i32 undef, i32 undef>
12015 // %a = binop <8 x i32> %op, %s
12016 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
12017 // we expect something like:
12018 // <4,5,6,7,u,u,u,u>
12019 // <2,3,u,u,u,u,u,u>
12020 // <1,u,u,u,u,u,u,u>
12021 // While a partial reduction match would be:
12022 // <2,3,u,u,u,u,u,u>
12023 // <1,u,u,u,u,u,u,u>
12024 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
12025 SDValue PrevOp;
12026 for (unsigned i = 0; i < Stages; ++i) {
12027 unsigned MaskEnd = (1 << i);
12028
12029 if (Op.getOpcode() != CandidateBinOp)
12030 return PartialReduction(PrevOp, MaskEnd);
12031
12032 SDValue Op0 = Op.getOperand(i: 0);
12033 SDValue Op1 = Op.getOperand(i: 1);
12034
12035 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
12036 if (Shuffle) {
12037 Op = Op1;
12038 } else {
12039 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
12040 Op = Op0;
12041 }
12042
12043 // The first operand of the shuffle should be the same as the other operand
12044 // of the binop.
12045 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
12046 return PartialReduction(PrevOp, MaskEnd);
12047
12048 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12049 for (int Index = 0; Index < (int)MaskEnd; ++Index)
12050 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
12051 return PartialReduction(PrevOp, MaskEnd);
12052
12053 PrevOp = Op;
12054 }
12055
12056 // Handle subvector reductions, which tend to appear after the shuffle
12057 // reduction stages.
12058 while (Op.getOpcode() == CandidateBinOp) {
12059 unsigned NumElts = Op.getValueType().getVectorNumElements();
12060 SDValue Op0 = Op.getOperand(i: 0);
12061 SDValue Op1 = Op.getOperand(i: 1);
12062 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12063 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12064 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
12065 break;
12066 SDValue Src = Op0.getOperand(i: 0);
12067 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12068 if (NumSrcElts != (2 * NumElts))
12069 break;
12070 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
12071 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
12072 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
12073 Op0.getConstantOperandAPInt(i: 1) == NumElts))
12074 break;
12075 Op = Src;
12076 }
12077
12078 BinOp = (ISD::NodeType)CandidateBinOp;
12079 return Op;
12080}
12081
12082SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
12083 EVT VT = N->getValueType(ResNo: 0);
12084 EVT EltVT = VT.getVectorElementType();
12085 unsigned NE = VT.getVectorNumElements();
12086
12087 SDLoc dl(N);
12088
12089 // If ResNE is 0, fully unroll the vector op.
12090 if (ResNE == 0)
12091 ResNE = NE;
12092 else if (NE > ResNE)
12093 NE = ResNE;
12094
12095 if (N->getNumValues() == 2) {
12096 SmallVector<SDValue, 8> Scalars0, Scalars1;
12097 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12098 EVT VT1 = N->getValueType(ResNo: 1);
12099 EVT EltVT1 = VT1.getVectorElementType();
12100
12101 unsigned i;
12102 for (i = 0; i != NE; ++i) {
12103 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12104 SDValue Operand = N->getOperand(Num: j);
12105 EVT OperandVT = Operand.getValueType();
12106
12107 // A vector operand; extract a single element.
12108 EVT OperandEltVT = OperandVT.getVectorElementType();
12109 Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
12110 N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl));
12111 }
12112
12113 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
12114 Scalars0.push_back(Elt: EltOp);
12115 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
12116 }
12117
12118 SDValue Vec0 = getBuildVector(VT, DL: dl, Ops: Scalars0);
12119 SDValue Vec1 = getBuildVector(VT: VT1, DL: dl, Ops: Scalars1);
12120 return getMergeValues(Ops: {Vec0, Vec1}, dl);
12121 }
12122
12123 assert(N->getNumValues() == 1 &&
12124 "Can't unroll a vector with multiple results!");
12125
12126 SmallVector<SDValue, 8> Scalars;
12127 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12128
12129 unsigned i;
12130 for (i= 0; i != NE; ++i) {
12131 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12132 SDValue Operand = N->getOperand(Num: j);
12133 EVT OperandVT = Operand.getValueType();
12134 if (OperandVT.isVector()) {
12135 // A vector operand; extract a single element.
12136 EVT OperandEltVT = OperandVT.getVectorElementType();
12137 Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT,
12138 N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl));
12139 } else {
12140 // A scalar operand; just use it as is.
12141 Operands[j] = Operand;
12142 }
12143 }
12144
12145 switch (N->getOpcode()) {
12146 default: {
12147 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
12148 Flags: N->getFlags()));
12149 break;
12150 }
12151 case ISD::VSELECT:
12152 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
12153 break;
12154 case ISD::SHL:
12155 case ISD::SRA:
12156 case ISD::SRL:
12157 case ISD::ROTL:
12158 case ISD::ROTR:
12159 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
12160 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
12161 Op: Operands[1])));
12162 break;
12163 case ISD::SIGN_EXTEND_INREG: {
12164 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
12165 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
12166 N1: Operands[0],
12167 N2: getValueType(VT: ExtVT)));
12168 }
12169 }
12170 }
12171
12172 for (; i < ResNE; ++i)
12173 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
12174
12175 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
12176 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
12177}
12178
12179std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12180 SDNode *N, unsigned ResNE) {
12181 unsigned Opcode = N->getOpcode();
12182 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12183 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12184 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12185 "Expected an overflow opcode");
12186
12187 EVT ResVT = N->getValueType(ResNo: 0);
12188 EVT OvVT = N->getValueType(ResNo: 1);
12189 EVT ResEltVT = ResVT.getVectorElementType();
12190 EVT OvEltVT = OvVT.getVectorElementType();
12191 SDLoc dl(N);
12192
12193 // If ResNE is 0, fully unroll the vector op.
12194 unsigned NE = ResVT.getVectorNumElements();
12195 if (ResNE == 0)
12196 ResNE = NE;
12197 else if (NE > ResNE)
12198 NE = ResNE;
12199
12200 SmallVector<SDValue, 8> LHSScalars;
12201 SmallVector<SDValue, 8> RHSScalars;
12202 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
12203 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
12204
12205 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
12206 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
12207 SmallVector<SDValue, 8> ResScalars;
12208 SmallVector<SDValue, 8> OvScalars;
12209 for (unsigned i = 0; i < NE; ++i) {
12210 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
12211 SDValue Ov =
12212 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
12213 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
12214 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
12215
12216 ResScalars.push_back(Elt: Res);
12217 OvScalars.push_back(Elt: Ov);
12218 }
12219
12220 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
12221 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
12222
12223 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
12224 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
12225 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
12226 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
12227}
12228
12229bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
12230 LoadSDNode *Base,
12231 unsigned Bytes,
12232 int Dist) const {
12233 if (LD->isVolatile() || Base->isVolatile())
12234 return false;
12235 // TODO: probably too restrictive for atomics, revisit
12236 if (!LD->isSimple())
12237 return false;
12238 if (LD->isIndexed() || Base->isIndexed())
12239 return false;
12240 if (LD->getChain() != Base->getChain())
12241 return false;
12242 EVT VT = LD->getMemoryVT();
12243 if (VT.getSizeInBits() / 8 != Bytes)
12244 return false;
12245
12246 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
12247 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
12248
12249 int64_t Offset = 0;
12250 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
12251 return (Dist * (int64_t)Bytes == Offset);
12252 return false;
12253}
12254
12255/// InferPtrAlignment - Infer alignment of a load / store address. Return
12256/// std::nullopt if it cannot be inferred.
12257MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
12258 // If this is a GlobalAddress + cst, return the alignment.
12259 const GlobalValue *GV = nullptr;
12260 int64_t GVOffset = 0;
12261 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
12262 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12263 KnownBits Known(PtrWidth);
12264 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
12265 unsigned AlignBits = Known.countMinTrailingZeros();
12266 if (AlignBits)
12267 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
12268 }
12269
12270 // If this is a direct reference to a stack slot, use information about the
12271 // stack slot's alignment.
12272 int FrameIdx = INT_MIN;
12273 int64_t FrameOffset = 0;
12274 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
12275 FrameIdx = FI->getIndex();
12276 } else if (isBaseWithConstantOffset(Op: Ptr) &&
12277 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
12278 // Handle FI+Cst
12279 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
12280 FrameOffset = Ptr.getConstantOperandVal(i: 1);
12281 }
12282
12283 if (FrameIdx != INT_MIN) {
12284 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
12285 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
12286 }
12287
12288 return std::nullopt;
12289}
12290
12291/// Split the scalar node with EXTRACT_ELEMENT using the provided
12292/// VTs and return the low/high part.
12293std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12294 const SDLoc &DL,
12295 const EVT &LoVT,
12296 const EVT &HiVT) {
12297 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12298 "Split node must be a scalar type");
12299 SDValue Lo =
12300 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
12301 SDValue Hi =
12302 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
12303 return std::make_pair(x&: Lo, y&: Hi);
12304}
12305
12306/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12307/// which is split (or expanded) into two not necessarily identical pieces.
12308std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12309 // Currently all types are split in half.
12310 EVT LoVT, HiVT;
12311 if (!VT.isVector())
12312 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
12313 else
12314 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
12315
12316 return std::make_pair(x&: LoVT, y&: HiVT);
12317}
12318
12319/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12320/// type, dependent on an enveloping VT that has been split into two identical
12321/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12322std::pair<EVT, EVT>
12323SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
12324 bool *HiIsEmpty) const {
12325 EVT EltTp = VT.getVectorElementType();
12326 // Examples:
12327 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12328 // custom VL=9 with enveloping VL=8/8 yields 8/1
12329 // custom VL=10 with enveloping VL=8/8 yields 8/2
12330 // etc.
12331 ElementCount VTNumElts = VT.getVectorElementCount();
12332 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12333 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12334 "Mixing fixed width and scalable vectors when enveloping a type");
12335 EVT LoVT, HiVT;
12336 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12337 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
12338 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
12339 *HiIsEmpty = false;
12340 } else {
12341 // Flag that hi type has zero storage size, but return split envelop type
12342 // (this would be easier if vector types with zero elements were allowed).
12343 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
12344 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
12345 *HiIsEmpty = true;
12346 }
12347 return std::make_pair(x&: LoVT, y&: HiVT);
12348}
12349
12350/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12351/// low/high part.
12352std::pair<SDValue, SDValue>
12353SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12354 const EVT &HiVT) {
12355 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12356 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12357 "Splitting vector with an invalid mixture of fixed and scalable "
12358 "vector types");
12359 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
12360 N.getValueType().getVectorMinNumElements() &&
12361 "More vector elements requested than available!");
12362 SDValue Lo, Hi;
12363 Lo =
12364 getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N, N2: getVectorIdxConstant(Val: 0, DL));
12365 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12366 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12367 // IDX with the runtime scaling factor of the result vector type. For
12368 // fixed-width result vectors, that runtime scaling factor is 1.
12369 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
12370 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
12371 return std::make_pair(x&: Lo, y&: Hi);
12372}
12373
12374std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12375 const SDLoc &DL) {
12376 // Split the vector length parameter.
12377 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12378 EVT VT = N.getValueType();
12379 assert(VecVT.getVectorElementCount().isKnownEven() &&
12380 "Expecting the mask to be an evenly-sized vector");
12381 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12382 SDValue HalfNumElts =
12383 VecVT.isFixedLengthVector()
12384 ? getConstant(Val: HalfMinNumElts, DL, VT)
12385 : getVScale(DL, VT, MulImm: APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12386 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
12387 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
12388 return std::make_pair(x&: Lo, y&: Hi);
12389}
12390
12391/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
12392SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
12393 EVT VT = N.getValueType();
12394 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
12395 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
12396 return getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: getUNDEF(VT: WideVT), N2: N,
12397 N3: getVectorIdxConstant(Val: 0, DL));
12398}
12399
12400void SelectionDAG::ExtractVectorElements(SDValue Op,
12401 SmallVectorImpl<SDValue> &Args,
12402 unsigned Start, unsigned Count,
12403 EVT EltVT) {
12404 EVT VT = Op.getValueType();
12405 if (Count == 0)
12406 Count = VT.getVectorNumElements();
12407 if (EltVT == EVT())
12408 EltVT = VT.getVectorElementType();
12409 SDLoc SL(Op);
12410 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12411 Args.push_back(Elt: getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Op,
12412 N2: getVectorIdxConstant(Val: i, DL: SL)));
12413 }
12414}
12415
12416// getAddressSpace - Return the address space this GlobalAddress belongs to.
12417unsigned GlobalAddressSDNode::getAddressSpace() const {
12418 return getGlobal()->getType()->getAddressSpace();
12419}
12420
12421Type *ConstantPoolSDNode::getType() const {
12422 if (isMachineConstantPoolEntry())
12423 return Val.MachineCPVal->getType();
12424 return Val.ConstVal->getType();
12425}
12426
12427bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12428 unsigned &SplatBitSize,
12429 bool &HasAnyUndefs,
12430 unsigned MinSplatBits,
12431 bool IsBigEndian) const {
12432 EVT VT = getValueType(ResNo: 0);
12433 assert(VT.isVector() && "Expected a vector type");
12434 unsigned VecWidth = VT.getSizeInBits();
12435 if (MinSplatBits > VecWidth)
12436 return false;
12437
12438 // FIXME: The widths are based on this node's type, but build vectors can
12439 // truncate their operands.
12440 SplatValue = APInt(VecWidth, 0);
12441 SplatUndef = APInt(VecWidth, 0);
12442
12443 // Get the bits. Bits with undefined values (when the corresponding element
12444 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12445 // in SplatValue. If any of the values are not constant, give up and return
12446 // false.
12447 unsigned int NumOps = getNumOperands();
12448 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12449 unsigned EltWidth = VT.getScalarSizeInBits();
12450
12451 for (unsigned j = 0; j < NumOps; ++j) {
12452 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12453 SDValue OpVal = getOperand(Num: i);
12454 unsigned BitPos = j * EltWidth;
12455
12456 if (OpVal.isUndef())
12457 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
12458 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
12459 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
12460 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
12461 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
12462 else
12463 return false;
12464 }
12465
12466 // The build_vector is all constants or undefs. Find the smallest element
12467 // size that splats the vector.
12468 HasAnyUndefs = (SplatUndef != 0);
12469
12470 // FIXME: This does not work for vectors with elements less than 8 bits.
12471 while (VecWidth > 8) {
12472 // If we can't split in half, stop here.
12473 if (VecWidth & 1)
12474 break;
12475
12476 unsigned HalfSize = VecWidth / 2;
12477 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
12478 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
12479 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
12480 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
12481
12482 // If the two halves do not match (ignoring undef bits), stop here.
12483 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12484 MinSplatBits > HalfSize)
12485 break;
12486
12487 SplatValue = HighValue | LowValue;
12488 SplatUndef = HighUndef & LowUndef;
12489
12490 VecWidth = HalfSize;
12491 }
12492
12493 // FIXME: The loop above only tries to split in halves. But if the input
12494 // vector for example is <3 x i16> it wouldn't be able to detect a
12495 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
12496 // optimizations. I guess that back in the days when this helper was created
12497 // vectors normally was power-of-2 sized.
12498
12499 SplatBitSize = VecWidth;
12500 return true;
12501}
12502
12503SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
12504 BitVector *UndefElements) const {
12505 unsigned NumOps = getNumOperands();
12506 if (UndefElements) {
12507 UndefElements->clear();
12508 UndefElements->resize(N: NumOps);
12509 }
12510 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12511 if (!DemandedElts)
12512 return SDValue();
12513 SDValue Splatted;
12514 for (unsigned i = 0; i != NumOps; ++i) {
12515 if (!DemandedElts[i])
12516 continue;
12517 SDValue Op = getOperand(Num: i);
12518 if (Op.isUndef()) {
12519 if (UndefElements)
12520 (*UndefElements)[i] = true;
12521 } else if (!Splatted) {
12522 Splatted = Op;
12523 } else if (Splatted != Op) {
12524 return SDValue();
12525 }
12526 }
12527
12528 if (!Splatted) {
12529 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12530 assert(getOperand(FirstDemandedIdx).isUndef() &&
12531 "Can only have a splat without a constant for all undefs.");
12532 return getOperand(Num: FirstDemandedIdx);
12533 }
12534
12535 return Splatted;
12536}
12537
12538SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
12539 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
12540 return getSplatValue(DemandedElts, UndefElements);
12541}
12542
12543bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
12544 SmallVectorImpl<SDValue> &Sequence,
12545 BitVector *UndefElements) const {
12546 unsigned NumOps = getNumOperands();
12547 Sequence.clear();
12548 if (UndefElements) {
12549 UndefElements->clear();
12550 UndefElements->resize(N: NumOps);
12551 }
12552 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12553 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
12554 return false;
12555
12556 // Set the undefs even if we don't find a sequence (like getSplatValue).
12557 if (UndefElements)
12558 for (unsigned I = 0; I != NumOps; ++I)
12559 if (DemandedElts[I] && getOperand(Num: I).isUndef())
12560 (*UndefElements)[I] = true;
12561
12562 // Iteratively widen the sequence length looking for repetitions.
12563 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12564 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
12565 for (unsigned I = 0; I != NumOps; ++I) {
12566 if (!DemandedElts[I])
12567 continue;
12568 SDValue &SeqOp = Sequence[I % SeqLen];
12569 SDValue Op = getOperand(Num: I);
12570 if (Op.isUndef()) {
12571 if (!SeqOp)
12572 SeqOp = Op;
12573 continue;
12574 }
12575 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12576 Sequence.clear();
12577 break;
12578 }
12579 SeqOp = Op;
12580 }
12581 if (!Sequence.empty())
12582 return true;
12583 }
12584
12585 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12586 return false;
12587}
12588
12589bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
12590 BitVector *UndefElements) const {
12591 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
12592 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12593}
12594
12595ConstantSDNode *
12596BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
12597 BitVector *UndefElements) const {
12598 return dyn_cast_or_null<ConstantSDNode>(
12599 Val: getSplatValue(DemandedElts, UndefElements));
12600}
12601
12602ConstantSDNode *
12603BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
12604 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
12605}
12606
12607ConstantFPSDNode *
12608BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
12609 BitVector *UndefElements) const {
12610 return dyn_cast_or_null<ConstantFPSDNode>(
12611 Val: getSplatValue(DemandedElts, UndefElements));
12612}
12613
12614ConstantFPSDNode *
12615BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
12616 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
12617}
12618
12619int32_t
12620BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
12621 uint32_t BitWidth) const {
12622 if (ConstantFPSDNode *CN =
12623 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
12624 bool IsExact;
12625 APSInt IntVal(BitWidth);
12626 const APFloat &APF = CN->getValueAPF();
12627 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
12628 APFloat::opOK ||
12629 !IsExact)
12630 return -1;
12631
12632 return IntVal.exactLogBase2();
12633 }
12634 return -1;
12635}
12636
12637bool BuildVectorSDNode::getConstantRawBits(
12638 bool IsLittleEndian, unsigned DstEltSizeInBits,
12639 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12640 // Early-out if this contains anything but Undef/Constant/ConstantFP.
12641 if (!isConstant())
12642 return false;
12643
12644 unsigned NumSrcOps = getNumOperands();
12645 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
12646 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12647 "Invalid bitcast scale");
12648
12649 // Extract raw src bits.
12650 SmallVector<APInt> SrcBitElements(NumSrcOps,
12651 APInt::getZero(numBits: SrcEltSizeInBits));
12652 BitVector SrcUndeElements(NumSrcOps, false);
12653
12654 for (unsigned I = 0; I != NumSrcOps; ++I) {
12655 SDValue Op = getOperand(Num: I);
12656 if (Op.isUndef()) {
12657 SrcUndeElements.set(I);
12658 continue;
12659 }
12660 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
12661 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
12662 assert((CInt || CFP) && "Unknown constant");
12663 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
12664 : CFP->getValueAPF().bitcastToAPInt();
12665 }
12666
12667 // Recast to dst width.
12668 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
12669 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
12670 return true;
12671}
12672
12673void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12674 unsigned DstEltSizeInBits,
12675 SmallVectorImpl<APInt> &DstBitElements,
12676 ArrayRef<APInt> SrcBitElements,
12677 BitVector &DstUndefElements,
12678 const BitVector &SrcUndefElements) {
12679 unsigned NumSrcOps = SrcBitElements.size();
12680 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12681 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12682 "Invalid bitcast scale");
12683 assert(NumSrcOps == SrcUndefElements.size() &&
12684 "Vector size mismatch");
12685
12686 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12687 DstUndefElements.clear();
12688 DstUndefElements.resize(N: NumDstOps, t: false);
12689 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
12690
12691 // Concatenate src elements constant bits together into dst element.
12692 if (SrcEltSizeInBits <= DstEltSizeInBits) {
12693 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12694 for (unsigned I = 0; I != NumDstOps; ++I) {
12695 DstUndefElements.set(I);
12696 APInt &DstBits = DstBitElements[I];
12697 for (unsigned J = 0; J != Scale; ++J) {
12698 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12699 if (SrcUndefElements[Idx])
12700 continue;
12701 DstUndefElements.reset(Idx: I);
12702 const APInt &SrcBits = SrcBitElements[Idx];
12703 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12704 "Illegal constant bitwidths");
12705 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
12706 }
12707 }
12708 return;
12709 }
12710
12711 // Split src element constant bits into dst elements.
12712 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12713 for (unsigned I = 0; I != NumSrcOps; ++I) {
12714 if (SrcUndefElements[I]) {
12715 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
12716 continue;
12717 }
12718 const APInt &SrcBits = SrcBitElements[I];
12719 for (unsigned J = 0; J != Scale; ++J) {
12720 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12721 APInt &DstBits = DstBitElements[Idx];
12722 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
12723 }
12724 }
12725}
12726
12727bool BuildVectorSDNode::isConstant() const {
12728 for (const SDValue &Op : op_values()) {
12729 unsigned Opc = Op.getOpcode();
12730 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12731 return false;
12732 }
12733 return true;
12734}
12735
12736std::optional<std::pair<APInt, APInt>>
12737BuildVectorSDNode::isConstantSequence() const {
12738 unsigned NumOps = getNumOperands();
12739 if (NumOps < 2)
12740 return std::nullopt;
12741
12742 if (!isa<ConstantSDNode>(Val: getOperand(Num: 0)) ||
12743 !isa<ConstantSDNode>(Val: getOperand(Num: 1)))
12744 return std::nullopt;
12745
12746 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
12747 APInt Start = getConstantOperandAPInt(Num: 0).trunc(width: EltSize);
12748 APInt Stride = getConstantOperandAPInt(Num: 1).trunc(width: EltSize) - Start;
12749
12750 if (Stride.isZero())
12751 return std::nullopt;
12752
12753 for (unsigned i = 2; i < NumOps; ++i) {
12754 if (!isa<ConstantSDNode>(Val: getOperand(Num: i)))
12755 return std::nullopt;
12756
12757 APInt Val = getConstantOperandAPInt(Num: i).trunc(width: EltSize);
12758 if (Val != (Start + (Stride * i)))
12759 return std::nullopt;
12760 }
12761
12762 return std::make_pair(x&: Start, y&: Stride);
12763}
12764
12765bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12766 // Find the first non-undef value in the shuffle mask.
12767 unsigned i, e;
12768 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12769 /* search */;
12770
12771 // If all elements are undefined, this shuffle can be considered a splat
12772 // (although it should eventually get simplified away completely).
12773 if (i == e)
12774 return true;
12775
12776 // Make sure all remaining elements are either undef or the same as the first
12777 // non-undef value.
12778 for (int Idx = Mask[i]; i != e; ++i)
12779 if (Mask[i] >= 0 && Mask[i] != Idx)
12780 return false;
12781 return true;
12782}
12783
12784// Returns the SDNode if it is a constant integer BuildVector
12785// or constant integer.
12786SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
12787 if (isa<ConstantSDNode>(Val: N))
12788 return N.getNode();
12789 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
12790 return N.getNode();
12791 // Treat a GlobalAddress supporting constant offset folding as a
12792 // constant integer.
12793 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
12794 if (GA->getOpcode() == ISD::GlobalAddress &&
12795 TLI->isOffsetFoldingLegal(GA))
12796 return GA;
12797 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12798 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
12799 return N.getNode();
12800 return nullptr;
12801}
12802
12803// Returns the SDNode if it is a constant float BuildVector
12804// or constant float.
12805SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
12806 if (isa<ConstantFPSDNode>(Val: N))
12807 return N.getNode();
12808
12809 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
12810 return N.getNode();
12811
12812 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12813 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
12814 return N.getNode();
12815
12816 return nullptr;
12817}
12818
12819void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12820 assert(!Node->OperandList && "Node already has operands");
12821 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
12822 "too many operands to fit into SDNode");
12823 SDUse *Ops = OperandRecycler.allocate(
12824 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
12825
12826 bool IsDivergent = false;
12827 for (unsigned I = 0; I != Vals.size(); ++I) {
12828 Ops[I].setUser(Node);
12829 Ops[I].setInitial(Vals[I]);
12830 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12831 IsDivergent |= Ops[I].getNode()->isDivergent();
12832 }
12833 Node->NumOperands = Vals.size();
12834 Node->OperandList = Ops;
12835 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
12836 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
12837 Node->SDNodeBits.IsDivergent = IsDivergent;
12838 }
12839 checkForCycles(N: Node);
12840}
12841
12842SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
12843 SmallVectorImpl<SDValue> &Vals) {
12844 size_t Limit = SDNode::getMaxNumOperands();
12845 while (Vals.size() > Limit) {
12846 unsigned SliceIdx = Vals.size() - Limit;
12847 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
12848 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12849 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
12850 Vals.emplace_back(Args&: NewTF);
12851 }
12852 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12853}
12854
12855SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
12856 EVT VT, SDNodeFlags Flags) {
12857 switch (Opcode) {
12858 default:
12859 return SDValue();
12860 case ISD::ADD:
12861 case ISD::OR:
12862 case ISD::XOR:
12863 case ISD::UMAX:
12864 return getConstant(Val: 0, DL, VT);
12865 case ISD::MUL:
12866 return getConstant(Val: 1, DL, VT);
12867 case ISD::AND:
12868 case ISD::UMIN:
12869 return getAllOnesConstant(DL, VT);
12870 case ISD::SMAX:
12871 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
12872 case ISD::SMIN:
12873 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
12874 case ISD::FADD:
12875 return getConstantFP(Val: -0.0, DL, VT);
12876 case ISD::FMUL:
12877 return getConstantFP(Val: 1.0, DL, VT);
12878 case ISD::FMINNUM:
12879 case ISD::FMAXNUM: {
12880 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12881 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12882 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
12883 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
12884 APFloat::getLargest(Sem: Semantics);
12885 if (Opcode == ISD::FMAXNUM)
12886 NeutralAF.changeSign();
12887
12888 return getConstantFP(V: NeutralAF, DL, VT);
12889 }
12890 case ISD::FMINIMUM:
12891 case ISD::FMAXIMUM: {
12892 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12893 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12894 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
12895 : APFloat::getLargest(Sem: Semantics);
12896 if (Opcode == ISD::FMAXIMUM)
12897 NeutralAF.changeSign();
12898
12899 return getConstantFP(V: NeutralAF, DL, VT);
12900 }
12901
12902 }
12903}
12904
12905/// Helper used to make a call to a library function that has one argument of
12906/// pointer type.
12907///
12908/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12909/// used to get or set floating-point state. They have one argument of pointer
12910/// type, which points to the memory region containing bits of the
12911/// floating-point state. The value returned by such function is ignored in the
12912/// created call.
12913///
12914/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12915/// \param Ptr Pointer used to save/load state.
12916/// \param InChain Ingoing token chain.
12917/// \returns Outgoing chain token.
12918SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
12919 SDValue InChain,
12920 const SDLoc &DLoc) {
12921 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12922 TargetLowering::ArgListTy Args;
12923 TargetLowering::ArgListEntry Entry;
12924 Entry.Node = Ptr;
12925 Entry.Ty = Ptr.getValueType().getTypeForEVT(Context&: *getContext());
12926 Args.push_back(x: Entry);
12927 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12928 SDValue Callee = getExternalSymbol(Sym: TLI->getLibcallName(Call: LC),
12929 VT: TLI->getPointerTy(DL: getDataLayout()));
12930 TargetLowering::CallLoweringInfo CLI(*this);
12931 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12932 CC: TLI->getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee,
12933 ArgsList: std::move(Args));
12934 return TLI->LowerCallTo(CLI).second;
12935}
12936
12937void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
12938 assert(From && To && "Invalid SDNode; empty source SDValue?");
12939 auto I = SDEI.find(Val: From);
12940 if (I == SDEI.end())
12941 return;
12942
12943 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
12944 // the iterator, hence the need to make a copy to prevent a use-after-free.
12945 NodeExtraInfo NEI = I->second;
12946 if (LLVM_LIKELY(!NEI.PCSections)) {
12947 // No deep copy required for the types of extra info set.
12948 //
12949 // FIXME: Investigate if other types of extra info also need deep copy. This
12950 // depends on the types of nodes they can be attached to: if some extra info
12951 // is only ever attached to nodes where a replacement To node is always the
12952 // node where later use and propagation of the extra info has the intended
12953 // semantics, no deep copy is required.
12954 SDEI[To] = std::move(NEI);
12955 return;
12956 }
12957
12958 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12959 // through the replacement of From with To. Otherwise, replacements of a node
12960 // (From) with more complex nodes (To and its operands) may result in lost
12961 // extra info where the root node (To) is insignificant in further propagating
12962 // and using extra info when further lowering to MIR.
12963 //
12964 // In the first step pre-populate the visited set with the nodes reachable
12965 // from the old From node. This avoids copying NodeExtraInfo to parts of the
12966 // DAG that is not new and should be left untouched.
12967 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12968 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12969 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12970 if (MaxDepth == 0) {
12971 // Remember this node in case we need to increase MaxDepth and continue
12972 // populating FromReach from this node.
12973 Leafs.emplace_back(Args&: N);
12974 return;
12975 }
12976 if (!FromReach.insert(V: N).second)
12977 return;
12978 for (const SDValue &Op : N->op_values())
12979 Self(Self, Op.getNode(), MaxDepth - 1);
12980 };
12981
12982 // Copy extra info to To and all its transitive operands (that are new).
12983 SmallPtrSet<const SDNode *, 8> Visited;
12984 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
12985 if (FromReach.contains(V: N))
12986 return true;
12987 if (!Visited.insert(Ptr: N).second)
12988 return true;
12989 if (getEntryNode().getNode() == N)
12990 return false;
12991 for (const SDValue &Op : N->op_values()) {
12992 if (!Self(Self, Op.getNode()))
12993 return false;
12994 }
12995 // Copy only if entry node was not reached.
12996 SDEI[N] = NEI;
12997 return true;
12998 };
12999
13000 // We first try with a lower MaxDepth, assuming that the path to common
13001 // operands between From and To is relatively short. This significantly
13002 // improves performance in the common case. The initial MaxDepth is big
13003 // enough to avoid retry in the common case; the last MaxDepth is large
13004 // enough to avoid having to use the fallback below (and protects from
13005 // potential stack exhaustion from recursion).
13006 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
13007 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
13008 // StartFrom is the previous (or initial) set of leafs reachable at the
13009 // previous maximum depth.
13010 SmallVector<const SDNode *> StartFrom;
13011 std::swap(LHS&: StartFrom, RHS&: Leafs);
13012 for (const SDNode *N : StartFrom)
13013 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
13014 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
13015 return;
13016 // This should happen very rarely (reached the entry node).
13017 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
13018 assert(!Leafs.empty());
13019 }
13020
13021 // This should not happen - but if it did, that means the subgraph reachable
13022 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
13023 // could not visit all reachable common operands. Consequently, we were able
13024 // to reach the entry node.
13025 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13026 assert(false && "From subgraph too complex - increase max. MaxDepth?");
13027 // Best-effort fallback if assertions disabled.
13028 SDEI[To] = std::move(NEI);
13029}
13030
13031#ifndef NDEBUG
13032static void checkForCyclesHelper(const SDNode *N,
13033 SmallPtrSetImpl<const SDNode*> &Visited,
13034 SmallPtrSetImpl<const SDNode*> &Checked,
13035 const llvm::SelectionDAG *DAG) {
13036 // If this node has already been checked, don't check it again.
13037 if (Checked.count(Ptr: N))
13038 return;
13039
13040 // If a node has already been visited on this depth-first walk, reject it as
13041 // a cycle.
13042 if (!Visited.insert(Ptr: N).second) {
13043 errs() << "Detected cycle in SelectionDAG\n";
13044 dbgs() << "Offending node:\n";
13045 N->dumprFull(G: DAG); dbgs() << "\n";
13046 abort();
13047 }
13048
13049 for (const SDValue &Op : N->op_values())
13050 checkForCyclesHelper(N: Op.getNode(), Visited, Checked, DAG);
13051
13052 Checked.insert(Ptr: N);
13053 Visited.erase(Ptr: N);
13054}
13055#endif
13056
13057void llvm::checkForCycles(const llvm::SDNode *N,
13058 const llvm::SelectionDAG *DAG,
13059 bool force) {
13060#ifndef NDEBUG
13061 bool check = force;
13062#ifdef EXPENSIVE_CHECKS
13063 check = true;
13064#endif // EXPENSIVE_CHECKS
13065 if (check) {
13066 assert(N && "Checking nonexistent SDNode");
13067 SmallPtrSet<const SDNode*, 32> visited;
13068 SmallPtrSet<const SDNode*, 32> checked;
13069 checkForCyclesHelper(N, Visited&: visited, Checked&: checked, DAG);
13070 }
13071#endif // !NDEBUG
13072}
13073
13074void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13075 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
13076}
13077

source code of llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp