1 | //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This implements the SelectionDAG class. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/CodeGen/SelectionDAG.h" |
14 | #include "SDNodeDbgValue.h" |
15 | #include "llvm/ADT/APFloat.h" |
16 | #include "llvm/ADT/APInt.h" |
17 | #include "llvm/ADT/APSInt.h" |
18 | #include "llvm/ADT/ArrayRef.h" |
19 | #include "llvm/ADT/BitVector.h" |
20 | #include "llvm/ADT/DenseSet.h" |
21 | #include "llvm/ADT/FoldingSet.h" |
22 | #include "llvm/ADT/STLExtras.h" |
23 | #include "llvm/ADT/SmallPtrSet.h" |
24 | #include "llvm/ADT/SmallVector.h" |
25 | #include "llvm/ADT/Twine.h" |
26 | #include "llvm/Analysis/AliasAnalysis.h" |
27 | #include "llvm/Analysis/MemoryLocation.h" |
28 | #include "llvm/Analysis/ValueTracking.h" |
29 | #include "llvm/Analysis/VectorUtils.h" |
30 | #include "llvm/BinaryFormat/Dwarf.h" |
31 | #include "llvm/CodeGen/Analysis.h" |
32 | #include "llvm/CodeGen/FunctionLoweringInfo.h" |
33 | #include "llvm/CodeGen/ISDOpcodes.h" |
34 | #include "llvm/CodeGen/MachineBasicBlock.h" |
35 | #include "llvm/CodeGen/MachineConstantPool.h" |
36 | #include "llvm/CodeGen/MachineFrameInfo.h" |
37 | #include "llvm/CodeGen/MachineFunction.h" |
38 | #include "llvm/CodeGen/MachineMemOperand.h" |
39 | #include "llvm/CodeGen/RuntimeLibcalls.h" |
40 | #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" |
41 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
42 | #include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
43 | #include "llvm/CodeGen/TargetFrameLowering.h" |
44 | #include "llvm/CodeGen/TargetLowering.h" |
45 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
46 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
47 | #include "llvm/CodeGen/ValueTypes.h" |
48 | #include "llvm/CodeGenTypes/MachineValueType.h" |
49 | #include "llvm/IR/Constant.h" |
50 | #include "llvm/IR/ConstantRange.h" |
51 | #include "llvm/IR/Constants.h" |
52 | #include "llvm/IR/DataLayout.h" |
53 | #include "llvm/IR/DebugInfoMetadata.h" |
54 | #include "llvm/IR/DebugLoc.h" |
55 | #include "llvm/IR/DerivedTypes.h" |
56 | #include "llvm/IR/Function.h" |
57 | #include "llvm/IR/GlobalValue.h" |
58 | #include "llvm/IR/Metadata.h" |
59 | #include "llvm/IR/Type.h" |
60 | #include "llvm/Support/Casting.h" |
61 | #include "llvm/Support/CodeGen.h" |
62 | #include "llvm/Support/Compiler.h" |
63 | #include "llvm/Support/Debug.h" |
64 | #include "llvm/Support/ErrorHandling.h" |
65 | #include "llvm/Support/KnownBits.h" |
66 | #include "llvm/Support/MathExtras.h" |
67 | #include "llvm/Support/Mutex.h" |
68 | #include "llvm/Support/raw_ostream.h" |
69 | #include "llvm/Target/TargetMachine.h" |
70 | #include "llvm/Target/TargetOptions.h" |
71 | #include "llvm/TargetParser/Triple.h" |
72 | #include "llvm/Transforms/Utils/SizeOpts.h" |
73 | #include <algorithm> |
74 | #include <cassert> |
75 | #include <cstdint> |
76 | #include <cstdlib> |
77 | #include <limits> |
78 | #include <set> |
79 | #include <string> |
80 | #include <utility> |
81 | #include <vector> |
82 | |
83 | using namespace llvm; |
84 | |
85 | /// makeVTList - Return an instance of the SDVTList struct initialized with the |
86 | /// specified members. |
87 | static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { |
88 | SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs}; |
89 | return Res; |
90 | } |
91 | |
92 | // Default null implementations of the callbacks. |
93 | void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} |
94 | void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} |
95 | void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} |
96 | |
97 | void SelectionDAG::DAGNodeDeletedListener::anchor() {} |
98 | void SelectionDAG::DAGNodeInsertedListener::anchor() {} |
99 | |
100 | #define DEBUG_TYPE "selectiondag" |
101 | |
102 | static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt" , |
103 | cl::Hidden, cl::init(Val: true), |
104 | cl::desc("Gang up loads and stores generated by inlining of memcpy" )); |
105 | |
106 | static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max" , |
107 | cl::desc("Number limit for gluing ld/st of memcpy." ), |
108 | cl::Hidden, cl::init(Val: 0)); |
109 | |
110 | static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { |
111 | LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); |
112 | } |
113 | |
114 | //===----------------------------------------------------------------------===// |
115 | // ConstantFPSDNode Class |
116 | //===----------------------------------------------------------------------===// |
117 | |
118 | /// isExactlyValue - We don't rely on operator== working on double values, as |
119 | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
120 | /// As such, this method can be used to do an exact bit-for-bit comparison of |
121 | /// two floating point values. |
122 | bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { |
123 | return getValueAPF().bitwiseIsEqual(RHS: V); |
124 | } |
125 | |
126 | bool ConstantFPSDNode::isValueValidForType(EVT VT, |
127 | const APFloat& Val) { |
128 | assert(VT.isFloatingPoint() && "Can only convert between FP types" ); |
129 | |
130 | // convert modifies in place, so make a copy. |
131 | APFloat Val2 = APFloat(Val); |
132 | bool losesInfo; |
133 | (void) Val2.convert(ToSemantics: SelectionDAG::EVTToAPFloatSemantics(VT), |
134 | RM: APFloat::rmNearestTiesToEven, |
135 | losesInfo: &losesInfo); |
136 | return !losesInfo; |
137 | } |
138 | |
139 | //===----------------------------------------------------------------------===// |
140 | // ISD Namespace |
141 | //===----------------------------------------------------------------------===// |
142 | |
143 | bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { |
144 | if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
145 | unsigned EltSize = |
146 | N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits(); |
147 | if (auto *Op0 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) { |
148 | SplatVal = Op0->getAPIntValue().trunc(width: EltSize); |
149 | return true; |
150 | } |
151 | if (auto *Op0 = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0))) { |
152 | SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(width: EltSize); |
153 | return true; |
154 | } |
155 | } |
156 | |
157 | auto *BV = dyn_cast<BuildVectorSDNode>(Val: N); |
158 | if (!BV) |
159 | return false; |
160 | |
161 | APInt SplatUndef; |
162 | unsigned SplatBitSize; |
163 | bool HasUndefs; |
164 | unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits(); |
165 | // Endianness does not matter here. We are checking for a splat given the |
166 | // element size of the vector, and if we find such a splat for little endian |
167 | // layout, then that should be valid also for big endian (as the full vector |
168 | // size is known to be a multiple of the element size). |
169 | const bool IsBigEndian = false; |
170 | return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs, |
171 | MinSplatBits: EltSize, isBigEndian: IsBigEndian) && |
172 | EltSize == SplatBitSize; |
173 | } |
174 | |
175 | // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be |
176 | // specializations of the more general isConstantSplatVector()? |
177 | |
178 | bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { |
179 | // Look through a bit convert. |
180 | while (N->getOpcode() == ISD::BITCAST) |
181 | N = N->getOperand(Num: 0).getNode(); |
182 | |
183 | if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
184 | APInt SplatVal; |
185 | return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes(); |
186 | } |
187 | |
188 | if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
189 | |
190 | unsigned i = 0, e = N->getNumOperands(); |
191 | |
192 | // Skip over all of the undef values. |
193 | while (i != e && N->getOperand(Num: i).isUndef()) |
194 | ++i; |
195 | |
196 | // Do not accept an all-undef vector. |
197 | if (i == e) return false; |
198 | |
199 | // Do not accept build_vectors that aren't all constants or which have non-~0 |
200 | // elements. We have to be a bit careful here, as the type of the constant |
201 | // may not be the same as the type of the vector elements due to type |
202 | // legalization (the elements are promoted to a legal type for the target and |
203 | // a vector of a type may be legal when the base element type is not). |
204 | // We only want to check enough bits to cover the vector elements, because |
205 | // we care if the resultant vector is all ones, not whether the individual |
206 | // constants are. |
207 | SDValue NotZero = N->getOperand(Num: i); |
208 | unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits(); |
209 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: NotZero)) { |
210 | if (CN->getAPIntValue().countr_one() < EltSize) |
211 | return false; |
212 | } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val&: NotZero)) { |
213 | if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize) |
214 | return false; |
215 | } else |
216 | return false; |
217 | |
218 | // Okay, we have at least one ~0 value, check to see if the rest match or are |
219 | // undefs. Even with the above element type twiddling, this should be OK, as |
220 | // the same type legalization should have applied to all the elements. |
221 | for (++i; i != e; ++i) |
222 | if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef()) |
223 | return false; |
224 | return true; |
225 | } |
226 | |
227 | bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { |
228 | // Look through a bit convert. |
229 | while (N->getOpcode() == ISD::BITCAST) |
230 | N = N->getOperand(Num: 0).getNode(); |
231 | |
232 | if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
233 | APInt SplatVal; |
234 | return isConstantSplatVector(N, SplatVal) && SplatVal.isZero(); |
235 | } |
236 | |
237 | if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
238 | |
239 | bool IsAllUndef = true; |
240 | for (const SDValue &Op : N->op_values()) { |
241 | if (Op.isUndef()) |
242 | continue; |
243 | IsAllUndef = false; |
244 | // Do not accept build_vectors that aren't all constants or which have non-0 |
245 | // elements. We have to be a bit careful here, as the type of the constant |
246 | // may not be the same as the type of the vector elements due to type |
247 | // legalization (the elements are promoted to a legal type for the target |
248 | // and a vector of a type may be legal when the base element type is not). |
249 | // We only want to check enough bits to cover the vector elements, because |
250 | // we care if the resultant vector is all zeros, not whether the individual |
251 | // constants are. |
252 | unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits(); |
253 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: Op)) { |
254 | if (CN->getAPIntValue().countr_zero() < EltSize) |
255 | return false; |
256 | } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Val: Op)) { |
257 | if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize) |
258 | return false; |
259 | } else |
260 | return false; |
261 | } |
262 | |
263 | // Do not accept an all-undef vector. |
264 | if (IsAllUndef) |
265 | return false; |
266 | return true; |
267 | } |
268 | |
269 | bool ISD::isBuildVectorAllOnes(const SDNode *N) { |
270 | return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); |
271 | } |
272 | |
273 | bool ISD::isBuildVectorAllZeros(const SDNode *N) { |
274 | return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); |
275 | } |
276 | |
277 | bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { |
278 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
279 | return false; |
280 | |
281 | for (const SDValue &Op : N->op_values()) { |
282 | if (Op.isUndef()) |
283 | continue; |
284 | if (!isa<ConstantSDNode>(Val: Op)) |
285 | return false; |
286 | } |
287 | return true; |
288 | } |
289 | |
290 | bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { |
291 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
292 | return false; |
293 | |
294 | for (const SDValue &Op : N->op_values()) { |
295 | if (Op.isUndef()) |
296 | continue; |
297 | if (!isa<ConstantFPSDNode>(Val: Op)) |
298 | return false; |
299 | } |
300 | return true; |
301 | } |
302 | |
303 | bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize, |
304 | bool Signed) { |
305 | assert(N->getValueType(0).isVector() && "Expected a vector!" ); |
306 | |
307 | unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits(); |
308 | if (EltSize <= NewEltSize) |
309 | return false; |
310 | |
311 | if (N->getOpcode() == ISD::ZERO_EXTEND) { |
312 | return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <= |
313 | NewEltSize) && |
314 | !Signed; |
315 | } |
316 | if (N->getOpcode() == ISD::SIGN_EXTEND) { |
317 | return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <= |
318 | NewEltSize) && |
319 | Signed; |
320 | } |
321 | if (N->getOpcode() != ISD::BUILD_VECTOR) |
322 | return false; |
323 | |
324 | for (const SDValue &Op : N->op_values()) { |
325 | if (Op.isUndef()) |
326 | continue; |
327 | if (!isa<ConstantSDNode>(Val: Op)) |
328 | return false; |
329 | |
330 | APInt C = Op->getAsAPIntVal().trunc(width: EltSize); |
331 | if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C) |
332 | return false; |
333 | if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C) |
334 | return false; |
335 | } |
336 | |
337 | return true; |
338 | } |
339 | |
340 | bool ISD::allOperandsUndef(const SDNode *N) { |
341 | // Return false if the node has no operands. |
342 | // This is "logically inconsistent" with the definition of "all" but |
343 | // is probably the desired behavior. |
344 | if (N->getNumOperands() == 0) |
345 | return false; |
346 | return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); }); |
347 | } |
348 | |
349 | bool ISD::isFreezeUndef(const SDNode *N) { |
350 | return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef(); |
351 | } |
352 | |
353 | template <typename ConstNodeType> |
354 | bool ISD::matchUnaryPredicateImpl(SDValue Op, |
355 | std::function<bool(ConstNodeType *)> Match, |
356 | bool AllowUndefs) { |
357 | // FIXME: Add support for scalar UNDEF cases? |
358 | if (auto *C = dyn_cast<ConstNodeType>(Op)) |
359 | return Match(C); |
360 | |
361 | // FIXME: Add support for vector UNDEF cases? |
362 | if (ISD::BUILD_VECTOR != Op.getOpcode() && |
363 | ISD::SPLAT_VECTOR != Op.getOpcode()) |
364 | return false; |
365 | |
366 | EVT SVT = Op.getValueType().getScalarType(); |
367 | for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
368 | if (AllowUndefs && Op.getOperand(i).isUndef()) { |
369 | if (!Match(nullptr)) |
370 | return false; |
371 | continue; |
372 | } |
373 | |
374 | auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i)); |
375 | if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) |
376 | return false; |
377 | } |
378 | return true; |
379 | } |
380 | // Build used template types. |
381 | template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>( |
382 | SDValue, std::function<bool(ConstantSDNode *)>, bool); |
383 | template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>( |
384 | SDValue, std::function<bool(ConstantFPSDNode *)>, bool); |
385 | |
386 | bool ISD::matchBinaryPredicate( |
387 | SDValue LHS, SDValue RHS, |
388 | std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match, |
389 | bool AllowUndefs, bool AllowTypeMismatch) { |
390 | if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) |
391 | return false; |
392 | |
393 | // TODO: Add support for scalar UNDEF cases? |
394 | if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS)) |
395 | if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS)) |
396 | return Match(LHSCst, RHSCst); |
397 | |
398 | // TODO: Add support for vector UNDEF cases? |
399 | if (LHS.getOpcode() != RHS.getOpcode() || |
400 | (LHS.getOpcode() != ISD::BUILD_VECTOR && |
401 | LHS.getOpcode() != ISD::SPLAT_VECTOR)) |
402 | return false; |
403 | |
404 | EVT SVT = LHS.getValueType().getScalarType(); |
405 | for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { |
406 | SDValue LHSOp = LHS.getOperand(i); |
407 | SDValue RHSOp = RHS.getOperand(i); |
408 | bool LHSUndef = AllowUndefs && LHSOp.isUndef(); |
409 | bool RHSUndef = AllowUndefs && RHSOp.isUndef(); |
410 | auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp); |
411 | auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp); |
412 | if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) |
413 | return false; |
414 | if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || |
415 | LHSOp.getValueType() != RHSOp.getValueType())) |
416 | return false; |
417 | if (!Match(LHSCst, RHSCst)) |
418 | return false; |
419 | } |
420 | return true; |
421 | } |
422 | |
423 | ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { |
424 | switch (VecReduceOpcode) { |
425 | default: |
426 | llvm_unreachable("Expected VECREDUCE opcode" ); |
427 | case ISD::VECREDUCE_FADD: |
428 | case ISD::VECREDUCE_SEQ_FADD: |
429 | case ISD::VP_REDUCE_FADD: |
430 | case ISD::VP_REDUCE_SEQ_FADD: |
431 | return ISD::FADD; |
432 | case ISD::VECREDUCE_FMUL: |
433 | case ISD::VECREDUCE_SEQ_FMUL: |
434 | case ISD::VP_REDUCE_FMUL: |
435 | case ISD::VP_REDUCE_SEQ_FMUL: |
436 | return ISD::FMUL; |
437 | case ISD::VECREDUCE_ADD: |
438 | case ISD::VP_REDUCE_ADD: |
439 | return ISD::ADD; |
440 | case ISD::VECREDUCE_MUL: |
441 | case ISD::VP_REDUCE_MUL: |
442 | return ISD::MUL; |
443 | case ISD::VECREDUCE_AND: |
444 | case ISD::VP_REDUCE_AND: |
445 | return ISD::AND; |
446 | case ISD::VECREDUCE_OR: |
447 | case ISD::VP_REDUCE_OR: |
448 | return ISD::OR; |
449 | case ISD::VECREDUCE_XOR: |
450 | case ISD::VP_REDUCE_XOR: |
451 | return ISD::XOR; |
452 | case ISD::VECREDUCE_SMAX: |
453 | case ISD::VP_REDUCE_SMAX: |
454 | return ISD::SMAX; |
455 | case ISD::VECREDUCE_SMIN: |
456 | case ISD::VP_REDUCE_SMIN: |
457 | return ISD::SMIN; |
458 | case ISD::VECREDUCE_UMAX: |
459 | case ISD::VP_REDUCE_UMAX: |
460 | return ISD::UMAX; |
461 | case ISD::VECREDUCE_UMIN: |
462 | case ISD::VP_REDUCE_UMIN: |
463 | return ISD::UMIN; |
464 | case ISD::VECREDUCE_FMAX: |
465 | case ISD::VP_REDUCE_FMAX: |
466 | return ISD::FMAXNUM; |
467 | case ISD::VECREDUCE_FMIN: |
468 | case ISD::VP_REDUCE_FMIN: |
469 | return ISD::FMINNUM; |
470 | case ISD::VECREDUCE_FMAXIMUM: |
471 | return ISD::FMAXIMUM; |
472 | case ISD::VECREDUCE_FMINIMUM: |
473 | return ISD::FMINIMUM; |
474 | } |
475 | } |
476 | |
477 | bool ISD::isVPOpcode(unsigned Opcode) { |
478 | switch (Opcode) { |
479 | default: |
480 | return false; |
481 | #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \ |
482 | case ISD::VPSD: \ |
483 | return true; |
484 | #include "llvm/IR/VPIntrinsics.def" |
485 | } |
486 | } |
487 | |
488 | bool ISD::isVPBinaryOp(unsigned Opcode) { |
489 | switch (Opcode) { |
490 | default: |
491 | break; |
492 | #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
493 | #define VP_PROPERTY_BINARYOP return true; |
494 | #define END_REGISTER_VP_SDNODE(VPSD) break; |
495 | #include "llvm/IR/VPIntrinsics.def" |
496 | } |
497 | return false; |
498 | } |
499 | |
500 | bool ISD::isVPReduction(unsigned Opcode) { |
501 | switch (Opcode) { |
502 | default: |
503 | break; |
504 | #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
505 | #define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true; |
506 | #define END_REGISTER_VP_SDNODE(VPSD) break; |
507 | #include "llvm/IR/VPIntrinsics.def" |
508 | } |
509 | return false; |
510 | } |
511 | |
512 | /// The operand position of the vector mask. |
513 | std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { |
514 | switch (Opcode) { |
515 | default: |
516 | return std::nullopt; |
517 | #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \ |
518 | case ISD::VPSD: \ |
519 | return MASKPOS; |
520 | #include "llvm/IR/VPIntrinsics.def" |
521 | } |
522 | } |
523 | |
524 | /// The operand position of the explicit vector length parameter. |
525 | std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { |
526 | switch (Opcode) { |
527 | default: |
528 | return std::nullopt; |
529 | #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ |
530 | case ISD::VPSD: \ |
531 | return EVLPOS; |
532 | #include "llvm/IR/VPIntrinsics.def" |
533 | } |
534 | } |
535 | |
536 | std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode, |
537 | bool hasFPExcept) { |
538 | // FIXME: Return strict opcodes in case of fp exceptions. |
539 | switch (VPOpcode) { |
540 | default: |
541 | return std::nullopt; |
542 | #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC: |
543 | #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC; |
544 | #define END_REGISTER_VP_SDNODE(VPOPC) break; |
545 | #include "llvm/IR/VPIntrinsics.def" |
546 | } |
547 | return std::nullopt; |
548 | } |
549 | |
550 | unsigned ISD::getVPForBaseOpcode(unsigned Opcode) { |
551 | switch (Opcode) { |
552 | default: |
553 | llvm_unreachable("can not translate this Opcode to VP." ); |
554 | #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break; |
555 | #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC: |
556 | #define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC; |
557 | #include "llvm/IR/VPIntrinsics.def" |
558 | } |
559 | } |
560 | |
561 | ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { |
562 | switch (ExtType) { |
563 | case ISD::EXTLOAD: |
564 | return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; |
565 | case ISD::SEXTLOAD: |
566 | return ISD::SIGN_EXTEND; |
567 | case ISD::ZEXTLOAD: |
568 | return ISD::ZERO_EXTEND; |
569 | default: |
570 | break; |
571 | } |
572 | |
573 | llvm_unreachable("Invalid LoadExtType" ); |
574 | } |
575 | |
576 | ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { |
577 | // To perform this operation, we just need to swap the L and G bits of the |
578 | // operation. |
579 | unsigned OldL = (Operation >> 2) & 1; |
580 | unsigned OldG = (Operation >> 1) & 1; |
581 | return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits |
582 | (OldL << 1) | // New G bit |
583 | (OldG << 2)); // New L bit. |
584 | } |
585 | |
586 | static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { |
587 | unsigned Operation = Op; |
588 | if (isIntegerLike) |
589 | Operation ^= 7; // Flip L, G, E bits, but not U. |
590 | else |
591 | Operation ^= 15; // Flip all of the condition bits. |
592 | |
593 | if (Operation > ISD::SETTRUE2) |
594 | Operation &= ~8; // Don't let N and U bits get set. |
595 | |
596 | return ISD::CondCode(Operation); |
597 | } |
598 | |
599 | ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { |
600 | return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger()); |
601 | } |
602 | |
603 | ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, |
604 | bool isIntegerLike) { |
605 | return getSetCCInverseImpl(Op, isIntegerLike); |
606 | } |
607 | |
608 | /// For an integer comparison, return 1 if the comparison is a signed operation |
609 | /// and 2 if the result is an unsigned comparison. Return zero if the operation |
610 | /// does not depend on the sign of the input (setne and seteq). |
611 | static int isSignedOp(ISD::CondCode Opcode) { |
612 | switch (Opcode) { |
613 | default: llvm_unreachable("Illegal integer setcc operation!" ); |
614 | case ISD::SETEQ: |
615 | case ISD::SETNE: return 0; |
616 | case ISD::SETLT: |
617 | case ISD::SETLE: |
618 | case ISD::SETGT: |
619 | case ISD::SETGE: return 1; |
620 | case ISD::SETULT: |
621 | case ISD::SETULE: |
622 | case ISD::SETUGT: |
623 | case ISD::SETUGE: return 2; |
624 | } |
625 | } |
626 | |
627 | ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
628 | EVT Type) { |
629 | bool IsInteger = Type.isInteger(); |
630 | if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3) |
631 | // Cannot fold a signed integer setcc with an unsigned integer setcc. |
632 | return ISD::SETCC_INVALID; |
633 | |
634 | unsigned Op = Op1 | Op2; // Combine all of the condition bits. |
635 | |
636 | // If the N and U bits get set, then the resultant comparison DOES suddenly |
637 | // care about orderedness, and it is true when ordered. |
638 | if (Op > ISD::SETTRUE2) |
639 | Op &= ~16; // Clear the U bit if the N bit is set. |
640 | |
641 | // Canonicalize illegal integer setcc's. |
642 | if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT |
643 | Op = ISD::SETNE; |
644 | |
645 | return ISD::CondCode(Op); |
646 | } |
647 | |
648 | ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
649 | EVT Type) { |
650 | bool IsInteger = Type.isInteger(); |
651 | if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3) |
652 | // Cannot fold a signed setcc with an unsigned setcc. |
653 | return ISD::SETCC_INVALID; |
654 | |
655 | // Combine all of the condition bits. |
656 | ISD::CondCode Result = ISD::CondCode(Op1 & Op2); |
657 | |
658 | // Canonicalize illegal integer setcc's. |
659 | if (IsInteger) { |
660 | switch (Result) { |
661 | default: break; |
662 | case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT |
663 | case ISD::SETOEQ: // SETEQ & SETU[LG]E |
664 | case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE |
665 | case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE |
666 | case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE |
667 | } |
668 | } |
669 | |
670 | return Result; |
671 | } |
672 | |
673 | //===----------------------------------------------------------------------===// |
674 | // SDNode Profile Support |
675 | //===----------------------------------------------------------------------===// |
676 | |
677 | /// AddNodeIDOpcode - Add the node opcode to the NodeID data. |
678 | static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { |
679 | ID.AddInteger(I: OpC); |
680 | } |
681 | |
682 | /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them |
683 | /// solely with their pointer. |
684 | static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { |
685 | ID.AddPointer(Ptr: VTList.VTs); |
686 | } |
687 | |
688 | /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
689 | static void AddNodeIDOperands(FoldingSetNodeID &ID, |
690 | ArrayRef<SDValue> Ops) { |
691 | for (const auto &Op : Ops) { |
692 | ID.AddPointer(Ptr: Op.getNode()); |
693 | ID.AddInteger(I: Op.getResNo()); |
694 | } |
695 | } |
696 | |
697 | /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
698 | static void AddNodeIDOperands(FoldingSetNodeID &ID, |
699 | ArrayRef<SDUse> Ops) { |
700 | for (const auto &Op : Ops) { |
701 | ID.AddPointer(Ptr: Op.getNode()); |
702 | ID.AddInteger(I: Op.getResNo()); |
703 | } |
704 | } |
705 | |
706 | static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, |
707 | SDVTList VTList, ArrayRef<SDValue> OpList) { |
708 | AddNodeIDOpcode(ID, OpC); |
709 | AddNodeIDValueTypes(ID, VTList); |
710 | AddNodeIDOperands(ID, Ops: OpList); |
711 | } |
712 | |
713 | /// If this is an SDNode with special info, add this info to the NodeID data. |
714 | static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { |
715 | switch (N->getOpcode()) { |
716 | case ISD::TargetExternalSymbol: |
717 | case ISD::ExternalSymbol: |
718 | case ISD::MCSymbol: |
719 | llvm_unreachable("Should only be used on nodes with operands" ); |
720 | default: break; // Normal nodes don't need extra info. |
721 | case ISD::TargetConstant: |
722 | case ISD::Constant: { |
723 | const ConstantSDNode *C = cast<ConstantSDNode>(Val: N); |
724 | ID.AddPointer(Ptr: C->getConstantIntValue()); |
725 | ID.AddBoolean(B: C->isOpaque()); |
726 | break; |
727 | } |
728 | case ISD::TargetConstantFP: |
729 | case ISD::ConstantFP: |
730 | ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue()); |
731 | break; |
732 | case ISD::TargetGlobalAddress: |
733 | case ISD::GlobalAddress: |
734 | case ISD::TargetGlobalTLSAddress: |
735 | case ISD::GlobalTLSAddress: { |
736 | const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N); |
737 | ID.AddPointer(Ptr: GA->getGlobal()); |
738 | ID.AddInteger(I: GA->getOffset()); |
739 | ID.AddInteger(I: GA->getTargetFlags()); |
740 | break; |
741 | } |
742 | case ISD::BasicBlock: |
743 | ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock()); |
744 | break; |
745 | case ISD::Register: |
746 | ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg()); |
747 | break; |
748 | case ISD::RegisterMask: |
749 | ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask()); |
750 | break; |
751 | case ISD::SRCVALUE: |
752 | ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue()); |
753 | break; |
754 | case ISD::FrameIndex: |
755 | case ISD::TargetFrameIndex: |
756 | ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex()); |
757 | break; |
758 | case ISD::LIFETIME_START: |
759 | case ISD::LIFETIME_END: |
760 | if (cast<LifetimeSDNode>(Val: N)->hasOffset()) { |
761 | ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getSize()); |
762 | ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getOffset()); |
763 | } |
764 | break; |
765 | case ISD::PSEUDO_PROBE: |
766 | ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid()); |
767 | ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex()); |
768 | ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes()); |
769 | break; |
770 | case ISD::JumpTable: |
771 | case ISD::TargetJumpTable: |
772 | ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex()); |
773 | ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags()); |
774 | break; |
775 | case ISD::ConstantPool: |
776 | case ISD::TargetConstantPool: { |
777 | const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N); |
778 | ID.AddInteger(I: CP->getAlign().value()); |
779 | ID.AddInteger(I: CP->getOffset()); |
780 | if (CP->isMachineConstantPoolEntry()) |
781 | CP->getMachineCPVal()->addSelectionDAGCSEId(ID); |
782 | else |
783 | ID.AddPointer(Ptr: CP->getConstVal()); |
784 | ID.AddInteger(I: CP->getTargetFlags()); |
785 | break; |
786 | } |
787 | case ISD::TargetIndex: { |
788 | const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N); |
789 | ID.AddInteger(I: TI->getIndex()); |
790 | ID.AddInteger(I: TI->getOffset()); |
791 | ID.AddInteger(I: TI->getTargetFlags()); |
792 | break; |
793 | } |
794 | case ISD::LOAD: { |
795 | const LoadSDNode *LD = cast<LoadSDNode>(Val: N); |
796 | ID.AddInteger(I: LD->getMemoryVT().getRawBits()); |
797 | ID.AddInteger(I: LD->getRawSubclassData()); |
798 | ID.AddInteger(I: LD->getPointerInfo().getAddrSpace()); |
799 | ID.AddInteger(I: LD->getMemOperand()->getFlags()); |
800 | break; |
801 | } |
802 | case ISD::STORE: { |
803 | const StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
804 | ID.AddInteger(I: ST->getMemoryVT().getRawBits()); |
805 | ID.AddInteger(I: ST->getRawSubclassData()); |
806 | ID.AddInteger(I: ST->getPointerInfo().getAddrSpace()); |
807 | ID.AddInteger(I: ST->getMemOperand()->getFlags()); |
808 | break; |
809 | } |
810 | case ISD::VP_LOAD: { |
811 | const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N); |
812 | ID.AddInteger(I: ELD->getMemoryVT().getRawBits()); |
813 | ID.AddInteger(I: ELD->getRawSubclassData()); |
814 | ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace()); |
815 | ID.AddInteger(I: ELD->getMemOperand()->getFlags()); |
816 | break; |
817 | } |
818 | case ISD::VP_STORE: { |
819 | const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N); |
820 | ID.AddInteger(I: EST->getMemoryVT().getRawBits()); |
821 | ID.AddInteger(I: EST->getRawSubclassData()); |
822 | ID.AddInteger(I: EST->getPointerInfo().getAddrSpace()); |
823 | ID.AddInteger(I: EST->getMemOperand()->getFlags()); |
824 | break; |
825 | } |
826 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: { |
827 | const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N); |
828 | ID.AddInteger(I: SLD->getMemoryVT().getRawBits()); |
829 | ID.AddInteger(I: SLD->getRawSubclassData()); |
830 | ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace()); |
831 | break; |
832 | } |
833 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: { |
834 | const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N); |
835 | ID.AddInteger(I: SST->getMemoryVT().getRawBits()); |
836 | ID.AddInteger(I: SST->getRawSubclassData()); |
837 | ID.AddInteger(I: SST->getPointerInfo().getAddrSpace()); |
838 | break; |
839 | } |
840 | case ISD::VP_GATHER: { |
841 | const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N); |
842 | ID.AddInteger(I: EG->getMemoryVT().getRawBits()); |
843 | ID.AddInteger(I: EG->getRawSubclassData()); |
844 | ID.AddInteger(I: EG->getPointerInfo().getAddrSpace()); |
845 | ID.AddInteger(I: EG->getMemOperand()->getFlags()); |
846 | break; |
847 | } |
848 | case ISD::VP_SCATTER: { |
849 | const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N); |
850 | ID.AddInteger(I: ES->getMemoryVT().getRawBits()); |
851 | ID.AddInteger(I: ES->getRawSubclassData()); |
852 | ID.AddInteger(I: ES->getPointerInfo().getAddrSpace()); |
853 | ID.AddInteger(I: ES->getMemOperand()->getFlags()); |
854 | break; |
855 | } |
856 | case ISD::MLOAD: { |
857 | const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N); |
858 | ID.AddInteger(I: MLD->getMemoryVT().getRawBits()); |
859 | ID.AddInteger(I: MLD->getRawSubclassData()); |
860 | ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace()); |
861 | ID.AddInteger(I: MLD->getMemOperand()->getFlags()); |
862 | break; |
863 | } |
864 | case ISD::MSTORE: { |
865 | const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N); |
866 | ID.AddInteger(I: MST->getMemoryVT().getRawBits()); |
867 | ID.AddInteger(I: MST->getRawSubclassData()); |
868 | ID.AddInteger(I: MST->getPointerInfo().getAddrSpace()); |
869 | ID.AddInteger(I: MST->getMemOperand()->getFlags()); |
870 | break; |
871 | } |
872 | case ISD::MGATHER: { |
873 | const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N); |
874 | ID.AddInteger(I: MG->getMemoryVT().getRawBits()); |
875 | ID.AddInteger(I: MG->getRawSubclassData()); |
876 | ID.AddInteger(I: MG->getPointerInfo().getAddrSpace()); |
877 | ID.AddInteger(I: MG->getMemOperand()->getFlags()); |
878 | break; |
879 | } |
880 | case ISD::MSCATTER: { |
881 | const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N); |
882 | ID.AddInteger(I: MS->getMemoryVT().getRawBits()); |
883 | ID.AddInteger(I: MS->getRawSubclassData()); |
884 | ID.AddInteger(I: MS->getPointerInfo().getAddrSpace()); |
885 | ID.AddInteger(I: MS->getMemOperand()->getFlags()); |
886 | break; |
887 | } |
888 | case ISD::ATOMIC_CMP_SWAP: |
889 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
890 | case ISD::ATOMIC_SWAP: |
891 | case ISD::ATOMIC_LOAD_ADD: |
892 | case ISD::ATOMIC_LOAD_SUB: |
893 | case ISD::ATOMIC_LOAD_AND: |
894 | case ISD::ATOMIC_LOAD_CLR: |
895 | case ISD::ATOMIC_LOAD_OR: |
896 | case ISD::ATOMIC_LOAD_XOR: |
897 | case ISD::ATOMIC_LOAD_NAND: |
898 | case ISD::ATOMIC_LOAD_MIN: |
899 | case ISD::ATOMIC_LOAD_MAX: |
900 | case ISD::ATOMIC_LOAD_UMIN: |
901 | case ISD::ATOMIC_LOAD_UMAX: |
902 | case ISD::ATOMIC_LOAD: |
903 | case ISD::ATOMIC_STORE: { |
904 | const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N); |
905 | ID.AddInteger(I: AT->getMemoryVT().getRawBits()); |
906 | ID.AddInteger(I: AT->getRawSubclassData()); |
907 | ID.AddInteger(I: AT->getPointerInfo().getAddrSpace()); |
908 | ID.AddInteger(I: AT->getMemOperand()->getFlags()); |
909 | break; |
910 | } |
911 | case ISD::VECTOR_SHUFFLE: { |
912 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask(); |
913 | for (int M : Mask) |
914 | ID.AddInteger(I: M); |
915 | break; |
916 | } |
917 | case ISD::TargetBlockAddress: |
918 | case ISD::BlockAddress: { |
919 | const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N); |
920 | ID.AddPointer(Ptr: BA->getBlockAddress()); |
921 | ID.AddInteger(I: BA->getOffset()); |
922 | ID.AddInteger(I: BA->getTargetFlags()); |
923 | break; |
924 | } |
925 | case ISD::AssertAlign: |
926 | ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value()); |
927 | break; |
928 | case ISD::PREFETCH: |
929 | case ISD::INTRINSIC_VOID: |
930 | case ISD::INTRINSIC_W_CHAIN: |
931 | // Handled by MemIntrinsicSDNode check after the switch. |
932 | break; |
933 | } // end switch (N->getOpcode()) |
934 | |
935 | // MemIntrinsic nodes could also have subclass data, address spaces, and flags |
936 | // to check. |
937 | if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) { |
938 | ID.AddInteger(I: MN->getRawSubclassData()); |
939 | ID.AddInteger(I: MN->getPointerInfo().getAddrSpace()); |
940 | ID.AddInteger(I: MN->getMemOperand()->getFlags()); |
941 | ID.AddInteger(I: MN->getMemoryVT().getRawBits()); |
942 | } |
943 | } |
944 | |
945 | /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID |
946 | /// data. |
947 | static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { |
948 | AddNodeIDOpcode(ID, OpC: N->getOpcode()); |
949 | // Add the return value info. |
950 | AddNodeIDValueTypes(ID, VTList: N->getVTList()); |
951 | // Add the operand info. |
952 | AddNodeIDOperands(ID, Ops: N->ops()); |
953 | |
954 | // Handle SDNode leafs with special info. |
955 | AddNodeIDCustom(ID, N); |
956 | } |
957 | |
958 | //===----------------------------------------------------------------------===// |
959 | // SelectionDAG Class |
960 | //===----------------------------------------------------------------------===// |
961 | |
962 | /// doNotCSE - Return true if CSE should not be performed for this node. |
963 | static bool doNotCSE(SDNode *N) { |
964 | if (N->getValueType(ResNo: 0) == MVT::Glue) |
965 | return true; // Never CSE anything that produces a glue result. |
966 | |
967 | switch (N->getOpcode()) { |
968 | default: break; |
969 | case ISD::HANDLENODE: |
970 | case ISD::EH_LABEL: |
971 | return true; // Never CSE these nodes. |
972 | } |
973 | |
974 | // Check that remaining values produced are not flags. |
975 | for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) |
976 | if (N->getValueType(ResNo: i) == MVT::Glue) |
977 | return true; // Never CSE anything that produces a glue result. |
978 | |
979 | return false; |
980 | } |
981 | |
982 | /// RemoveDeadNodes - This method deletes all unreachable nodes in the |
983 | /// SelectionDAG. |
984 | void SelectionDAG::RemoveDeadNodes() { |
985 | // Create a dummy node (which is not added to allnodes), that adds a reference |
986 | // to the root node, preventing it from being deleted. |
987 | HandleSDNode Dummy(getRoot()); |
988 | |
989 | SmallVector<SDNode*, 128> DeadNodes; |
990 | |
991 | // Add all obviously-dead nodes to the DeadNodes worklist. |
992 | for (SDNode &Node : allnodes()) |
993 | if (Node.use_empty()) |
994 | DeadNodes.push_back(Elt: &Node); |
995 | |
996 | RemoveDeadNodes(DeadNodes); |
997 | |
998 | // If the root changed (e.g. it was a dead load, update the root). |
999 | setRoot(Dummy.getValue()); |
1000 | } |
1001 | |
1002 | /// RemoveDeadNodes - This method deletes the unreachable nodes in the |
1003 | /// given list, and any nodes that become unreachable as a result. |
1004 | void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { |
1005 | |
1006 | // Process the worklist, deleting the nodes and adding their uses to the |
1007 | // worklist. |
1008 | while (!DeadNodes.empty()) { |
1009 | SDNode *N = DeadNodes.pop_back_val(); |
1010 | // Skip to next node if we've already managed to delete the node. This could |
1011 | // happen if replacing a node causes a node previously added to the node to |
1012 | // be deleted. |
1013 | if (N->getOpcode() == ISD::DELETED_NODE) |
1014 | continue; |
1015 | |
1016 | for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
1017 | DUL->NodeDeleted(N, nullptr); |
1018 | |
1019 | // Take the node out of the appropriate CSE map. |
1020 | RemoveNodeFromCSEMaps(N); |
1021 | |
1022 | // Next, brutally remove the operand list. This is safe to do, as there are |
1023 | // no cycles in the graph. |
1024 | for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
1025 | SDUse &Use = *I++; |
1026 | SDNode *Operand = Use.getNode(); |
1027 | Use.set(SDValue()); |
1028 | |
1029 | // Now that we removed this operand, see if there are no uses of it left. |
1030 | if (Operand->use_empty()) |
1031 | DeadNodes.push_back(Elt: Operand); |
1032 | } |
1033 | |
1034 | DeallocateNode(N); |
1035 | } |
1036 | } |
1037 | |
1038 | void SelectionDAG::RemoveDeadNode(SDNode *N){ |
1039 | SmallVector<SDNode*, 16> DeadNodes(1, N); |
1040 | |
1041 | // Create a dummy node that adds a reference to the root node, preventing |
1042 | // it from being deleted. (This matters if the root is an operand of the |
1043 | // dead node.) |
1044 | HandleSDNode Dummy(getRoot()); |
1045 | |
1046 | RemoveDeadNodes(DeadNodes); |
1047 | } |
1048 | |
1049 | void SelectionDAG::DeleteNode(SDNode *N) { |
1050 | // First take this out of the appropriate CSE map. |
1051 | RemoveNodeFromCSEMaps(N); |
1052 | |
1053 | // Finally, remove uses due to operands of this node, remove from the |
1054 | // AllNodes list, and delete the node. |
1055 | DeleteNodeNotInCSEMaps(N); |
1056 | } |
1057 | |
1058 | void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { |
1059 | assert(N->getIterator() != AllNodes.begin() && |
1060 | "Cannot delete the entry node!" ); |
1061 | assert(N->use_empty() && "Cannot delete a node that is not dead!" ); |
1062 | |
1063 | // Drop all of the operands and decrement used node's use counts. |
1064 | N->DropOperands(); |
1065 | |
1066 | DeallocateNode(N); |
1067 | } |
1068 | |
1069 | void SDDbgInfo::add(SDDbgValue *V, bool isParameter) { |
1070 | assert(!(V->isVariadic() && isParameter)); |
1071 | if (isParameter) |
1072 | ByvalParmDbgValues.push_back(Elt: V); |
1073 | else |
1074 | DbgValues.push_back(Elt: V); |
1075 | for (const SDNode *Node : V->getSDNodes()) |
1076 | if (Node) |
1077 | DbgValMap[Node].push_back(Elt: V); |
1078 | } |
1079 | |
1080 | void SDDbgInfo::erase(const SDNode *Node) { |
1081 | DbgValMapType::iterator I = DbgValMap.find(Val: Node); |
1082 | if (I == DbgValMap.end()) |
1083 | return; |
1084 | for (auto &Val: I->second) |
1085 | Val->setIsInvalidated(); |
1086 | DbgValMap.erase(I); |
1087 | } |
1088 | |
1089 | void SelectionDAG::DeallocateNode(SDNode *N) { |
1090 | // If we have operands, deallocate them. |
1091 | removeOperands(Node: N); |
1092 | |
1093 | NodeAllocator.Deallocate(E: AllNodes.remove(IT: N)); |
1094 | |
1095 | // Set the opcode to DELETED_NODE to help catch bugs when node |
1096 | // memory is reallocated. |
1097 | // FIXME: There are places in SDag that have grown a dependency on the opcode |
1098 | // value in the released node. |
1099 | __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); |
1100 | N->NodeType = ISD::DELETED_NODE; |
1101 | |
1102 | // If any of the SDDbgValue nodes refer to this SDNode, invalidate |
1103 | // them and forget about that node. |
1104 | DbgInfo->erase(Node: N); |
1105 | |
1106 | // Invalidate extra info. |
1107 | SDEI.erase(Val: N); |
1108 | } |
1109 | |
1110 | #ifndef NDEBUG |
1111 | /// VerifySDNode - Check the given SDNode. Aborts if it is invalid. |
1112 | static void VerifySDNode(SDNode *N) { |
1113 | switch (N->getOpcode()) { |
1114 | default: |
1115 | break; |
1116 | case ISD::BUILD_PAIR: { |
1117 | EVT VT = N->getValueType(ResNo: 0); |
1118 | assert(N->getNumValues() == 1 && "Too many results!" ); |
1119 | assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && |
1120 | "Wrong return type!" ); |
1121 | assert(N->getNumOperands() == 2 && "Wrong number of operands!" ); |
1122 | assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && |
1123 | "Mismatched operand types!" ); |
1124 | assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && |
1125 | "Wrong operand type!" ); |
1126 | assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && |
1127 | "Wrong return type size" ); |
1128 | break; |
1129 | } |
1130 | case ISD::BUILD_VECTOR: { |
1131 | assert(N->getNumValues() == 1 && "Too many results!" ); |
1132 | assert(N->getValueType(0).isVector() && "Wrong return type!" ); |
1133 | assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && |
1134 | "Wrong number of operands!" ); |
1135 | EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
1136 | for (const SDUse &Op : N->ops()) { |
1137 | assert((Op.getValueType() == EltVT || |
1138 | (EltVT.isInteger() && Op.getValueType().isInteger() && |
1139 | EltVT.bitsLE(Op.getValueType()))) && |
1140 | "Wrong operand type!" ); |
1141 | assert(Op.getValueType() == N->getOperand(0).getValueType() && |
1142 | "Operands must all have the same type" ); |
1143 | } |
1144 | break; |
1145 | } |
1146 | } |
1147 | } |
1148 | #endif // NDEBUG |
1149 | |
1150 | /// Insert a newly allocated node into the DAG. |
1151 | /// |
1152 | /// Handles insertion into the all nodes list and CSE map, as well as |
1153 | /// verification and other common operations when a new node is allocated. |
1154 | void SelectionDAG::InsertNode(SDNode *N) { |
1155 | AllNodes.push_back(val: N); |
1156 | #ifndef NDEBUG |
1157 | N->PersistentId = NextPersistentId++; |
1158 | VerifySDNode(N); |
1159 | #endif |
1160 | for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
1161 | DUL->NodeInserted(N); |
1162 | } |
1163 | |
1164 | /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that |
1165 | /// correspond to it. This is useful when we're about to delete or repurpose |
1166 | /// the node. We don't want future request for structurally identical nodes |
1167 | /// to return N anymore. |
1168 | bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { |
1169 | bool Erased = false; |
1170 | switch (N->getOpcode()) { |
1171 | case ISD::HANDLENODE: return false; // noop. |
1172 | case ISD::CONDCODE: |
1173 | assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && |
1174 | "Cond code doesn't exist!" ); |
1175 | Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr; |
1176 | CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr; |
1177 | break; |
1178 | case ISD::ExternalSymbol: |
1179 | Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol()); |
1180 | break; |
1181 | case ISD::TargetExternalSymbol: { |
1182 | ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N); |
1183 | Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>( |
1184 | ESN->getSymbol(), ESN->getTargetFlags())); |
1185 | break; |
1186 | } |
1187 | case ISD::MCSymbol: { |
1188 | auto *MCSN = cast<MCSymbolSDNode>(Val: N); |
1189 | Erased = MCSymbols.erase(Val: MCSN->getMCSymbol()); |
1190 | break; |
1191 | } |
1192 | case ISD::VALUETYPE: { |
1193 | EVT VT = cast<VTSDNode>(Val: N)->getVT(); |
1194 | if (VT.isExtended()) { |
1195 | Erased = ExtendedValueTypeNodes.erase(x: VT); |
1196 | } else { |
1197 | Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; |
1198 | ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; |
1199 | } |
1200 | break; |
1201 | } |
1202 | default: |
1203 | // Remove it from the CSE Map. |
1204 | assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!" ); |
1205 | assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!" ); |
1206 | Erased = CSEMap.RemoveNode(N); |
1207 | break; |
1208 | } |
1209 | #ifndef NDEBUG |
1210 | // Verify that the node was actually in one of the CSE maps, unless it has a |
1211 | // glue result (which cannot be CSE'd) or is one of the special cases that are |
1212 | // not subject to CSE. |
1213 | if (!Erased && N->getValueType(ResNo: N->getNumValues()-1) != MVT::Glue && |
1214 | !N->isMachineOpcode() && !doNotCSE(N)) { |
1215 | N->dump(G: this); |
1216 | dbgs() << "\n" ; |
1217 | llvm_unreachable("Node is not in map!" ); |
1218 | } |
1219 | #endif |
1220 | return Erased; |
1221 | } |
1222 | |
1223 | /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE |
1224 | /// maps and modified in place. Add it back to the CSE maps, unless an identical |
1225 | /// node already exists, in which case transfer all its users to the existing |
1226 | /// node. This transfer can potentially trigger recursive merging. |
1227 | void |
1228 | SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { |
1229 | // For node types that aren't CSE'd, just act as if no identical node |
1230 | // already exists. |
1231 | if (!doNotCSE(N)) { |
1232 | SDNode *Existing = CSEMap.GetOrInsertNode(N); |
1233 | if (Existing != N) { |
1234 | // If there was already an existing matching node, use ReplaceAllUsesWith |
1235 | // to replace the dead one with the existing one. This can cause |
1236 | // recursive merging of other unrelated nodes down the line. |
1237 | ReplaceAllUsesWith(From: N, To: Existing); |
1238 | |
1239 | // N is now dead. Inform the listeners and delete it. |
1240 | for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
1241 | DUL->NodeDeleted(N, Existing); |
1242 | DeleteNodeNotInCSEMaps(N); |
1243 | return; |
1244 | } |
1245 | } |
1246 | |
1247 | // If the node doesn't already exist, we updated it. Inform listeners. |
1248 | for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
1249 | DUL->NodeUpdated(N); |
1250 | } |
1251 | |
1252 | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
1253 | /// were replaced with those specified. If this node is never memoized, |
1254 | /// return null, otherwise return a pointer to the slot it would take. If a |
1255 | /// node already exists with these operands, the slot will be non-null. |
1256 | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, |
1257 | void *&InsertPos) { |
1258 | if (doNotCSE(N)) |
1259 | return nullptr; |
1260 | |
1261 | SDValue Ops[] = { Op }; |
1262 | FoldingSetNodeID ID; |
1263 | AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops); |
1264 | AddNodeIDCustom(ID, N); |
1265 | SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos); |
1266 | if (Node) |
1267 | Node->intersectFlagsWith(Flags: N->getFlags()); |
1268 | return Node; |
1269 | } |
1270 | |
1271 | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
1272 | /// were replaced with those specified. If this node is never memoized, |
1273 | /// return null, otherwise return a pointer to the slot it would take. If a |
1274 | /// node already exists with these operands, the slot will be non-null. |
1275 | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, |
1276 | SDValue Op1, SDValue Op2, |
1277 | void *&InsertPos) { |
1278 | if (doNotCSE(N)) |
1279 | return nullptr; |
1280 | |
1281 | SDValue Ops[] = { Op1, Op2 }; |
1282 | FoldingSetNodeID ID; |
1283 | AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops); |
1284 | AddNodeIDCustom(ID, N); |
1285 | SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos); |
1286 | if (Node) |
1287 | Node->intersectFlagsWith(Flags: N->getFlags()); |
1288 | return Node; |
1289 | } |
1290 | |
1291 | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
1292 | /// were replaced with those specified. If this node is never memoized, |
1293 | /// return null, otherwise return a pointer to the slot it would take. If a |
1294 | /// node already exists with these operands, the slot will be non-null. |
1295 | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, |
1296 | void *&InsertPos) { |
1297 | if (doNotCSE(N)) |
1298 | return nullptr; |
1299 | |
1300 | FoldingSetNodeID ID; |
1301 | AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops); |
1302 | AddNodeIDCustom(ID, N); |
1303 | SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos); |
1304 | if (Node) |
1305 | Node->intersectFlagsWith(Flags: N->getFlags()); |
1306 | return Node; |
1307 | } |
1308 | |
1309 | Align SelectionDAG::getEVTAlign(EVT VT) const { |
1310 | Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0) |
1311 | : VT.getTypeForEVT(Context&: *getContext()); |
1312 | |
1313 | return getDataLayout().getABITypeAlign(Ty); |
1314 | } |
1315 | |
1316 | // EntryNode could meaningfully have debug info if we can find it... |
1317 | SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL) |
1318 | : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(), |
1319 | getVTList(MVT::Other, MVT::Glue)), |
1320 | Root(getEntryNode()) { |
1321 | InsertNode(N: &EntryNode); |
1322 | DbgInfo = new SDDbgInfo(); |
1323 | } |
1324 | |
1325 | void SelectionDAG::(MachineFunction &NewMF, |
1326 | OptimizationRemarkEmitter &NewORE, Pass *PassPtr, |
1327 | const TargetLibraryInfo *LibraryInfo, |
1328 | UniformityInfo *NewUA, ProfileSummaryInfo *PSIin, |
1329 | BlockFrequencyInfo *BFIin, |
1330 | FunctionVarLocs const *VarLocs) { |
1331 | MF = &NewMF; |
1332 | SDAGISelPass = PassPtr; |
1333 | ORE = &NewORE; |
1334 | TLI = getSubtarget().getTargetLowering(); |
1335 | TSI = getSubtarget().getSelectionDAGInfo(); |
1336 | LibInfo = LibraryInfo; |
1337 | Context = &MF->getFunction().getContext(); |
1338 | UA = NewUA; |
1339 | PSI = PSIin; |
1340 | BFI = BFIin; |
1341 | FnVarLocs = VarLocs; |
1342 | } |
1343 | |
1344 | SelectionDAG::~SelectionDAG() { |
1345 | assert(!UpdateListeners && "Dangling registered DAGUpdateListeners" ); |
1346 | allnodes_clear(); |
1347 | OperandRecycler.clear(OperandAllocator); |
1348 | delete DbgInfo; |
1349 | } |
1350 | |
1351 | bool SelectionDAG::shouldOptForSize() const { |
1352 | return MF->getFunction().hasOptSize() || |
1353 | llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI); |
1354 | } |
1355 | |
1356 | void SelectionDAG::allnodes_clear() { |
1357 | assert(&*AllNodes.begin() == &EntryNode); |
1358 | AllNodes.remove(IT: AllNodes.begin()); |
1359 | while (!AllNodes.empty()) |
1360 | DeallocateNode(N: &AllNodes.front()); |
1361 | #ifndef NDEBUG |
1362 | NextPersistentId = 0; |
1363 | #endif |
1364 | } |
1365 | |
1366 | SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
1367 | void *&InsertPos) { |
1368 | SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
1369 | if (N) { |
1370 | switch (N->getOpcode()) { |
1371 | default: break; |
1372 | case ISD::Constant: |
1373 | case ISD::ConstantFP: |
1374 | llvm_unreachable("Querying for Constant and ConstantFP nodes requires " |
1375 | "debug location. Use another overload." ); |
1376 | } |
1377 | } |
1378 | return N; |
1379 | } |
1380 | |
1381 | SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
1382 | const SDLoc &DL, void *&InsertPos) { |
1383 | SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
1384 | if (N) { |
1385 | switch (N->getOpcode()) { |
1386 | case ISD::Constant: |
1387 | case ISD::ConstantFP: |
1388 | // Erase debug location from the node if the node is used at several |
1389 | // different places. Do not propagate one location to all uses as it |
1390 | // will cause a worse single stepping debugging experience. |
1391 | if (N->getDebugLoc() != DL.getDebugLoc()) |
1392 | N->setDebugLoc(DebugLoc()); |
1393 | break; |
1394 | default: |
1395 | // When the node's point of use is located earlier in the instruction |
1396 | // sequence than its prior point of use, update its debug info to the |
1397 | // earlier location. |
1398 | if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) |
1399 | N->setDebugLoc(DL.getDebugLoc()); |
1400 | break; |
1401 | } |
1402 | } |
1403 | return N; |
1404 | } |
1405 | |
1406 | void SelectionDAG::clear() { |
1407 | allnodes_clear(); |
1408 | OperandRecycler.clear(OperandAllocator); |
1409 | OperandAllocator.Reset(); |
1410 | CSEMap.clear(); |
1411 | |
1412 | ExtendedValueTypeNodes.clear(); |
1413 | ExternalSymbols.clear(); |
1414 | TargetExternalSymbols.clear(); |
1415 | MCSymbols.clear(); |
1416 | SDEI.clear(); |
1417 | std::fill(first: CondCodeNodes.begin(), last: CondCodeNodes.end(), |
1418 | value: static_cast<CondCodeSDNode*>(nullptr)); |
1419 | std::fill(first: ValueTypeNodes.begin(), last: ValueTypeNodes.end(), |
1420 | value: static_cast<SDNode*>(nullptr)); |
1421 | |
1422 | EntryNode.UseList = nullptr; |
1423 | InsertNode(N: &EntryNode); |
1424 | Root = getEntryNode(); |
1425 | DbgInfo->clear(); |
1426 | } |
1427 | |
1428 | SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { |
1429 | return VT.bitsGT(VT: Op.getValueType()) |
1430 | ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op) |
1431 | : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op, |
1432 | N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)); |
1433 | } |
1434 | |
1435 | std::pair<SDValue, SDValue> |
1436 | SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, |
1437 | const SDLoc &DL, EVT VT) { |
1438 | assert(!VT.bitsEq(Op.getValueType()) && |
1439 | "Strict no-op FP extend/round not allowed." ); |
1440 | SDValue Res = |
1441 | VT.bitsGT(Op.getValueType()) |
1442 | ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) |
1443 | : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, |
1444 | {Chain, Op, getIntPtrConstant(0, DL)}); |
1445 | |
1446 | return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1)); |
1447 | } |
1448 | |
1449 | SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1450 | return VT.bitsGT(VT: Op.getValueType()) ? |
1451 | getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) : |
1452 | getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op); |
1453 | } |
1454 | |
1455 | SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1456 | return VT.bitsGT(VT: Op.getValueType()) ? |
1457 | getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) : |
1458 | getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op); |
1459 | } |
1460 | |
1461 | SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1462 | return VT.bitsGT(VT: Op.getValueType()) ? |
1463 | getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) : |
1464 | getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op); |
1465 | } |
1466 | |
1467 | SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, |
1468 | EVT VT) { |
1469 | assert(!VT.isVector()); |
1470 | auto Type = Op.getValueType(); |
1471 | SDValue DestOp; |
1472 | if (Type == VT) |
1473 | return Op; |
1474 | auto Size = Op.getValueSizeInBits(); |
1475 | DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op); |
1476 | if (DestOp.getValueType() == VT) |
1477 | return DestOp; |
1478 | |
1479 | return getAnyExtOrTrunc(Op: DestOp, DL, VT); |
1480 | } |
1481 | |
1482 | SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, |
1483 | EVT VT) { |
1484 | assert(!VT.isVector()); |
1485 | auto Type = Op.getValueType(); |
1486 | SDValue DestOp; |
1487 | if (Type == VT) |
1488 | return Op; |
1489 | auto Size = Op.getValueSizeInBits(); |
1490 | DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op); |
1491 | if (DestOp.getValueType() == VT) |
1492 | return DestOp; |
1493 | |
1494 | return getSExtOrTrunc(Op: DestOp, DL, VT); |
1495 | } |
1496 | |
1497 | SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, |
1498 | EVT VT) { |
1499 | assert(!VT.isVector()); |
1500 | auto Type = Op.getValueType(); |
1501 | SDValue DestOp; |
1502 | if (Type == VT) |
1503 | return Op; |
1504 | auto Size = Op.getValueSizeInBits(); |
1505 | DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op); |
1506 | if (DestOp.getValueType() == VT) |
1507 | return DestOp; |
1508 | |
1509 | return getZExtOrTrunc(Op: DestOp, DL, VT); |
1510 | } |
1511 | |
1512 | SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, |
1513 | EVT OpVT) { |
1514 | if (VT.bitsLE(VT: Op.getValueType())) |
1515 | return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op); |
1516 | |
1517 | TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT); |
1518 | return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op); |
1519 | } |
1520 | |
1521 | SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
1522 | EVT OpVT = Op.getValueType(); |
1523 | assert(VT.isInteger() && OpVT.isInteger() && |
1524 | "Cannot getZeroExtendInReg FP types" ); |
1525 | assert(VT.isVector() == OpVT.isVector() && |
1526 | "getZeroExtendInReg type should be vector iff the operand " |
1527 | "type is vector!" ); |
1528 | assert((!VT.isVector() || |
1529 | VT.getVectorElementCount() == OpVT.getVectorElementCount()) && |
1530 | "Vector element counts must match in getZeroExtendInReg" ); |
1531 | assert(VT.bitsLE(OpVT) && "Not extending!" ); |
1532 | if (OpVT == VT) |
1533 | return Op; |
1534 | APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(), |
1535 | loBitsSet: VT.getScalarSizeInBits()); |
1536 | return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT)); |
1537 | } |
1538 | |
1539 | SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1540 | // Only unsigned pointer semantics are supported right now. In the future this |
1541 | // might delegate to TLI to check pointer signedness. |
1542 | return getZExtOrTrunc(Op, DL, VT); |
1543 | } |
1544 | |
1545 | SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
1546 | // Only unsigned pointer semantics are supported right now. In the future this |
1547 | // might delegate to TLI to check pointer signedness. |
1548 | return getZeroExtendInReg(Op, DL, VT); |
1549 | } |
1550 | |
1551 | SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) { |
1552 | return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val); |
1553 | } |
1554 | |
1555 | /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). |
1556 | SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
1557 | return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT)); |
1558 | } |
1559 | |
1560 | SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
1561 | SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT); |
1562 | return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue); |
1563 | } |
1564 | |
1565 | SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val, |
1566 | SDValue Mask, SDValue EVL, EVT VT) { |
1567 | SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT); |
1568 | return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL); |
1569 | } |
1570 | |
1571 | SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
1572 | SDValue Mask, SDValue EVL) { |
1573 | return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL); |
1574 | } |
1575 | |
1576 | SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
1577 | SDValue Mask, SDValue EVL) { |
1578 | if (VT.bitsGT(VT: Op.getValueType())) |
1579 | return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL); |
1580 | if (VT.bitsLT(VT: Op.getValueType())) |
1581 | return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL); |
1582 | return Op; |
1583 | } |
1584 | |
1585 | SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, |
1586 | EVT OpVT) { |
1587 | if (!V) |
1588 | return getConstant(Val: 0, DL, VT); |
1589 | |
1590 | switch (TLI->getBooleanContents(Type: OpVT)) { |
1591 | case TargetLowering::ZeroOrOneBooleanContent: |
1592 | case TargetLowering::UndefinedBooleanContent: |
1593 | return getConstant(Val: 1, DL, VT); |
1594 | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
1595 | return getAllOnesConstant(DL, VT); |
1596 | } |
1597 | llvm_unreachable("Unexpected boolean content enum!" ); |
1598 | } |
1599 | |
1600 | SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
1601 | bool isT, bool isO) { |
1602 | EVT EltVT = VT.getScalarType(); |
1603 | assert((EltVT.getSizeInBits() >= 64 || |
1604 | (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && |
1605 | "getConstant with a uint64_t value that doesn't fit in the type!" ); |
1606 | return getConstant(Val: APInt(EltVT.getSizeInBits(), Val), DL, VT, isTarget: isT, isOpaque: isO); |
1607 | } |
1608 | |
1609 | SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
1610 | bool isT, bool isO) { |
1611 | return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO); |
1612 | } |
1613 | |
1614 | SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, |
1615 | EVT VT, bool isT, bool isO) { |
1616 | assert(VT.isInteger() && "Cannot create FP integer constant!" ); |
1617 | |
1618 | EVT EltVT = VT.getScalarType(); |
1619 | const ConstantInt *Elt = &Val; |
1620 | |
1621 | // In some cases the vector type is legal but the element type is illegal and |
1622 | // needs to be promoted, for example v8i8 on ARM. In this case, promote the |
1623 | // inserted value (the type does not need to match the vector element type). |
1624 | // Any extra bits introduced will be truncated away. |
1625 | if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) == |
1626 | TargetLowering::TypePromoteInteger) { |
1627 | EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT); |
1628 | APInt NewVal; |
1629 | if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT)) |
1630 | NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits()); |
1631 | else |
1632 | NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits()); |
1633 | Elt = ConstantInt::get(Context&: *getContext(), V: NewVal); |
1634 | } |
1635 | // In other cases the element type is illegal and needs to be expanded, for |
1636 | // example v2i64 on MIPS32. In this case, find the nearest legal type, split |
1637 | // the value into n parts and use a vector type with n-times the elements. |
1638 | // Then bitcast to the type requested. |
1639 | // Legalizing constants too early makes the DAGCombiner's job harder so we |
1640 | // only legalize if the DAG tells us we must produce legal types. |
1641 | else if (NewNodesMustHaveLegalTypes && VT.isVector() && |
1642 | TLI->getTypeAction(Context&: *getContext(), VT: EltVT) == |
1643 | TargetLowering::TypeExpandInteger) { |
1644 | const APInt &NewVal = Elt->getValue(); |
1645 | EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT); |
1646 | unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); |
1647 | |
1648 | // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node. |
1649 | if (VT.isScalableVector() || |
1650 | TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) { |
1651 | assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && |
1652 | "Can only handle an even split!" ); |
1653 | unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits; |
1654 | |
1655 | SmallVector<SDValue, 2> ScalarParts; |
1656 | for (unsigned i = 0; i != Parts; ++i) |
1657 | ScalarParts.push_back(Elt: getConstant( |
1658 | Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL, |
1659 | VT: ViaEltVT, isT, isO)); |
1660 | |
1661 | return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts); |
1662 | } |
1663 | |
1664 | unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; |
1665 | EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts); |
1666 | |
1667 | // Check the temporary vector is the correct size. If this fails then |
1668 | // getTypeToTransformTo() probably returned a type whose size (in bits) |
1669 | // isn't a power-of-2 factor of the requested type size. |
1670 | assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); |
1671 | |
1672 | SmallVector<SDValue, 2> EltParts; |
1673 | for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) |
1674 | EltParts.push_back(Elt: getConstant( |
1675 | Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL, |
1676 | VT: ViaEltVT, isT, isO)); |
1677 | |
1678 | // EltParts is currently in little endian order. If we actually want |
1679 | // big-endian order then reverse it now. |
1680 | if (getDataLayout().isBigEndian()) |
1681 | std::reverse(first: EltParts.begin(), last: EltParts.end()); |
1682 | |
1683 | // The elements must be reversed when the element order is different |
1684 | // to the endianness of the elements (because the BITCAST is itself a |
1685 | // vector shuffle in this situation). However, we do not need any code to |
1686 | // perform this reversal because getConstant() is producing a vector |
1687 | // splat. |
1688 | // This situation occurs in MIPS MSA. |
1689 | |
1690 | SmallVector<SDValue, 8> Ops; |
1691 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) |
1692 | llvm::append_range(C&: Ops, R&: EltParts); |
1693 | |
1694 | SDValue V = |
1695 | getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops)); |
1696 | return V; |
1697 | } |
1698 | |
1699 | assert(Elt->getBitWidth() == EltVT.getSizeInBits() && |
1700 | "APInt size does not match type size!" ); |
1701 | unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; |
1702 | FoldingSetNodeID ID; |
1703 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt); |
1704 | ID.AddPointer(Ptr: Elt); |
1705 | ID.AddBoolean(B: isO); |
1706 | void *IP = nullptr; |
1707 | SDNode *N = nullptr; |
1708 | if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))) |
1709 | if (!VT.isVector()) |
1710 | return SDValue(N, 0); |
1711 | |
1712 | if (!N) { |
1713 | N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: EltVT); |
1714 | CSEMap.InsertNode(N, InsertPos: IP); |
1715 | InsertNode(N); |
1716 | NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: " , G: this); |
1717 | } |
1718 | |
1719 | SDValue Result(N, 0); |
1720 | if (VT.isVector()) |
1721 | Result = getSplat(VT, DL, Op: Result); |
1722 | return Result; |
1723 | } |
1724 | |
1725 | SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, |
1726 | bool isTarget) { |
1727 | return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget); |
1728 | } |
1729 | |
1730 | SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, |
1731 | const SDLoc &DL, bool LegalTypes) { |
1732 | assert(VT.isInteger() && "Shift amount is not an integer type!" ); |
1733 | EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout(), LegalTypes); |
1734 | return getConstant(Val, DL, VT: ShiftVT); |
1735 | } |
1736 | |
1737 | SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT, |
1738 | const SDLoc &DL, bool LegalTypes) { |
1739 | assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift" ); |
1740 | return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL, LegalTypes); |
1741 | } |
1742 | |
1743 | SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, |
1744 | bool isTarget) { |
1745 | return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget); |
1746 | } |
1747 | |
1748 | SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, |
1749 | bool isTarget) { |
1750 | return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget); |
1751 | } |
1752 | |
1753 | SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, |
1754 | EVT VT, bool isTarget) { |
1755 | assert(VT.isFloatingPoint() && "Cannot create integer FP constant!" ); |
1756 | |
1757 | EVT EltVT = VT.getScalarType(); |
1758 | |
1759 | // Do the map lookup using the actual bit pattern for the floating point |
1760 | // value, so that we don't have problems with 0.0 comparing equal to -0.0, and |
1761 | // we don't have issues with SNANs. |
1762 | unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; |
1763 | FoldingSetNodeID ID; |
1764 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT: EltVT), OpList: std::nullopt); |
1765 | ID.AddPointer(Ptr: &V); |
1766 | void *IP = nullptr; |
1767 | SDNode *N = nullptr; |
1768 | if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))) |
1769 | if (!VT.isVector()) |
1770 | return SDValue(N, 0); |
1771 | |
1772 | if (!N) { |
1773 | N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args: &V, Args&: EltVT); |
1774 | CSEMap.InsertNode(N, InsertPos: IP); |
1775 | InsertNode(N); |
1776 | } |
1777 | |
1778 | SDValue Result(N, 0); |
1779 | if (VT.isVector()) |
1780 | Result = getSplat(VT, DL, Op: Result); |
1781 | NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: " , G: this); |
1782 | return Result; |
1783 | } |
1784 | |
1785 | SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, |
1786 | bool isTarget) { |
1787 | EVT EltVT = VT.getScalarType(); |
1788 | if (EltVT == MVT::f32) |
1789 | return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget); |
1790 | if (EltVT == MVT::f64) |
1791 | return getConstantFP(V: APFloat(Val), DL, VT, isTarget); |
1792 | if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || |
1793 | EltVT == MVT::f16 || EltVT == MVT::bf16) { |
1794 | bool Ignored; |
1795 | APFloat APF = APFloat(Val); |
1796 | APF.convert(ToSemantics: EVTToAPFloatSemantics(VT: EltVT), RM: APFloat::rmNearestTiesToEven, |
1797 | losesInfo: &Ignored); |
1798 | return getConstantFP(V: APF, DL, VT, isTarget); |
1799 | } |
1800 | llvm_unreachable("Unsupported type in getConstantFP" ); |
1801 | } |
1802 | |
1803 | SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, |
1804 | EVT VT, int64_t Offset, bool isTargetGA, |
1805 | unsigned TargetFlags) { |
1806 | assert((TargetFlags == 0 || isTargetGA) && |
1807 | "Cannot set target flags on target-independent globals" ); |
1808 | |
1809 | // Truncate (with sign-extension) the offset value to the pointer size. |
1810 | unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
1811 | if (BitWidth < 64) |
1812 | Offset = SignExtend64(X: Offset, B: BitWidth); |
1813 | |
1814 | unsigned Opc; |
1815 | if (GV->isThreadLocal()) |
1816 | Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; |
1817 | else |
1818 | Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; |
1819 | |
1820 | FoldingSetNodeID ID; |
1821 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
1822 | ID.AddPointer(Ptr: GV); |
1823 | ID.AddInteger(I: Offset); |
1824 | ID.AddInteger(I: TargetFlags); |
1825 | void *IP = nullptr; |
1826 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
1827 | return SDValue(E, 0); |
1828 | |
1829 | auto *N = newSDNode<GlobalAddressSDNode>( |
1830 | Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VT, Args&: Offset, Args&: TargetFlags); |
1831 | CSEMap.InsertNode(N, InsertPos: IP); |
1832 | InsertNode(N); |
1833 | return SDValue(N, 0); |
1834 | } |
1835 | |
1836 | SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { |
1837 | unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; |
1838 | FoldingSetNodeID ID; |
1839 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
1840 | ID.AddInteger(I: FI); |
1841 | void *IP = nullptr; |
1842 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
1843 | return SDValue(E, 0); |
1844 | |
1845 | auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VT, Args&: isTarget); |
1846 | CSEMap.InsertNode(N, InsertPos: IP); |
1847 | InsertNode(N); |
1848 | return SDValue(N, 0); |
1849 | } |
1850 | |
1851 | SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, |
1852 | unsigned TargetFlags) { |
1853 | assert((TargetFlags == 0 || isTarget) && |
1854 | "Cannot set target flags on target-independent jump tables" ); |
1855 | unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; |
1856 | FoldingSetNodeID ID; |
1857 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
1858 | ID.AddInteger(I: JTI); |
1859 | ID.AddInteger(I: TargetFlags); |
1860 | void *IP = nullptr; |
1861 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
1862 | return SDValue(E, 0); |
1863 | |
1864 | auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VT, Args&: isTarget, Args&: TargetFlags); |
1865 | CSEMap.InsertNode(N, InsertPos: IP); |
1866 | InsertNode(N); |
1867 | return SDValue(N, 0); |
1868 | } |
1869 | |
1870 | SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain, |
1871 | const SDLoc &DL) { |
1872 | EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout()); |
1873 | return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain, |
1874 | getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true)); |
1875 | } |
1876 | |
1877 | SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, |
1878 | MaybeAlign Alignment, int Offset, |
1879 | bool isTarget, unsigned TargetFlags) { |
1880 | assert((TargetFlags == 0 || isTarget) && |
1881 | "Cannot set target flags on target-independent globals" ); |
1882 | if (!Alignment) |
1883 | Alignment = shouldOptForSize() |
1884 | ? getDataLayout().getABITypeAlign(Ty: C->getType()) |
1885 | : getDataLayout().getPrefTypeAlign(Ty: C->getType()); |
1886 | unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
1887 | FoldingSetNodeID ID; |
1888 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
1889 | ID.AddInteger(I: Alignment->value()); |
1890 | ID.AddInteger(I: Offset); |
1891 | ID.AddPointer(Ptr: C); |
1892 | ID.AddInteger(I: TargetFlags); |
1893 | void *IP = nullptr; |
1894 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
1895 | return SDValue(E, 0); |
1896 | |
1897 | auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment, |
1898 | Args&: TargetFlags); |
1899 | CSEMap.InsertNode(N, InsertPos: IP); |
1900 | InsertNode(N); |
1901 | SDValue V = SDValue(N, 0); |
1902 | NewSDValueDbgMsg(V, Msg: "Creating new constant pool: " , G: this); |
1903 | return V; |
1904 | } |
1905 | |
1906 | SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, |
1907 | MaybeAlign Alignment, int Offset, |
1908 | bool isTarget, unsigned TargetFlags) { |
1909 | assert((TargetFlags == 0 || isTarget) && |
1910 | "Cannot set target flags on target-independent globals" ); |
1911 | if (!Alignment) |
1912 | Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType()); |
1913 | unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
1914 | FoldingSetNodeID ID; |
1915 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
1916 | ID.AddInteger(I: Alignment->value()); |
1917 | ID.AddInteger(I: Offset); |
1918 | C->addSelectionDAGCSEId(ID); |
1919 | ID.AddInteger(I: TargetFlags); |
1920 | void *IP = nullptr; |
1921 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
1922 | return SDValue(E, 0); |
1923 | |
1924 | auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VT, Args&: Offset, Args&: *Alignment, |
1925 | Args&: TargetFlags); |
1926 | CSEMap.InsertNode(N, InsertPos: IP); |
1927 | InsertNode(N); |
1928 | return SDValue(N, 0); |
1929 | } |
1930 | |
1931 | SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { |
1932 | FoldingSetNodeID ID; |
1933 | AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt); |
1934 | ID.AddPointer(Ptr: MBB); |
1935 | void *IP = nullptr; |
1936 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
1937 | return SDValue(E, 0); |
1938 | |
1939 | auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB); |
1940 | CSEMap.InsertNode(N, InsertPos: IP); |
1941 | InsertNode(N); |
1942 | return SDValue(N, 0); |
1943 | } |
1944 | |
1945 | SDValue SelectionDAG::getValueType(EVT VT) { |
1946 | if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= |
1947 | ValueTypeNodes.size()) |
1948 | ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1); |
1949 | |
1950 | SDNode *&N = VT.isExtended() ? |
1951 | ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; |
1952 | |
1953 | if (N) return SDValue(N, 0); |
1954 | N = newSDNode<VTSDNode>(Args&: VT); |
1955 | InsertNode(N); |
1956 | return SDValue(N, 0); |
1957 | } |
1958 | |
1959 | SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { |
1960 | SDNode *&N = ExternalSymbols[Sym]; |
1961 | if (N) return SDValue(N, 0); |
1962 | N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args&: VT); |
1963 | InsertNode(N); |
1964 | return SDValue(N, 0); |
1965 | } |
1966 | |
1967 | SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { |
1968 | SDNode *&N = MCSymbols[Sym]; |
1969 | if (N) |
1970 | return SDValue(N, 0); |
1971 | N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args&: VT); |
1972 | InsertNode(N); |
1973 | return SDValue(N, 0); |
1974 | } |
1975 | |
1976 | SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, |
1977 | unsigned TargetFlags) { |
1978 | SDNode *&N = |
1979 | TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)]; |
1980 | if (N) return SDValue(N, 0); |
1981 | N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args&: VT); |
1982 | InsertNode(N); |
1983 | return SDValue(N, 0); |
1984 | } |
1985 | |
1986 | SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { |
1987 | if ((unsigned)Cond >= CondCodeNodes.size()) |
1988 | CondCodeNodes.resize(new_size: Cond+1); |
1989 | |
1990 | if (!CondCodeNodes[Cond]) { |
1991 | auto *N = newSDNode<CondCodeSDNode>(Args&: Cond); |
1992 | CondCodeNodes[Cond] = N; |
1993 | InsertNode(N); |
1994 | } |
1995 | |
1996 | return SDValue(CondCodeNodes[Cond], 0); |
1997 | } |
1998 | |
1999 | SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm, |
2000 | bool ConstantFold) { |
2001 | assert(MulImm.getBitWidth() == VT.getSizeInBits() && |
2002 | "APInt size does not match type size!" ); |
2003 | |
2004 | if (MulImm == 0) |
2005 | return getConstant(Val: 0, DL, VT); |
2006 | |
2007 | if (ConstantFold) { |
2008 | const MachineFunction &MF = getMachineFunction(); |
2009 | const Function &F = MF.getFunction(); |
2010 | ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64); |
2011 | if (const APInt *C = CR.getSingleElement()) |
2012 | return getConstant(Val: MulImm * C->getZExtValue(), DL, VT); |
2013 | } |
2014 | |
2015 | return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT)); |
2016 | } |
2017 | |
2018 | SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, |
2019 | bool ConstantFold) { |
2020 | if (EC.isScalable()) |
2021 | return getVScale(DL, VT, |
2022 | MulImm: APInt(VT.getSizeInBits(), EC.getKnownMinValue())); |
2023 | |
2024 | return getConstant(Val: EC.getKnownMinValue(), DL, VT); |
2025 | } |
2026 | |
2027 | SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { |
2028 | APInt One(ResVT.getScalarSizeInBits(), 1); |
2029 | return getStepVector(DL, ResVT, StepVal: One); |
2030 | } |
2031 | |
2032 | SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { |
2033 | assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); |
2034 | if (ResVT.isScalableVector()) |
2035 | return getNode( |
2036 | Opcode: ISD::STEP_VECTOR, DL, VT: ResVT, |
2037 | Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType())); |
2038 | |
2039 | SmallVector<SDValue, 16> OpsStepConstants; |
2040 | for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++) |
2041 | OpsStepConstants.push_back( |
2042 | Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType())); |
2043 | return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants); |
2044 | } |
2045 | |
2046 | /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that |
2047 | /// point at N1 to point at N2 and indices that point at N2 to point at N1. |
2048 | static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { |
2049 | std::swap(a&: N1, b&: N2); |
2050 | ShuffleVectorSDNode::commuteMask(Mask: M); |
2051 | } |
2052 | |
2053 | SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, |
2054 | SDValue N2, ArrayRef<int> Mask) { |
2055 | assert(VT.getVectorNumElements() == Mask.size() && |
2056 | "Must have the same number of vector elements as mask elements!" ); |
2057 | assert(VT == N1.getValueType() && VT == N2.getValueType() && |
2058 | "Invalid VECTOR_SHUFFLE" ); |
2059 | |
2060 | // Canonicalize shuffle undef, undef -> undef |
2061 | if (N1.isUndef() && N2.isUndef()) |
2062 | return getUNDEF(VT); |
2063 | |
2064 | // Validate that all indices in Mask are within the range of the elements |
2065 | // input to the shuffle. |
2066 | int NElts = Mask.size(); |
2067 | assert(llvm::all_of(Mask, |
2068 | [&](int M) { return M < (NElts * 2) && M >= -1; }) && |
2069 | "Index out of range" ); |
2070 | |
2071 | // Copy the mask so we can do any needed cleanup. |
2072 | SmallVector<int, 8> MaskVec(Mask); |
2073 | |
2074 | // Canonicalize shuffle v, v -> v, undef |
2075 | if (N1 == N2) { |
2076 | N2 = getUNDEF(VT); |
2077 | for (int i = 0; i != NElts; ++i) |
2078 | if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; |
2079 | } |
2080 | |
2081 | // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. |
2082 | if (N1.isUndef()) |
2083 | commuteShuffle(N1, N2, M: MaskVec); |
2084 | |
2085 | if (TLI->hasVectorBlend()) { |
2086 | // If shuffling a splat, try to blend the splat instead. We do this here so |
2087 | // that even when this arises during lowering we don't have to re-handle it. |
2088 | auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { |
2089 | BitVector UndefElements; |
2090 | SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements); |
2091 | if (!Splat) |
2092 | return; |
2093 | |
2094 | for (int i = 0; i < NElts; ++i) { |
2095 | if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) |
2096 | continue; |
2097 | |
2098 | // If this input comes from undef, mark it as such. |
2099 | if (UndefElements[MaskVec[i] - Offset]) { |
2100 | MaskVec[i] = -1; |
2101 | continue; |
2102 | } |
2103 | |
2104 | // If we can blend a non-undef lane, use that instead. |
2105 | if (!UndefElements[i]) |
2106 | MaskVec[i] = i + Offset; |
2107 | } |
2108 | }; |
2109 | if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1)) |
2110 | BlendSplat(N1BV, 0); |
2111 | if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2)) |
2112 | BlendSplat(N2BV, NElts); |
2113 | } |
2114 | |
2115 | // Canonicalize all index into lhs, -> shuffle lhs, undef |
2116 | // Canonicalize all index into rhs, -> shuffle rhs, undef |
2117 | bool AllLHS = true, AllRHS = true; |
2118 | bool N2Undef = N2.isUndef(); |
2119 | for (int i = 0; i != NElts; ++i) { |
2120 | if (MaskVec[i] >= NElts) { |
2121 | if (N2Undef) |
2122 | MaskVec[i] = -1; |
2123 | else |
2124 | AllLHS = false; |
2125 | } else if (MaskVec[i] >= 0) { |
2126 | AllRHS = false; |
2127 | } |
2128 | } |
2129 | if (AllLHS && AllRHS) |
2130 | return getUNDEF(VT); |
2131 | if (AllLHS && !N2Undef) |
2132 | N2 = getUNDEF(VT); |
2133 | if (AllRHS) { |
2134 | N1 = getUNDEF(VT); |
2135 | commuteShuffle(N1, N2, M: MaskVec); |
2136 | } |
2137 | // Reset our undef status after accounting for the mask. |
2138 | N2Undef = N2.isUndef(); |
2139 | // Re-check whether both sides ended up undef. |
2140 | if (N1.isUndef() && N2Undef) |
2141 | return getUNDEF(VT); |
2142 | |
2143 | // If Identity shuffle return that node. |
2144 | bool Identity = true, AllSame = true; |
2145 | for (int i = 0; i != NElts; ++i) { |
2146 | if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; |
2147 | if (MaskVec[i] != MaskVec[0]) AllSame = false; |
2148 | } |
2149 | if (Identity && NElts) |
2150 | return N1; |
2151 | |
2152 | // Shuffling a constant splat doesn't change the result. |
2153 | if (N2Undef) { |
2154 | SDValue V = N1; |
2155 | |
2156 | // Look through any bitcasts. We check that these don't change the number |
2157 | // (and size) of elements and just changes their types. |
2158 | while (V.getOpcode() == ISD::BITCAST) |
2159 | V = V->getOperand(Num: 0); |
2160 | |
2161 | // A splat should always show up as a build vector node. |
2162 | if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) { |
2163 | BitVector UndefElements; |
2164 | SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements); |
2165 | // If this is a splat of an undef, shuffling it is also undef. |
2166 | if (Splat && Splat.isUndef()) |
2167 | return getUNDEF(VT); |
2168 | |
2169 | bool = |
2170 | V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); |
2171 | |
2172 | // We only have a splat which can skip shuffles if there is a splatted |
2173 | // value and no undef lanes rearranged by the shuffle. |
2174 | if (Splat && UndefElements.none()) { |
2175 | // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the |
2176 | // number of elements match or the value splatted is a zero constant. |
2177 | if (SameNumElts || isNullConstant(V: Splat)) |
2178 | return N1; |
2179 | } |
2180 | |
2181 | // If the shuffle itself creates a splat, build the vector directly. |
2182 | if (AllSame && SameNumElts) { |
2183 | EVT BuildVT = BV->getValueType(ResNo: 0); |
2184 | const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]); |
2185 | SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted); |
2186 | |
2187 | // We may have jumped through bitcasts, so the type of the |
2188 | // BUILD_VECTOR may not match the type of the shuffle. |
2189 | if (BuildVT != VT) |
2190 | NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV); |
2191 | return NewBV; |
2192 | } |
2193 | } |
2194 | } |
2195 | |
2196 | FoldingSetNodeID ID; |
2197 | SDValue Ops[2] = { N1, N2 }; |
2198 | AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: getVTList(VT), OpList: Ops); |
2199 | for (int i = 0; i != NElts; ++i) |
2200 | ID.AddInteger(I: MaskVec[i]); |
2201 | |
2202 | void* IP = nullptr; |
2203 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
2204 | return SDValue(E, 0); |
2205 | |
2206 | // Allocate the mask array for the node out of the BumpPtrAllocator, since |
2207 | // SDNode doesn't have access to it. This memory will be "leaked" when |
2208 | // the node is deallocated, but recovered when the NodeAllocator is released. |
2209 | int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts); |
2210 | llvm::copy(Range&: MaskVec, Out: MaskAlloc); |
2211 | |
2212 | auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VT, Args: dl.getIROrder(), |
2213 | Args: dl.getDebugLoc(), Args&: MaskAlloc); |
2214 | createOperands(Node: N, Vals: Ops); |
2215 | |
2216 | CSEMap.InsertNode(N, InsertPos: IP); |
2217 | InsertNode(N); |
2218 | SDValue V = SDValue(N, 0); |
2219 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
2220 | return V; |
2221 | } |
2222 | |
2223 | SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { |
2224 | EVT VT = SV.getValueType(ResNo: 0); |
2225 | SmallVector<int, 8> MaskVec(SV.getMask()); |
2226 | ShuffleVectorSDNode::commuteMask(Mask: MaskVec); |
2227 | |
2228 | SDValue Op0 = SV.getOperand(Num: 0); |
2229 | SDValue Op1 = SV.getOperand(Num: 1); |
2230 | return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec); |
2231 | } |
2232 | |
2233 | SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { |
2234 | FoldingSetNodeID ID; |
2235 | AddNodeIDNode(ID, OpC: ISD::Register, VTList: getVTList(VT), OpList: std::nullopt); |
2236 | ID.AddInteger(I: RegNo); |
2237 | void *IP = nullptr; |
2238 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2239 | return SDValue(E, 0); |
2240 | |
2241 | auto *N = newSDNode<RegisterSDNode>(Args&: RegNo, Args&: VT); |
2242 | N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA); |
2243 | CSEMap.InsertNode(N, InsertPos: IP); |
2244 | InsertNode(N); |
2245 | return SDValue(N, 0); |
2246 | } |
2247 | |
2248 | SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { |
2249 | FoldingSetNodeID ID; |
2250 | AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt); |
2251 | ID.AddPointer(Ptr: RegMask); |
2252 | void *IP = nullptr; |
2253 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2254 | return SDValue(E, 0); |
2255 | |
2256 | auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask); |
2257 | CSEMap.InsertNode(N, InsertPos: IP); |
2258 | InsertNode(N); |
2259 | return SDValue(N, 0); |
2260 | } |
2261 | |
2262 | SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, |
2263 | MCSymbol *Label) { |
2264 | return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label); |
2265 | } |
2266 | |
2267 | SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, |
2268 | SDValue Root, MCSymbol *Label) { |
2269 | FoldingSetNodeID ID; |
2270 | SDValue Ops[] = { Root }; |
2271 | AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); |
2272 | ID.AddPointer(Ptr: Label); |
2273 | void *IP = nullptr; |
2274 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2275 | return SDValue(E, 0); |
2276 | |
2277 | auto *N = |
2278 | newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label); |
2279 | createOperands(Node: N, Vals: Ops); |
2280 | |
2281 | CSEMap.InsertNode(N, InsertPos: IP); |
2282 | InsertNode(N); |
2283 | return SDValue(N, 0); |
2284 | } |
2285 | |
2286 | SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, |
2287 | int64_t Offset, bool isTarget, |
2288 | unsigned TargetFlags) { |
2289 | unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; |
2290 | |
2291 | FoldingSetNodeID ID; |
2292 | AddNodeIDNode(ID, OpC: Opc, VTList: getVTList(VT), OpList: std::nullopt); |
2293 | ID.AddPointer(Ptr: BA); |
2294 | ID.AddInteger(I: Offset); |
2295 | ID.AddInteger(I: TargetFlags); |
2296 | void *IP = nullptr; |
2297 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2298 | return SDValue(E, 0); |
2299 | |
2300 | auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VT, Args&: BA, Args&: Offset, Args&: TargetFlags); |
2301 | CSEMap.InsertNode(N, InsertPos: IP); |
2302 | InsertNode(N); |
2303 | return SDValue(N, 0); |
2304 | } |
2305 | |
2306 | SDValue SelectionDAG::getSrcValue(const Value *V) { |
2307 | FoldingSetNodeID ID; |
2308 | AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt); |
2309 | ID.AddPointer(Ptr: V); |
2310 | |
2311 | void *IP = nullptr; |
2312 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2313 | return SDValue(E, 0); |
2314 | |
2315 | auto *N = newSDNode<SrcValueSDNode>(Args&: V); |
2316 | CSEMap.InsertNode(N, InsertPos: IP); |
2317 | InsertNode(N); |
2318 | return SDValue(N, 0); |
2319 | } |
2320 | |
2321 | SDValue SelectionDAG::getMDNode(const MDNode *MD) { |
2322 | FoldingSetNodeID ID; |
2323 | AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt); |
2324 | ID.AddPointer(Ptr: MD); |
2325 | |
2326 | void *IP = nullptr; |
2327 | if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) |
2328 | return SDValue(E, 0); |
2329 | |
2330 | auto *N = newSDNode<MDNodeSDNode>(Args&: MD); |
2331 | CSEMap.InsertNode(N, InsertPos: IP); |
2332 | InsertNode(N); |
2333 | return SDValue(N, 0); |
2334 | } |
2335 | |
2336 | SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { |
2337 | if (VT == V.getValueType()) |
2338 | return V; |
2339 | |
2340 | return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V); |
2341 | } |
2342 | |
2343 | SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, |
2344 | unsigned SrcAS, unsigned DestAS) { |
2345 | SDValue Ops[] = {Ptr}; |
2346 | FoldingSetNodeID ID; |
2347 | AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: getVTList(VT), OpList: Ops); |
2348 | ID.AddInteger(I: SrcAS); |
2349 | ID.AddInteger(I: DestAS); |
2350 | |
2351 | void *IP = nullptr; |
2352 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
2353 | return SDValue(E, 0); |
2354 | |
2355 | auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
2356 | Args&: VT, Args&: SrcAS, Args&: DestAS); |
2357 | createOperands(Node: N, Vals: Ops); |
2358 | |
2359 | CSEMap.InsertNode(N, InsertPos: IP); |
2360 | InsertNode(N); |
2361 | return SDValue(N, 0); |
2362 | } |
2363 | |
2364 | SDValue SelectionDAG::getFreeze(SDValue V) { |
2365 | return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V); |
2366 | } |
2367 | |
2368 | /// getShiftAmountOperand - Return the specified value casted to |
2369 | /// the target's desired shift amount type. |
2370 | SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { |
2371 | EVT OpTy = Op.getValueType(); |
2372 | EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout()); |
2373 | if (OpTy == ShTy || OpTy.isVector()) return Op; |
2374 | |
2375 | return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy); |
2376 | } |
2377 | |
2378 | SDValue SelectionDAG::expandVAArg(SDNode *Node) { |
2379 | SDLoc dl(Node); |
2380 | const TargetLowering &TLI = getTargetLoweringInfo(); |
2381 | const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue(); |
2382 | EVT VT = Node->getValueType(ResNo: 0); |
2383 | SDValue Tmp1 = Node->getOperand(Num: 0); |
2384 | SDValue Tmp2 = Node->getOperand(Num: 1); |
2385 | const MaybeAlign MA(Node->getConstantOperandVal(Num: 3)); |
2386 | |
2387 | SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1, |
2388 | Ptr: Tmp2, PtrInfo: MachinePointerInfo(V)); |
2389 | SDValue VAList = VAListLoad; |
2390 | |
2391 | if (MA && *MA > TLI.getMinStackArgumentAlignment()) { |
2392 | VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList, |
2393 | N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType())); |
2394 | |
2395 | VAList = |
2396 | getNode(Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList, |
2397 | N2: getConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType())); |
2398 | } |
2399 | |
2400 | // Increment the pointer, VAList, to the next vaarg |
2401 | Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList, |
2402 | N2: getConstant(Val: getDataLayout().getTypeAllocSize( |
2403 | Ty: VT.getTypeForEVT(Context&: *getContext())), |
2404 | DL: dl, VT: VAList.getValueType())); |
2405 | // Store the incremented VAList to the legalized pointer |
2406 | Tmp1 = |
2407 | getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V)); |
2408 | // Load the actual argument out of the pointer VAList |
2409 | return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo()); |
2410 | } |
2411 | |
2412 | SDValue SelectionDAG::expandVACopy(SDNode *Node) { |
2413 | SDLoc dl(Node); |
2414 | const TargetLowering &TLI = getTargetLoweringInfo(); |
2415 | // This defaults to loading a pointer from the input and storing it to the |
2416 | // output, returning the chain. |
2417 | const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue(); |
2418 | const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue(); |
2419 | SDValue Tmp1 = |
2420 | getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0), |
2421 | Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS)); |
2422 | return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1), |
2423 | PtrInfo: MachinePointerInfo(VD)); |
2424 | } |
2425 | |
2426 | Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { |
2427 | const DataLayout &DL = getDataLayout(); |
2428 | Type *Ty = VT.getTypeForEVT(Context&: *getContext()); |
2429 | Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
2430 | |
2431 | if (TLI->isTypeLegal(VT) || !VT.isVector()) |
2432 | return RedAlign; |
2433 | |
2434 | const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
2435 | const Align StackAlign = TFI->getStackAlign(); |
2436 | |
2437 | // See if we can choose a smaller ABI alignment in cases where it's an |
2438 | // illegal vector type that will get broken down. |
2439 | if (RedAlign > StackAlign) { |
2440 | EVT IntermediateVT; |
2441 | MVT RegisterVT; |
2442 | unsigned NumIntermediates; |
2443 | TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT, |
2444 | NumIntermediates, RegisterVT); |
2445 | Ty = IntermediateVT.getTypeForEVT(Context&: *getContext()); |
2446 | Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
2447 | if (RedAlign2 < RedAlign) |
2448 | RedAlign = RedAlign2; |
2449 | } |
2450 | |
2451 | return RedAlign; |
2452 | } |
2453 | |
2454 | SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { |
2455 | MachineFrameInfo &MFI = MF->getFrameInfo(); |
2456 | const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
2457 | int StackID = 0; |
2458 | if (Bytes.isScalable()) |
2459 | StackID = TFI->getStackIDForScalableVectors(); |
2460 | // The stack id gives an indication of whether the object is scalable or |
2461 | // not, so it's safe to pass in the minimum size here. |
2462 | int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment, |
2463 | isSpillSlot: false, Alloca: nullptr, ID: StackID); |
2464 | return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout())); |
2465 | } |
2466 | |
2467 | SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { |
2468 | Type *Ty = VT.getTypeForEVT(Context&: *getContext()); |
2469 | Align StackAlign = |
2470 | std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign)); |
2471 | return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign); |
2472 | } |
2473 | |
2474 | SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { |
2475 | TypeSize VT1Size = VT1.getStoreSize(); |
2476 | TypeSize VT2Size = VT2.getStoreSize(); |
2477 | assert(VT1Size.isScalable() == VT2Size.isScalable() && |
2478 | "Don't know how to choose the maximum size when creating a stack " |
2479 | "temporary" ); |
2480 | TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue() |
2481 | ? VT1Size |
2482 | : VT2Size; |
2483 | |
2484 | Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext()); |
2485 | Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext()); |
2486 | const DataLayout &DL = getDataLayout(); |
2487 | Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2)); |
2488 | return CreateStackTemporary(Bytes, Alignment: Align); |
2489 | } |
2490 | |
2491 | SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, |
2492 | ISD::CondCode Cond, const SDLoc &dl) { |
2493 | EVT OpVT = N1.getValueType(); |
2494 | |
2495 | auto GetUndefBooleanConstant = [&]() { |
2496 | if (VT.getScalarType() == MVT::i1 || |
2497 | TLI->getBooleanContents(Type: OpVT) == |
2498 | TargetLowering::UndefinedBooleanContent) |
2499 | return getUNDEF(VT); |
2500 | // ZeroOrOne / ZeroOrNegative require specific values for the high bits, |
2501 | // so we cannot use getUNDEF(). Return zero instead. |
2502 | return getConstant(Val: 0, DL: dl, VT); |
2503 | }; |
2504 | |
2505 | // These setcc operations always fold. |
2506 | switch (Cond) { |
2507 | default: break; |
2508 | case ISD::SETFALSE: |
2509 | case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT); |
2510 | case ISD::SETTRUE: |
2511 | case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT); |
2512 | |
2513 | case ISD::SETOEQ: |
2514 | case ISD::SETOGT: |
2515 | case ISD::SETOGE: |
2516 | case ISD::SETOLT: |
2517 | case ISD::SETOLE: |
2518 | case ISD::SETONE: |
2519 | case ISD::SETO: |
2520 | case ISD::SETUO: |
2521 | case ISD::SETUEQ: |
2522 | case ISD::SETUNE: |
2523 | assert(!OpVT.isInteger() && "Illegal setcc for integer!" ); |
2524 | break; |
2525 | } |
2526 | |
2527 | if (OpVT.isInteger()) { |
2528 | // For EQ and NE, we can always pick a value for the undef to make the |
2529 | // predicate pass or fail, so we can return undef. |
2530 | // Matches behavior in llvm::ConstantFoldCompareInstruction. |
2531 | // icmp eq/ne X, undef -> undef. |
2532 | if ((N1.isUndef() || N2.isUndef()) && |
2533 | (Cond == ISD::SETEQ || Cond == ISD::SETNE)) |
2534 | return GetUndefBooleanConstant(); |
2535 | |
2536 | // If both operands are undef, we can return undef for int comparison. |
2537 | // icmp undef, undef -> undef. |
2538 | if (N1.isUndef() && N2.isUndef()) |
2539 | return GetUndefBooleanConstant(); |
2540 | |
2541 | // icmp X, X -> true/false |
2542 | // icmp X, undef -> true/false because undef could be X. |
2543 | if (N1.isUndef() || N2.isUndef() || N1 == N2) |
2544 | return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT); |
2545 | } |
2546 | |
2547 | if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) { |
2548 | const APInt &C2 = N2C->getAPIntValue(); |
2549 | if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) { |
2550 | const APInt &C1 = N1C->getAPIntValue(); |
2551 | |
2552 | return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)), |
2553 | DL: dl, VT, OpVT); |
2554 | } |
2555 | } |
2556 | |
2557 | auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1); |
2558 | auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2); |
2559 | |
2560 | if (N1CFP && N2CFP) { |
2561 | APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF()); |
2562 | switch (Cond) { |
2563 | default: break; |
2564 | case ISD::SETEQ: if (R==APFloat::cmpUnordered) |
2565 | return GetUndefBooleanConstant(); |
2566 | [[fallthrough]]; |
2567 | case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT, |
2568 | OpVT); |
2569 | case ISD::SETNE: if (R==APFloat::cmpUnordered) |
2570 | return GetUndefBooleanConstant(); |
2571 | [[fallthrough]]; |
2572 | case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan || |
2573 | R==APFloat::cmpLessThan, DL: dl, VT, |
2574 | OpVT); |
2575 | case ISD::SETLT: if (R==APFloat::cmpUnordered) |
2576 | return GetUndefBooleanConstant(); |
2577 | [[fallthrough]]; |
2578 | case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT, |
2579 | OpVT); |
2580 | case ISD::SETGT: if (R==APFloat::cmpUnordered) |
2581 | return GetUndefBooleanConstant(); |
2582 | [[fallthrough]]; |
2583 | case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl, |
2584 | VT, OpVT); |
2585 | case ISD::SETLE: if (R==APFloat::cmpUnordered) |
2586 | return GetUndefBooleanConstant(); |
2587 | [[fallthrough]]; |
2588 | case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan || |
2589 | R==APFloat::cmpEqual, DL: dl, VT, |
2590 | OpVT); |
2591 | case ISD::SETGE: if (R==APFloat::cmpUnordered) |
2592 | return GetUndefBooleanConstant(); |
2593 | [[fallthrough]]; |
2594 | case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan || |
2595 | R==APFloat::cmpEqual, DL: dl, VT, OpVT); |
2596 | case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT, |
2597 | OpVT); |
2598 | case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT, |
2599 | OpVT); |
2600 | case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered || |
2601 | R==APFloat::cmpEqual, DL: dl, VT, |
2602 | OpVT); |
2603 | case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT, |
2604 | OpVT); |
2605 | case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered || |
2606 | R==APFloat::cmpLessThan, DL: dl, VT, |
2607 | OpVT); |
2608 | case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan || |
2609 | R==APFloat::cmpUnordered, DL: dl, VT, |
2610 | OpVT); |
2611 | case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl, |
2612 | VT, OpVT); |
2613 | case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT, |
2614 | OpVT); |
2615 | } |
2616 | } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { |
2617 | // Ensure that the constant occurs on the RHS. |
2618 | ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond); |
2619 | if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT())) |
2620 | return SDValue(); |
2621 | return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond); |
2622 | } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || |
2623 | (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { |
2624 | // If an operand is known to be a nan (or undef that could be a nan), we can |
2625 | // fold it. |
2626 | // Choosing NaN for the undef will always make unordered comparison succeed |
2627 | // and ordered comparison fails. |
2628 | // Matches behavior in llvm::ConstantFoldCompareInstruction. |
2629 | switch (ISD::getUnorderedFlavor(Cond)) { |
2630 | default: |
2631 | llvm_unreachable("Unknown flavor!" ); |
2632 | case 0: // Known false. |
2633 | return getBoolConstant(V: false, DL: dl, VT, OpVT); |
2634 | case 1: // Known true. |
2635 | return getBoolConstant(V: true, DL: dl, VT, OpVT); |
2636 | case 2: // Undefined. |
2637 | return GetUndefBooleanConstant(); |
2638 | } |
2639 | } |
2640 | |
2641 | // Could not fold it. |
2642 | return SDValue(); |
2643 | } |
2644 | |
2645 | /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We |
2646 | /// use this predicate to simplify operations downstream. |
2647 | bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { |
2648 | unsigned BitWidth = Op.getScalarValueSizeInBits(); |
2649 | return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth); |
2650 | } |
2651 | |
2652 | /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use |
2653 | /// this predicate to simplify operations downstream. Mask is known to be zero |
2654 | /// for bits that V cannot have. |
2655 | bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
2656 | unsigned Depth) const { |
2657 | return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero); |
2658 | } |
2659 | |
2660 | /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in |
2661 | /// DemandedElts. We use this predicate to simplify operations downstream. |
2662 | /// Mask is known to be zero for bits that V cannot have. |
2663 | bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
2664 | const APInt &DemandedElts, |
2665 | unsigned Depth) const { |
2666 | return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero); |
2667 | } |
2668 | |
2669 | /// MaskedVectorIsZero - Return true if 'Op' is known to be zero in |
2670 | /// DemandedElts. We use this predicate to simplify operations downstream. |
2671 | bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts, |
2672 | unsigned Depth /* = 0 */) const { |
2673 | return computeKnownBits(Op: V, DemandedElts, Depth).isZero(); |
2674 | } |
2675 | |
2676 | /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. |
2677 | bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, |
2678 | unsigned Depth) const { |
2679 | return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One); |
2680 | } |
2681 | |
2682 | APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op, |
2683 | const APInt &DemandedElts, |
2684 | unsigned Depth) const { |
2685 | EVT VT = Op.getValueType(); |
2686 | assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!" ); |
2687 | |
2688 | unsigned NumElts = VT.getVectorNumElements(); |
2689 | assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask." ); |
2690 | |
2691 | APInt KnownZeroElements = APInt::getZero(numBits: NumElts); |
2692 | for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) { |
2693 | if (!DemandedElts[EltIdx]) |
2694 | continue; // Don't query elements that are not demanded. |
2695 | APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx); |
2696 | if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth)) |
2697 | KnownZeroElements.setBit(EltIdx); |
2698 | } |
2699 | return KnownZeroElements; |
2700 | } |
2701 | |
2702 | /// isSplatValue - Return true if the vector V has the same value |
2703 | /// across all DemandedElts. For scalable vectors, we don't know the |
2704 | /// number of lanes at compile time. Instead, we use a 1 bit APInt |
2705 | /// to represent a conservative value for all lanes; that is, that |
2706 | /// one bit value is implicitly splatted across all lanes. |
2707 | bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, |
2708 | APInt &UndefElts, unsigned Depth) const { |
2709 | unsigned Opcode = V.getOpcode(); |
2710 | EVT VT = V.getValueType(); |
2711 | assert(VT.isVector() && "Vector type expected" ); |
2712 | assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) && |
2713 | "scalable demanded bits are ignored" ); |
2714 | |
2715 | if (!DemandedElts) |
2716 | return false; // No demanded elts, better to assume we don't know anything. |
2717 | |
2718 | if (Depth >= MaxRecursionDepth) |
2719 | return false; // Limit search depth. |
2720 | |
2721 | // Deal with some common cases here that work for both fixed and scalable |
2722 | // vector types. |
2723 | switch (Opcode) { |
2724 | case ISD::SPLAT_VECTOR: |
2725 | UndefElts = V.getOperand(i: 0).isUndef() |
2726 | ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth()) |
2727 | : APInt(DemandedElts.getBitWidth(), 0); |
2728 | return true; |
2729 | case ISD::ADD: |
2730 | case ISD::SUB: |
2731 | case ISD::AND: |
2732 | case ISD::XOR: |
2733 | case ISD::OR: { |
2734 | APInt UndefLHS, UndefRHS; |
2735 | SDValue LHS = V.getOperand(i: 0); |
2736 | SDValue RHS = V.getOperand(i: 1); |
2737 | if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) && |
2738 | isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1)) { |
2739 | UndefElts = UndefLHS | UndefRHS; |
2740 | return true; |
2741 | } |
2742 | return false; |
2743 | } |
2744 | case ISD::ABS: |
2745 | case ISD::TRUNCATE: |
2746 | case ISD::SIGN_EXTEND: |
2747 | case ISD::ZERO_EXTEND: |
2748 | return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1); |
2749 | default: |
2750 | if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
2751 | Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
2752 | return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this, |
2753 | Depth); |
2754 | break; |
2755 | } |
2756 | |
2757 | // We don't support other cases than those above for scalable vectors at |
2758 | // the moment. |
2759 | if (VT.isScalableVector()) |
2760 | return false; |
2761 | |
2762 | unsigned NumElts = VT.getVectorNumElements(); |
2763 | assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch" ); |
2764 | UndefElts = APInt::getZero(numBits: NumElts); |
2765 | |
2766 | switch (Opcode) { |
2767 | case ISD::BUILD_VECTOR: { |
2768 | SDValue Scl; |
2769 | for (unsigned i = 0; i != NumElts; ++i) { |
2770 | SDValue Op = V.getOperand(i); |
2771 | if (Op.isUndef()) { |
2772 | UndefElts.setBit(i); |
2773 | continue; |
2774 | } |
2775 | if (!DemandedElts[i]) |
2776 | continue; |
2777 | if (Scl && Scl != Op) |
2778 | return false; |
2779 | Scl = Op; |
2780 | } |
2781 | return true; |
2782 | } |
2783 | case ISD::VECTOR_SHUFFLE: { |
2784 | // Check if this is a shuffle node doing a splat or a shuffle of a splat. |
2785 | APInt DemandedLHS = APInt::getZero(numBits: NumElts); |
2786 | APInt DemandedRHS = APInt::getZero(numBits: NumElts); |
2787 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask(); |
2788 | for (int i = 0; i != (int)NumElts; ++i) { |
2789 | int M = Mask[i]; |
2790 | if (M < 0) { |
2791 | UndefElts.setBit(i); |
2792 | continue; |
2793 | } |
2794 | if (!DemandedElts[i]) |
2795 | continue; |
2796 | if (M < (int)NumElts) |
2797 | DemandedLHS.setBit(M); |
2798 | else |
2799 | DemandedRHS.setBit(M - NumElts); |
2800 | } |
2801 | |
2802 | // If we aren't demanding either op, assume there's no splat. |
2803 | // If we are demanding both ops, assume there's no splat. |
2804 | if ((DemandedLHS.isZero() && DemandedRHS.isZero()) || |
2805 | (!DemandedLHS.isZero() && !DemandedRHS.isZero())) |
2806 | return false; |
2807 | |
2808 | // See if the demanded elts of the source op is a splat or we only demand |
2809 | // one element, which should always be a splat. |
2810 | // TODO: Handle source ops splats with undefs. |
2811 | auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) { |
2812 | APInt SrcUndefs; |
2813 | return (SrcElts.popcount() == 1) || |
2814 | (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) && |
2815 | (SrcElts & SrcUndefs).isZero()); |
2816 | }; |
2817 | if (!DemandedLHS.isZero()) |
2818 | return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS); |
2819 | return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS); |
2820 | } |
2821 | case ISD::EXTRACT_SUBVECTOR: { |
2822 | // Offset the demanded elts by the subvector index. |
2823 | SDValue Src = V.getOperand(i: 0); |
2824 | // We don't support scalable vectors at the moment. |
2825 | if (Src.getValueType().isScalableVector()) |
2826 | return false; |
2827 | uint64_t Idx = V.getConstantOperandVal(i: 1); |
2828 | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
2829 | APInt UndefSrcElts; |
2830 | APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx); |
2831 | if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) { |
2832 | UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx); |
2833 | return true; |
2834 | } |
2835 | break; |
2836 | } |
2837 | case ISD::ANY_EXTEND_VECTOR_INREG: |
2838 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
2839 | case ISD::ZERO_EXTEND_VECTOR_INREG: { |
2840 | // Widen the demanded elts by the src element count. |
2841 | SDValue Src = V.getOperand(i: 0); |
2842 | // We don't support scalable vectors at the moment. |
2843 | if (Src.getValueType().isScalableVector()) |
2844 | return false; |
2845 | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
2846 | APInt UndefSrcElts; |
2847 | APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts); |
2848 | if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) { |
2849 | UndefElts = UndefSrcElts.trunc(width: NumElts); |
2850 | return true; |
2851 | } |
2852 | break; |
2853 | } |
2854 | case ISD::BITCAST: { |
2855 | SDValue Src = V.getOperand(i: 0); |
2856 | EVT SrcVT = Src.getValueType(); |
2857 | unsigned SrcBitWidth = SrcVT.getScalarSizeInBits(); |
2858 | unsigned BitWidth = VT.getScalarSizeInBits(); |
2859 | |
2860 | // Ignore bitcasts from unsupported types. |
2861 | // TODO: Add fp support? |
2862 | if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger()) |
2863 | break; |
2864 | |
2865 | // Bitcast 'small element' vector to 'large element' vector. |
2866 | if ((BitWidth % SrcBitWidth) == 0) { |
2867 | // See if each sub element is a splat. |
2868 | unsigned Scale = BitWidth / SrcBitWidth; |
2869 | unsigned NumSrcElts = SrcVT.getVectorNumElements(); |
2870 | APInt ScaledDemandedElts = |
2871 | APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts); |
2872 | for (unsigned I = 0; I != Scale; ++I) { |
2873 | APInt SubUndefElts; |
2874 | APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I); |
2875 | APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt); |
2876 | SubDemandedElts &= ScaledDemandedElts; |
2877 | if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1)) |
2878 | return false; |
2879 | // TODO: Add support for merging sub undef elements. |
2880 | if (!SubUndefElts.isZero()) |
2881 | return false; |
2882 | } |
2883 | return true; |
2884 | } |
2885 | break; |
2886 | } |
2887 | } |
2888 | |
2889 | return false; |
2890 | } |
2891 | |
2892 | /// Helper wrapper to main isSplatValue function. |
2893 | bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const { |
2894 | EVT VT = V.getValueType(); |
2895 | assert(VT.isVector() && "Vector type expected" ); |
2896 | |
2897 | APInt UndefElts; |
2898 | // Since the number of lanes in a scalable vector is unknown at compile time, |
2899 | // we track one bit which is implicitly broadcast to all lanes. This means |
2900 | // that all lanes in a scalable vector are considered demanded. |
2901 | APInt DemandedElts |
2902 | = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
2903 | return isSplatValue(V, DemandedElts, UndefElts) && |
2904 | (AllowUndefs || !UndefElts); |
2905 | } |
2906 | |
2907 | SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { |
2908 | V = peekThroughExtractSubvectors(V); |
2909 | |
2910 | EVT VT = V.getValueType(); |
2911 | unsigned Opcode = V.getOpcode(); |
2912 | switch (Opcode) { |
2913 | default: { |
2914 | APInt UndefElts; |
2915 | // Since the number of lanes in a scalable vector is unknown at compile time, |
2916 | // we track one bit which is implicitly broadcast to all lanes. This means |
2917 | // that all lanes in a scalable vector are considered demanded. |
2918 | APInt DemandedElts |
2919 | = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
2920 | |
2921 | if (isSplatValue(V, DemandedElts, UndefElts)) { |
2922 | if (VT.isScalableVector()) { |
2923 | // DemandedElts and UndefElts are ignored for scalable vectors, since |
2924 | // the only supported cases are SPLAT_VECTOR nodes. |
2925 | SplatIdx = 0; |
2926 | } else { |
2927 | // Handle case where all demanded elements are UNDEF. |
2928 | if (DemandedElts.isSubsetOf(RHS: UndefElts)) { |
2929 | SplatIdx = 0; |
2930 | return getUNDEF(VT); |
2931 | } |
2932 | SplatIdx = (UndefElts & DemandedElts).countr_one(); |
2933 | } |
2934 | return V; |
2935 | } |
2936 | break; |
2937 | } |
2938 | case ISD::SPLAT_VECTOR: |
2939 | SplatIdx = 0; |
2940 | return V; |
2941 | case ISD::VECTOR_SHUFFLE: { |
2942 | assert(!VT.isScalableVector()); |
2943 | // Check if this is a shuffle node doing a splat. |
2944 | // TODO - remove this and rely purely on SelectionDAG::isSplatValue, |
2945 | // getTargetVShiftNode currently struggles without the splat source. |
2946 | auto *SVN = cast<ShuffleVectorSDNode>(Val&: V); |
2947 | if (!SVN->isSplat()) |
2948 | break; |
2949 | int Idx = SVN->getSplatIndex(); |
2950 | int NumElts = V.getValueType().getVectorNumElements(); |
2951 | SplatIdx = Idx % NumElts; |
2952 | return V.getOperand(i: Idx / NumElts); |
2953 | } |
2954 | } |
2955 | |
2956 | return SDValue(); |
2957 | } |
2958 | |
2959 | SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) { |
2960 | int SplatIdx; |
2961 | if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) { |
2962 | EVT SVT = SrcVector.getValueType().getScalarType(); |
2963 | EVT LegalSVT = SVT; |
2964 | if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) { |
2965 | if (!SVT.isInteger()) |
2966 | return SDValue(); |
2967 | LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT); |
2968 | if (LegalSVT.bitsLT(VT: SVT)) |
2969 | return SDValue(); |
2970 | } |
2971 | return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(V), VT: LegalSVT, N1: SrcVector, |
2972 | N2: getVectorIdxConstant(Val: SplatIdx, DL: SDLoc(V))); |
2973 | } |
2974 | return SDValue(); |
2975 | } |
2976 | |
2977 | const APInt * |
2978 | SelectionDAG::getValidShiftAmountConstant(SDValue V, |
2979 | const APInt &DemandedElts) const { |
2980 | assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
2981 | V.getOpcode() == ISD::SRA) && |
2982 | "Unknown shift node" ); |
2983 | unsigned BitWidth = V.getScalarValueSizeInBits(); |
2984 | if (ConstantSDNode *SA = isConstOrConstSplat(N: V.getOperand(i: 1), DemandedElts)) { |
2985 | // Shifting more than the bitwidth is not valid. |
2986 | const APInt &ShAmt = SA->getAPIntValue(); |
2987 | if (ShAmt.ult(RHS: BitWidth)) |
2988 | return &ShAmt; |
2989 | } |
2990 | return nullptr; |
2991 | } |
2992 | |
2993 | const APInt *SelectionDAG::getValidMinimumShiftAmountConstant( |
2994 | SDValue V, const APInt &DemandedElts) const { |
2995 | assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
2996 | V.getOpcode() == ISD::SRA) && |
2997 | "Unknown shift node" ); |
2998 | if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
2999 | return ValidAmt; |
3000 | unsigned BitWidth = V.getScalarValueSizeInBits(); |
3001 | auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1)); |
3002 | if (!BV) |
3003 | return nullptr; |
3004 | const APInt *MinShAmt = nullptr; |
3005 | for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
3006 | if (!DemandedElts[i]) |
3007 | continue; |
3008 | auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i)); |
3009 | if (!SA) |
3010 | return nullptr; |
3011 | // Shifting more than the bitwidth is not valid. |
3012 | const APInt &ShAmt = SA->getAPIntValue(); |
3013 | if (ShAmt.uge(RHS: BitWidth)) |
3014 | return nullptr; |
3015 | if (MinShAmt && MinShAmt->ule(RHS: ShAmt)) |
3016 | continue; |
3017 | MinShAmt = &ShAmt; |
3018 | } |
3019 | return MinShAmt; |
3020 | } |
3021 | |
3022 | const APInt *SelectionDAG::getValidMaximumShiftAmountConstant( |
3023 | SDValue V, const APInt &DemandedElts) const { |
3024 | assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
3025 | V.getOpcode() == ISD::SRA) && |
3026 | "Unknown shift node" ); |
3027 | if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
3028 | return ValidAmt; |
3029 | unsigned BitWidth = V.getScalarValueSizeInBits(); |
3030 | auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1)); |
3031 | if (!BV) |
3032 | return nullptr; |
3033 | const APInt *MaxShAmt = nullptr; |
3034 | for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
3035 | if (!DemandedElts[i]) |
3036 | continue; |
3037 | auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i)); |
3038 | if (!SA) |
3039 | return nullptr; |
3040 | // Shifting more than the bitwidth is not valid. |
3041 | const APInt &ShAmt = SA->getAPIntValue(); |
3042 | if (ShAmt.uge(RHS: BitWidth)) |
3043 | return nullptr; |
3044 | if (MaxShAmt && MaxShAmt->uge(RHS: ShAmt)) |
3045 | continue; |
3046 | MaxShAmt = &ShAmt; |
3047 | } |
3048 | return MaxShAmt; |
3049 | } |
3050 | |
3051 | /// Determine which bits of Op are known to be either zero or one and return |
3052 | /// them in Known. For vectors, the known bits are those that are shared by |
3053 | /// every vector element. |
3054 | KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { |
3055 | EVT VT = Op.getValueType(); |
3056 | |
3057 | // Since the number of lanes in a scalable vector is unknown at compile time, |
3058 | // we track one bit which is implicitly broadcast to all lanes. This means |
3059 | // that all lanes in a scalable vector are considered demanded. |
3060 | APInt DemandedElts = VT.isFixedLengthVector() |
3061 | ? APInt::getAllOnes(numBits: VT.getVectorNumElements()) |
3062 | : APInt(1, 1); |
3063 | return computeKnownBits(Op, DemandedElts, Depth); |
3064 | } |
3065 | |
3066 | /// Determine which bits of Op are known to be either zero or one and return |
3067 | /// them in Known. The DemandedElts argument allows us to only collect the known |
3068 | /// bits that are shared by the requested vector elements. |
3069 | KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, |
3070 | unsigned Depth) const { |
3071 | unsigned BitWidth = Op.getScalarValueSizeInBits(); |
3072 | |
3073 | KnownBits Known(BitWidth); // Don't know anything. |
3074 | |
3075 | if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) { |
3076 | // We know all of the bits for a constant! |
3077 | return KnownBits::makeConstant(C: C->getAPIntValue()); |
3078 | } |
3079 | if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) { |
3080 | // We know all of the bits for a constant fp! |
3081 | return KnownBits::makeConstant(C: C->getValueAPF().bitcastToAPInt()); |
3082 | } |
3083 | |
3084 | if (Depth >= MaxRecursionDepth) |
3085 | return Known; // Limit search depth. |
3086 | |
3087 | KnownBits Known2; |
3088 | unsigned NumElts = DemandedElts.getBitWidth(); |
3089 | assert((!Op.getValueType().isFixedLengthVector() || |
3090 | NumElts == Op.getValueType().getVectorNumElements()) && |
3091 | "Unexpected vector size" ); |
3092 | |
3093 | if (!DemandedElts) |
3094 | return Known; // No demanded elts, better to assume we don't know anything. |
3095 | |
3096 | unsigned Opcode = Op.getOpcode(); |
3097 | switch (Opcode) { |
3098 | case ISD::MERGE_VALUES: |
3099 | return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts, |
3100 | Depth: Depth + 1); |
3101 | case ISD::SPLAT_VECTOR: { |
3102 | SDValue SrcOp = Op.getOperand(i: 0); |
3103 | assert(SrcOp.getValueSizeInBits() >= BitWidth && |
3104 | "Expected SPLAT_VECTOR implicit truncation" ); |
3105 | // Implicitly truncate the bits to match the official semantics of |
3106 | // SPLAT_VECTOR. |
3107 | Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth); |
3108 | break; |
3109 | } |
3110 | case ISD::SPLAT_VECTOR_PARTS: { |
3111 | unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits(); |
3112 | assert(ScalarSize * Op.getNumOperands() == BitWidth && |
3113 | "Expected SPLAT_VECTOR_PARTS scalars to cover element width" ); |
3114 | for (auto [I, SrcOp] : enumerate(First: Op->ops())) { |
3115 | Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I); |
3116 | } |
3117 | break; |
3118 | } |
3119 | case ISD::STEP_VECTOR: { |
3120 | const APInt &Step = Op.getConstantOperandAPInt(i: 0); |
3121 | |
3122 | if (Step.isPowerOf2()) |
3123 | Known.Zero.setLowBits(Step.logBase2()); |
3124 | |
3125 | const Function &F = getMachineFunction().getFunction(); |
3126 | |
3127 | if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements())) |
3128 | break; |
3129 | const APInt MinNumElts = |
3130 | APInt(BitWidth, Op.getValueType().getVectorMinNumElements()); |
3131 | |
3132 | bool Overflow; |
3133 | const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth) |
3134 | .getUnsignedMax() |
3135 | .umul_ov(RHS: MinNumElts, Overflow); |
3136 | if (Overflow) |
3137 | break; |
3138 | |
3139 | const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow); |
3140 | if (Overflow) |
3141 | break; |
3142 | |
3143 | Known.Zero.setHighBits(MaxValue.countl_zero()); |
3144 | break; |
3145 | } |
3146 | case ISD::BUILD_VECTOR: |
3147 | assert(!Op.getValueType().isScalableVector()); |
3148 | // Collect the known bits that are shared by every demanded vector element. |
3149 | Known.Zero.setAllBits(); Known.One.setAllBits(); |
3150 | for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
3151 | if (!DemandedElts[i]) |
3152 | continue; |
3153 | |
3154 | SDValue SrcOp = Op.getOperand(i); |
3155 | Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1); |
3156 | |
3157 | // BUILD_VECTOR can implicitly truncate sources, we must handle this. |
3158 | if (SrcOp.getValueSizeInBits() != BitWidth) { |
3159 | assert(SrcOp.getValueSizeInBits() > BitWidth && |
3160 | "Expected BUILD_VECTOR implicit truncation" ); |
3161 | Known2 = Known2.trunc(BitWidth); |
3162 | } |
3163 | |
3164 | // Known bits are the values that are shared by every demanded element. |
3165 | Known = Known.intersectWith(RHS: Known2); |
3166 | |
3167 | // If we don't know any bits, early out. |
3168 | if (Known.isUnknown()) |
3169 | break; |
3170 | } |
3171 | break; |
3172 | case ISD::VECTOR_SHUFFLE: { |
3173 | assert(!Op.getValueType().isScalableVector()); |
3174 | // Collect the known bits that are shared by every vector element referenced |
3175 | // by the shuffle. |
3176 | APInt DemandedLHS, DemandedRHS; |
3177 | const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op); |
3178 | assert(NumElts == SVN->getMask().size() && "Unexpected vector size" ); |
3179 | if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts, |
3180 | DemandedLHS, DemandedRHS)) |
3181 | break; |
3182 | |
3183 | // Known bits are the values that are shared by every demanded element. |
3184 | Known.Zero.setAllBits(); Known.One.setAllBits(); |
3185 | if (!!DemandedLHS) { |
3186 | SDValue LHS = Op.getOperand(i: 0); |
3187 | Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1); |
3188 | Known = Known.intersectWith(RHS: Known2); |
3189 | } |
3190 | // If we don't know any bits, early out. |
3191 | if (Known.isUnknown()) |
3192 | break; |
3193 | if (!!DemandedRHS) { |
3194 | SDValue RHS = Op.getOperand(i: 1); |
3195 | Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1); |
3196 | Known = Known.intersectWith(RHS: Known2); |
3197 | } |
3198 | break; |
3199 | } |
3200 | case ISD::VSCALE: { |
3201 | const Function &F = getMachineFunction().getFunction(); |
3202 | const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0); |
3203 | Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits(); |
3204 | break; |
3205 | } |
3206 | case ISD::CONCAT_VECTORS: { |
3207 | if (Op.getValueType().isScalableVector()) |
3208 | break; |
3209 | // Split DemandedElts and test each of the demanded subvectors. |
3210 | Known.Zero.setAllBits(); Known.One.setAllBits(); |
3211 | EVT SubVectorVT = Op.getOperand(i: 0).getValueType(); |
3212 | unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
3213 | unsigned NumSubVectors = Op.getNumOperands(); |
3214 | for (unsigned i = 0; i != NumSubVectors; ++i) { |
3215 | APInt DemandedSub = |
3216 | DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts); |
3217 | if (!!DemandedSub) { |
3218 | SDValue Sub = Op.getOperand(i); |
3219 | Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1); |
3220 | Known = Known.intersectWith(RHS: Known2); |
3221 | } |
3222 | // If we don't know any bits, early out. |
3223 | if (Known.isUnknown()) |
3224 | break; |
3225 | } |
3226 | break; |
3227 | } |
3228 | case ISD::INSERT_SUBVECTOR: { |
3229 | if (Op.getValueType().isScalableVector()) |
3230 | break; |
3231 | // Demand any elements from the subvector and the remainder from the src its |
3232 | // inserted into. |
3233 | SDValue Src = Op.getOperand(i: 0); |
3234 | SDValue Sub = Op.getOperand(i: 1); |
3235 | uint64_t Idx = Op.getConstantOperandVal(i: 2); |
3236 | unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
3237 | APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx); |
3238 | APInt DemandedSrcElts = DemandedElts; |
3239 | DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx); |
3240 | |
3241 | Known.One.setAllBits(); |
3242 | Known.Zero.setAllBits(); |
3243 | if (!!DemandedSubElts) { |
3244 | Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1); |
3245 | if (Known.isUnknown()) |
3246 | break; // early-out. |
3247 | } |
3248 | if (!!DemandedSrcElts) { |
3249 | Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
3250 | Known = Known.intersectWith(RHS: Known2); |
3251 | } |
3252 | break; |
3253 | } |
3254 | case ISD::EXTRACT_SUBVECTOR: { |
3255 | // Offset the demanded elts by the subvector index. |
3256 | SDValue Src = Op.getOperand(i: 0); |
3257 | // Bail until we can represent demanded elements for scalable vectors. |
3258 | if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector()) |
3259 | break; |
3260 | uint64_t Idx = Op.getConstantOperandVal(i: 1); |
3261 | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
3262 | APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx); |
3263 | Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
3264 | break; |
3265 | } |
3266 | case ISD::SCALAR_TO_VECTOR: { |
3267 | if (Op.getValueType().isScalableVector()) |
3268 | break; |
3269 | // We know about scalar_to_vector as much as we know about it source, |
3270 | // which becomes the first element of otherwise unknown vector. |
3271 | if (DemandedElts != 1) |
3272 | break; |
3273 | |
3274 | SDValue N0 = Op.getOperand(i: 0); |
3275 | Known = computeKnownBits(Op: N0, Depth: Depth + 1); |
3276 | if (N0.getValueSizeInBits() != BitWidth) |
3277 | Known = Known.trunc(BitWidth); |
3278 | |
3279 | break; |
3280 | } |
3281 | case ISD::BITCAST: { |
3282 | if (Op.getValueType().isScalableVector()) |
3283 | break; |
3284 | |
3285 | SDValue N0 = Op.getOperand(i: 0); |
3286 | EVT SubVT = N0.getValueType(); |
3287 | unsigned SubBitWidth = SubVT.getScalarSizeInBits(); |
3288 | |
3289 | // Ignore bitcasts from unsupported types. |
3290 | if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) |
3291 | break; |
3292 | |
3293 | // Fast handling of 'identity' bitcasts. |
3294 | if (BitWidth == SubBitWidth) { |
3295 | Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1); |
3296 | break; |
3297 | } |
3298 | |
3299 | bool IsLE = getDataLayout().isLittleEndian(); |
3300 | |
3301 | // Bitcast 'small element' vector to 'large element' scalar/vector. |
3302 | if ((BitWidth % SubBitWidth) == 0) { |
3303 | assert(N0.getValueType().isVector() && "Expected bitcast from vector" ); |
3304 | |
3305 | // Collect known bits for the (larger) output by collecting the known |
3306 | // bits from each set of sub elements and shift these into place. |
3307 | // We need to separately call computeKnownBits for each set of |
3308 | // sub elements as the knownbits for each is likely to be different. |
3309 | unsigned SubScale = BitWidth / SubBitWidth; |
3310 | APInt SubDemandedElts(NumElts * SubScale, 0); |
3311 | for (unsigned i = 0; i != NumElts; ++i) |
3312 | if (DemandedElts[i]) |
3313 | SubDemandedElts.setBit(i * SubScale); |
3314 | |
3315 | for (unsigned i = 0; i != SubScale; ++i) { |
3316 | Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i), |
3317 | Depth: Depth + 1); |
3318 | unsigned Shifts = IsLE ? i : SubScale - 1 - i; |
3319 | Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts); |
3320 | } |
3321 | } |
3322 | |
3323 | // Bitcast 'large element' scalar/vector to 'small element' vector. |
3324 | if ((SubBitWidth % BitWidth) == 0) { |
3325 | assert(Op.getValueType().isVector() && "Expected bitcast to vector" ); |
3326 | |
3327 | // Collect known bits for the (smaller) output by collecting the known |
3328 | // bits from the overlapping larger input elements and extracting the |
3329 | // sub sections we actually care about. |
3330 | unsigned SubScale = SubBitWidth / BitWidth; |
3331 | APInt SubDemandedElts = |
3332 | APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale); |
3333 | Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1); |
3334 | |
3335 | Known.Zero.setAllBits(); Known.One.setAllBits(); |
3336 | for (unsigned i = 0; i != NumElts; ++i) |
3337 | if (DemandedElts[i]) { |
3338 | unsigned Shifts = IsLE ? i : NumElts - 1 - i; |
3339 | unsigned Offset = (Shifts % SubScale) * BitWidth; |
3340 | Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset)); |
3341 | // If we don't know any bits, early out. |
3342 | if (Known.isUnknown()) |
3343 | break; |
3344 | } |
3345 | } |
3346 | break; |
3347 | } |
3348 | case ISD::AND: |
3349 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3350 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3351 | |
3352 | Known &= Known2; |
3353 | break; |
3354 | case ISD::OR: |
3355 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3356 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3357 | |
3358 | Known |= Known2; |
3359 | break; |
3360 | case ISD::XOR: |
3361 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3362 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3363 | |
3364 | Known ^= Known2; |
3365 | break; |
3366 | case ISD::MUL: { |
3367 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3368 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3369 | bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1); |
3370 | // TODO: SelfMultiply can be poison, but not undef. |
3371 | if (SelfMultiply) |
3372 | SelfMultiply &= isGuaranteedNotToBeUndefOrPoison( |
3373 | Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1); |
3374 | Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply); |
3375 | |
3376 | // If the multiplication is known not to overflow, the product of a number |
3377 | // with itself is non-negative. Only do this if we didn't already computed |
3378 | // the opposite value for the sign bit. |
3379 | if (Op->getFlags().hasNoSignedWrap() && |
3380 | Op.getOperand(i: 0) == Op.getOperand(i: 1) && |
3381 | !Known.isNegative()) |
3382 | Known.makeNonNegative(); |
3383 | break; |
3384 | } |
3385 | case ISD::MULHU: { |
3386 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3387 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3388 | Known = KnownBits::mulhu(LHS: Known, RHS: Known2); |
3389 | break; |
3390 | } |
3391 | case ISD::MULHS: { |
3392 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3393 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3394 | Known = KnownBits::mulhs(LHS: Known, RHS: Known2); |
3395 | break; |
3396 | } |
3397 | case ISD::UMUL_LOHI: { |
3398 | assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result" ); |
3399 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3400 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3401 | bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1); |
3402 | if (Op.getResNo() == 0) |
3403 | Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply); |
3404 | else |
3405 | Known = KnownBits::mulhu(LHS: Known, RHS: Known2); |
3406 | break; |
3407 | } |
3408 | case ISD::SMUL_LOHI: { |
3409 | assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result" ); |
3410 | Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3411 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3412 | bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1); |
3413 | if (Op.getResNo() == 0) |
3414 | Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply); |
3415 | else |
3416 | Known = KnownBits::mulhs(LHS: Known, RHS: Known2); |
3417 | break; |
3418 | } |
3419 | case ISD::AVGCEILU: { |
3420 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3421 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3422 | Known = Known.zext(BitWidth: BitWidth + 1); |
3423 | Known2 = Known2.zext(BitWidth: BitWidth + 1); |
3424 | KnownBits One = KnownBits::makeConstant(C: APInt(1, 1)); |
3425 | Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry: One); |
3426 | Known = Known.extractBits(NumBits: BitWidth, BitPosition: 1); |
3427 | break; |
3428 | } |
3429 | case ISD::SELECT: |
3430 | case ISD::VSELECT: |
3431 | Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1); |
3432 | // If we don't know any bits, early out. |
3433 | if (Known.isUnknown()) |
3434 | break; |
3435 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1); |
3436 | |
3437 | // Only known if known in both the LHS and RHS. |
3438 | Known = Known.intersectWith(RHS: Known2); |
3439 | break; |
3440 | case ISD::SELECT_CC: |
3441 | Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1); |
3442 | // If we don't know any bits, early out. |
3443 | if (Known.isUnknown()) |
3444 | break; |
3445 | Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1); |
3446 | |
3447 | // Only known if known in both the LHS and RHS. |
3448 | Known = Known.intersectWith(RHS: Known2); |
3449 | break; |
3450 | case ISD::SMULO: |
3451 | case ISD::UMULO: |
3452 | if (Op.getResNo() != 1) |
3453 | break; |
3454 | // The boolean result conforms to getBooleanContents. |
3455 | // If we know the result of a setcc has the top bits zero, use this info. |
3456 | // We know that we have an integer-based boolean since these operations |
3457 | // are only available for integer. |
3458 | if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) == |
3459 | TargetLowering::ZeroOrOneBooleanContent && |
3460 | BitWidth > 1) |
3461 | Known.Zero.setBitsFrom(1); |
3462 | break; |
3463 | case ISD::SETCC: |
3464 | case ISD::SETCCCARRY: |
3465 | case ISD::STRICT_FSETCC: |
3466 | case ISD::STRICT_FSETCCS: { |
3467 | unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
3468 | // If we know the result of a setcc has the top bits zero, use this info. |
3469 | if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) == |
3470 | TargetLowering::ZeroOrOneBooleanContent && |
3471 | BitWidth > 1) |
3472 | Known.Zero.setBitsFrom(1); |
3473 | break; |
3474 | } |
3475 | case ISD::SHL: |
3476 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3477 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3478 | Known = KnownBits::shl(LHS: Known, RHS: Known2); |
3479 | |
3480 | // Minimum shift low bits are known zero. |
3481 | if (const APInt *ShMinAmt = |
3482 | getValidMinimumShiftAmountConstant(V: Op, DemandedElts)) |
3483 | Known.Zero.setLowBits(ShMinAmt->getZExtValue()); |
3484 | break; |
3485 | case ISD::SRL: |
3486 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3487 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3488 | Known = KnownBits::lshr(LHS: Known, RHS: Known2); |
3489 | |
3490 | // Minimum shift high bits are known zero. |
3491 | if (const APInt *ShMinAmt = |
3492 | getValidMinimumShiftAmountConstant(V: Op, DemandedElts)) |
3493 | Known.Zero.setHighBits(ShMinAmt->getZExtValue()); |
3494 | break; |
3495 | case ISD::SRA: |
3496 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3497 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3498 | Known = KnownBits::ashr(LHS: Known, RHS: Known2); |
3499 | break; |
3500 | case ISD::FSHL: |
3501 | case ISD::FSHR: |
3502 | if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) { |
3503 | unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth); |
3504 | |
3505 | // For fshl, 0-shift returns the 1st arg. |
3506 | // For fshr, 0-shift returns the 2nd arg. |
3507 | if (Amt == 0) { |
3508 | Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1), |
3509 | DemandedElts, Depth: Depth + 1); |
3510 | break; |
3511 | } |
3512 | |
3513 | // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) |
3514 | // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) |
3515 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3516 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3517 | if (Opcode == ISD::FSHL) { |
3518 | Known.One <<= Amt; |
3519 | Known.Zero <<= Amt; |
3520 | Known2.One.lshrInPlace(ShiftAmt: BitWidth - Amt); |
3521 | Known2.Zero.lshrInPlace(ShiftAmt: BitWidth - Amt); |
3522 | } else { |
3523 | Known.One <<= BitWidth - Amt; |
3524 | Known.Zero <<= BitWidth - Amt; |
3525 | Known2.One.lshrInPlace(ShiftAmt: Amt); |
3526 | Known2.Zero.lshrInPlace(ShiftAmt: Amt); |
3527 | } |
3528 | Known = Known.unionWith(RHS: Known2); |
3529 | } |
3530 | break; |
3531 | case ISD::SHL_PARTS: |
3532 | case ISD::SRA_PARTS: |
3533 | case ISD::SRL_PARTS: { |
3534 | assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result" ); |
3535 | |
3536 | // Collect lo/hi source values and concatenate. |
3537 | unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits(); |
3538 | unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits(); |
3539 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3540 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3541 | Known = Known2.concat(Lo: Known); |
3542 | |
3543 | // Collect shift amount. |
3544 | Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1); |
3545 | |
3546 | if (Opcode == ISD::SHL_PARTS) |
3547 | Known = KnownBits::shl(LHS: Known, RHS: Known2); |
3548 | else if (Opcode == ISD::SRA_PARTS) |
3549 | Known = KnownBits::ashr(LHS: Known, RHS: Known2); |
3550 | else // if (Opcode == ISD::SRL_PARTS) |
3551 | Known = KnownBits::lshr(LHS: Known, RHS: Known2); |
3552 | |
3553 | // TODO: Minimum shift low/high bits are known zero. |
3554 | |
3555 | if (Op.getResNo() == 0) |
3556 | Known = Known.extractBits(NumBits: LoBits, BitPosition: 0); |
3557 | else |
3558 | Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits); |
3559 | break; |
3560 | } |
3561 | case ISD::SIGN_EXTEND_INREG: { |
3562 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3563 | EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT(); |
3564 | Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits()); |
3565 | break; |
3566 | } |
3567 | case ISD::CTTZ: |
3568 | case ISD::CTTZ_ZERO_UNDEF: { |
3569 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3570 | // If we have a known 1, its position is our upper bound. |
3571 | unsigned PossibleTZ = Known2.countMaxTrailingZeros(); |
3572 | unsigned LowBits = llvm::bit_width(Value: PossibleTZ); |
3573 | Known.Zero.setBitsFrom(LowBits); |
3574 | break; |
3575 | } |
3576 | case ISD::CTLZ: |
3577 | case ISD::CTLZ_ZERO_UNDEF: { |
3578 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3579 | // If we have a known 1, its position is our upper bound. |
3580 | unsigned PossibleLZ = Known2.countMaxLeadingZeros(); |
3581 | unsigned LowBits = llvm::bit_width(Value: PossibleLZ); |
3582 | Known.Zero.setBitsFrom(LowBits); |
3583 | break; |
3584 | } |
3585 | case ISD::CTPOP: { |
3586 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3587 | // If we know some of the bits are zero, they can't be one. |
3588 | unsigned PossibleOnes = Known2.countMaxPopulation(); |
3589 | Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes)); |
3590 | break; |
3591 | } |
3592 | case ISD::PARITY: { |
3593 | // Parity returns 0 everywhere but the LSB. |
3594 | Known.Zero.setBitsFrom(1); |
3595 | break; |
3596 | } |
3597 | case ISD::LOAD: { |
3598 | LoadSDNode *LD = cast<LoadSDNode>(Val&: Op); |
3599 | const Constant *Cst = TLI->getTargetConstantFromLoad(LD); |
3600 | if (ISD::isNON_EXTLoad(N: LD) && Cst) { |
3601 | // Determine any common known bits from the loaded constant pool value. |
3602 | Type *CstTy = Cst->getType(); |
3603 | if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() && |
3604 | !Op.getValueType().isScalableVector()) { |
3605 | // If its a vector splat, then we can (quickly) reuse the scalar path. |
3606 | // NOTE: We assume all elements match and none are UNDEF. |
3607 | if (CstTy->isVectorTy()) { |
3608 | if (const Constant *Splat = Cst->getSplatValue()) { |
3609 | Cst = Splat; |
3610 | CstTy = Cst->getType(); |
3611 | } |
3612 | } |
3613 | // TODO - do we need to handle different bitwidths? |
3614 | if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { |
3615 | // Iterate across all vector elements finding common known bits. |
3616 | Known.One.setAllBits(); |
3617 | Known.Zero.setAllBits(); |
3618 | for (unsigned i = 0; i != NumElts; ++i) { |
3619 | if (!DemandedElts[i]) |
3620 | continue; |
3621 | if (Constant *Elt = Cst->getAggregateElement(Elt: i)) { |
3622 | if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) { |
3623 | const APInt &Value = CInt->getValue(); |
3624 | Known.One &= Value; |
3625 | Known.Zero &= ~Value; |
3626 | continue; |
3627 | } |
3628 | if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) { |
3629 | APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
3630 | Known.One &= Value; |
3631 | Known.Zero &= ~Value; |
3632 | continue; |
3633 | } |
3634 | } |
3635 | Known.One.clearAllBits(); |
3636 | Known.Zero.clearAllBits(); |
3637 | break; |
3638 | } |
3639 | } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { |
3640 | if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) { |
3641 | Known = KnownBits::makeConstant(C: CInt->getValue()); |
3642 | } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) { |
3643 | Known = |
3644 | KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt()); |
3645 | } |
3646 | } |
3647 | } |
3648 | } else if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) { |
3649 | // If this is a ZEXTLoad and we are looking at the loaded value. |
3650 | EVT VT = LD->getMemoryVT(); |
3651 | unsigned MemBits = VT.getScalarSizeInBits(); |
3652 | Known.Zero.setBitsFrom(MemBits); |
3653 | } else if (const MDNode *Ranges = LD->getRanges()) { |
3654 | EVT VT = LD->getValueType(ResNo: 0); |
3655 | |
3656 | // TODO: Handle for extending loads |
3657 | if (LD->getExtensionType() == ISD::NON_EXTLOAD) { |
3658 | if (VT.isVector()) { |
3659 | // Handle truncation to the first demanded element. |
3660 | // TODO: Figure out which demanded elements are covered |
3661 | if (DemandedElts != 1 || !getDataLayout().isLittleEndian()) |
3662 | break; |
3663 | |
3664 | // Handle the case where a load has a vector type, but scalar memory |
3665 | // with an attached range. |
3666 | EVT MemVT = LD->getMemoryVT(); |
3667 | KnownBits KnownFull(MemVT.getSizeInBits()); |
3668 | |
3669 | computeKnownBitsFromRangeMetadata(Ranges: *Ranges, Known&: KnownFull); |
3670 | Known = KnownFull.trunc(BitWidth); |
3671 | } else |
3672 | computeKnownBitsFromRangeMetadata(Ranges: *Ranges, Known); |
3673 | } |
3674 | } |
3675 | break; |
3676 | } |
3677 | case ISD::ZERO_EXTEND_VECTOR_INREG: { |
3678 | if (Op.getValueType().isScalableVector()) |
3679 | break; |
3680 | EVT InVT = Op.getOperand(i: 0).getValueType(); |
3681 | APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements()); |
3682 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1); |
3683 | Known = Known.zext(BitWidth); |
3684 | break; |
3685 | } |
3686 | case ISD::ZERO_EXTEND: { |
3687 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3688 | Known = Known.zext(BitWidth); |
3689 | break; |
3690 | } |
3691 | case ISD::SIGN_EXTEND_VECTOR_INREG: { |
3692 | if (Op.getValueType().isScalableVector()) |
3693 | break; |
3694 | EVT InVT = Op.getOperand(i: 0).getValueType(); |
3695 | APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements()); |
3696 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1); |
3697 | // If the sign bit is known to be zero or one, then sext will extend |
3698 | // it to the top bits, else it will just zext. |
3699 | Known = Known.sext(BitWidth); |
3700 | break; |
3701 | } |
3702 | case ISD::SIGN_EXTEND: { |
3703 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3704 | // If the sign bit is known to be zero or one, then sext will extend |
3705 | // it to the top bits, else it will just zext. |
3706 | Known = Known.sext(BitWidth); |
3707 | break; |
3708 | } |
3709 | case ISD::ANY_EXTEND_VECTOR_INREG: { |
3710 | if (Op.getValueType().isScalableVector()) |
3711 | break; |
3712 | EVT InVT = Op.getOperand(i: 0).getValueType(); |
3713 | APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements()); |
3714 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1); |
3715 | Known = Known.anyext(BitWidth); |
3716 | break; |
3717 | } |
3718 | case ISD::ANY_EXTEND: { |
3719 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3720 | Known = Known.anyext(BitWidth); |
3721 | break; |
3722 | } |
3723 | case ISD::TRUNCATE: { |
3724 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3725 | Known = Known.trunc(BitWidth); |
3726 | break; |
3727 | } |
3728 | case ISD::AssertZext: { |
3729 | EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT(); |
3730 | APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits()); |
3731 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3732 | Known.Zero |= (~InMask); |
3733 | Known.One &= (~Known.Zero); |
3734 | break; |
3735 | } |
3736 | case ISD::AssertAlign: { |
3737 | unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign()); |
3738 | assert(LogOfAlign != 0); |
3739 | |
3740 | // TODO: Should use maximum with source |
3741 | // If a node is guaranteed to be aligned, set low zero bits accordingly as |
3742 | // well as clearing one bits. |
3743 | Known.Zero.setLowBits(LogOfAlign); |
3744 | Known.One.clearLowBits(loBits: LogOfAlign); |
3745 | break; |
3746 | } |
3747 | case ISD::FGETSIGN: |
3748 | // All bits are zero except the low bit. |
3749 | Known.Zero.setBitsFrom(1); |
3750 | break; |
3751 | case ISD::ADD: |
3752 | case ISD::SUB: { |
3753 | SDNodeFlags Flags = Op.getNode()->getFlags(); |
3754 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3755 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3756 | Known = KnownBits::computeForAddSub(Add: Op.getOpcode() == ISD::ADD, |
3757 | NSW: Flags.hasNoSignedWrap(), LHS: Known, RHS: Known2); |
3758 | break; |
3759 | } |
3760 | case ISD::USUBO: |
3761 | case ISD::SSUBO: |
3762 | case ISD::USUBO_CARRY: |
3763 | case ISD::SSUBO_CARRY: |
3764 | if (Op.getResNo() == 1) { |
3765 | // If we know the result of a setcc has the top bits zero, use this info. |
3766 | if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) == |
3767 | TargetLowering::ZeroOrOneBooleanContent && |
3768 | BitWidth > 1) |
3769 | Known.Zero.setBitsFrom(1); |
3770 | break; |
3771 | } |
3772 | [[fallthrough]]; |
3773 | case ISD::SUBC: { |
3774 | assert(Op.getResNo() == 0 && |
3775 | "We only compute knownbits for the difference here." ); |
3776 | |
3777 | // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in. |
3778 | KnownBits Borrow(1); |
3779 | if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) { |
3780 | Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1); |
3781 | // Borrow has bit width 1 |
3782 | Borrow = Borrow.trunc(BitWidth: 1); |
3783 | } else { |
3784 | Borrow.setAllZero(); |
3785 | } |
3786 | |
3787 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3788 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3789 | Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow); |
3790 | break; |
3791 | } |
3792 | case ISD::UADDO: |
3793 | case ISD::SADDO: |
3794 | case ISD::UADDO_CARRY: |
3795 | case ISD::SADDO_CARRY: |
3796 | if (Op.getResNo() == 1) { |
3797 | // If we know the result of a setcc has the top bits zero, use this info. |
3798 | if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) == |
3799 | TargetLowering::ZeroOrOneBooleanContent && |
3800 | BitWidth > 1) |
3801 | Known.Zero.setBitsFrom(1); |
3802 | break; |
3803 | } |
3804 | [[fallthrough]]; |
3805 | case ISD::ADDC: |
3806 | case ISD::ADDE: { |
3807 | assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here." ); |
3808 | |
3809 | // With ADDE and UADDO_CARRY, a carry bit may be added in. |
3810 | KnownBits Carry(1); |
3811 | if (Opcode == ISD::ADDE) |
3812 | // Can't track carry from glue, set carry to unknown. |
3813 | Carry.resetAll(); |
3814 | else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) { |
3815 | Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1); |
3816 | // Carry has bit width 1 |
3817 | Carry = Carry.trunc(BitWidth: 1); |
3818 | } else { |
3819 | Carry.setAllZero(); |
3820 | } |
3821 | |
3822 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3823 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3824 | Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry); |
3825 | break; |
3826 | } |
3827 | case ISD::UDIV: { |
3828 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3829 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3830 | Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact()); |
3831 | break; |
3832 | } |
3833 | case ISD::SDIV: { |
3834 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3835 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3836 | Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact()); |
3837 | break; |
3838 | } |
3839 | case ISD::SREM: { |
3840 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3841 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3842 | Known = KnownBits::srem(LHS: Known, RHS: Known2); |
3843 | break; |
3844 | } |
3845 | case ISD::UREM: { |
3846 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3847 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3848 | Known = KnownBits::urem(LHS: Known, RHS: Known2); |
3849 | break; |
3850 | } |
3851 | case ISD::EXTRACT_ELEMENT: { |
3852 | Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1); |
3853 | const unsigned Index = Op.getConstantOperandVal(i: 1); |
3854 | const unsigned EltBitWidth = Op.getValueSizeInBits(); |
3855 | |
3856 | // Remove low part of known bits mask |
3857 | Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth); |
3858 | Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth); |
3859 | |
3860 | // Remove high part of known bit mask |
3861 | Known = Known.trunc(BitWidth: EltBitWidth); |
3862 | break; |
3863 | } |
3864 | case ISD::EXTRACT_VECTOR_ELT: { |
3865 | SDValue InVec = Op.getOperand(i: 0); |
3866 | SDValue EltNo = Op.getOperand(i: 1); |
3867 | EVT VecVT = InVec.getValueType(); |
3868 | // computeKnownBits not yet implemented for scalable vectors. |
3869 | if (VecVT.isScalableVector()) |
3870 | break; |
3871 | const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); |
3872 | const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
3873 | |
3874 | // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know |
3875 | // anything about the extended bits. |
3876 | if (BitWidth > EltBitWidth) |
3877 | Known = Known.trunc(BitWidth: EltBitWidth); |
3878 | |
3879 | // If we know the element index, just demand that vector element, else for |
3880 | // an unknown element index, ignore DemandedElts and demand them all. |
3881 | APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts); |
3882 | auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo); |
3883 | if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)) |
3884 | DemandedSrcElts = |
3885 | APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue()); |
3886 | |
3887 | Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
3888 | if (BitWidth > EltBitWidth) |
3889 | Known = Known.anyext(BitWidth); |
3890 | break; |
3891 | } |
3892 | case ISD::INSERT_VECTOR_ELT: { |
3893 | if (Op.getValueType().isScalableVector()) |
3894 | break; |
3895 | |
3896 | // If we know the element index, split the demand between the |
3897 | // source vector and the inserted element, otherwise assume we need |
3898 | // the original demanded vector elements and the value. |
3899 | SDValue InVec = Op.getOperand(i: 0); |
3900 | SDValue InVal = Op.getOperand(i: 1); |
3901 | SDValue EltNo = Op.getOperand(i: 2); |
3902 | bool DemandedVal = true; |
3903 | APInt DemandedVecElts = DemandedElts; |
3904 | auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo); |
3905 | if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) { |
3906 | unsigned EltIdx = CEltNo->getZExtValue(); |
3907 | DemandedVal = !!DemandedElts[EltIdx]; |
3908 | DemandedVecElts.clearBit(BitPosition: EltIdx); |
3909 | } |
3910 | Known.One.setAllBits(); |
3911 | Known.Zero.setAllBits(); |
3912 | if (DemandedVal) { |
3913 | Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1); |
3914 | Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth)); |
3915 | } |
3916 | if (!!DemandedVecElts) { |
3917 | Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1); |
3918 | Known = Known.intersectWith(RHS: Known2); |
3919 | } |
3920 | break; |
3921 | } |
3922 | case ISD::BITREVERSE: { |
3923 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3924 | Known = Known2.reverseBits(); |
3925 | break; |
3926 | } |
3927 | case ISD::BSWAP: { |
3928 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3929 | Known = Known2.byteSwap(); |
3930 | break; |
3931 | } |
3932 | case ISD::ABS: { |
3933 | Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3934 | Known = Known2.abs(); |
3935 | break; |
3936 | } |
3937 | case ISD::USUBSAT: { |
3938 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3939 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3940 | Known = KnownBits::usub_sat(LHS: Known, RHS: Known2); |
3941 | break; |
3942 | } |
3943 | case ISD::UMIN: { |
3944 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3945 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3946 | Known = KnownBits::umin(LHS: Known, RHS: Known2); |
3947 | break; |
3948 | } |
3949 | case ISD::UMAX: { |
3950 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3951 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3952 | Known = KnownBits::umax(LHS: Known, RHS: Known2); |
3953 | break; |
3954 | } |
3955 | case ISD::SMIN: |
3956 | case ISD::SMAX: { |
3957 | // If we have a clamp pattern, we know that the number of sign bits will be |
3958 | // the minimum of the clamp min/max range. |
3959 | bool IsMax = (Opcode == ISD::SMAX); |
3960 | ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
3961 | if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))) |
3962 | if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
3963 | CstHigh = |
3964 | isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts); |
3965 | if (CstLow && CstHigh) { |
3966 | if (!IsMax) |
3967 | std::swap(a&: CstLow, b&: CstHigh); |
3968 | |
3969 | const APInt &ValueLow = CstLow->getAPIntValue(); |
3970 | const APInt &ValueHigh = CstHigh->getAPIntValue(); |
3971 | if (ValueLow.sle(RHS: ValueHigh)) { |
3972 | unsigned LowSignBits = ValueLow.getNumSignBits(); |
3973 | unsigned HighSignBits = ValueHigh.getNumSignBits(); |
3974 | unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits); |
3975 | if (ValueLow.isNegative() && ValueHigh.isNegative()) { |
3976 | Known.One.setHighBits(MinSignBits); |
3977 | break; |
3978 | } |
3979 | if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { |
3980 | Known.Zero.setHighBits(MinSignBits); |
3981 | break; |
3982 | } |
3983 | } |
3984 | } |
3985 | |
3986 | Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
3987 | Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
3988 | if (IsMax) |
3989 | Known = KnownBits::smax(LHS: Known, RHS: Known2); |
3990 | else |
3991 | Known = KnownBits::smin(LHS: Known, RHS: Known2); |
3992 | |
3993 | // For SMAX, if CstLow is non-negative we know the result will be |
3994 | // non-negative and thus all sign bits are 0. |
3995 | // TODO: There's an equivalent of this for smin with negative constant for |
3996 | // known ones. |
3997 | if (IsMax && CstLow) { |
3998 | const APInt &ValueLow = CstLow->getAPIntValue(); |
3999 | if (ValueLow.isNonNegative()) { |
4000 | unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
4001 | Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits())); |
4002 | } |
4003 | } |
4004 | |
4005 | break; |
4006 | } |
4007 | case ISD::FP_TO_UINT_SAT: { |
4008 | // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT. |
4009 | EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT(); |
4010 | Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits()); |
4011 | break; |
4012 | } |
4013 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
4014 | if (Op.getResNo() == 1) { |
4015 | // The boolean result conforms to getBooleanContents. |
4016 | // If we know the result of a setcc has the top bits zero, use this info. |
4017 | // We know that we have an integer-based boolean since these operations |
4018 | // are only available for integer. |
4019 | if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) == |
4020 | TargetLowering::ZeroOrOneBooleanContent && |
4021 | BitWidth > 1) |
4022 | Known.Zero.setBitsFrom(1); |
4023 | break; |
4024 | } |
4025 | [[fallthrough]]; |
4026 | case ISD::ATOMIC_CMP_SWAP: |
4027 | case ISD::ATOMIC_SWAP: |
4028 | case ISD::ATOMIC_LOAD_ADD: |
4029 | case ISD::ATOMIC_LOAD_SUB: |
4030 | case ISD::ATOMIC_LOAD_AND: |
4031 | case ISD::ATOMIC_LOAD_CLR: |
4032 | case ISD::ATOMIC_LOAD_OR: |
4033 | case ISD::ATOMIC_LOAD_XOR: |
4034 | case ISD::ATOMIC_LOAD_NAND: |
4035 | case ISD::ATOMIC_LOAD_MIN: |
4036 | case ISD::ATOMIC_LOAD_MAX: |
4037 | case ISD::ATOMIC_LOAD_UMIN: |
4038 | case ISD::ATOMIC_LOAD_UMAX: |
4039 | case ISD::ATOMIC_LOAD: { |
4040 | unsigned MemBits = |
4041 | cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits(); |
4042 | // If we are looking at the loaded value. |
4043 | if (Op.getResNo() == 0) { |
4044 | if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
4045 | Known.Zero.setBitsFrom(MemBits); |
4046 | } |
4047 | break; |
4048 | } |
4049 | case ISD::FrameIndex: |
4050 | case ISD::TargetFrameIndex: |
4051 | TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(), |
4052 | Known, MF: getMachineFunction()); |
4053 | break; |
4054 | |
4055 | default: |
4056 | if (Opcode < ISD::BUILTIN_OP_END) |
4057 | break; |
4058 | [[fallthrough]]; |
4059 | case ISD::INTRINSIC_WO_CHAIN: |
4060 | case ISD::INTRINSIC_W_CHAIN: |
4061 | case ISD::INTRINSIC_VOID: |
4062 | // TODO: Probably okay to remove after audit; here to reduce change size |
4063 | // in initial enablement patch for scalable vectors |
4064 | if (Op.getValueType().isScalableVector()) |
4065 | break; |
4066 | |
4067 | // Allow the target to implement this method for its nodes. |
4068 | TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth); |
4069 | break; |
4070 | } |
4071 | |
4072 | assert(!Known.hasConflict() && "Bits known to be one AND zero?" ); |
4073 | return Known; |
4074 | } |
4075 | |
4076 | /// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind. |
4077 | static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) { |
4078 | switch (OR) { |
4079 | case ConstantRange::OverflowResult::MayOverflow: |
4080 | return SelectionDAG::OFK_Sometime; |
4081 | case ConstantRange::OverflowResult::AlwaysOverflowsLow: |
4082 | case ConstantRange::OverflowResult::AlwaysOverflowsHigh: |
4083 | return SelectionDAG::OFK_Always; |
4084 | case ConstantRange::OverflowResult::NeverOverflows: |
4085 | return SelectionDAG::OFK_Never; |
4086 | } |
4087 | llvm_unreachable("Unknown OverflowResult" ); |
4088 | } |
4089 | |
4090 | SelectionDAG::OverflowKind |
4091 | SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const { |
4092 | // X + 0 never overflow |
4093 | if (isNullConstant(V: N1)) |
4094 | return OFK_Never; |
4095 | |
4096 | // If both operands each have at least two sign bits, the addition |
4097 | // cannot overflow. |
4098 | if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1) |
4099 | return OFK_Never; |
4100 | |
4101 | // TODO: Add ConstantRange::signedAddMayOverflow handling. |
4102 | return OFK_Sometime; |
4103 | } |
4104 | |
4105 | SelectionDAG::OverflowKind |
4106 | SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const { |
4107 | // X + 0 never overflow |
4108 | if (isNullConstant(V: N1)) |
4109 | return OFK_Never; |
4110 | |
4111 | // mulhi + 1 never overflow |
4112 | KnownBits N1Known = computeKnownBits(Op: N1); |
4113 | if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && |
4114 | N1Known.getMaxValue().ult(RHS: 2)) |
4115 | return OFK_Never; |
4116 | |
4117 | KnownBits N0Known = computeKnownBits(Op: N0); |
4118 | if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 && |
4119 | N0Known.getMaxValue().ult(RHS: 2)) |
4120 | return OFK_Never; |
4121 | |
4122 | // Fallback to ConstantRange::unsignedAddMayOverflow handling. |
4123 | ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false); |
4124 | ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false); |
4125 | return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range)); |
4126 | } |
4127 | |
4128 | SelectionDAG::OverflowKind |
4129 | SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const { |
4130 | // X - 0 never overflow |
4131 | if (isNullConstant(V: N1)) |
4132 | return OFK_Never; |
4133 | |
4134 | // If both operands each have at least two sign bits, the subtraction |
4135 | // cannot overflow. |
4136 | if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1) |
4137 | return OFK_Never; |
4138 | |
4139 | KnownBits N0Known = computeKnownBits(Op: N0); |
4140 | KnownBits N1Known = computeKnownBits(Op: N1); |
4141 | ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true); |
4142 | ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true); |
4143 | return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range)); |
4144 | } |
4145 | |
4146 | SelectionDAG::OverflowKind |
4147 | SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const { |
4148 | // X - 0 never overflow |
4149 | if (isNullConstant(V: N1)) |
4150 | return OFK_Never; |
4151 | |
4152 | KnownBits N0Known = computeKnownBits(Op: N0); |
4153 | KnownBits N1Known = computeKnownBits(Op: N1); |
4154 | ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false); |
4155 | ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false); |
4156 | return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range)); |
4157 | } |
4158 | |
4159 | SelectionDAG::OverflowKind |
4160 | SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const { |
4161 | // X * 0 and X * 1 never overflow. |
4162 | if (isNullConstant(V: N1) || isOneConstant(V: N1)) |
4163 | return OFK_Never; |
4164 | |
4165 | KnownBits N0Known = computeKnownBits(Op: N0); |
4166 | KnownBits N1Known = computeKnownBits(Op: N1); |
4167 | ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false); |
4168 | ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false); |
4169 | return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range)); |
4170 | } |
4171 | |
4172 | SelectionDAG::OverflowKind |
4173 | SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const { |
4174 | // X * 0 and X * 1 never overflow. |
4175 | if (isNullConstant(V: N1) || isOneConstant(V: N1)) |
4176 | return OFK_Never; |
4177 | |
4178 | // Get the size of the result. |
4179 | unsigned BitWidth = N0.getScalarValueSizeInBits(); |
4180 | |
4181 | // Sum of the sign bits. |
4182 | unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1); |
4183 | |
4184 | // If we have enough sign bits, then there's no overflow. |
4185 | if (SignBits > BitWidth + 1) |
4186 | return OFK_Never; |
4187 | |
4188 | if (SignBits == BitWidth + 1) { |
4189 | // The overflow occurs when the true multiplication of the |
4190 | // the operands is the minimum negative number. |
4191 | KnownBits N0Known = computeKnownBits(Op: N0); |
4192 | KnownBits N1Known = computeKnownBits(Op: N1); |
4193 | // If one of the operands is non-negative, then there's no |
4194 | // overflow. |
4195 | if (N0Known.isNonNegative() || N1Known.isNonNegative()) |
4196 | return OFK_Never; |
4197 | } |
4198 | |
4199 | return OFK_Sometime; |
4200 | } |
4201 | |
4202 | bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const { |
4203 | if (Depth >= MaxRecursionDepth) |
4204 | return false; // Limit search depth. |
4205 | |
4206 | EVT OpVT = Val.getValueType(); |
4207 | unsigned BitWidth = OpVT.getScalarSizeInBits(); |
4208 | |
4209 | // Is the constant a known power of 2? |
4210 | if (ISD::matchUnaryPredicate(Op: Val, Match: [BitWidth](ConstantSDNode *C) { |
4211 | return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2(); |
4212 | })) |
4213 | return true; |
4214 | |
4215 | // A left-shift of a constant one will have exactly one bit set because |
4216 | // shifting the bit off the end is undefined. |
4217 | if (Val.getOpcode() == ISD::SHL) { |
4218 | auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0)); |
4219 | if (C && C->getAPIntValue() == 1) |
4220 | return true; |
4221 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) && |
4222 | isKnownNeverZero(Op: Val, Depth); |
4223 | } |
4224 | |
4225 | // Similarly, a logical right-shift of a constant sign-bit will have exactly |
4226 | // one bit set. |
4227 | if (Val.getOpcode() == ISD::SRL) { |
4228 | auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0)); |
4229 | if (C && C->getAPIntValue().isSignMask()) |
4230 | return true; |
4231 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) && |
4232 | isKnownNeverZero(Op: Val, Depth); |
4233 | } |
4234 | |
4235 | if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR) |
4236 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1); |
4237 | |
4238 | // Are all operands of a build vector constant powers of two? |
4239 | if (Val.getOpcode() == ISD::BUILD_VECTOR) |
4240 | if (llvm::all_of(Range: Val->ops(), P: [BitWidth](SDValue E) { |
4241 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: E)) |
4242 | return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2(); |
4243 | return false; |
4244 | })) |
4245 | return true; |
4246 | |
4247 | // Is the operand of a splat vector a constant power of two? |
4248 | if (Val.getOpcode() == ISD::SPLAT_VECTOR) |
4249 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0))) |
4250 | if (C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2()) |
4251 | return true; |
4252 | |
4253 | // vscale(power-of-two) is a power-of-two for some targets |
4254 | if (Val.getOpcode() == ISD::VSCALE && |
4255 | getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() && |
4256 | isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1)) |
4257 | return true; |
4258 | |
4259 | if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX || |
4260 | Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX) |
4261 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1) && |
4262 | isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1); |
4263 | |
4264 | if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT) |
4265 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), Depth: Depth + 1) && |
4266 | isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1); |
4267 | |
4268 | if (Val.getOpcode() == ISD::AND) { |
4269 | // Looking for `x & -x` pattern: |
4270 | // If x == 0: |
4271 | // x & -x -> 0 |
4272 | // If x != 0: |
4273 | // x & -x -> non-zero pow2 |
4274 | // so if we find the pattern return whether we know `x` is non-zero. |
4275 | for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) { |
4276 | SDValue NegOp = Val.getOperand(i: OpIdx); |
4277 | if (NegOp.getOpcode() == ISD::SUB && |
4278 | NegOp.getOperand(i: 1) == Val.getOperand(i: 1 - OpIdx) && |
4279 | isNullOrNullSplat(V: NegOp.getOperand(i: 0))) |
4280 | return isKnownNeverZero(Op: Val.getOperand(i: 1 - OpIdx), Depth); |
4281 | } |
4282 | } |
4283 | |
4284 | if (Val.getOpcode() == ISD::ZERO_EXTEND) |
4285 | return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1); |
4286 | |
4287 | // More could be done here, though the above checks are enough |
4288 | // to handle some common cases. |
4289 | return false; |
4290 | } |
4291 | |
4292 | unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { |
4293 | EVT VT = Op.getValueType(); |
4294 | |
4295 | // Since the number of lanes in a scalable vector is unknown at compile time, |
4296 | // we track one bit which is implicitly broadcast to all lanes. This means |
4297 | // that all lanes in a scalable vector are considered demanded. |
4298 | APInt DemandedElts = VT.isFixedLengthVector() |
4299 | ? APInt::getAllOnes(numBits: VT.getVectorNumElements()) |
4300 | : APInt(1, 1); |
4301 | return ComputeNumSignBits(Op, DemandedElts, Depth); |
4302 | } |
4303 | |
4304 | unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, |
4305 | unsigned Depth) const { |
4306 | EVT VT = Op.getValueType(); |
4307 | assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!" ); |
4308 | unsigned VTBits = VT.getScalarSizeInBits(); |
4309 | unsigned NumElts = DemandedElts.getBitWidth(); |
4310 | unsigned Tmp, Tmp2; |
4311 | unsigned FirstAnswer = 1; |
4312 | |
4313 | if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) { |
4314 | const APInt &Val = C->getAPIntValue(); |
4315 | return Val.getNumSignBits(); |
4316 | } |
4317 | |
4318 | if (Depth >= MaxRecursionDepth) |
4319 | return 1; // Limit search depth. |
4320 | |
4321 | if (!DemandedElts) |
4322 | return 1; // No demanded elts, better to assume we don't know anything. |
4323 | |
4324 | unsigned Opcode = Op.getOpcode(); |
4325 | switch (Opcode) { |
4326 | default: break; |
4327 | case ISD::AssertSext: |
4328 | Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits(); |
4329 | return VTBits-Tmp+1; |
4330 | case ISD::AssertZext: |
4331 | Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits(); |
4332 | return VTBits-Tmp; |
4333 | case ISD::MERGE_VALUES: |
4334 | return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts, |
4335 | Depth: Depth + 1); |
4336 | case ISD::SPLAT_VECTOR: { |
4337 | // Check if the sign bits of source go down as far as the truncated value. |
4338 | unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits(); |
4339 | unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
4340 | if (NumSrcSignBits > (NumSrcBits - VTBits)) |
4341 | return NumSrcSignBits - (NumSrcBits - VTBits); |
4342 | break; |
4343 | } |
4344 | case ISD::BUILD_VECTOR: |
4345 | assert(!VT.isScalableVector()); |
4346 | Tmp = VTBits; |
4347 | for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { |
4348 | if (!DemandedElts[i]) |
4349 | continue; |
4350 | |
4351 | SDValue SrcOp = Op.getOperand(i); |
4352 | // BUILD_VECTOR can implicitly truncate sources, we handle this specially |
4353 | // for constant nodes to ensure we only look at the sign bits. |
4354 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) { |
4355 | APInt T = C->getAPIntValue().trunc(width: VTBits); |
4356 | Tmp2 = T.getNumSignBits(); |
4357 | } else { |
4358 | Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1); |
4359 | |
4360 | if (SrcOp.getValueSizeInBits() != VTBits) { |
4361 | assert(SrcOp.getValueSizeInBits() > VTBits && |
4362 | "Expected BUILD_VECTOR implicit truncation" ); |
4363 | unsigned = SrcOp.getValueSizeInBits() - VTBits; |
4364 | Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); |
4365 | } |
4366 | } |
4367 | Tmp = std::min(a: Tmp, b: Tmp2); |
4368 | } |
4369 | return Tmp; |
4370 | |
4371 | case ISD::VECTOR_SHUFFLE: { |
4372 | // Collect the minimum number of sign bits that are shared by every vector |
4373 | // element referenced by the shuffle. |
4374 | APInt DemandedLHS, DemandedRHS; |
4375 | const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op); |
4376 | assert(NumElts == SVN->getMask().size() && "Unexpected vector size" ); |
4377 | if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts, |
4378 | DemandedLHS, DemandedRHS)) |
4379 | return 1; |
4380 | |
4381 | Tmp = std::numeric_limits<unsigned>::max(); |
4382 | if (!!DemandedLHS) |
4383 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1); |
4384 | if (!!DemandedRHS) { |
4385 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1); |
4386 | Tmp = std::min(a: Tmp, b: Tmp2); |
4387 | } |
4388 | // If we don't know anything, early out and try computeKnownBits fall-back. |
4389 | if (Tmp == 1) |
4390 | break; |
4391 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits" ); |
4392 | return Tmp; |
4393 | } |
4394 | |
4395 | case ISD::BITCAST: { |
4396 | if (VT.isScalableVector()) |
4397 | break; |
4398 | SDValue N0 = Op.getOperand(i: 0); |
4399 | EVT SrcVT = N0.getValueType(); |
4400 | unsigned SrcBits = SrcVT.getScalarSizeInBits(); |
4401 | |
4402 | // Ignore bitcasts from unsupported types.. |
4403 | if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) |
4404 | break; |
4405 | |
4406 | // Fast handling of 'identity' bitcasts. |
4407 | if (VTBits == SrcBits) |
4408 | return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1); |
4409 | |
4410 | bool IsLE = getDataLayout().isLittleEndian(); |
4411 | |
4412 | // Bitcast 'large element' scalar/vector to 'small element' vector. |
4413 | if ((SrcBits % VTBits) == 0) { |
4414 | assert(VT.isVector() && "Expected bitcast to vector" ); |
4415 | |
4416 | unsigned Scale = SrcBits / VTBits; |
4417 | APInt SrcDemandedElts = |
4418 | APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale); |
4419 | |
4420 | // Fast case - sign splat can be simply split across the small elements. |
4421 | Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1); |
4422 | if (Tmp == SrcBits) |
4423 | return VTBits; |
4424 | |
4425 | // Slow case - determine how far the sign extends into each sub-element. |
4426 | Tmp2 = VTBits; |
4427 | for (unsigned i = 0; i != NumElts; ++i) |
4428 | if (DemandedElts[i]) { |
4429 | unsigned SubOffset = i % Scale; |
4430 | SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset); |
4431 | SubOffset = SubOffset * VTBits; |
4432 | if (Tmp <= SubOffset) |
4433 | return 1; |
4434 | Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset); |
4435 | } |
4436 | return Tmp2; |
4437 | } |
4438 | break; |
4439 | } |
4440 | |
4441 | case ISD::FP_TO_SINT_SAT: |
4442 | // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT. |
4443 | Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits(); |
4444 | return VTBits - Tmp + 1; |
4445 | case ISD::SIGN_EXTEND: |
4446 | Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits(); |
4447 | return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp; |
4448 | case ISD::SIGN_EXTEND_INREG: |
4449 | // Max of the input and what this extends. |
4450 | Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits(); |
4451 | Tmp = VTBits-Tmp+1; |
4452 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1); |
4453 | return std::max(a: Tmp, b: Tmp2); |
4454 | case ISD::SIGN_EXTEND_VECTOR_INREG: { |
4455 | if (VT.isScalableVector()) |
4456 | break; |
4457 | SDValue Src = Op.getOperand(i: 0); |
4458 | EVT SrcVT = Src.getValueType(); |
4459 | APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements()); |
4460 | Tmp = VTBits - SrcVT.getScalarSizeInBits(); |
4461 | return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp; |
4462 | } |
4463 | case ISD::SRA: |
4464 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4465 | // SRA X, C -> adds C sign bits. |
4466 | if (const APInt *ShAmt = |
4467 | getValidMinimumShiftAmountConstant(V: Op, DemandedElts)) |
4468 | Tmp = std::min<uint64_t>(a: Tmp + ShAmt->getZExtValue(), b: VTBits); |
4469 | return Tmp; |
4470 | case ISD::SHL: |
4471 | if (const APInt *ShAmt = |
4472 | getValidMaximumShiftAmountConstant(V: Op, DemandedElts)) { |
4473 | // shl destroys sign bits, ensure it doesn't shift out all sign bits. |
4474 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4475 | if (ShAmt->ult(RHS: Tmp)) |
4476 | return Tmp - ShAmt->getZExtValue(); |
4477 | } |
4478 | break; |
4479 | case ISD::AND: |
4480 | case ISD::OR: |
4481 | case ISD::XOR: // NOT is handled here. |
4482 | // Logical binary ops preserve the number of sign bits at the worst. |
4483 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1); |
4484 | if (Tmp != 1) { |
4485 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1); |
4486 | FirstAnswer = std::min(a: Tmp, b: Tmp2); |
4487 | // We computed what we know about the sign bits as our first |
4488 | // answer. Now proceed to the generic code that uses |
4489 | // computeKnownBits, and pick whichever answer is better. |
4490 | } |
4491 | break; |
4492 | |
4493 | case ISD::SELECT: |
4494 | case ISD::VSELECT: |
4495 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1); |
4496 | if (Tmp == 1) return 1; // Early out. |
4497 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1); |
4498 | return std::min(a: Tmp, b: Tmp2); |
4499 | case ISD::SELECT_CC: |
4500 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1); |
4501 | if (Tmp == 1) return 1; // Early out. |
4502 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1); |
4503 | return std::min(a: Tmp, b: Tmp2); |
4504 | |
4505 | case ISD::SMIN: |
4506 | case ISD::SMAX: { |
4507 | // If we have a clamp pattern, we know that the number of sign bits will be |
4508 | // the minimum of the clamp min/max range. |
4509 | bool IsMax = (Opcode == ISD::SMAX); |
4510 | ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
4511 | if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))) |
4512 | if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
4513 | CstHigh = |
4514 | isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts); |
4515 | if (CstLow && CstHigh) { |
4516 | if (!IsMax) |
4517 | std::swap(a&: CstLow, b&: CstHigh); |
4518 | if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) { |
4519 | Tmp = CstLow->getAPIntValue().getNumSignBits(); |
4520 | Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); |
4521 | return std::min(a: Tmp, b: Tmp2); |
4522 | } |
4523 | } |
4524 | |
4525 | // Fallback - just get the minimum number of sign bits of the operands. |
4526 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4527 | if (Tmp == 1) |
4528 | return 1; // Early out. |
4529 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
4530 | return std::min(a: Tmp, b: Tmp2); |
4531 | } |
4532 | case ISD::UMIN: |
4533 | case ISD::UMAX: |
4534 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4535 | if (Tmp == 1) |
4536 | return 1; // Early out. |
4537 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
4538 | return std::min(a: Tmp, b: Tmp2); |
4539 | case ISD::SADDO: |
4540 | case ISD::UADDO: |
4541 | case ISD::SADDO_CARRY: |
4542 | case ISD::UADDO_CARRY: |
4543 | case ISD::SSUBO: |
4544 | case ISD::USUBO: |
4545 | case ISD::SSUBO_CARRY: |
4546 | case ISD::USUBO_CARRY: |
4547 | case ISD::SMULO: |
4548 | case ISD::UMULO: |
4549 | if (Op.getResNo() != 1) |
4550 | break; |
4551 | // The boolean result conforms to getBooleanContents. Fall through. |
4552 | // If setcc returns 0/-1, all bits are sign bits. |
4553 | // We know that we have an integer-based boolean since these operations |
4554 | // are only available for integer. |
4555 | if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) == |
4556 | TargetLowering::ZeroOrNegativeOneBooleanContent) |
4557 | return VTBits; |
4558 | break; |
4559 | case ISD::SETCC: |
4560 | case ISD::SETCCCARRY: |
4561 | case ISD::STRICT_FSETCC: |
4562 | case ISD::STRICT_FSETCCS: { |
4563 | unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
4564 | // If setcc returns 0/-1, all bits are sign bits. |
4565 | if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) == |
4566 | TargetLowering::ZeroOrNegativeOneBooleanContent) |
4567 | return VTBits; |
4568 | break; |
4569 | } |
4570 | case ISD::ROTL: |
4571 | case ISD::ROTR: |
4572 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4573 | |
4574 | // If we're rotating an 0/-1 value, then it stays an 0/-1 value. |
4575 | if (Tmp == VTBits) |
4576 | return VTBits; |
4577 | |
4578 | if (ConstantSDNode *C = |
4579 | isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) { |
4580 | unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits); |
4581 | |
4582 | // Handle rotate right by N like a rotate left by 32-N. |
4583 | if (Opcode == ISD::ROTR) |
4584 | RotAmt = (VTBits - RotAmt) % VTBits; |
4585 | |
4586 | // If we aren't rotating out all of the known-in sign bits, return the |
4587 | // number that are left. This handles rotl(sext(x), 1) for example. |
4588 | if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); |
4589 | } |
4590 | break; |
4591 | case ISD::ADD: |
4592 | case ISD::ADDC: |
4593 | // Add can have at most one carry bit. Thus we know that the output |
4594 | // is, at worst, one more bit than the inputs. |
4595 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4596 | if (Tmp == 1) return 1; // Early out. |
4597 | |
4598 | // Special case decrementing a value (ADD X, -1): |
4599 | if (ConstantSDNode *CRHS = |
4600 | isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) |
4601 | if (CRHS->isAllOnes()) { |
4602 | KnownBits Known = |
4603 | computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4604 | |
4605 | // If the input is known to be 0 or 1, the output is 0/-1, which is all |
4606 | // sign bits set. |
4607 | if ((Known.Zero | 1).isAllOnes()) |
4608 | return VTBits; |
4609 | |
4610 | // If we are subtracting one from a positive number, there is no carry |
4611 | // out of the result. |
4612 | if (Known.isNonNegative()) |
4613 | return Tmp; |
4614 | } |
4615 | |
4616 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
4617 | if (Tmp2 == 1) return 1; // Early out. |
4618 | return std::min(a: Tmp, b: Tmp2) - 1; |
4619 | case ISD::SUB: |
4620 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
4621 | if (Tmp2 == 1) return 1; // Early out. |
4622 | |
4623 | // Handle NEG. |
4624 | if (ConstantSDNode *CLHS = |
4625 | isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts)) |
4626 | if (CLHS->isZero()) { |
4627 | KnownBits Known = |
4628 | computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1); |
4629 | // If the input is known to be 0 or 1, the output is 0/-1, which is all |
4630 | // sign bits set. |
4631 | if ((Known.Zero | 1).isAllOnes()) |
4632 | return VTBits; |
4633 | |
4634 | // If the input is known to be positive (the sign bit is known clear), |
4635 | // the output of the NEG has the same number of sign bits as the input. |
4636 | if (Known.isNonNegative()) |
4637 | return Tmp2; |
4638 | |
4639 | // Otherwise, we treat this like a SUB. |
4640 | } |
4641 | |
4642 | // Sub can have at most one carry bit. Thus we know that the output |
4643 | // is, at worst, one more bit than the inputs. |
4644 | Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4645 | if (Tmp == 1) return 1; // Early out. |
4646 | return std::min(a: Tmp, b: Tmp2) - 1; |
4647 | case ISD::MUL: { |
4648 | // The output of the Mul can be at most twice the valid bits in the inputs. |
4649 | unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
4650 | if (SignBitsOp0 == 1) |
4651 | break; |
4652 | unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1); |
4653 | if (SignBitsOp1 == 1) |
4654 | break; |
4655 | unsigned OutValidBits = |
4656 | (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1); |
4657 | return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1; |
4658 | } |
4659 | case ISD::SREM: |
4660 | // The sign bit is the LHS's sign bit, except when the result of the |
4661 | // remainder is zero. The magnitude of the result should be less than or |
4662 | // equal to the magnitude of the LHS. Therefore, the result should have |
4663 | // at least as many sign bits as the left hand side. |
4664 | return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1); |
4665 | case ISD::TRUNCATE: { |
4666 | // Check if the sign bits of source go down as far as the truncated value. |
4667 | unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits(); |
4668 | unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
4669 | if (NumSrcSignBits > (NumSrcBits - VTBits)) |
4670 | return NumSrcSignBits - (NumSrcBits - VTBits); |
4671 | break; |
4672 | } |
4673 | case ISD::EXTRACT_ELEMENT: { |
4674 | if (VT.isScalableVector()) |
4675 | break; |
4676 | const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1); |
4677 | const int BitWidth = Op.getValueSizeInBits(); |
4678 | const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth; |
4679 | |
4680 | // Get reverse index (starting from 1), Op1 value indexes elements from |
4681 | // little end. Sign starts at big end. |
4682 | const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1); |
4683 | |
4684 | // If the sign portion ends in our element the subtraction gives correct |
4685 | // result. Otherwise it gives either negative or > bitwidth result |
4686 | return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 0, hi: BitWidth); |
4687 | } |
4688 | case ISD::INSERT_VECTOR_ELT: { |
4689 | if (VT.isScalableVector()) |
4690 | break; |
4691 | // If we know the element index, split the demand between the |
4692 | // source vector and the inserted element, otherwise assume we need |
4693 | // the original demanded vector elements and the value. |
4694 | SDValue InVec = Op.getOperand(i: 0); |
4695 | SDValue InVal = Op.getOperand(i: 1); |
4696 | SDValue EltNo = Op.getOperand(i: 2); |
4697 | bool DemandedVal = true; |
4698 | APInt DemandedVecElts = DemandedElts; |
4699 | auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo); |
4700 | if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) { |
4701 | unsigned EltIdx = CEltNo->getZExtValue(); |
4702 | DemandedVal = !!DemandedElts[EltIdx]; |
4703 | DemandedVecElts.clearBit(BitPosition: EltIdx); |
4704 | } |
4705 | Tmp = std::numeric_limits<unsigned>::max(); |
4706 | if (DemandedVal) { |
4707 | // TODO - handle implicit truncation of inserted elements. |
4708 | if (InVal.getScalarValueSizeInBits() != VTBits) |
4709 | break; |
4710 | Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1); |
4711 | Tmp = std::min(a: Tmp, b: Tmp2); |
4712 | } |
4713 | if (!!DemandedVecElts) { |
4714 | Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1); |
4715 | Tmp = std::min(a: Tmp, b: Tmp2); |
4716 | } |
4717 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits" ); |
4718 | return Tmp; |
4719 | } |
4720 | case ISD::EXTRACT_VECTOR_ELT: { |
4721 | assert(!VT.isScalableVector()); |
4722 | SDValue InVec = Op.getOperand(i: 0); |
4723 | SDValue EltNo = Op.getOperand(i: 1); |
4724 | EVT VecVT = InVec.getValueType(); |
4725 | // ComputeNumSignBits not yet implemented for scalable vectors. |
4726 | if (VecVT.isScalableVector()) |
4727 | break; |
4728 | const unsigned BitWidth = Op.getValueSizeInBits(); |
4729 | const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits(); |
4730 | const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
4731 | |
4732 | // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know |
4733 | // anything about sign bits. But if the sizes match we can derive knowledge |
4734 | // about sign bits from the vector operand. |
4735 | if (BitWidth != EltBitWidth) |
4736 | break; |
4737 | |
4738 | // If we know the element index, just demand that vector element, else for |
4739 | // an unknown element index, ignore DemandedElts and demand them all. |
4740 | APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts); |
4741 | auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo); |
4742 | if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)) |
4743 | DemandedSrcElts = |
4744 | APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue()); |
4745 | |
4746 | return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
4747 | } |
4748 | case ISD::EXTRACT_SUBVECTOR: { |
4749 | // Offset the demanded elts by the subvector index. |
4750 | SDValue Src = Op.getOperand(i: 0); |
4751 | // Bail until we can represent demanded elements for scalable vectors. |
4752 | if (Src.getValueType().isScalableVector()) |
4753 | break; |
4754 | uint64_t Idx = Op.getConstantOperandVal(i: 1); |
4755 | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
4756 | APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx); |
4757 | return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
4758 | } |
4759 | case ISD::CONCAT_VECTORS: { |
4760 | if (VT.isScalableVector()) |
4761 | break; |
4762 | // Determine the minimum number of sign bits across all demanded |
4763 | // elts of the input vectors. Early out if the result is already 1. |
4764 | Tmp = std::numeric_limits<unsigned>::max(); |
4765 | EVT SubVectorVT = Op.getOperand(i: 0).getValueType(); |
4766 | unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
4767 | unsigned NumSubVectors = Op.getNumOperands(); |
4768 | for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { |
4769 | APInt DemandedSub = |
4770 | DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts); |
4771 | if (!DemandedSub) |
4772 | continue; |
4773 | Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1); |
4774 | Tmp = std::min(a: Tmp, b: Tmp2); |
4775 | } |
4776 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits" ); |
4777 | return Tmp; |
4778 | } |
4779 | case ISD::INSERT_SUBVECTOR: { |
4780 | if (VT.isScalableVector()) |
4781 | break; |
4782 | // Demand any elements from the subvector and the remainder from the src its |
4783 | // inserted into. |
4784 | SDValue Src = Op.getOperand(i: 0); |
4785 | SDValue Sub = Op.getOperand(i: 1); |
4786 | uint64_t Idx = Op.getConstantOperandVal(i: 2); |
4787 | unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
4788 | APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx); |
4789 | APInt DemandedSrcElts = DemandedElts; |
4790 | DemandedSrcElts.insertBits(SubBits: APInt::getZero(numBits: NumSubElts), bitPosition: Idx); |
4791 | |
4792 | Tmp = std::numeric_limits<unsigned>::max(); |
4793 | if (!!DemandedSubElts) { |
4794 | Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1); |
4795 | if (Tmp == 1) |
4796 | return 1; // early-out |
4797 | } |
4798 | if (!!DemandedSrcElts) { |
4799 | Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1); |
4800 | Tmp = std::min(a: Tmp, b: Tmp2); |
4801 | } |
4802 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits" ); |
4803 | return Tmp; |
4804 | } |
4805 | case ISD::LOAD: { |
4806 | LoadSDNode *LD = cast<LoadSDNode>(Val&: Op); |
4807 | if (const MDNode *Ranges = LD->getRanges()) { |
4808 | if (DemandedElts != 1) |
4809 | break; |
4810 | |
4811 | ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges); |
4812 | if (VTBits > CR.getBitWidth()) { |
4813 | switch (LD->getExtensionType()) { |
4814 | case ISD::SEXTLOAD: |
4815 | CR = CR.signExtend(BitWidth: VTBits); |
4816 | break; |
4817 | case ISD::ZEXTLOAD: |
4818 | CR = CR.zeroExtend(BitWidth: VTBits); |
4819 | break; |
4820 | default: |
4821 | break; |
4822 | } |
4823 | } |
4824 | |
4825 | if (VTBits != CR.getBitWidth()) |
4826 | break; |
4827 | return std::min(a: CR.getSignedMin().getNumSignBits(), |
4828 | b: CR.getSignedMax().getNumSignBits()); |
4829 | } |
4830 | |
4831 | break; |
4832 | } |
4833 | case ISD::ATOMIC_CMP_SWAP: |
4834 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
4835 | case ISD::ATOMIC_SWAP: |
4836 | case ISD::ATOMIC_LOAD_ADD: |
4837 | case ISD::ATOMIC_LOAD_SUB: |
4838 | case ISD::ATOMIC_LOAD_AND: |
4839 | case ISD::ATOMIC_LOAD_CLR: |
4840 | case ISD::ATOMIC_LOAD_OR: |
4841 | case ISD::ATOMIC_LOAD_XOR: |
4842 | case ISD::ATOMIC_LOAD_NAND: |
4843 | case ISD::ATOMIC_LOAD_MIN: |
4844 | case ISD::ATOMIC_LOAD_MAX: |
4845 | case ISD::ATOMIC_LOAD_UMIN: |
4846 | case ISD::ATOMIC_LOAD_UMAX: |
4847 | case ISD::ATOMIC_LOAD: { |
4848 | Tmp = cast<AtomicSDNode>(Val&: Op)->getMemoryVT().getScalarSizeInBits(); |
4849 | // If we are looking at the loaded value. |
4850 | if (Op.getResNo() == 0) { |
4851 | if (Tmp == VTBits) |
4852 | return 1; // early-out |
4853 | if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) |
4854 | return VTBits - Tmp + 1; |
4855 | if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
4856 | return VTBits - Tmp; |
4857 | } |
4858 | break; |
4859 | } |
4860 | } |
4861 | |
4862 | // If we are looking at the loaded value of the SDNode. |
4863 | if (Op.getResNo() == 0) { |
4864 | // Handle LOADX separately here. EXTLOAD case will fallthrough. |
4865 | if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op)) { |
4866 | unsigned ExtType = LD->getExtensionType(); |
4867 | switch (ExtType) { |
4868 | default: break; |
4869 | case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. |
4870 | Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
4871 | return VTBits - Tmp + 1; |
4872 | case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. |
4873 | Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
4874 | return VTBits - Tmp; |
4875 | case ISD::NON_EXTLOAD: |
4876 | if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { |
4877 | // We only need to handle vectors - computeKnownBits should handle |
4878 | // scalar cases. |
4879 | Type *CstTy = Cst->getType(); |
4880 | if (CstTy->isVectorTy() && !VT.isScalableVector() && |
4881 | (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() && |
4882 | VTBits == CstTy->getScalarSizeInBits()) { |
4883 | Tmp = VTBits; |
4884 | for (unsigned i = 0; i != NumElts; ++i) { |
4885 | if (!DemandedElts[i]) |
4886 | continue; |
4887 | if (Constant *Elt = Cst->getAggregateElement(Elt: i)) { |
4888 | if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) { |
4889 | const APInt &Value = CInt->getValue(); |
4890 | Tmp = std::min(a: Tmp, b: Value.getNumSignBits()); |
4891 | continue; |
4892 | } |
4893 | if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) { |
4894 | APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
4895 | Tmp = std::min(a: Tmp, b: Value.getNumSignBits()); |
4896 | continue; |
4897 | } |
4898 | } |
4899 | // Unknown type. Conservatively assume no bits match sign bit. |
4900 | return 1; |
4901 | } |
4902 | return Tmp; |
4903 | } |
4904 | } |
4905 | break; |
4906 | } |
4907 | } |
4908 | } |
4909 | |
4910 | // Allow the target to implement this method for its nodes. |
4911 | if (Opcode >= ISD::BUILTIN_OP_END || |
4912 | Opcode == ISD::INTRINSIC_WO_CHAIN || |
4913 | Opcode == ISD::INTRINSIC_W_CHAIN || |
4914 | Opcode == ISD::INTRINSIC_VOID) { |
4915 | // TODO: This can probably be removed once target code is audited. This |
4916 | // is here purely to reduce patch size and review complexity. |
4917 | if (!VT.isScalableVector()) { |
4918 | unsigned NumBits = |
4919 | TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth); |
4920 | if (NumBits > 1) |
4921 | FirstAnswer = std::max(a: FirstAnswer, b: NumBits); |
4922 | } |
4923 | } |
4924 | |
4925 | // Finally, if we can prove that the top bits of the result are 0's or 1's, |
4926 | // use this information. |
4927 | KnownBits Known = computeKnownBits(Op, DemandedElts, Depth); |
4928 | return std::max(a: FirstAnswer, b: Known.countMinSignBits()); |
4929 | } |
4930 | |
4931 | unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
4932 | unsigned Depth) const { |
4933 | unsigned SignBits = ComputeNumSignBits(Op, Depth); |
4934 | return Op.getScalarValueSizeInBits() - SignBits + 1; |
4935 | } |
4936 | |
4937 | unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
4938 | const APInt &DemandedElts, |
4939 | unsigned Depth) const { |
4940 | unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth); |
4941 | return Op.getScalarValueSizeInBits() - SignBits + 1; |
4942 | } |
4943 | |
4944 | bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly, |
4945 | unsigned Depth) const { |
4946 | // Early out for FREEZE. |
4947 | if (Op.getOpcode() == ISD::FREEZE) |
4948 | return true; |
4949 | |
4950 | // TODO: Assume we don't know anything for now. |
4951 | EVT VT = Op.getValueType(); |
4952 | if (VT.isScalableVector()) |
4953 | return false; |
4954 | |
4955 | APInt DemandedElts = VT.isVector() |
4956 | ? APInt::getAllOnes(numBits: VT.getVectorNumElements()) |
4957 | : APInt(1, 1); |
4958 | return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth); |
4959 | } |
4960 | |
4961 | bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, |
4962 | const APInt &DemandedElts, |
4963 | bool PoisonOnly, |
4964 | unsigned Depth) const { |
4965 | unsigned Opcode = Op.getOpcode(); |
4966 | |
4967 | // Early out for FREEZE. |
4968 | if (Opcode == ISD::FREEZE) |
4969 | return true; |
4970 | |
4971 | if (Depth >= MaxRecursionDepth) |
4972 | return false; // Limit search depth. |
4973 | |
4974 | if (isIntOrFPConstant(V: Op)) |
4975 | return true; |
4976 | |
4977 | switch (Opcode) { |
4978 | case ISD::VALUETYPE: |
4979 | case ISD::FrameIndex: |
4980 | case ISD::TargetFrameIndex: |
4981 | return true; |
4982 | |
4983 | case ISD::UNDEF: |
4984 | return PoisonOnly; |
4985 | |
4986 | case ISD::BUILD_VECTOR: |
4987 | // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements - |
4988 | // this shouldn't affect the result. |
4989 | for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) { |
4990 | if (!DemandedElts[i]) |
4991 | continue; |
4992 | if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly, |
4993 | Depth: Depth + 1)) |
4994 | return false; |
4995 | } |
4996 | return true; |
4997 | |
4998 | // TODO: Search for noundef attributes from library functions. |
4999 | |
5000 | // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef. |
5001 | |
5002 | default: |
5003 | // Allow the target to implement this method for its nodes. |
5004 | if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
5005 | Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
5006 | return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode( |
5007 | Op, DemandedElts, DAG: *this, PoisonOnly, Depth); |
5008 | break; |
5009 | } |
5010 | |
5011 | // If Op can't create undef/poison and none of its operands are undef/poison |
5012 | // then Op is never undef/poison. |
5013 | // NOTE: TargetNodes should handle this in themselves in |
5014 | // isGuaranteedNotToBeUndefOrPoisonForTargetNode. |
5015 | return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true, |
5016 | Depth) && |
5017 | all_of(Range: Op->ops(), P: [&](SDValue V) { |
5018 | return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1); |
5019 | }); |
5020 | } |
5021 | |
5022 | bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly, |
5023 | bool ConsiderFlags, |
5024 | unsigned Depth) const { |
5025 | // TODO: Assume we don't know anything for now. |
5026 | EVT VT = Op.getValueType(); |
5027 | if (VT.isScalableVector()) |
5028 | return true; |
5029 | |
5030 | APInt DemandedElts = VT.isVector() |
5031 | ? APInt::getAllOnes(numBits: VT.getVectorNumElements()) |
5032 | : APInt(1, 1); |
5033 | return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags, |
5034 | Depth); |
5035 | } |
5036 | |
5037 | bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, |
5038 | bool PoisonOnly, bool ConsiderFlags, |
5039 | unsigned Depth) const { |
5040 | // TODO: Assume we don't know anything for now. |
5041 | EVT VT = Op.getValueType(); |
5042 | if (VT.isScalableVector()) |
5043 | return true; |
5044 | |
5045 | unsigned Opcode = Op.getOpcode(); |
5046 | switch (Opcode) { |
5047 | case ISD::FREEZE: |
5048 | case ISD::CONCAT_VECTORS: |
5049 | case ISD::INSERT_SUBVECTOR: |
5050 | case ISD::AND: |
5051 | case ISD::XOR: |
5052 | case ISD::ROTL: |
5053 | case ISD::ROTR: |
5054 | case ISD::FSHL: |
5055 | case ISD::FSHR: |
5056 | case ISD::BSWAP: |
5057 | case ISD::CTPOP: |
5058 | case ISD::BITREVERSE: |
5059 | case ISD::PARITY: |
5060 | case ISD::SIGN_EXTEND: |
5061 | case ISD::TRUNCATE: |
5062 | case ISD::SIGN_EXTEND_INREG: |
5063 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
5064 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
5065 | case ISD::BITCAST: |
5066 | case ISD::BUILD_VECTOR: |
5067 | case ISD::BUILD_PAIR: |
5068 | return false; |
5069 | |
5070 | // Matches hasPoisonGeneratingFlags(). |
5071 | case ISD::ZERO_EXTEND: |
5072 | return ConsiderFlags && Op->getFlags().hasNonNeg(); |
5073 | |
5074 | case ISD::ADD: |
5075 | case ISD::SUB: |
5076 | case ISD::MUL: |
5077 | // Matches hasPoisonGeneratingFlags(). |
5078 | return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
5079 | Op->getFlags().hasNoUnsignedWrap()); |
5080 | |
5081 | case ISD::SHL: |
5082 | // If the max shift amount isn't in range, then the shift can create poison. |
5083 | if (!getValidMaximumShiftAmountConstant(V: Op, DemandedElts)) |
5084 | return true; |
5085 | |
5086 | // Matches hasPoisonGeneratingFlags(). |
5087 | return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
5088 | Op->getFlags().hasNoUnsignedWrap()); |
5089 | |
5090 | // Matches hasPoisonGeneratingFlags(). |
5091 | case ISD::OR: |
5092 | return ConsiderFlags && Op->getFlags().hasDisjoint(); |
5093 | |
5094 | case ISD::INSERT_VECTOR_ELT:{ |
5095 | // Ensure that the element index is in bounds. |
5096 | EVT VecVT = Op.getOperand(i: 0).getValueType(); |
5097 | KnownBits KnownIdx = computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1); |
5098 | return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements()); |
5099 | } |
5100 | |
5101 | default: |
5102 | // Allow the target to implement this method for its nodes. |
5103 | if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
5104 | Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
5105 | return TLI->canCreateUndefOrPoisonForTargetNode( |
5106 | Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth); |
5107 | break; |
5108 | } |
5109 | |
5110 | // Be conservative and return true. |
5111 | return true; |
5112 | } |
5113 | |
5114 | bool SelectionDAG::isADDLike(SDValue Op) const { |
5115 | unsigned Opcode = Op.getOpcode(); |
5116 | if (Opcode == ISD::OR) |
5117 | return Op->getFlags().hasDisjoint() || |
5118 | haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1)); |
5119 | if (Opcode == ISD::XOR) |
5120 | return isMinSignedConstant(V: Op.getOperand(i: 1)); |
5121 | return false; |
5122 | } |
5123 | |
5124 | bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { |
5125 | if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || |
5126 | !isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) |
5127 | return false; |
5128 | |
5129 | if (Op.getOpcode() == ISD::OR && |
5130 | !MaskedValueIsZero(V: Op.getOperand(i: 0), Mask: Op.getConstantOperandAPInt(i: 1))) |
5131 | return false; |
5132 | |
5133 | return true; |
5134 | } |
5135 | |
5136 | bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { |
5137 | // If we're told that NaNs won't happen, assume they won't. |
5138 | if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs()) |
5139 | return true; |
5140 | |
5141 | if (Depth >= MaxRecursionDepth) |
5142 | return false; // Limit search depth. |
5143 | |
5144 | // If the value is a constant, we can obviously see if it is a NaN or not. |
5145 | if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) { |
5146 | return !C->getValueAPF().isNaN() || |
5147 | (SNaN && !C->getValueAPF().isSignaling()); |
5148 | } |
5149 | |
5150 | unsigned Opcode = Op.getOpcode(); |
5151 | switch (Opcode) { |
5152 | case ISD::FADD: |
5153 | case ISD::FSUB: |
5154 | case ISD::FMUL: |
5155 | case ISD::FDIV: |
5156 | case ISD::FREM: |
5157 | case ISD::FSIN: |
5158 | case ISD::FCOS: |
5159 | case ISD::FMA: |
5160 | case ISD::FMAD: { |
5161 | if (SNaN) |
5162 | return true; |
5163 | // TODO: Need isKnownNeverInfinity |
5164 | return false; |
5165 | } |
5166 | case ISD::FCANONICALIZE: |
5167 | case ISD::FEXP: |
5168 | case ISD::FEXP2: |
5169 | case ISD::FEXP10: |
5170 | case ISD::FTRUNC: |
5171 | case ISD::FFLOOR: |
5172 | case ISD::FCEIL: |
5173 | case ISD::FROUND: |
5174 | case ISD::FROUNDEVEN: |
5175 | case ISD::FRINT: |
5176 | case ISD::LRINT: |
5177 | case ISD::LLRINT: |
5178 | case ISD::FNEARBYINT: |
5179 | case ISD::FLDEXP: { |
5180 | if (SNaN) |
5181 | return true; |
5182 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1); |
5183 | } |
5184 | case ISD::FABS: |
5185 | case ISD::FNEG: |
5186 | case ISD::FCOPYSIGN: { |
5187 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1); |
5188 | } |
5189 | case ISD::SELECT: |
5190 | return isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) && |
5191 | isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1); |
5192 | case ISD::FP_EXTEND: |
5193 | case ISD::FP_ROUND: { |
5194 | if (SNaN) |
5195 | return true; |
5196 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1); |
5197 | } |
5198 | case ISD::SINT_TO_FP: |
5199 | case ISD::UINT_TO_FP: |
5200 | return true; |
5201 | case ISD::FSQRT: // Need is known positive |
5202 | case ISD::FLOG: |
5203 | case ISD::FLOG2: |
5204 | case ISD::FLOG10: |
5205 | case ISD::FPOWI: |
5206 | case ISD::FPOW: { |
5207 | if (SNaN) |
5208 | return true; |
5209 | // TODO: Refine on operand |
5210 | return false; |
5211 | } |
5212 | case ISD::FMINNUM: |
5213 | case ISD::FMAXNUM: { |
5214 | // Only one needs to be known not-nan, since it will be returned if the |
5215 | // other ends up being one. |
5216 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) || |
5217 | isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1); |
5218 | } |
5219 | case ISD::FMINNUM_IEEE: |
5220 | case ISD::FMAXNUM_IEEE: { |
5221 | if (SNaN) |
5222 | return true; |
5223 | // This can return a NaN if either operand is an sNaN, or if both operands |
5224 | // are NaN. |
5225 | return (isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN: false, Depth: Depth + 1) && |
5226 | isKnownNeverSNaN(Op: Op.getOperand(i: 1), Depth: Depth + 1)) || |
5227 | (isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN: false, Depth: Depth + 1) && |
5228 | isKnownNeverSNaN(Op: Op.getOperand(i: 0), Depth: Depth + 1)); |
5229 | } |
5230 | case ISD::FMINIMUM: |
5231 | case ISD::FMAXIMUM: { |
5232 | // TODO: Does this quiet or return the origina NaN as-is? |
5233 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) && |
5234 | isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1); |
5235 | } |
5236 | case ISD::EXTRACT_VECTOR_ELT: { |
5237 | return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1); |
5238 | } |
5239 | case ISD::BUILD_VECTOR: { |
5240 | for (const SDValue &Opnd : Op->ops()) |
5241 | if (!isKnownNeverNaN(Op: Opnd, SNaN, Depth: Depth + 1)) |
5242 | return false; |
5243 | return true; |
5244 | } |
5245 | default: |
5246 | if (Opcode >= ISD::BUILTIN_OP_END || |
5247 | Opcode == ISD::INTRINSIC_WO_CHAIN || |
5248 | Opcode == ISD::INTRINSIC_W_CHAIN || |
5249 | Opcode == ISD::INTRINSIC_VOID) { |
5250 | return TLI->isKnownNeverNaNForTargetNode(Op, DAG: *this, SNaN, Depth); |
5251 | } |
5252 | |
5253 | return false; |
5254 | } |
5255 | } |
5256 | |
5257 | bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { |
5258 | assert(Op.getValueType().isFloatingPoint() && |
5259 | "Floating point type expected" ); |
5260 | |
5261 | // If the value is a constant, we can obviously see if it is a zero or not. |
5262 | return ISD::matchUnaryFpPredicate( |
5263 | Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); }); |
5264 | } |
5265 | |
5266 | bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const { |
5267 | if (Depth >= MaxRecursionDepth) |
5268 | return false; // Limit search depth. |
5269 | |
5270 | assert(!Op.getValueType().isFloatingPoint() && |
5271 | "Floating point types unsupported - use isKnownNeverZeroFloat" ); |
5272 | |
5273 | // If the value is a constant, we can obviously see if it is a zero or not. |
5274 | if (ISD::matchUnaryPredicate(Op, |
5275 | Match: [](ConstantSDNode *C) { return !C->isZero(); })) |
5276 | return true; |
5277 | |
5278 | // TODO: Recognize more cases here. Most of the cases are also incomplete to |
5279 | // some degree. |
5280 | switch (Op.getOpcode()) { |
5281 | default: |
5282 | break; |
5283 | |
5284 | case ISD::OR: |
5285 | return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) || |
5286 | isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5287 | |
5288 | case ISD::VSELECT: |
5289 | case ISD::SELECT: |
5290 | return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) && |
5291 | isKnownNeverZero(Op: Op.getOperand(i: 2), Depth: Depth + 1); |
5292 | |
5293 | case ISD::SHL: { |
5294 | if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
5295 | return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5296 | KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5297 | // 1 << X is never zero. |
5298 | if (ValKnown.One[0]) |
5299 | return true; |
5300 | // If max shift cnt of known ones is non-zero, result is non-zero. |
5301 | APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue(); |
5302 | if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) && |
5303 | !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero()) |
5304 | return true; |
5305 | break; |
5306 | } |
5307 | case ISD::UADDSAT: |
5308 | case ISD::UMAX: |
5309 | return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) || |
5310 | isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5311 | |
5312 | // TODO for smin/smax: If either operand is known negative/positive |
5313 | // respectively we don't need the other to be known at all. |
5314 | case ISD::SMAX: |
5315 | case ISD::SMIN: |
5316 | case ISD::UMIN: |
5317 | return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) && |
5318 | isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5319 | |
5320 | case ISD::ROTL: |
5321 | case ISD::ROTR: |
5322 | case ISD::BITREVERSE: |
5323 | case ISD::BSWAP: |
5324 | case ISD::CTPOP: |
5325 | case ISD::ABS: |
5326 | return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5327 | |
5328 | case ISD::SRA: |
5329 | case ISD::SRL: { |
5330 | if (Op->getFlags().hasExact()) |
5331 | return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5332 | KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5333 | if (ValKnown.isNegative()) |
5334 | return true; |
5335 | // If max shift cnt of known ones is non-zero, result is non-zero. |
5336 | APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue(); |
5337 | if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) && |
5338 | !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero()) |
5339 | return true; |
5340 | break; |
5341 | } |
5342 | case ISD::UDIV: |
5343 | case ISD::SDIV: |
5344 | // div exact can only produce a zero if the dividend is zero. |
5345 | // TODO: For udiv this is also true if Op1 u<= Op0 |
5346 | if (Op->getFlags().hasExact()) |
5347 | return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5348 | break; |
5349 | |
5350 | case ISD::ADD: |
5351 | if (Op->getFlags().hasNoUnsignedWrap()) |
5352 | if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) || |
5353 | isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1)) |
5354 | return true; |
5355 | // TODO: There are a lot more cases we can prove for add. |
5356 | break; |
5357 | |
5358 | case ISD::SUB: { |
5359 | if (isNullConstant(V: Op.getOperand(i: 0))) |
5360 | return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1); |
5361 | |
5362 | std::optional<bool> ne = |
5363 | KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1), |
5364 | RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1)); |
5365 | return ne && *ne; |
5366 | } |
5367 | |
5368 | case ISD::MUL: |
5369 | if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
5370 | if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) && |
5371 | isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1)) |
5372 | return true; |
5373 | break; |
5374 | |
5375 | case ISD::ZERO_EXTEND: |
5376 | case ISD::SIGN_EXTEND: |
5377 | return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1); |
5378 | } |
5379 | |
5380 | return computeKnownBits(Op, Depth).isNonZero(); |
5381 | } |
5382 | |
5383 | bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { |
5384 | // Check the obvious case. |
5385 | if (A == B) return true; |
5386 | |
5387 | // For negative and positive zero. |
5388 | if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) |
5389 | if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) |
5390 | if (CA->isZero() && CB->isZero()) return true; |
5391 | |
5392 | // Otherwise they may not be equal. |
5393 | return false; |
5394 | } |
5395 | |
5396 | // Only bits set in Mask must be negated, other bits may be arbitrary. |
5397 | SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) { |
5398 | if (isBitwiseNot(V, AllowUndefs)) |
5399 | return V.getOperand(i: 0); |
5400 | |
5401 | // Handle any_extend (not (truncate X)) pattern, where Mask only sets |
5402 | // bits in the non-extended part. |
5403 | ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask); |
5404 | if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND) |
5405 | return SDValue(); |
5406 | SDValue ExtArg = V.getOperand(i: 0); |
5407 | if (ExtArg.getScalarValueSizeInBits() >= |
5408 | MaskC->getAPIntValue().getActiveBits() && |
5409 | isBitwiseNot(V: ExtArg, AllowUndefs) && |
5410 | ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE && |
5411 | ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType()) |
5412 | return ExtArg.getOperand(i: 0).getOperand(i: 0); |
5413 | return SDValue(); |
5414 | } |
5415 | |
5416 | static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) { |
5417 | // Match masked merge pattern (X & ~M) op (Y & M) |
5418 | // Including degenerate case (X & ~M) op M |
5419 | auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask, |
5420 | SDValue Other) { |
5421 | if (SDValue NotOperand = |
5422 | getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) { |
5423 | if (NotOperand->getOpcode() == ISD::ZERO_EXTEND || |
5424 | NotOperand->getOpcode() == ISD::TRUNCATE) |
5425 | NotOperand = NotOperand->getOperand(Num: 0); |
5426 | |
5427 | if (Other == NotOperand) |
5428 | return true; |
5429 | if (Other->getOpcode() == ISD::AND) |
5430 | return NotOperand == Other->getOperand(Num: 0) || |
5431 | NotOperand == Other->getOperand(Num: 1); |
5432 | } |
5433 | return false; |
5434 | }; |
5435 | |
5436 | if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE) |
5437 | A = A->getOperand(Num: 0); |
5438 | |
5439 | if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE) |
5440 | B = B->getOperand(Num: 0); |
5441 | |
5442 | if (A->getOpcode() == ISD::AND) |
5443 | return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) || |
5444 | MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B); |
5445 | return false; |
5446 | } |
5447 | |
5448 | // FIXME: unify with llvm::haveNoCommonBitsSet. |
5449 | bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { |
5450 | assert(A.getValueType() == B.getValueType() && |
5451 | "Values must have the same type" ); |
5452 | if (haveNoCommonBitsSetCommutative(A, B) || |
5453 | haveNoCommonBitsSetCommutative(A: B, B: A)) |
5454 | return true; |
5455 | return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A), |
5456 | RHS: computeKnownBits(Op: B)); |
5457 | } |
5458 | |
5459 | static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, |
5460 | SelectionDAG &DAG) { |
5461 | if (cast<ConstantSDNode>(Val&: Step)->isZero()) |
5462 | return DAG.getConstant(Val: 0, DL, VT); |
5463 | |
5464 | return SDValue(); |
5465 | } |
5466 | |
5467 | static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, |
5468 | ArrayRef<SDValue> Ops, |
5469 | SelectionDAG &DAG) { |
5470 | int NumOps = Ops.size(); |
5471 | assert(NumOps != 0 && "Can't build an empty vector!" ); |
5472 | assert(!VT.isScalableVector() && |
5473 | "BUILD_VECTOR cannot be used with scalable types" ); |
5474 | assert(VT.getVectorNumElements() == (unsigned)NumOps && |
5475 | "Incorrect element count in BUILD_VECTOR!" ); |
5476 | |
5477 | // BUILD_VECTOR of UNDEFs is UNDEF. |
5478 | if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); })) |
5479 | return DAG.getUNDEF(VT); |
5480 | |
5481 | // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. |
5482 | SDValue IdentitySrc; |
5483 | bool IsIdentity = true; |
5484 | for (int i = 0; i != NumOps; ++i) { |
5485 | if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
5486 | Ops[i].getOperand(i: 0).getValueType() != VT || |
5487 | (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) || |
5488 | !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) || |
5489 | Ops[i].getConstantOperandAPInt(i: 1) != i) { |
5490 | IsIdentity = false; |
5491 | break; |
5492 | } |
5493 | IdentitySrc = Ops[i].getOperand(i: 0); |
5494 | } |
5495 | if (IsIdentity) |
5496 | return IdentitySrc; |
5497 | |
5498 | return SDValue(); |
5499 | } |
5500 | |
5501 | /// Try to simplify vector concatenation to an input value, undef, or build |
5502 | /// vector. |
5503 | static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, |
5504 | ArrayRef<SDValue> Ops, |
5505 | SelectionDAG &DAG) { |
5506 | assert(!Ops.empty() && "Can't concatenate an empty list of vectors!" ); |
5507 | assert(llvm::all_of(Ops, |
5508 | [Ops](SDValue Op) { |
5509 | return Ops[0].getValueType() == Op.getValueType(); |
5510 | }) && |
5511 | "Concatenation of vectors with inconsistent value types!" ); |
5512 | assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) == |
5513 | VT.getVectorElementCount() && |
5514 | "Incorrect element count in vector concatenation!" ); |
5515 | |
5516 | if (Ops.size() == 1) |
5517 | return Ops[0]; |
5518 | |
5519 | // Concat of UNDEFs is UNDEF. |
5520 | if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); })) |
5521 | return DAG.getUNDEF(VT); |
5522 | |
5523 | // Scan the operands and look for extract operations from a single source |
5524 | // that correspond to insertion at the same location via this concatenation: |
5525 | // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... |
5526 | SDValue IdentitySrc; |
5527 | bool IsIdentity = true; |
5528 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) { |
5529 | SDValue Op = Ops[i]; |
5530 | unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements(); |
5531 | if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
5532 | Op.getOperand(i: 0).getValueType() != VT || |
5533 | (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) || |
5534 | Op.getConstantOperandVal(i: 1) != IdentityIndex) { |
5535 | IsIdentity = false; |
5536 | break; |
5537 | } |
5538 | assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && |
5539 | "Unexpected identity source vector for concat of extracts" ); |
5540 | IdentitySrc = Op.getOperand(i: 0); |
5541 | } |
5542 | if (IsIdentity) { |
5543 | assert(IdentitySrc && "Failed to set source vector of extracts" ); |
5544 | return IdentitySrc; |
5545 | } |
5546 | |
5547 | // The code below this point is only designed to work for fixed width |
5548 | // vectors, so we bail out for now. |
5549 | if (VT.isScalableVector()) |
5550 | return SDValue(); |
5551 | |
5552 | // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be |
5553 | // simplified to one big BUILD_VECTOR. |
5554 | // FIXME: Add support for SCALAR_TO_VECTOR as well. |
5555 | EVT SVT = VT.getScalarType(); |
5556 | SmallVector<SDValue, 16> Elts; |
5557 | for (SDValue Op : Ops) { |
5558 | EVT OpVT = Op.getValueType(); |
5559 | if (Op.isUndef()) |
5560 | Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT)); |
5561 | else if (Op.getOpcode() == ISD::BUILD_VECTOR) |
5562 | Elts.append(in_start: Op->op_begin(), in_end: Op->op_end()); |
5563 | else |
5564 | return SDValue(); |
5565 | } |
5566 | |
5567 | // BUILD_VECTOR requires all inputs to be of the same type, find the |
5568 | // maximum type and extend them all. |
5569 | for (SDValue Op : Elts) |
5570 | SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT); |
5571 | |
5572 | if (SVT.bitsGT(VT: VT.getScalarType())) { |
5573 | for (SDValue &Op : Elts) { |
5574 | if (Op.isUndef()) |
5575 | Op = DAG.getUNDEF(VT: SVT); |
5576 | else |
5577 | Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT) |
5578 | ? DAG.getZExtOrTrunc(Op, DL, VT: SVT) |
5579 | : DAG.getSExtOrTrunc(Op, DL, VT: SVT); |
5580 | } |
5581 | } |
5582 | |
5583 | SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts); |
5584 | NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: " , G: &DAG); |
5585 | return V; |
5586 | } |
5587 | |
5588 | /// Gets or creates the specified node. |
5589 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { |
5590 | FoldingSetNodeID ID; |
5591 | AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT), OpList: std::nullopt); |
5592 | void *IP = nullptr; |
5593 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
5594 | return SDValue(E, 0); |
5595 | |
5596 | auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), |
5597 | Args: getVTList(VT)); |
5598 | CSEMap.InsertNode(N, InsertPos: IP); |
5599 | |
5600 | InsertNode(N); |
5601 | SDValue V = SDValue(N, 0); |
5602 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
5603 | return V; |
5604 | } |
5605 | |
5606 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
5607 | SDValue N1) { |
5608 | SDNodeFlags Flags; |
5609 | if (Inserter) |
5610 | Flags = Inserter->getFlags(); |
5611 | return getNode(Opcode, DL, VT, Operand: N1, Flags); |
5612 | } |
5613 | |
5614 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
5615 | SDValue N1, const SDNodeFlags Flags) { |
5616 | assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!" ); |
5617 | |
5618 | // Constant fold unary operations with a vector integer or float operand. |
5619 | switch (Opcode) { |
5620 | default: |
5621 | // FIXME: Entirely reasonable to perform folding of other unary |
5622 | // operations here as the need arises. |
5623 | break; |
5624 | case ISD::FNEG: |
5625 | case ISD::FABS: |
5626 | case ISD::FCEIL: |
5627 | case ISD::FTRUNC: |
5628 | case ISD::FFLOOR: |
5629 | case ISD::FP_EXTEND: |
5630 | case ISD::FP_TO_SINT: |
5631 | case ISD::FP_TO_UINT: |
5632 | case ISD::FP_TO_FP16: |
5633 | case ISD::FP_TO_BF16: |
5634 | case ISD::TRUNCATE: |
5635 | case ISD::ANY_EXTEND: |
5636 | case ISD::ZERO_EXTEND: |
5637 | case ISD::SIGN_EXTEND: |
5638 | case ISD::UINT_TO_FP: |
5639 | case ISD::SINT_TO_FP: |
5640 | case ISD::FP16_TO_FP: |
5641 | case ISD::BF16_TO_FP: |
5642 | case ISD::BITCAST: |
5643 | case ISD::ABS: |
5644 | case ISD::BITREVERSE: |
5645 | case ISD::BSWAP: |
5646 | case ISD::CTLZ: |
5647 | case ISD::CTLZ_ZERO_UNDEF: |
5648 | case ISD::CTTZ: |
5649 | case ISD::CTTZ_ZERO_UNDEF: |
5650 | case ISD::CTPOP: |
5651 | case ISD::STEP_VECTOR: { |
5652 | SDValue Ops = {N1}; |
5653 | if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops)) |
5654 | return Fold; |
5655 | } |
5656 | } |
5657 | |
5658 | unsigned OpOpcode = N1.getNode()->getOpcode(); |
5659 | switch (Opcode) { |
5660 | case ISD::STEP_VECTOR: |
5661 | assert(VT.isScalableVector() && |
5662 | "STEP_VECTOR can only be used with scalable types" ); |
5663 | assert(OpOpcode == ISD::TargetConstant && |
5664 | VT.getVectorElementType() == N1.getValueType() && |
5665 | "Unexpected step operand" ); |
5666 | break; |
5667 | case ISD::FREEZE: |
5668 | assert(VT == N1.getValueType() && "Unexpected VT!" ); |
5669 | if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly*/ false, |
5670 | /*Depth*/ 1)) |
5671 | return N1; |
5672 | break; |
5673 | case ISD::TokenFactor: |
5674 | case ISD::MERGE_VALUES: |
5675 | case ISD::CONCAT_VECTORS: |
5676 | return N1; // Factor, merge or concat of one node? No need. |
5677 | case ISD::BUILD_VECTOR: { |
5678 | // Attempt to simplify BUILD_VECTOR. |
5679 | SDValue Ops[] = {N1}; |
5680 | if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this)) |
5681 | return V; |
5682 | break; |
5683 | } |
5684 | case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node" ); |
5685 | case ISD::FP_EXTEND: |
5686 | assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && |
5687 | "Invalid FP cast!" ); |
5688 | if (N1.getValueType() == VT) return N1; // noop conversion. |
5689 | assert((!VT.isVector() || VT.getVectorElementCount() == |
5690 | N1.getValueType().getVectorElementCount()) && |
5691 | "Vector element count mismatch!" ); |
5692 | assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!" ); |
5693 | if (N1.isUndef()) |
5694 | return getUNDEF(VT); |
5695 | break; |
5696 | case ISD::FP_TO_SINT: |
5697 | case ISD::FP_TO_UINT: |
5698 | if (N1.isUndef()) |
5699 | return getUNDEF(VT); |
5700 | break; |
5701 | case ISD::SINT_TO_FP: |
5702 | case ISD::UINT_TO_FP: |
5703 | // [us]itofp(undef) = 0, because the result value is bounded. |
5704 | if (N1.isUndef()) |
5705 | return getConstantFP(Val: 0.0, DL, VT); |
5706 | break; |
5707 | case ISD::SIGN_EXTEND: |
5708 | assert(VT.isInteger() && N1.getValueType().isInteger() && |
5709 | "Invalid SIGN_EXTEND!" ); |
5710 | assert(VT.isVector() == N1.getValueType().isVector() && |
5711 | "SIGN_EXTEND result type type should be vector iff the operand " |
5712 | "type is vector!" ); |
5713 | if (N1.getValueType() == VT) return N1; // noop extension |
5714 | assert((!VT.isVector() || VT.getVectorElementCount() == |
5715 | N1.getValueType().getVectorElementCount()) && |
5716 | "Vector element count mismatch!" ); |
5717 | assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!" ); |
5718 | if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) |
5719 | return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0)); |
5720 | if (OpOpcode == ISD::UNDEF) |
5721 | // sext(undef) = 0, because the top bits will all be the same. |
5722 | return getConstant(Val: 0, DL, VT); |
5723 | break; |
5724 | case ISD::ZERO_EXTEND: |
5725 | assert(VT.isInteger() && N1.getValueType().isInteger() && |
5726 | "Invalid ZERO_EXTEND!" ); |
5727 | assert(VT.isVector() == N1.getValueType().isVector() && |
5728 | "ZERO_EXTEND result type type should be vector iff the operand " |
5729 | "type is vector!" ); |
5730 | if (N1.getValueType() == VT) return N1; // noop extension |
5731 | assert((!VT.isVector() || VT.getVectorElementCount() == |
5732 | N1.getValueType().getVectorElementCount()) && |
5733 | "Vector element count mismatch!" ); |
5734 | assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!" ); |
5735 | if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) |
5736 | return getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0)); |
5737 | if (OpOpcode == ISD::UNDEF) |
5738 | // zext(undef) = 0, because the top bits will be zero. |
5739 | return getConstant(Val: 0, DL, VT); |
5740 | |
5741 | // Skip unnecessary zext_inreg pattern: |
5742 | // (zext (trunc x)) -> x iff the upper bits are known zero. |
5743 | // TODO: Remove (zext (trunc (and x, c))) exception which some targets |
5744 | // use to recognise zext_inreg patterns. |
5745 | if (OpOpcode == ISD::TRUNCATE) { |
5746 | SDValue OpOp = N1.getOperand(i: 0); |
5747 | if (OpOp.getValueType() == VT) { |
5748 | if (OpOp.getOpcode() != ISD::AND) { |
5749 | APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(), |
5750 | loBit: N1.getScalarValueSizeInBits()); |
5751 | if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) { |
5752 | transferDbgValues(From: N1, To: OpOp); |
5753 | return OpOp; |
5754 | } |
5755 | } |
5756 | } |
5757 | } |
5758 | break; |
5759 | case ISD::ANY_EXTEND: |
5760 | assert(VT.isInteger() && N1.getValueType().isInteger() && |
5761 | "Invalid ANY_EXTEND!" ); |
5762 | assert(VT.isVector() == N1.getValueType().isVector() && |
5763 | "ANY_EXTEND result type type should be vector iff the operand " |
5764 | "type is vector!" ); |
5765 | if (N1.getValueType() == VT) return N1; // noop extension |
5766 | assert((!VT.isVector() || VT.getVectorElementCount() == |
5767 | N1.getValueType().getVectorElementCount()) && |
5768 | "Vector element count mismatch!" ); |
5769 | assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!" ); |
5770 | |
5771 | if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
5772 | OpOpcode == ISD::ANY_EXTEND) |
5773 | // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) |
5774 | return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0)); |
5775 | if (OpOpcode == ISD::UNDEF) |
5776 | return getUNDEF(VT); |
5777 | |
5778 | // (ext (trunc x)) -> x |
5779 | if (OpOpcode == ISD::TRUNCATE) { |
5780 | SDValue OpOp = N1.getOperand(i: 0); |
5781 | if (OpOp.getValueType() == VT) { |
5782 | transferDbgValues(From: N1, To: OpOp); |
5783 | return OpOp; |
5784 | } |
5785 | } |
5786 | break; |
5787 | case ISD::TRUNCATE: |
5788 | assert(VT.isInteger() && N1.getValueType().isInteger() && |
5789 | "Invalid TRUNCATE!" ); |
5790 | assert(VT.isVector() == N1.getValueType().isVector() && |
5791 | "TRUNCATE result type type should be vector iff the operand " |
5792 | "type is vector!" ); |
5793 | if (N1.getValueType() == VT) return N1; // noop truncate |
5794 | assert((!VT.isVector() || VT.getVectorElementCount() == |
5795 | N1.getValueType().getVectorElementCount()) && |
5796 | "Vector element count mismatch!" ); |
5797 | assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!" ); |
5798 | if (OpOpcode == ISD::TRUNCATE) |
5799 | return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0)); |
5800 | if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
5801 | OpOpcode == ISD::ANY_EXTEND) { |
5802 | // If the source is smaller than the dest, we still need an extend. |
5803 | if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT( |
5804 | VT: VT.getScalarType())) |
5805 | return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0)); |
5806 | if (N1.getOperand(i: 0).getValueType().bitsGT(VT)) |
5807 | return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0)); |
5808 | return N1.getOperand(i: 0); |
5809 | } |
5810 | if (OpOpcode == ISD::UNDEF) |
5811 | return getUNDEF(VT); |
5812 | if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes) |
5813 | return getVScale(DL, VT, |
5814 | MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits())); |
5815 | break; |
5816 | case ISD::ANY_EXTEND_VECTOR_INREG: |
5817 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
5818 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
5819 | assert(VT.isVector() && "This DAG node is restricted to vector types." ); |
5820 | assert(N1.getValueType().bitsLE(VT) && |
5821 | "The input must be the same size or smaller than the result." ); |
5822 | assert(VT.getVectorMinNumElements() < |
5823 | N1.getValueType().getVectorMinNumElements() && |
5824 | "The destination vector type must have fewer lanes than the input." ); |
5825 | break; |
5826 | case ISD::ABS: |
5827 | assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!" ); |
5828 | if (OpOpcode == ISD::UNDEF) |
5829 | return getConstant(Val: 0, DL, VT); |
5830 | break; |
5831 | case ISD::BSWAP: |
5832 | assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!" ); |
5833 | assert((VT.getScalarSizeInBits() % 16 == 0) && |
5834 | "BSWAP types must be a multiple of 16 bits!" ); |
5835 | if (OpOpcode == ISD::UNDEF) |
5836 | return getUNDEF(VT); |
5837 | // bswap(bswap(X)) -> X. |
5838 | if (OpOpcode == ISD::BSWAP) |
5839 | return N1.getOperand(i: 0); |
5840 | break; |
5841 | case ISD::BITREVERSE: |
5842 | assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!" ); |
5843 | if (OpOpcode == ISD::UNDEF) |
5844 | return getUNDEF(VT); |
5845 | break; |
5846 | case ISD::BITCAST: |
5847 | assert(VT.getSizeInBits() == N1.getValueSizeInBits() && |
5848 | "Cannot BITCAST between types of different sizes!" ); |
5849 | if (VT == N1.getValueType()) return N1; // noop conversion. |
5850 | if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) |
5851 | return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0)); |
5852 | if (OpOpcode == ISD::UNDEF) |
5853 | return getUNDEF(VT); |
5854 | break; |
5855 | case ISD::SCALAR_TO_VECTOR: |
5856 | assert(VT.isVector() && !N1.getValueType().isVector() && |
5857 | (VT.getVectorElementType() == N1.getValueType() || |
5858 | (VT.getVectorElementType().isInteger() && |
5859 | N1.getValueType().isInteger() && |
5860 | VT.getVectorElementType().bitsLE(N1.getValueType()))) && |
5861 | "Illegal SCALAR_TO_VECTOR node!" ); |
5862 | if (OpOpcode == ISD::UNDEF) |
5863 | return getUNDEF(VT); |
5864 | // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. |
5865 | if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && |
5866 | isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) && |
5867 | N1.getConstantOperandVal(i: 1) == 0 && |
5868 | N1.getOperand(i: 0).getValueType() == VT) |
5869 | return N1.getOperand(i: 0); |
5870 | break; |
5871 | case ISD::FNEG: |
5872 | // Negation of an unknown bag of bits is still completely undefined. |
5873 | if (OpOpcode == ISD::UNDEF) |
5874 | return getUNDEF(VT); |
5875 | |
5876 | if (OpOpcode == ISD::FNEG) // --X -> X |
5877 | return N1.getOperand(i: 0); |
5878 | break; |
5879 | case ISD::FABS: |
5880 | if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) |
5881 | return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0)); |
5882 | break; |
5883 | case ISD::VSCALE: |
5884 | assert(VT == N1.getValueType() && "Unexpected VT!" ); |
5885 | break; |
5886 | case ISD::CTPOP: |
5887 | if (N1.getValueType().getScalarType() == MVT::i1) |
5888 | return N1; |
5889 | break; |
5890 | case ISD::CTLZ: |
5891 | case ISD::CTTZ: |
5892 | if (N1.getValueType().getScalarType() == MVT::i1) |
5893 | return getNOT(DL, Val: N1, VT: N1.getValueType()); |
5894 | break; |
5895 | case ISD::VECREDUCE_ADD: |
5896 | if (N1.getValueType().getScalarType() == MVT::i1) |
5897 | return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1); |
5898 | break; |
5899 | case ISD::VECREDUCE_SMIN: |
5900 | case ISD::VECREDUCE_UMAX: |
5901 | if (N1.getValueType().getScalarType() == MVT::i1) |
5902 | return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1); |
5903 | break; |
5904 | case ISD::VECREDUCE_SMAX: |
5905 | case ISD::VECREDUCE_UMIN: |
5906 | if (N1.getValueType().getScalarType() == MVT::i1) |
5907 | return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1); |
5908 | break; |
5909 | } |
5910 | |
5911 | SDNode *N; |
5912 | SDVTList VTs = getVTList(VT); |
5913 | SDValue Ops[] = {N1}; |
5914 | if (VT != MVT::Glue) { // Don't CSE glue producing nodes |
5915 | FoldingSetNodeID ID; |
5916 | AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops); |
5917 | void *IP = nullptr; |
5918 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
5919 | E->intersectFlagsWith(Flags); |
5920 | return SDValue(E, 0); |
5921 | } |
5922 | |
5923 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
5924 | N->setFlags(Flags); |
5925 | createOperands(Node: N, Vals: Ops); |
5926 | CSEMap.InsertNode(N, InsertPos: IP); |
5927 | } else { |
5928 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
5929 | createOperands(Node: N, Vals: Ops); |
5930 | } |
5931 | |
5932 | InsertNode(N); |
5933 | SDValue V = SDValue(N, 0); |
5934 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
5935 | return V; |
5936 | } |
5937 | |
5938 | static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1, |
5939 | const APInt &C2) { |
5940 | switch (Opcode) { |
5941 | case ISD::ADD: return C1 + C2; |
5942 | case ISD::SUB: return C1 - C2; |
5943 | case ISD::MUL: return C1 * C2; |
5944 | case ISD::AND: return C1 & C2; |
5945 | case ISD::OR: return C1 | C2; |
5946 | case ISD::XOR: return C1 ^ C2; |
5947 | case ISD::SHL: return C1 << C2; |
5948 | case ISD::SRL: return C1.lshr(ShiftAmt: C2); |
5949 | case ISD::SRA: return C1.ashr(ShiftAmt: C2); |
5950 | case ISD::ROTL: return C1.rotl(rotateAmt: C2); |
5951 | case ISD::ROTR: return C1.rotr(rotateAmt: C2); |
5952 | case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2; |
5953 | case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2; |
5954 | case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2; |
5955 | case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2; |
5956 | case ISD::SADDSAT: return C1.sadd_sat(RHS: C2); |
5957 | case ISD::UADDSAT: return C1.uadd_sat(RHS: C2); |
5958 | case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2); |
5959 | case ISD::USUBSAT: return C1.usub_sat(RHS: C2); |
5960 | case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2); |
5961 | case ISD::USHLSAT: return C1.ushl_sat(RHS: C2); |
5962 | case ISD::UDIV: |
5963 | if (!C2.getBoolValue()) |
5964 | break; |
5965 | return C1.udiv(RHS: C2); |
5966 | case ISD::UREM: |
5967 | if (!C2.getBoolValue()) |
5968 | break; |
5969 | return C1.urem(RHS: C2); |
5970 | case ISD::SDIV: |
5971 | if (!C2.getBoolValue()) |
5972 | break; |
5973 | return C1.sdiv(RHS: C2); |
5974 | case ISD::SREM: |
5975 | if (!C2.getBoolValue()) |
5976 | break; |
5977 | return C1.srem(RHS: C2); |
5978 | case ISD::MULHS: { |
5979 | unsigned FullWidth = C1.getBitWidth() * 2; |
5980 | APInt C1Ext = C1.sext(width: FullWidth); |
5981 | APInt C2Ext = C2.sext(width: FullWidth); |
5982 | return (C1Ext * C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: C1.getBitWidth()); |
5983 | } |
5984 | case ISD::MULHU: { |
5985 | unsigned FullWidth = C1.getBitWidth() * 2; |
5986 | APInt C1Ext = C1.zext(width: FullWidth); |
5987 | APInt C2Ext = C2.zext(width: FullWidth); |
5988 | return (C1Ext * C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: C1.getBitWidth()); |
5989 | } |
5990 | case ISD::AVGFLOORS: { |
5991 | unsigned FullWidth = C1.getBitWidth() + 1; |
5992 | APInt C1Ext = C1.sext(width: FullWidth); |
5993 | APInt C2Ext = C2.sext(width: FullWidth); |
5994 | return (C1Ext + C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: 1); |
5995 | } |
5996 | case ISD::AVGFLOORU: { |
5997 | unsigned FullWidth = C1.getBitWidth() + 1; |
5998 | APInt C1Ext = C1.zext(width: FullWidth); |
5999 | APInt C2Ext = C2.zext(width: FullWidth); |
6000 | return (C1Ext + C2Ext).extractBits(numBits: C1.getBitWidth(), bitPosition: 1); |
6001 | } |
6002 | case ISD::AVGCEILS: { |
6003 | unsigned FullWidth = C1.getBitWidth() + 1; |
6004 | APInt C1Ext = C1.sext(width: FullWidth); |
6005 | APInt C2Ext = C2.sext(width: FullWidth); |
6006 | return (C1Ext + C2Ext + 1).extractBits(numBits: C1.getBitWidth(), bitPosition: 1); |
6007 | } |
6008 | case ISD::AVGCEILU: { |
6009 | unsigned FullWidth = C1.getBitWidth() + 1; |
6010 | APInt C1Ext = C1.zext(width: FullWidth); |
6011 | APInt C2Ext = C2.zext(width: FullWidth); |
6012 | return (C1Ext + C2Ext + 1).extractBits(numBits: C1.getBitWidth(), bitPosition: 1); |
6013 | } |
6014 | case ISD::ABDS: |
6015 | return APIntOps::smax(A: C1, B: C2) - APIntOps::smin(A: C1, B: C2); |
6016 | case ISD::ABDU: |
6017 | return APIntOps::umax(A: C1, B: C2) - APIntOps::umin(A: C1, B: C2); |
6018 | } |
6019 | return std::nullopt; |
6020 | } |
6021 | |
6022 | // Handle constant folding with UNDEF. |
6023 | // TODO: Handle more cases. |
6024 | static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1, |
6025 | bool IsUndef1, const APInt &C2, |
6026 | bool IsUndef2) { |
6027 | if (!(IsUndef1 || IsUndef2)) |
6028 | return FoldValue(Opcode, C1, C2); |
6029 | |
6030 | // Fold and(x, undef) -> 0 |
6031 | // Fold mul(x, undef) -> 0 |
6032 | if (Opcode == ISD::AND || Opcode == ISD::MUL) |
6033 | return APInt::getZero(numBits: C1.getBitWidth()); |
6034 | |
6035 | return std::nullopt; |
6036 | } |
6037 | |
6038 | SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, |
6039 | const GlobalAddressSDNode *GA, |
6040 | const SDNode *N2) { |
6041 | if (GA->getOpcode() != ISD::GlobalAddress) |
6042 | return SDValue(); |
6043 | if (!TLI->isOffsetFoldingLegal(GA)) |
6044 | return SDValue(); |
6045 | auto *C2 = dyn_cast<ConstantSDNode>(Val: N2); |
6046 | if (!C2) |
6047 | return SDValue(); |
6048 | int64_t Offset = C2->getSExtValue(); |
6049 | switch (Opcode) { |
6050 | case ISD::ADD: break; |
6051 | case ISD::SUB: Offset = -uint64_t(Offset); break; |
6052 | default: return SDValue(); |
6053 | } |
6054 | return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT, |
6055 | Offset: GA->getOffset() + uint64_t(Offset)); |
6056 | } |
6057 | |
6058 | bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { |
6059 | switch (Opcode) { |
6060 | case ISD::SDIV: |
6061 | case ISD::UDIV: |
6062 | case ISD::SREM: |
6063 | case ISD::UREM: { |
6064 | // If a divisor is zero/undef or any element of a divisor vector is |
6065 | // zero/undef, the whole op is undef. |
6066 | assert(Ops.size() == 2 && "Div/rem should have 2 operands" ); |
6067 | SDValue Divisor = Ops[1]; |
6068 | if (Divisor.isUndef() || isNullConstant(V: Divisor)) |
6069 | return true; |
6070 | |
6071 | return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) && |
6072 | llvm::any_of(Range: Divisor->op_values(), |
6073 | P: [](SDValue V) { return V.isUndef() || |
6074 | isNullConstant(V); }); |
6075 | // TODO: Handle signed overflow. |
6076 | } |
6077 | // TODO: Handle oversized shifts. |
6078 | default: |
6079 | return false; |
6080 | } |
6081 | } |
6082 | |
6083 | SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, |
6084 | EVT VT, ArrayRef<SDValue> Ops) { |
6085 | // If the opcode is a target-specific ISD node, there's nothing we can |
6086 | // do here and the operand rules may not line up with the below, so |
6087 | // bail early. |
6088 | // We can't create a scalar CONCAT_VECTORS so skip it. It will break |
6089 | // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by |
6090 | // foldCONCAT_VECTORS in getNode before this is called. |
6091 | if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) |
6092 | return SDValue(); |
6093 | |
6094 | unsigned NumOps = Ops.size(); |
6095 | if (NumOps == 0) |
6096 | return SDValue(); |
6097 | |
6098 | if (isUndef(Opcode, Ops)) |
6099 | return getUNDEF(VT); |
6100 | |
6101 | // Handle unary special cases. |
6102 | if (NumOps == 1) { |
6103 | SDValue N1 = Ops[0]; |
6104 | |
6105 | // Constant fold unary operations with an integer constant operand. Even |
6106 | // opaque constant will be folded, because the folding of unary operations |
6107 | // doesn't create new constants with different values. Nevertheless, the |
6108 | // opaque flag is preserved during folding to prevent future folding with |
6109 | // other constants. |
6110 | if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) { |
6111 | const APInt &Val = C->getAPIntValue(); |
6112 | switch (Opcode) { |
6113 | case ISD::SIGN_EXTEND: |
6114 | return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT, |
6115 | isT: C->isTargetOpcode(), isO: C->isOpaque()); |
6116 | case ISD::TRUNCATE: |
6117 | if (C->isOpaque()) |
6118 | break; |
6119 | [[fallthrough]]; |
6120 | case ISD::ZERO_EXTEND: |
6121 | return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT, |
6122 | isT: C->isTargetOpcode(), isO: C->isOpaque()); |
6123 | case ISD::ANY_EXTEND: |
6124 | // Some targets like RISCV prefer to sign extend some types. |
6125 | if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT)) |
6126 | return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT, |
6127 | isT: C->isTargetOpcode(), isO: C->isOpaque()); |
6128 | return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT, |
6129 | isT: C->isTargetOpcode(), isO: C->isOpaque()); |
6130 | case ISD::ABS: |
6131 | return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(), |
6132 | isO: C->isOpaque()); |
6133 | case ISD::BITREVERSE: |
6134 | return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(), |
6135 | isO: C->isOpaque()); |
6136 | case ISD::BSWAP: |
6137 | return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(), |
6138 | isO: C->isOpaque()); |
6139 | case ISD::CTPOP: |
6140 | return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(), |
6141 | isO: C->isOpaque()); |
6142 | case ISD::CTLZ: |
6143 | case ISD::CTLZ_ZERO_UNDEF: |
6144 | return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(), |
6145 | isO: C->isOpaque()); |
6146 | case ISD::CTTZ: |
6147 | case ISD::CTTZ_ZERO_UNDEF: |
6148 | return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(), |
6149 | isO: C->isOpaque()); |
6150 | case ISD::UINT_TO_FP: |
6151 | case ISD::SINT_TO_FP: { |
6152 | APFloat apf(EVTToAPFloatSemantics(VT), |
6153 | APInt::getZero(numBits: VT.getSizeInBits())); |
6154 | (void)apf.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP, |
6155 | RM: APFloat::rmNearestTiesToEven); |
6156 | return getConstantFP(V: apf, DL, VT); |
6157 | } |
6158 | case ISD::FP16_TO_FP: |
6159 | case ISD::BF16_TO_FP: { |
6160 | bool Ignored; |
6161 | APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf() |
6162 | : APFloat::BFloat(), |
6163 | (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16)); |
6164 | |
6165 | // This can return overflow, underflow, or inexact; we don't care. |
6166 | // FIXME need to be more flexible about rounding mode. |
6167 | (void)FPV.convert(ToSemantics: EVTToAPFloatSemantics(VT), |
6168 | RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored); |
6169 | return getConstantFP(V: FPV, DL, VT); |
6170 | } |
6171 | case ISD::STEP_VECTOR: |
6172 | if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this)) |
6173 | return V; |
6174 | break; |
6175 | case ISD::BITCAST: |
6176 | if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16) |
6177 | return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT); |
6178 | if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32) |
6179 | return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT); |
6180 | if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64) |
6181 | return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT); |
6182 | if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128) |
6183 | return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT); |
6184 | break; |
6185 | } |
6186 | } |
6187 | |
6188 | // Constant fold unary operations with a floating point constant operand. |
6189 | if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) { |
6190 | APFloat V = C->getValueAPF(); // make copy |
6191 | switch (Opcode) { |
6192 | case ISD::FNEG: |
6193 | V.changeSign(); |
6194 | return getConstantFP(V, DL, VT); |
6195 | case ISD::FABS: |
6196 | V.clearSign(); |
6197 | return getConstantFP(V, DL, VT); |
6198 | case ISD::FCEIL: { |
6199 | APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive); |
6200 | if (fs == APFloat::opOK || fs == APFloat::opInexact) |
6201 | return getConstantFP(V, DL, VT); |
6202 | return SDValue(); |
6203 | } |
6204 | case ISD::FTRUNC: { |
6205 | APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero); |
6206 | if (fs == APFloat::opOK || fs == APFloat::opInexact) |
6207 | return getConstantFP(V, DL, VT); |
6208 | return SDValue(); |
6209 | } |
6210 | case ISD::FFLOOR: { |
6211 | APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative); |
6212 | if (fs == APFloat::opOK || fs == APFloat::opInexact) |
6213 | return getConstantFP(V, DL, VT); |
6214 | return SDValue(); |
6215 | } |
6216 | case ISD::FP_EXTEND: { |
6217 | bool ignored; |
6218 | // This can return overflow, underflow, or inexact; we don't care. |
6219 | // FIXME need to be more flexible about rounding mode. |
6220 | (void)V.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven, |
6221 | losesInfo: &ignored); |
6222 | return getConstantFP(V, DL, VT); |
6223 | } |
6224 | case ISD::FP_TO_SINT: |
6225 | case ISD::FP_TO_UINT: { |
6226 | bool ignored; |
6227 | APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); |
6228 | // FIXME need to be more flexible about rounding mode. |
6229 | APFloat::opStatus s = |
6230 | V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored); |
6231 | if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual |
6232 | break; |
6233 | return getConstant(Val: IntVal, DL, VT); |
6234 | } |
6235 | case ISD::FP_TO_FP16: |
6236 | case ISD::FP_TO_BF16: { |
6237 | bool Ignored; |
6238 | // This can return overflow, underflow, or inexact; we don't care. |
6239 | // FIXME need to be more flexible about rounding mode. |
6240 | (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf() |
6241 | : APFloat::BFloat(), |
6242 | RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored); |
6243 | return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT); |
6244 | } |
6245 | case ISD::BITCAST: |
6246 | if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16) |
6247 | return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL, |
6248 | VT); |
6249 | if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16) |
6250 | return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL, |
6251 | VT); |
6252 | if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32) |
6253 | return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL, |
6254 | VT); |
6255 | if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64) |
6256 | return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT); |
6257 | break; |
6258 | } |
6259 | } |
6260 | |
6261 | // Early-out if we failed to constant fold a bitcast. |
6262 | if (Opcode == ISD::BITCAST) |
6263 | return SDValue(); |
6264 | } |
6265 | |
6266 | // Handle binops special cases. |
6267 | if (NumOps == 2) { |
6268 | if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops)) |
6269 | return CFP; |
6270 | |
6271 | if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) { |
6272 | if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) { |
6273 | if (C1->isOpaque() || C2->isOpaque()) |
6274 | return SDValue(); |
6275 | |
6276 | std::optional<APInt> FoldAttempt = |
6277 | FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue()); |
6278 | if (!FoldAttempt) |
6279 | return SDValue(); |
6280 | |
6281 | SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT); |
6282 | assert((!Folded || !VT.isVector()) && |
6283 | "Can't fold vectors ops with scalar operands" ); |
6284 | return Folded; |
6285 | } |
6286 | } |
6287 | |
6288 | // fold (add Sym, c) -> Sym+c |
6289 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0])) |
6290 | return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode()); |
6291 | if (TLI->isCommutativeBinOp(Opcode)) |
6292 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1])) |
6293 | return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode()); |
6294 | } |
6295 | |
6296 | // This is for vector folding only from here on. |
6297 | if (!VT.isVector()) |
6298 | return SDValue(); |
6299 | |
6300 | ElementCount NumElts = VT.getVectorElementCount(); |
6301 | |
6302 | // See if we can fold through bitcasted integer ops. |
6303 | if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() && |
6304 | Ops[0].getValueType() == VT && Ops[1].getValueType() == VT && |
6305 | Ops[0].getOpcode() == ISD::BITCAST && |
6306 | Ops[1].getOpcode() == ISD::BITCAST) { |
6307 | SDValue N1 = peekThroughBitcasts(V: Ops[0]); |
6308 | SDValue N2 = peekThroughBitcasts(V: Ops[1]); |
6309 | auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1); |
6310 | auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2); |
6311 | EVT BVVT = N1.getValueType(); |
6312 | if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) { |
6313 | bool IsLE = getDataLayout().isLittleEndian(); |
6314 | unsigned EltBits = VT.getScalarSizeInBits(); |
6315 | SmallVector<APInt> RawBits1, RawBits2; |
6316 | BitVector UndefElts1, UndefElts2; |
6317 | if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) && |
6318 | BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) { |
6319 | SmallVector<APInt> RawBits; |
6320 | for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) { |
6321 | std::optional<APInt> Fold = FoldValueWithUndef( |
6322 | Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]); |
6323 | if (!Fold) |
6324 | break; |
6325 | RawBits.push_back(Elt: *Fold); |
6326 | } |
6327 | if (RawBits.size() == NumElts.getFixedValue()) { |
6328 | // We have constant folded, but we need to cast this again back to |
6329 | // the original (possibly legalized) type. |
6330 | SmallVector<APInt> DstBits; |
6331 | BitVector DstUndefs; |
6332 | BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(), |
6333 | DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs, |
6334 | SrcUndefElements: BitVector(RawBits.size(), false)); |
6335 | EVT BVEltVT = BV1->getOperand(Num: 0).getValueType(); |
6336 | unsigned BVEltBits = BVEltVT.getSizeInBits(); |
6337 | SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT)); |
6338 | for (unsigned I = 0, E = DstBits.size(); I != E; ++I) { |
6339 | if (DstUndefs[I]) |
6340 | continue; |
6341 | Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT); |
6342 | } |
6343 | return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops)); |
6344 | } |
6345 | } |
6346 | } |
6347 | } |
6348 | |
6349 | // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)). |
6350 | // (shl step_vector(C0), C1) -> (step_vector(C0 << C1)) |
6351 | if ((Opcode == ISD::MUL || Opcode == ISD::SHL) && |
6352 | Ops[0].getOpcode() == ISD::STEP_VECTOR) { |
6353 | APInt RHSVal; |
6354 | if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) { |
6355 | APInt NewStep = Opcode == ISD::MUL |
6356 | ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal |
6357 | : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal; |
6358 | return getStepVector(DL, ResVT: VT, StepVal: NewStep); |
6359 | } |
6360 | } |
6361 | |
6362 | auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) { |
6363 | return !Op.getValueType().isVector() || |
6364 | Op.getValueType().getVectorElementCount() == NumElts; |
6365 | }; |
6366 | |
6367 | auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) { |
6368 | return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || |
6369 | Op.getOpcode() == ISD::BUILD_VECTOR || |
6370 | Op.getOpcode() == ISD::SPLAT_VECTOR; |
6371 | }; |
6372 | |
6373 | // All operands must be vector types with the same number of elements as |
6374 | // the result type and must be either UNDEF or a build/splat vector |
6375 | // or UNDEF scalars. |
6376 | if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) || |
6377 | !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize)) |
6378 | return SDValue(); |
6379 | |
6380 | // If we are comparing vectors, then the result needs to be a i1 boolean that |
6381 | // is then extended back to the legal result type depending on how booleans |
6382 | // are represented. |
6383 | EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); |
6384 | ISD::NodeType ExtendCode = |
6385 | (Opcode == ISD::SETCC && SVT != VT.getScalarType()) |
6386 | ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT)) |
6387 | : ISD::SIGN_EXTEND; |
6388 | |
6389 | // Find legal integer scalar type for constant promotion and |
6390 | // ensure that its scalar size is at least as large as source. |
6391 | EVT LegalSVT = VT.getScalarType(); |
6392 | if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { |
6393 | LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT); |
6394 | if (LegalSVT.bitsLT(VT: VT.getScalarType())) |
6395 | return SDValue(); |
6396 | } |
6397 | |
6398 | // For scalable vector types we know we're dealing with SPLAT_VECTORs. We |
6399 | // only have one operand to check. For fixed-length vector types we may have |
6400 | // a combination of BUILD_VECTOR and SPLAT_VECTOR. |
6401 | unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue(); |
6402 | |
6403 | // Constant fold each scalar lane separately. |
6404 | SmallVector<SDValue, 4> ScalarResults; |
6405 | for (unsigned I = 0; I != NumVectorElts; I++) { |
6406 | SmallVector<SDValue, 4> ScalarOps; |
6407 | for (SDValue Op : Ops) { |
6408 | EVT InSVT = Op.getValueType().getScalarType(); |
6409 | if (Op.getOpcode() != ISD::BUILD_VECTOR && |
6410 | Op.getOpcode() != ISD::SPLAT_VECTOR) { |
6411 | if (Op.isUndef()) |
6412 | ScalarOps.push_back(Elt: getUNDEF(VT: InSVT)); |
6413 | else |
6414 | ScalarOps.push_back(Elt: Op); |
6415 | continue; |
6416 | } |
6417 | |
6418 | SDValue ScalarOp = |
6419 | Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); |
6420 | EVT ScalarVT = ScalarOp.getValueType(); |
6421 | |
6422 | // Build vector (integer) scalar operands may need implicit |
6423 | // truncation - do this before constant folding. |
6424 | if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) { |
6425 | // Don't create illegally-typed nodes unless they're constants or undef |
6426 | // - if we fail to constant fold we can't guarantee the (dead) nodes |
6427 | // we're creating will be cleaned up before being visited for |
6428 | // legalization. |
6429 | if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() && |
6430 | !isa<ConstantSDNode>(Val: ScalarOp) && |
6431 | TLI->getTypeAction(Context&: *getContext(), VT: InSVT) != |
6432 | TargetLowering::TypeLegal) |
6433 | return SDValue(); |
6434 | ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp); |
6435 | } |
6436 | |
6437 | ScalarOps.push_back(Elt: ScalarOp); |
6438 | } |
6439 | |
6440 | // Constant fold the scalar operands. |
6441 | SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps); |
6442 | |
6443 | // Legalize the (integer) scalar constant if necessary. |
6444 | if (LegalSVT != SVT) |
6445 | ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult); |
6446 | |
6447 | // Scalar folding only succeeded if the result is a constant or UNDEF. |
6448 | if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && |
6449 | ScalarResult.getOpcode() != ISD::ConstantFP) |
6450 | return SDValue(); |
6451 | ScalarResults.push_back(Elt: ScalarResult); |
6452 | } |
6453 | |
6454 | SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0]) |
6455 | : getBuildVector(VT, DL, Ops: ScalarResults); |
6456 | NewSDValueDbgMsg(V, Msg: "New node fold constant vector: " , G: this); |
6457 | return V; |
6458 | } |
6459 | |
6460 | SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, |
6461 | EVT VT, ArrayRef<SDValue> Ops) { |
6462 | // TODO: Add support for unary/ternary fp opcodes. |
6463 | if (Ops.size() != 2) |
6464 | return SDValue(); |
6465 | |
6466 | // TODO: We don't do any constant folding for strict FP opcodes here, but we |
6467 | // should. That will require dealing with a potentially non-default |
6468 | // rounding mode, checking the "opStatus" return value from the APFloat |
6469 | // math calculations, and possibly other variations. |
6470 | SDValue N1 = Ops[0]; |
6471 | SDValue N2 = Ops[1]; |
6472 | ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false); |
6473 | ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false); |
6474 | if (N1CFP && N2CFP) { |
6475 | APFloat C1 = N1CFP->getValueAPF(); // make copy |
6476 | const APFloat &C2 = N2CFP->getValueAPF(); |
6477 | switch (Opcode) { |
6478 | case ISD::FADD: |
6479 | C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven); |
6480 | return getConstantFP(V: C1, DL, VT); |
6481 | case ISD::FSUB: |
6482 | C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven); |
6483 | return getConstantFP(V: C1, DL, VT); |
6484 | case ISD::FMUL: |
6485 | C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven); |
6486 | return getConstantFP(V: C1, DL, VT); |
6487 | case ISD::FDIV: |
6488 | C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven); |
6489 | return getConstantFP(V: C1, DL, VT); |
6490 | case ISD::FREM: |
6491 | C1.mod(RHS: C2); |
6492 | return getConstantFP(V: C1, DL, VT); |
6493 | case ISD::FCOPYSIGN: |
6494 | C1.copySign(RHS: C2); |
6495 | return getConstantFP(V: C1, DL, VT); |
6496 | case ISD::FMINNUM: |
6497 | return getConstantFP(V: minnum(A: C1, B: C2), DL, VT); |
6498 | case ISD::FMAXNUM: |
6499 | return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT); |
6500 | case ISD::FMINIMUM: |
6501 | return getConstantFP(V: minimum(A: C1, B: C2), DL, VT); |
6502 | case ISD::FMAXIMUM: |
6503 | return getConstantFP(V: maximum(A: C1, B: C2), DL, VT); |
6504 | default: break; |
6505 | } |
6506 | } |
6507 | if (N1CFP && Opcode == ISD::FP_ROUND) { |
6508 | APFloat C1 = N1CFP->getValueAPF(); // make copy |
6509 | bool Unused; |
6510 | // This can return overflow, underflow, or inexact; we don't care. |
6511 | // FIXME need to be more flexible about rounding mode. |
6512 | (void) C1.convert(ToSemantics: EVTToAPFloatSemantics(VT), RM: APFloat::rmNearestTiesToEven, |
6513 | losesInfo: &Unused); |
6514 | return getConstantFP(V: C1, DL, VT); |
6515 | } |
6516 | |
6517 | switch (Opcode) { |
6518 | case ISD::FSUB: |
6519 | // -0.0 - undef --> undef (consistent with "fneg undef") |
6520 | if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true)) |
6521 | if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef()) |
6522 | return getUNDEF(VT); |
6523 | [[fallthrough]]; |
6524 | |
6525 | case ISD::FADD: |
6526 | case ISD::FMUL: |
6527 | case ISD::FDIV: |
6528 | case ISD::FREM: |
6529 | // If both operands are undef, the result is undef. If 1 operand is undef, |
6530 | // the result is NaN. This should match the behavior of the IR optimizer. |
6531 | if (N1.isUndef() && N2.isUndef()) |
6532 | return getUNDEF(VT); |
6533 | if (N1.isUndef() || N2.isUndef()) |
6534 | return getConstantFP(V: APFloat::getNaN(Sem: EVTToAPFloatSemantics(VT)), DL, VT); |
6535 | } |
6536 | return SDValue(); |
6537 | } |
6538 | |
6539 | SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { |
6540 | assert(Val.getValueType().isInteger() && "Invalid AssertAlign!" ); |
6541 | |
6542 | // There's no need to assert on a byte-aligned pointer. All pointers are at |
6543 | // least byte aligned. |
6544 | if (A == Align(1)) |
6545 | return Val; |
6546 | |
6547 | FoldingSetNodeID ID; |
6548 | AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: getVTList(VT: Val.getValueType()), OpList: {Val}); |
6549 | ID.AddInteger(I: A.value()); |
6550 | |
6551 | void *IP = nullptr; |
6552 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
6553 | return SDValue(E, 0); |
6554 | |
6555 | auto *N = newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), |
6556 | Args: Val.getValueType(), Args&: A); |
6557 | createOperands(Node: N, Vals: {Val}); |
6558 | |
6559 | CSEMap.InsertNode(N, InsertPos: IP); |
6560 | InsertNode(N); |
6561 | |
6562 | SDValue V(N, 0); |
6563 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
6564 | return V; |
6565 | } |
6566 | |
6567 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6568 | SDValue N1, SDValue N2) { |
6569 | SDNodeFlags Flags; |
6570 | if (Inserter) |
6571 | Flags = Inserter->getFlags(); |
6572 | return getNode(Opcode, DL, VT, N1, N2, Flags); |
6573 | } |
6574 | |
6575 | void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, |
6576 | SDValue &N2) const { |
6577 | if (!TLI->isCommutativeBinOp(Opcode)) |
6578 | return; |
6579 | |
6580 | // Canonicalize: |
6581 | // binop(const, nonconst) -> binop(nonconst, const) |
6582 | SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N: N1); |
6583 | SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N: N2); |
6584 | SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1); |
6585 | SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2); |
6586 | if ((N1C && !N2C) || (N1CFP && !N2CFP)) |
6587 | std::swap(a&: N1, b&: N2); |
6588 | |
6589 | // Canonicalize: |
6590 | // binop(splat(x), step_vector) -> binop(step_vector, splat(x)) |
6591 | else if (N1.getOpcode() == ISD::SPLAT_VECTOR && |
6592 | N2.getOpcode() == ISD::STEP_VECTOR) |
6593 | std::swap(a&: N1, b&: N2); |
6594 | } |
6595 | |
6596 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6597 | SDValue N1, SDValue N2, const SDNodeFlags Flags) { |
6598 | assert(N1.getOpcode() != ISD::DELETED_NODE && |
6599 | N2.getOpcode() != ISD::DELETED_NODE && |
6600 | "Operand is DELETED_NODE!" ); |
6601 | |
6602 | canonicalizeCommutativeBinop(Opcode, N1, N2); |
6603 | |
6604 | auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1); |
6605 | auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2); |
6606 | |
6607 | // Don't allow undefs in vector splats - we might be returning N2 when folding |
6608 | // to zero etc. |
6609 | ConstantSDNode *N2CV = |
6610 | isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true); |
6611 | |
6612 | switch (Opcode) { |
6613 | default: break; |
6614 | case ISD::TokenFactor: |
6615 | assert(VT == MVT::Other && N1.getValueType() == MVT::Other && |
6616 | N2.getValueType() == MVT::Other && "Invalid token factor!" ); |
6617 | // Fold trivial token factors. |
6618 | if (N1.getOpcode() == ISD::EntryToken) return N2; |
6619 | if (N2.getOpcode() == ISD::EntryToken) return N1; |
6620 | if (N1 == N2) return N1; |
6621 | break; |
6622 | case ISD::BUILD_VECTOR: { |
6623 | // Attempt to simplify BUILD_VECTOR. |
6624 | SDValue Ops[] = {N1, N2}; |
6625 | if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this)) |
6626 | return V; |
6627 | break; |
6628 | } |
6629 | case ISD::CONCAT_VECTORS: { |
6630 | SDValue Ops[] = {N1, N2}; |
6631 | if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this)) |
6632 | return V; |
6633 | break; |
6634 | } |
6635 | case ISD::AND: |
6636 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6637 | assert(N1.getValueType() == N2.getValueType() && |
6638 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6639 | // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's |
6640 | // worth handling here. |
6641 | if (N2CV && N2CV->isZero()) |
6642 | return N2; |
6643 | if (N2CV && N2CV->isAllOnes()) // X & -1 -> X |
6644 | return N1; |
6645 | break; |
6646 | case ISD::OR: |
6647 | case ISD::XOR: |
6648 | case ISD::ADD: |
6649 | case ISD::SUB: |
6650 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6651 | assert(N1.getValueType() == N2.getValueType() && |
6652 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6653 | // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so |
6654 | // it's worth handling here. |
6655 | if (N2CV && N2CV->isZero()) |
6656 | return N1; |
6657 | if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && |
6658 | VT.getVectorElementType() == MVT::i1) |
6659 | return getNode(Opcode: ISD::XOR, DL, VT, N1, N2); |
6660 | break; |
6661 | case ISD::MUL: |
6662 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6663 | assert(N1.getValueType() == N2.getValueType() && |
6664 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6665 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
6666 | return getNode(Opcode: ISD::AND, DL, VT, N1, N2); |
6667 | if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
6668 | const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0); |
6669 | const APInt &N2CImm = N2C->getAPIntValue(); |
6670 | return getVScale(DL, VT, MulImm: MulImm * N2CImm); |
6671 | } |
6672 | break; |
6673 | case ISD::UDIV: |
6674 | case ISD::UREM: |
6675 | case ISD::MULHU: |
6676 | case ISD::MULHS: |
6677 | case ISD::SDIV: |
6678 | case ISD::SREM: |
6679 | case ISD::SADDSAT: |
6680 | case ISD::SSUBSAT: |
6681 | case ISD::UADDSAT: |
6682 | case ISD::USUBSAT: |
6683 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6684 | assert(N1.getValueType() == N2.getValueType() && |
6685 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6686 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { |
6687 | // fold (add_sat x, y) -> (or x, y) for bool types. |
6688 | if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) |
6689 | return getNode(Opcode: ISD::OR, DL, VT, N1, N2); |
6690 | // fold (sub_sat x, y) -> (and x, ~y) for bool types. |
6691 | if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) |
6692 | return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT)); |
6693 | } |
6694 | break; |
6695 | case ISD::ABDS: |
6696 | case ISD::ABDU: |
6697 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6698 | assert(N1.getValueType() == N2.getValueType() && |
6699 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6700 | break; |
6701 | case ISD::SMIN: |
6702 | case ISD::UMAX: |
6703 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6704 | assert(N1.getValueType() == N2.getValueType() && |
6705 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6706 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
6707 | return getNode(Opcode: ISD::OR, DL, VT, N1, N2); |
6708 | break; |
6709 | case ISD::SMAX: |
6710 | case ISD::UMIN: |
6711 | assert(VT.isInteger() && "This operator does not apply to FP types!" ); |
6712 | assert(N1.getValueType() == N2.getValueType() && |
6713 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6714 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
6715 | return getNode(Opcode: ISD::AND, DL, VT, N1, N2); |
6716 | break; |
6717 | case ISD::FADD: |
6718 | case ISD::FSUB: |
6719 | case ISD::FMUL: |
6720 | case ISD::FDIV: |
6721 | case ISD::FREM: |
6722 | assert(VT.isFloatingPoint() && "This operator only applies to FP types!" ); |
6723 | assert(N1.getValueType() == N2.getValueType() && |
6724 | N1.getValueType() == VT && "Binary operator types must match!" ); |
6725 | if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags)) |
6726 | return V; |
6727 | break; |
6728 | case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. |
6729 | assert(N1.getValueType() == VT && |
6730 | N1.getValueType().isFloatingPoint() && |
6731 | N2.getValueType().isFloatingPoint() && |
6732 | "Invalid FCOPYSIGN!" ); |
6733 | break; |
6734 | case ISD::SHL: |
6735 | if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
6736 | const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0); |
6737 | const APInt &ShiftImm = N2C->getAPIntValue(); |
6738 | return getVScale(DL, VT, MulImm: MulImm << ShiftImm); |
6739 | } |
6740 | [[fallthrough]]; |
6741 | case ISD::SRA: |
6742 | case ISD::SRL: |
6743 | if (SDValue V = simplifyShift(X: N1, Y: N2)) |
6744 | return V; |
6745 | [[fallthrough]]; |
6746 | case ISD::ROTL: |
6747 | case ISD::ROTR: |
6748 | assert(VT == N1.getValueType() && |
6749 | "Shift operators return type must be the same as their first arg" ); |
6750 | assert(VT.isInteger() && N2.getValueType().isInteger() && |
6751 | "Shifts only work on integers" ); |
6752 | assert((!VT.isVector() || VT == N2.getValueType()) && |
6753 | "Vector shift amounts must be in the same as their first arg" ); |
6754 | // Verify that the shift amount VT is big enough to hold valid shift |
6755 | // amounts. This catches things like trying to shift an i1024 value by an |
6756 | // i8, which is easy to fall into in generic code that uses |
6757 | // TLI.getShiftAmount(). |
6758 | assert(N2.getValueType().getScalarSizeInBits() >= |
6759 | Log2_32_Ceil(VT.getScalarSizeInBits()) && |
6760 | "Invalid use of small shift amount with oversized value!" ); |
6761 | |
6762 | // Always fold shifts of i1 values so the code generator doesn't need to |
6763 | // handle them. Since we know the size of the shift has to be less than the |
6764 | // size of the value, the shift/rotate count is guaranteed to be zero. |
6765 | if (VT == MVT::i1) |
6766 | return N1; |
6767 | if (N2CV && N2CV->isZero()) |
6768 | return N1; |
6769 | break; |
6770 | case ISD::FP_ROUND: |
6771 | assert(VT.isFloatingPoint() && |
6772 | N1.getValueType().isFloatingPoint() && |
6773 | VT.bitsLE(N1.getValueType()) && |
6774 | N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && |
6775 | "Invalid FP_ROUND!" ); |
6776 | if (N1.getValueType() == VT) return N1; // noop conversion. |
6777 | break; |
6778 | case ISD::AssertSext: |
6779 | case ISD::AssertZext: { |
6780 | EVT EVT = cast<VTSDNode>(Val&: N2)->getVT(); |
6781 | assert(VT == N1.getValueType() && "Not an inreg extend!" ); |
6782 | assert(VT.isInteger() && EVT.isInteger() && |
6783 | "Cannot *_EXTEND_INREG FP types" ); |
6784 | assert(!EVT.isVector() && |
6785 | "AssertSExt/AssertZExt type should be the vector element type " |
6786 | "rather than the vector type!" ); |
6787 | assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!" ); |
6788 | if (VT.getScalarType() == EVT) return N1; // noop assertion. |
6789 | break; |
6790 | } |
6791 | case ISD::SIGN_EXTEND_INREG: { |
6792 | EVT EVT = cast<VTSDNode>(Val&: N2)->getVT(); |
6793 | assert(VT == N1.getValueType() && "Not an inreg extend!" ); |
6794 | assert(VT.isInteger() && EVT.isInteger() && |
6795 | "Cannot *_EXTEND_INREG FP types" ); |
6796 | assert(EVT.isVector() == VT.isVector() && |
6797 | "SIGN_EXTEND_INREG type should be vector iff the operand " |
6798 | "type is vector!" ); |
6799 | assert((!EVT.isVector() || |
6800 | EVT.getVectorElementCount() == VT.getVectorElementCount()) && |
6801 | "Vector element counts must match in SIGN_EXTEND_INREG" ); |
6802 | assert(EVT.bitsLE(VT) && "Not extending!" ); |
6803 | if (EVT == VT) return N1; // Not actually extending |
6804 | |
6805 | auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { |
6806 | unsigned FromBits = EVT.getScalarSizeInBits(); |
6807 | Val <<= Val.getBitWidth() - FromBits; |
6808 | Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits); |
6809 | return getConstant(Val, DL, VT: ConstantVT); |
6810 | }; |
6811 | |
6812 | if (N1C) { |
6813 | const APInt &Val = N1C->getAPIntValue(); |
6814 | return SignExtendInReg(Val, VT); |
6815 | } |
6816 | |
6817 | if (ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) { |
6818 | SmallVector<SDValue, 8> Ops; |
6819 | llvm::EVT OpVT = N1.getOperand(i: 0).getValueType(); |
6820 | for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { |
6821 | SDValue Op = N1.getOperand(i); |
6822 | if (Op.isUndef()) { |
6823 | Ops.push_back(Elt: getUNDEF(VT: OpVT)); |
6824 | continue; |
6825 | } |
6826 | ConstantSDNode *C = cast<ConstantSDNode>(Val&: Op); |
6827 | APInt Val = C->getAPIntValue(); |
6828 | Ops.push_back(Elt: SignExtendInReg(Val, OpVT)); |
6829 | } |
6830 | return getBuildVector(VT, DL, Ops); |
6831 | } |
6832 | |
6833 | if (N1.getOpcode() == ISD::SPLAT_VECTOR && |
6834 | isa<ConstantSDNode>(Val: N1.getOperand(i: 0))) |
6835 | return getNode( |
6836 | Opcode: ISD::SPLAT_VECTOR, DL, VT, |
6837 | N1: SignExtendInReg(N1.getConstantOperandAPInt(i: 0), |
6838 | N1.getOperand(i: 0).getValueType())); |
6839 | break; |
6840 | } |
6841 | case ISD::FP_TO_SINT_SAT: |
6842 | case ISD::FP_TO_UINT_SAT: { |
6843 | assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() && |
6844 | N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT" ); |
6845 | assert(N1.getValueType().isVector() == VT.isVector() && |
6846 | "FP_TO_*INT_SAT type should be vector iff the operand type is " |
6847 | "vector!" ); |
6848 | assert((!VT.isVector() || VT.getVectorElementCount() == |
6849 | N1.getValueType().getVectorElementCount()) && |
6850 | "Vector element counts must match in FP_TO_*INT_SAT" ); |
6851 | assert(!cast<VTSDNode>(N2)->getVT().isVector() && |
6852 | "Type to saturate to must be a scalar." ); |
6853 | assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) && |
6854 | "Not extending!" ); |
6855 | break; |
6856 | } |
6857 | case ISD::EXTRACT_VECTOR_ELT: |
6858 | assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && |
6859 | "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ |
6860 | element type of the vector." ); |
6861 | |
6862 | // Extract from an undefined value or using an undefined index is undefined. |
6863 | if (N1.isUndef() || N2.isUndef()) |
6864 | return getUNDEF(VT); |
6865 | |
6866 | // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length |
6867 | // vectors. For scalable vectors we will provide appropriate support for |
6868 | // dealing with arbitrary indices. |
6869 | if (N2C && N1.getValueType().isFixedLengthVector() && |
6870 | N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements())) |
6871 | return getUNDEF(VT); |
6872 | |
6873 | // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is |
6874 | // expanding copies of large vectors from registers. This only works for |
6875 | // fixed length vectors, since we need to know the exact number of |
6876 | // elements. |
6877 | if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS && |
6878 | N1.getOperand(i: 0).getValueType().isFixedLengthVector()) { |
6879 | unsigned Factor = |
6880 | N1.getOperand(i: 0).getValueType().getVectorNumElements(); |
6881 | return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, |
6882 | N1: N1.getOperand(i: N2C->getZExtValue() / Factor), |
6883 | N2: getVectorIdxConstant(Val: N2C->getZExtValue() % Factor, DL)); |
6884 | } |
6885 | |
6886 | // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while |
6887 | // lowering is expanding large vector constants. |
6888 | if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || |
6889 | N1.getOpcode() == ISD::SPLAT_VECTOR)) { |
6890 | assert((N1.getOpcode() != ISD::BUILD_VECTOR || |
6891 | N1.getValueType().isFixedLengthVector()) && |
6892 | "BUILD_VECTOR used for scalable vectors" ); |
6893 | unsigned Index = |
6894 | N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; |
6895 | SDValue Elt = N1.getOperand(i: Index); |
6896 | |
6897 | if (VT != Elt.getValueType()) |
6898 | // If the vector element type is not legal, the BUILD_VECTOR operands |
6899 | // are promoted and implicitly truncated, and the result implicitly |
6900 | // extended. Make that explicit here. |
6901 | Elt = getAnyExtOrTrunc(Op: Elt, DL, VT); |
6902 | |
6903 | return Elt; |
6904 | } |
6905 | |
6906 | // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector |
6907 | // operations are lowered to scalars. |
6908 | if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { |
6909 | // If the indices are the same, return the inserted element else |
6910 | // if the indices are known different, extract the element from |
6911 | // the original vector. |
6912 | SDValue N1Op2 = N1.getOperand(i: 2); |
6913 | ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2); |
6914 | |
6915 | if (N1Op2C && N2C) { |
6916 | if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { |
6917 | if (VT == N1.getOperand(i: 1).getValueType()) |
6918 | return N1.getOperand(i: 1); |
6919 | if (VT.isFloatingPoint()) { |
6920 | assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits()); |
6921 | return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT); |
6922 | } |
6923 | return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT); |
6924 | } |
6925 | return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2); |
6926 | } |
6927 | } |
6928 | |
6929 | // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed |
6930 | // when vector types are scalarized and v1iX is legal. |
6931 | // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). |
6932 | // Here we are completely ignoring the extract element index (N2), |
6933 | // which is fine for fixed width vectors, since any index other than 0 |
6934 | // is undefined anyway. However, this cannot be ignored for scalable |
6935 | // vectors - in theory we could support this, but we don't want to do this |
6936 | // without a profitability check. |
6937 | if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
6938 | N1.getValueType().isFixedLengthVector() && |
6939 | N1.getValueType().getVectorNumElements() == 1) { |
6940 | return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), |
6941 | N2: N1.getOperand(i: 1)); |
6942 | } |
6943 | break; |
6944 | case ISD::EXTRACT_ELEMENT: |
6945 | assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!" ); |
6946 | assert(!N1.getValueType().isVector() && !VT.isVector() && |
6947 | (N1.getValueType().isInteger() == VT.isInteger()) && |
6948 | N1.getValueType() != VT && |
6949 | "Wrong types for EXTRACT_ELEMENT!" ); |
6950 | |
6951 | // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding |
6952 | // 64-bit integers into 32-bit parts. Instead of building the extract of |
6953 | // the BUILD_PAIR, only to have legalize rip it apart, just do it now. |
6954 | if (N1.getOpcode() == ISD::BUILD_PAIR) |
6955 | return N1.getOperand(i: N2C->getZExtValue()); |
6956 | |
6957 | // EXTRACT_ELEMENT of a constant int is also very common. |
6958 | if (N1C) { |
6959 | unsigned ElementSize = VT.getSizeInBits(); |
6960 | unsigned Shift = ElementSize * N2C->getZExtValue(); |
6961 | const APInt &Val = N1C->getAPIntValue(); |
6962 | return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT); |
6963 | } |
6964 | break; |
6965 | case ISD::EXTRACT_SUBVECTOR: { |
6966 | EVT N1VT = N1.getValueType(); |
6967 | assert(VT.isVector() && N1VT.isVector() && |
6968 | "Extract subvector VTs must be vectors!" ); |
6969 | assert(VT.getVectorElementType() == N1VT.getVectorElementType() && |
6970 | "Extract subvector VTs must have the same element type!" ); |
6971 | assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) && |
6972 | "Cannot extract a scalable vector from a fixed length vector!" ); |
6973 | assert((VT.isScalableVector() != N1VT.isScalableVector() || |
6974 | VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && |
6975 | "Extract subvector must be from larger vector to smaller vector!" ); |
6976 | assert(N2C && "Extract subvector index must be a constant" ); |
6977 | assert((VT.isScalableVector() != N1VT.isScalableVector() || |
6978 | (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= |
6979 | N1VT.getVectorMinNumElements()) && |
6980 | "Extract subvector overflow!" ); |
6981 | assert(N2C->getAPIntValue().getBitWidth() == |
6982 | TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
6983 | "Constant index for EXTRACT_SUBVECTOR has an invalid size" ); |
6984 | |
6985 | // Trivial extraction. |
6986 | if (VT == N1VT) |
6987 | return N1; |
6988 | |
6989 | // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. |
6990 | if (N1.isUndef()) |
6991 | return getUNDEF(VT); |
6992 | |
6993 | // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of |
6994 | // the concat have the same type as the extract. |
6995 | if (N1.getOpcode() == ISD::CONCAT_VECTORS && |
6996 | VT == N1.getOperand(i: 0).getValueType()) { |
6997 | unsigned Factor = VT.getVectorMinNumElements(); |
6998 | return N1.getOperand(i: N2C->getZExtValue() / Factor); |
6999 | } |
7000 | |
7001 | // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created |
7002 | // during shuffle legalization. |
7003 | if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) && |
7004 | VT == N1.getOperand(i: 1).getValueType()) |
7005 | return N1.getOperand(i: 1); |
7006 | break; |
7007 | } |
7008 | } |
7009 | |
7010 | // Perform trivial constant folding. |
7011 | if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2})) |
7012 | return SV; |
7013 | |
7014 | // Canonicalize an UNDEF to the RHS, even over a constant. |
7015 | if (N1.isUndef()) { |
7016 | if (TLI->isCommutativeBinOp(Opcode)) { |
7017 | std::swap(a&: N1, b&: N2); |
7018 | } else { |
7019 | switch (Opcode) { |
7020 | case ISD::SUB: |
7021 | return getUNDEF(VT); // fold op(undef, arg2) -> undef |
7022 | case ISD::SIGN_EXTEND_INREG: |
7023 | case ISD::UDIV: |
7024 | case ISD::SDIV: |
7025 | case ISD::UREM: |
7026 | case ISD::SREM: |
7027 | case ISD::SSUBSAT: |
7028 | case ISD::USUBSAT: |
7029 | return getConstant(Val: 0, DL, VT); // fold op(undef, arg2) -> 0 |
7030 | } |
7031 | } |
7032 | } |
7033 | |
7034 | // Fold a bunch of operators when the RHS is undef. |
7035 | if (N2.isUndef()) { |
7036 | switch (Opcode) { |
7037 | case ISD::XOR: |
7038 | if (N1.isUndef()) |
7039 | // Handle undef ^ undef -> 0 special case. This is a common |
7040 | // idiom (misuse). |
7041 | return getConstant(Val: 0, DL, VT); |
7042 | [[fallthrough]]; |
7043 | case ISD::ADD: |
7044 | case ISD::SUB: |
7045 | case ISD::UDIV: |
7046 | case ISD::SDIV: |
7047 | case ISD::UREM: |
7048 | case ISD::SREM: |
7049 | return getUNDEF(VT); // fold op(arg1, undef) -> undef |
7050 | case ISD::MUL: |
7051 | case ISD::AND: |
7052 | case ISD::SSUBSAT: |
7053 | case ISD::USUBSAT: |
7054 | return getConstant(Val: 0, DL, VT); // fold op(arg1, undef) -> 0 |
7055 | case ISD::OR: |
7056 | case ISD::SADDSAT: |
7057 | case ISD::UADDSAT: |
7058 | return getAllOnesConstant(DL, VT); |
7059 | } |
7060 | } |
7061 | |
7062 | // Memoize this node if possible. |
7063 | SDNode *N; |
7064 | SDVTList VTs = getVTList(VT); |
7065 | SDValue Ops[] = {N1, N2}; |
7066 | if (VT != MVT::Glue) { |
7067 | FoldingSetNodeID ID; |
7068 | AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops); |
7069 | void *IP = nullptr; |
7070 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
7071 | E->intersectFlagsWith(Flags); |
7072 | return SDValue(E, 0); |
7073 | } |
7074 | |
7075 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
7076 | N->setFlags(Flags); |
7077 | createOperands(Node: N, Vals: Ops); |
7078 | CSEMap.InsertNode(N, InsertPos: IP); |
7079 | } else { |
7080 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
7081 | createOperands(Node: N, Vals: Ops); |
7082 | } |
7083 | |
7084 | InsertNode(N); |
7085 | SDValue V = SDValue(N, 0); |
7086 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
7087 | return V; |
7088 | } |
7089 | |
7090 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
7091 | SDValue N1, SDValue N2, SDValue N3) { |
7092 | SDNodeFlags Flags; |
7093 | if (Inserter) |
7094 | Flags = Inserter->getFlags(); |
7095 | return getNode(Opcode, DL, VT, N1, N2, N3, Flags); |
7096 | } |
7097 | |
7098 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
7099 | SDValue N1, SDValue N2, SDValue N3, |
7100 | const SDNodeFlags Flags) { |
7101 | assert(N1.getOpcode() != ISD::DELETED_NODE && |
7102 | N2.getOpcode() != ISD::DELETED_NODE && |
7103 | N3.getOpcode() != ISD::DELETED_NODE && |
7104 | "Operand is DELETED_NODE!" ); |
7105 | // Perform various simplifications. |
7106 | switch (Opcode) { |
7107 | case ISD::FMA: |
7108 | case ISD::FMAD: { |
7109 | assert(VT.isFloatingPoint() && "This operator only applies to FP types!" ); |
7110 | assert(N1.getValueType() == VT && N2.getValueType() == VT && |
7111 | N3.getValueType() == VT && "FMA types must match!" ); |
7112 | ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1); |
7113 | ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2); |
7114 | ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(Val&: N3); |
7115 | if (N1CFP && N2CFP && N3CFP) { |
7116 | APFloat V1 = N1CFP->getValueAPF(); |
7117 | const APFloat &V2 = N2CFP->getValueAPF(); |
7118 | const APFloat &V3 = N3CFP->getValueAPF(); |
7119 | if (Opcode == ISD::FMAD) { |
7120 | V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven); |
7121 | V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven); |
7122 | } else |
7123 | V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven); |
7124 | return getConstantFP(V: V1, DL, VT); |
7125 | } |
7126 | break; |
7127 | } |
7128 | case ISD::BUILD_VECTOR: { |
7129 | // Attempt to simplify BUILD_VECTOR. |
7130 | SDValue Ops[] = {N1, N2, N3}; |
7131 | if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this)) |
7132 | return V; |
7133 | break; |
7134 | } |
7135 | case ISD::CONCAT_VECTORS: { |
7136 | SDValue Ops[] = {N1, N2, N3}; |
7137 | if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this)) |
7138 | return V; |
7139 | break; |
7140 | } |
7141 | case ISD::SETCC: { |
7142 | assert(VT.isInteger() && "SETCC result type must be an integer!" ); |
7143 | assert(N1.getValueType() == N2.getValueType() && |
7144 | "SETCC operands must have the same type!" ); |
7145 | assert(VT.isVector() == N1.getValueType().isVector() && |
7146 | "SETCC type should be vector iff the operand type is vector!" ); |
7147 | assert((!VT.isVector() || VT.getVectorElementCount() == |
7148 | N1.getValueType().getVectorElementCount()) && |
7149 | "SETCC vector element counts must match!" ); |
7150 | // Use FoldSetCC to simplify SETCC's. |
7151 | if (SDValue V = FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL)) |
7152 | return V; |
7153 | // Vector constant folding. |
7154 | SDValue Ops[] = {N1, N2, N3}; |
7155 | if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) { |
7156 | NewSDValueDbgMsg(V, Msg: "New node vector constant folding: " , G: this); |
7157 | return V; |
7158 | } |
7159 | break; |
7160 | } |
7161 | case ISD::SELECT: |
7162 | case ISD::VSELECT: |
7163 | if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3)) |
7164 | return V; |
7165 | break; |
7166 | case ISD::VECTOR_SHUFFLE: |
7167 | llvm_unreachable("should use getVectorShuffle constructor!" ); |
7168 | case ISD::VECTOR_SPLICE: { |
7169 | if (cast<ConstantSDNode>(Val&: N3)->isZero()) |
7170 | return N1; |
7171 | break; |
7172 | } |
7173 | case ISD::INSERT_VECTOR_ELT: { |
7174 | ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(Val&: N3); |
7175 | // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except |
7176 | // for scalable vectors where we will generate appropriate code to |
7177 | // deal with out-of-bounds cases correctly. |
7178 | if (N3C && N1.getValueType().isFixedLengthVector() && |
7179 | N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) |
7180 | return getUNDEF(VT); |
7181 | |
7182 | // Undefined index can be assumed out-of-bounds, so that's UNDEF too. |
7183 | if (N3.isUndef()) |
7184 | return getUNDEF(VT); |
7185 | |
7186 | // If the inserted element is an UNDEF, just use the input vector. |
7187 | if (N2.isUndef()) |
7188 | return N1; |
7189 | |
7190 | break; |
7191 | } |
7192 | case ISD::INSERT_SUBVECTOR: { |
7193 | // Inserting undef into undef is still undef. |
7194 | if (N1.isUndef() && N2.isUndef()) |
7195 | return getUNDEF(VT); |
7196 | |
7197 | EVT N2VT = N2.getValueType(); |
7198 | assert(VT == N1.getValueType() && |
7199 | "Dest and insert subvector source types must match!" ); |
7200 | assert(VT.isVector() && N2VT.isVector() && |
7201 | "Insert subvector VTs must be vectors!" ); |
7202 | assert(VT.getVectorElementType() == N2VT.getVectorElementType() && |
7203 | "Insert subvector VTs must have the same element type!" ); |
7204 | assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) && |
7205 | "Cannot insert a scalable vector into a fixed length vector!" ); |
7206 | assert((VT.isScalableVector() != N2VT.isScalableVector() || |
7207 | VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && |
7208 | "Insert subvector must be from smaller vector to larger vector!" ); |
7209 | assert(isa<ConstantSDNode>(N3) && |
7210 | "Insert subvector index must be constant" ); |
7211 | assert((VT.isScalableVector() != N2VT.isScalableVector() || |
7212 | (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <= |
7213 | VT.getVectorMinNumElements()) && |
7214 | "Insert subvector overflow!" ); |
7215 | assert(N3->getAsAPIntVal().getBitWidth() == |
7216 | TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
7217 | "Constant index for INSERT_SUBVECTOR has an invalid size" ); |
7218 | |
7219 | // Trivial insertion. |
7220 | if (VT == N2VT) |
7221 | return N2; |
7222 | |
7223 | // If this is an insert of an extracted vector into an undef vector, we |
7224 | // can just use the input to the extract. |
7225 | if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
7226 | N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) |
7227 | return N2.getOperand(i: 0); |
7228 | break; |
7229 | } |
7230 | case ISD::BITCAST: |
7231 | // Fold bit_convert nodes from a type to themselves. |
7232 | if (N1.getValueType() == VT) |
7233 | return N1; |
7234 | break; |
7235 | case ISD::VP_TRUNCATE: |
7236 | case ISD::VP_SIGN_EXTEND: |
7237 | case ISD::VP_ZERO_EXTEND: |
7238 | // Don't create noop casts. |
7239 | if (N1.getValueType() == VT) |
7240 | return N1; |
7241 | break; |
7242 | } |
7243 | |
7244 | // Memoize node if it doesn't produce a glue result. |
7245 | SDNode *N; |
7246 | SDVTList VTs = getVTList(VT); |
7247 | SDValue Ops[] = {N1, N2, N3}; |
7248 | if (VT != MVT::Glue) { |
7249 | FoldingSetNodeID ID; |
7250 | AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops); |
7251 | void *IP = nullptr; |
7252 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
7253 | E->intersectFlagsWith(Flags); |
7254 | return SDValue(E, 0); |
7255 | } |
7256 | |
7257 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
7258 | N->setFlags(Flags); |
7259 | createOperands(Node: N, Vals: Ops); |
7260 | CSEMap.InsertNode(N, InsertPos: IP); |
7261 | } else { |
7262 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
7263 | createOperands(Node: N, Vals: Ops); |
7264 | } |
7265 | |
7266 | InsertNode(N); |
7267 | SDValue V = SDValue(N, 0); |
7268 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
7269 | return V; |
7270 | } |
7271 | |
7272 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
7273 | SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
7274 | SDValue Ops[] = { N1, N2, N3, N4 }; |
7275 | return getNode(Opcode, DL, VT, Ops); |
7276 | } |
7277 | |
7278 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
7279 | SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
7280 | SDValue N5) { |
7281 | SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
7282 | return getNode(Opcode, DL, VT, Ops); |
7283 | } |
7284 | |
7285 | /// getStackArgumentTokenFactor - Compute a TokenFactor to force all |
7286 | /// the incoming stack arguments to be loaded from the stack. |
7287 | SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { |
7288 | SmallVector<SDValue, 8> ArgChains; |
7289 | |
7290 | // Include the original chain at the beginning of the list. When this is |
7291 | // used by target LowerCall hooks, this helps legalize find the |
7292 | // CALLSEQ_BEGIN node. |
7293 | ArgChains.push_back(Elt: Chain); |
7294 | |
7295 | // Add a chain value for each stack argument. |
7296 | for (SDNode *U : getEntryNode().getNode()->uses()) |
7297 | if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) |
7298 | if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) |
7299 | if (FI->getIndex() < 0) |
7300 | ArgChains.push_back(Elt: SDValue(L, 1)); |
7301 | |
7302 | // Build a tokenfactor for all the chains. |
7303 | return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); |
7304 | } |
7305 | |
7306 | /// getMemsetValue - Vectorized representation of the memset value |
7307 | /// operand. |
7308 | static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, |
7309 | const SDLoc &dl) { |
7310 | assert(!Value.isUndef()); |
7311 | |
7312 | unsigned NumBits = VT.getScalarSizeInBits(); |
7313 | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) { |
7314 | assert(C->getAPIntValue().getBitWidth() == 8); |
7315 | APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue()); |
7316 | if (VT.isInteger()) { |
7317 | bool IsOpaque = VT.getSizeInBits() > 64 || |
7318 | !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue()); |
7319 | return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque); |
7320 | } |
7321 | return DAG.getConstantFP(V: APFloat(DAG.EVTToAPFloatSemantics(VT), Val), DL: dl, |
7322 | VT); |
7323 | } |
7324 | |
7325 | assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?" ); |
7326 | EVT IntVT = VT.getScalarType(); |
7327 | if (!IntVT.isInteger()) |
7328 | IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits()); |
7329 | |
7330 | Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value); |
7331 | if (NumBits > 8) { |
7332 | // Use a multiplication with 0x010101... to extend the input to the |
7333 | // required length. |
7334 | APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01)); |
7335 | Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value, |
7336 | N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT)); |
7337 | } |
7338 | |
7339 | if (VT != Value.getValueType() && !VT.isInteger()) |
7340 | Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value); |
7341 | if (VT != Value.getValueType()) |
7342 | Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value); |
7343 | |
7344 | return Value; |
7345 | } |
7346 | |
7347 | /// getMemsetStringVal - Similar to getMemsetValue. Except this is only |
7348 | /// used when a memcpy is turned into a memset when the source is a constant |
7349 | /// string ptr. |
7350 | static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, |
7351 | const TargetLowering &TLI, |
7352 | const ConstantDataArraySlice &Slice) { |
7353 | // Handle vector with all elements zero. |
7354 | if (Slice.Array == nullptr) { |
7355 | if (VT.isInteger()) |
7356 | return DAG.getConstant(Val: 0, DL: dl, VT); |
7357 | if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) |
7358 | return DAG.getConstantFP(Val: 0.0, DL: dl, VT); |
7359 | if (VT.isVector()) { |
7360 | unsigned NumElts = VT.getVectorNumElements(); |
7361 | MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; |
7362 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, |
7363 | N1: DAG.getConstant(Val: 0, DL: dl, |
7364 | VT: EVT::getVectorVT(Context&: *DAG.getContext(), |
7365 | VT: EltVT, NumElements: NumElts))); |
7366 | } |
7367 | llvm_unreachable("Expected type!" ); |
7368 | } |
7369 | |
7370 | assert(!VT.isVector() && "Can't handle vector type here!" ); |
7371 | unsigned NumVTBits = VT.getSizeInBits(); |
7372 | unsigned NumVTBytes = NumVTBits / 8; |
7373 | unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length)); |
7374 | |
7375 | APInt Val(NumVTBits, 0); |
7376 | if (DAG.getDataLayout().isLittleEndian()) { |
7377 | for (unsigned i = 0; i != NumBytes; ++i) |
7378 | Val |= (uint64_t)(unsigned char)Slice[i] << i*8; |
7379 | } else { |
7380 | for (unsigned i = 0; i != NumBytes; ++i) |
7381 | Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; |
7382 | } |
7383 | |
7384 | // If the "cost" of materializing the integer immediate is less than the cost |
7385 | // of a load, then it is cost effective to turn the load into the immediate. |
7386 | Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext()); |
7387 | if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty)) |
7388 | return DAG.getConstant(Val, DL: dl, VT); |
7389 | return SDValue(); |
7390 | } |
7391 | |
7392 | SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, |
7393 | const SDLoc &DL, |
7394 | const SDNodeFlags Flags) { |
7395 | EVT VT = Base.getValueType(); |
7396 | SDValue Index; |
7397 | |
7398 | if (Offset.isScalable()) |
7399 | Index = getVScale(DL, VT: Base.getValueType(), |
7400 | MulImm: APInt(Base.getValueSizeInBits().getFixedValue(), |
7401 | Offset.getKnownMinValue())); |
7402 | else |
7403 | Index = getConstant(Val: Offset.getFixedValue(), DL, VT); |
7404 | |
7405 | return getMemBasePlusOffset(Base, Offset: Index, DL, Flags); |
7406 | } |
7407 | |
7408 | SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, |
7409 | const SDLoc &DL, |
7410 | const SDNodeFlags Flags) { |
7411 | assert(Offset.getValueType().isInteger()); |
7412 | EVT BasePtrVT = Ptr.getValueType(); |
7413 | return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags); |
7414 | } |
7415 | |
7416 | /// Returns true if memcpy source is constant data. |
7417 | static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { |
7418 | uint64_t SrcDelta = 0; |
7419 | GlobalAddressSDNode *G = nullptr; |
7420 | if (Src.getOpcode() == ISD::GlobalAddress) |
7421 | G = cast<GlobalAddressSDNode>(Val&: Src); |
7422 | else if (Src.getOpcode() == ISD::ADD && |
7423 | Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress && |
7424 | Src.getOperand(i: 1).getOpcode() == ISD::Constant) { |
7425 | G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0)); |
7426 | SrcDelta = Src.getConstantOperandVal(i: 1); |
7427 | } |
7428 | if (!G) |
7429 | return false; |
7430 | |
7431 | return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8, |
7432 | Offset: SrcDelta + G->getOffset()); |
7433 | } |
7434 | |
7435 | static bool shouldLowerMemFuncForSize(const MachineFunction &MF, |
7436 | SelectionDAG &DAG) { |
7437 | // On Darwin, -Os means optimize for size without hurting performance, so |
7438 | // only really optimize for size when -Oz (MinSize) is used. |
7439 | if (MF.getTarget().getTargetTriple().isOSDarwin()) |
7440 | return MF.getFunction().hasMinSize(); |
7441 | return DAG.shouldOptForSize(); |
7442 | } |
7443 | |
7444 | static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, |
7445 | SmallVector<SDValue, 32> &OutChains, unsigned From, |
7446 | unsigned To, SmallVector<SDValue, 16> &OutLoadChains, |
7447 | SmallVector<SDValue, 16> &OutStoreChains) { |
7448 | assert(OutLoadChains.size() && "Missing loads in memcpy inlining" ); |
7449 | assert(OutStoreChains.size() && "Missing stores in memcpy inlining" ); |
7450 | SmallVector<SDValue, 16> GluedLoadChains; |
7451 | for (unsigned i = From; i < To; ++i) { |
7452 | OutChains.push_back(Elt: OutLoadChains[i]); |
7453 | GluedLoadChains.push_back(Elt: OutLoadChains[i]); |
7454 | } |
7455 | |
7456 | // Chain for all loads. |
7457 | SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
7458 | GluedLoadChains); |
7459 | |
7460 | for (unsigned i = From; i < To; ++i) { |
7461 | StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]); |
7462 | SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(), |
7463 | Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(), |
7464 | MMO: ST->getMemOperand()); |
7465 | OutChains.push_back(Elt: NewStore); |
7466 | } |
7467 | } |
7468 | |
7469 | static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
7470 | SDValue Chain, SDValue Dst, SDValue Src, |
7471 | uint64_t Size, Align Alignment, |
7472 | bool isVol, bool AlwaysInline, |
7473 | MachinePointerInfo DstPtrInfo, |
7474 | MachinePointerInfo SrcPtrInfo, |
7475 | const AAMDNodes &AAInfo, AAResults *AA) { |
7476 | // Turn a memcpy of undef to nop. |
7477 | // FIXME: We need to honor volatile even is Src is undef. |
7478 | if (Src.isUndef()) |
7479 | return Chain; |
7480 | |
7481 | // Expand memcpy to a series of load and store ops if the size operand falls |
7482 | // below a certain threshold. |
7483 | // TODO: In the AlwaysInline case, if the size is big then generate a loop |
7484 | // rather than maybe a humongous number of loads and stores. |
7485 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
7486 | const DataLayout &DL = DAG.getDataLayout(); |
7487 | LLVMContext &C = *DAG.getContext(); |
7488 | std::vector<EVT> MemOps; |
7489 | bool DstAlignCanChange = false; |
7490 | MachineFunction &MF = DAG.getMachineFunction(); |
7491 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
7492 | bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
7493 | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst); |
7494 | if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex())) |
7495 | DstAlignCanChange = true; |
7496 | MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src); |
7497 | if (!SrcAlign || Alignment > *SrcAlign) |
7498 | SrcAlign = Alignment; |
7499 | assert(SrcAlign && "SrcAlign must be set" ); |
7500 | ConstantDataArraySlice Slice; |
7501 | // If marked as volatile, perform a copy even when marked as constant. |
7502 | bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); |
7503 | bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; |
7504 | unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); |
7505 | const MemOp Op = isZeroConstant |
7506 | ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, |
7507 | /*IsZeroMemset*/ true, IsVolatile: isVol) |
7508 | : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, |
7509 | SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant); |
7510 | if (!TLI.findOptimalMemOpLowering( |
7511 | MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(), |
7512 | SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes())) |
7513 | return SDValue(); |
7514 | |
7515 | if (DstAlignCanChange) { |
7516 | Type *Ty = MemOps[0].getTypeForEVT(Context&: C); |
7517 | Align NewAlign = DL.getABITypeAlign(Ty); |
7518 | |
7519 | // Don't promote to an alignment that would require dynamic stack |
7520 | // realignment which may conflict with optimizations such as tail call |
7521 | // optimization. |
7522 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
7523 | if (!TRI->hasStackRealignment(MF)) |
7524 | while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign)) |
7525 | NewAlign = NewAlign.previous(); |
7526 | |
7527 | if (NewAlign > Alignment) { |
7528 | // Give the stack frame object a larger alignment if needed. |
7529 | if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign) |
7530 | MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign); |
7531 | Alignment = NewAlign; |
7532 | } |
7533 | } |
7534 | |
7535 | // Prepare AAInfo for loads/stores after lowering this memcpy. |
7536 | AAMDNodes NewAAInfo = AAInfo; |
7537 | NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
7538 | |
7539 | const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V); |
7540 | bool isConstant = |
7541 | AA && SrcVal && |
7542 | AA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo)); |
7543 | |
7544 | MachineMemOperand::Flags MMOFlags = |
7545 | isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
7546 | SmallVector<SDValue, 16> OutLoadChains; |
7547 | SmallVector<SDValue, 16> OutStoreChains; |
7548 | SmallVector<SDValue, 32> OutChains; |
7549 | unsigned NumMemOps = MemOps.size(); |
7550 | uint64_t SrcOff = 0, DstOff = 0; |
7551 | for (unsigned i = 0; i != NumMemOps; ++i) { |
7552 | EVT VT = MemOps[i]; |
7553 | unsigned VTSize = VT.getSizeInBits() / 8; |
7554 | SDValue Value, Store; |
7555 | |
7556 | if (VTSize > Size) { |
7557 | // Issuing an unaligned load / store pair that overlaps with the previous |
7558 | // pair. Adjust the offset accordingly. |
7559 | assert(i == NumMemOps-1 && i != 0); |
7560 | SrcOff -= VTSize - Size; |
7561 | DstOff -= VTSize - Size; |
7562 | } |
7563 | |
7564 | if (CopyFromConstant && |
7565 | (isZeroConstant || (VT.isInteger() && !VT.isVector()))) { |
7566 | // It's unlikely a store of a vector immediate can be done in a single |
7567 | // instruction. It would require a load from a constantpool first. |
7568 | // We only handle zero vectors here. |
7569 | // FIXME: Handle other cases where store of vector immediate is done in |
7570 | // a single instruction. |
7571 | ConstantDataArraySlice SubSlice; |
7572 | if (SrcOff < Slice.Length) { |
7573 | SubSlice = Slice; |
7574 | SubSlice.move(Delta: SrcOff); |
7575 | } else { |
7576 | // This is an out-of-bounds access and hence UB. Pretend we read zero. |
7577 | SubSlice.Array = nullptr; |
7578 | SubSlice.Offset = 0; |
7579 | SubSlice.Length = VTSize; |
7580 | } |
7581 | Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice); |
7582 | if (Value.getNode()) { |
7583 | Store = DAG.getStore( |
7584 | Chain, dl, Val: Value, |
7585 | Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl), |
7586 | PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo); |
7587 | OutChains.push_back(Elt: Store); |
7588 | } |
7589 | } |
7590 | |
7591 | if (!Store.getNode()) { |
7592 | // The type might not be legal for the target. This should only happen |
7593 | // if the type is smaller than a legal type, as on PPC, so the right |
7594 | // thing to do is generate a LoadExt/StoreTrunc pair. These simplify |
7595 | // to Load/Store if NVT==VT. |
7596 | // FIXME does the case above also need this? |
7597 | EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT); |
7598 | assert(NVT.bitsGE(VT)); |
7599 | |
7600 | bool isDereferenceable = |
7601 | SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL); |
7602 | MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
7603 | if (isDereferenceable) |
7604 | SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
7605 | if (isConstant) |
7606 | SrcMMOFlags |= MachineMemOperand::MOInvariant; |
7607 | |
7608 | Value = DAG.getExtLoad( |
7609 | ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain, |
7610 | Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl), |
7611 | PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT, |
7612 | Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo); |
7613 | OutLoadChains.push_back(Elt: Value.getValue(R: 1)); |
7614 | |
7615 | Store = DAG.getTruncStore( |
7616 | Chain, dl, Val: Value, |
7617 | Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl), |
7618 | PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo); |
7619 | OutStoreChains.push_back(Elt: Store); |
7620 | } |
7621 | SrcOff += VTSize; |
7622 | DstOff += VTSize; |
7623 | Size -= VTSize; |
7624 | } |
7625 | |
7626 | unsigned GluedLdStLimit = MaxLdStGlue == 0 ? |
7627 | TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; |
7628 | unsigned NumLdStInMemcpy = OutStoreChains.size(); |
7629 | |
7630 | if (NumLdStInMemcpy) { |
7631 | // It may be that memcpy might be converted to memset if it's memcpy |
7632 | // of constants. In such a case, we won't have loads and stores, but |
7633 | // just stores. In the absence of loads, there is nothing to gang up. |
7634 | if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { |
7635 | // If target does not care, just leave as it. |
7636 | for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { |
7637 | OutChains.push_back(Elt: OutLoadChains[i]); |
7638 | OutChains.push_back(Elt: OutStoreChains[i]); |
7639 | } |
7640 | } else { |
7641 | // Ld/St less than/equal limit set by target. |
7642 | if (NumLdStInMemcpy <= GluedLdStLimit) { |
7643 | chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0, |
7644 | To: NumLdStInMemcpy, OutLoadChains, |
7645 | OutStoreChains); |
7646 | } else { |
7647 | unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; |
7648 | unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; |
7649 | unsigned GlueIter = 0; |
7650 | |
7651 | for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { |
7652 | unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; |
7653 | unsigned IndexTo = NumLdStInMemcpy - GlueIter; |
7654 | |
7655 | chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo, |
7656 | OutLoadChains, OutStoreChains); |
7657 | GlueIter += GluedLdStLimit; |
7658 | } |
7659 | |
7660 | // Residual ld/st. |
7661 | if (RemainingLdStInMemcpy) { |
7662 | chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0, |
7663 | To: RemainingLdStInMemcpy, OutLoadChains, |
7664 | OutStoreChains); |
7665 | } |
7666 | } |
7667 | } |
7668 | } |
7669 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
7670 | } |
7671 | |
7672 | static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
7673 | SDValue Chain, SDValue Dst, SDValue Src, |
7674 | uint64_t Size, Align Alignment, |
7675 | bool isVol, bool AlwaysInline, |
7676 | MachinePointerInfo DstPtrInfo, |
7677 | MachinePointerInfo SrcPtrInfo, |
7678 | const AAMDNodes &AAInfo) { |
7679 | // Turn a memmove of undef to nop. |
7680 | // FIXME: We need to honor volatile even is Src is undef. |
7681 | if (Src.isUndef()) |
7682 | return Chain; |
7683 | |
7684 | // Expand memmove to a series of load and store ops if the size operand falls |
7685 | // below a certain threshold. |
7686 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
7687 | const DataLayout &DL = DAG.getDataLayout(); |
7688 | LLVMContext &C = *DAG.getContext(); |
7689 | std::vector<EVT> MemOps; |
7690 | bool DstAlignCanChange = false; |
7691 | MachineFunction &MF = DAG.getMachineFunction(); |
7692 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
7693 | bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
7694 | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst); |
7695 | if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex())) |
7696 | DstAlignCanChange = true; |
7697 | MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src); |
7698 | if (!SrcAlign || Alignment > *SrcAlign) |
7699 | SrcAlign = Alignment; |
7700 | assert(SrcAlign && "SrcAlign must be set" ); |
7701 | unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); |
7702 | if (!TLI.findOptimalMemOpLowering( |
7703 | MemOps, Limit, |
7704 | Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign, |
7705 | /*IsVolatile*/ true), |
7706 | DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(), |
7707 | FuncAttributes: MF.getFunction().getAttributes())) |
7708 | return SDValue(); |
7709 | |
7710 | if (DstAlignCanChange) { |
7711 | Type *Ty = MemOps[0].getTypeForEVT(Context&: C); |
7712 | Align NewAlign = DL.getABITypeAlign(Ty); |
7713 | |
7714 | // Don't promote to an alignment that would require dynamic stack |
7715 | // realignment which may conflict with optimizations such as tail call |
7716 | // optimization. |
7717 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
7718 | if (!TRI->hasStackRealignment(MF)) |
7719 | while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign)) |
7720 | NewAlign = NewAlign.previous(); |
7721 | |
7722 | if (NewAlign > Alignment) { |
7723 | // Give the stack frame object a larger alignment if needed. |
7724 | if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign) |
7725 | MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign); |
7726 | Alignment = NewAlign; |
7727 | } |
7728 | } |
7729 | |
7730 | // Prepare AAInfo for loads/stores after lowering this memmove. |
7731 | AAMDNodes NewAAInfo = AAInfo; |
7732 | NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
7733 | |
7734 | MachineMemOperand::Flags MMOFlags = |
7735 | isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
7736 | uint64_t SrcOff = 0, DstOff = 0; |
7737 | SmallVector<SDValue, 8> LoadValues; |
7738 | SmallVector<SDValue, 8> LoadChains; |
7739 | SmallVector<SDValue, 8> OutChains; |
7740 | unsigned NumMemOps = MemOps.size(); |
7741 | for (unsigned i = 0; i < NumMemOps; i++) { |
7742 | EVT VT = MemOps[i]; |
7743 | unsigned VTSize = VT.getSizeInBits() / 8; |
7744 | SDValue Value; |
7745 | |
7746 | bool isDereferenceable = |
7747 | SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL); |
7748 | MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
7749 | if (isDereferenceable) |
7750 | SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
7751 | |
7752 | Value = DAG.getLoad( |
7753 | VT, dl, Chain, |
7754 | Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl), |
7755 | PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo); |
7756 | LoadValues.push_back(Elt: Value); |
7757 | LoadChains.push_back(Elt: Value.getValue(R: 1)); |
7758 | SrcOff += VTSize; |
7759 | } |
7760 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); |
7761 | OutChains.clear(); |
7762 | for (unsigned i = 0; i < NumMemOps; i++) { |
7763 | EVT VT = MemOps[i]; |
7764 | unsigned VTSize = VT.getSizeInBits() / 8; |
7765 | SDValue Store; |
7766 | |
7767 | Store = DAG.getStore( |
7768 | Chain, dl, Val: LoadValues[i], |
7769 | Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl), |
7770 | PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo); |
7771 | OutChains.push_back(Elt: Store); |
7772 | DstOff += VTSize; |
7773 | } |
7774 | |
7775 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
7776 | } |
7777 | |
7778 | /// Lower the call to 'memset' intrinsic function into a series of store |
7779 | /// operations. |
7780 | /// |
7781 | /// \param DAG Selection DAG where lowered code is placed. |
7782 | /// \param dl Link to corresponding IR location. |
7783 | /// \param Chain Control flow dependency. |
7784 | /// \param Dst Pointer to destination memory location. |
7785 | /// \param Src Value of byte to write into the memory. |
7786 | /// \param Size Number of bytes to write. |
7787 | /// \param Alignment Alignment of the destination in bytes. |
7788 | /// \param isVol True if destination is volatile. |
7789 | /// \param AlwaysInline Makes sure no function call is generated. |
7790 | /// \param DstPtrInfo IR information on the memory pointer. |
7791 | /// \returns New head in the control flow, if lowering was successful, empty |
7792 | /// SDValue otherwise. |
7793 | /// |
7794 | /// The function tries to replace 'llvm.memset' intrinsic with several store |
7795 | /// operations and value calculation code. This is usually profitable for small |
7796 | /// memory size or when the semantic requires inlining. |
7797 | static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, |
7798 | SDValue Chain, SDValue Dst, SDValue Src, |
7799 | uint64_t Size, Align Alignment, bool isVol, |
7800 | bool AlwaysInline, MachinePointerInfo DstPtrInfo, |
7801 | const AAMDNodes &AAInfo) { |
7802 | // Turn a memset of undef to nop. |
7803 | // FIXME: We need to honor volatile even is Src is undef. |
7804 | if (Src.isUndef()) |
7805 | return Chain; |
7806 | |
7807 | // Expand memset to a series of load/store ops if the size operand |
7808 | // falls below a certain threshold. |
7809 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
7810 | std::vector<EVT> MemOps; |
7811 | bool DstAlignCanChange = false; |
7812 | MachineFunction &MF = DAG.getMachineFunction(); |
7813 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
7814 | bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
7815 | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst); |
7816 | if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex())) |
7817 | DstAlignCanChange = true; |
7818 | bool IsZeroVal = isNullConstant(V: Src); |
7819 | unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize); |
7820 | |
7821 | if (!TLI.findOptimalMemOpLowering( |
7822 | MemOps, Limit, |
7823 | Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol), |
7824 | DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes())) |
7825 | return SDValue(); |
7826 | |
7827 | if (DstAlignCanChange) { |
7828 | Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext()); |
7829 | const DataLayout &DL = DAG.getDataLayout(); |
7830 | Align NewAlign = DL.getABITypeAlign(Ty); |
7831 | |
7832 | // Don't promote to an alignment that would require dynamic stack |
7833 | // realignment which may conflict with optimizations such as tail call |
7834 | // optimization. |
7835 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
7836 | if (!TRI->hasStackRealignment(MF)) |
7837 | while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(Alignment: NewAlign)) |
7838 | NewAlign = NewAlign.previous(); |
7839 | |
7840 | if (NewAlign > Alignment) { |
7841 | // Give the stack frame object a larger alignment if needed. |
7842 | if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign) |
7843 | MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign); |
7844 | Alignment = NewAlign; |
7845 | } |
7846 | } |
7847 | |
7848 | SmallVector<SDValue, 8> OutChains; |
7849 | uint64_t DstOff = 0; |
7850 | unsigned NumMemOps = MemOps.size(); |
7851 | |
7852 | // Find the largest store and generate the bit pattern for it. |
7853 | EVT LargestVT = MemOps[0]; |
7854 | for (unsigned i = 1; i < NumMemOps; i++) |
7855 | if (MemOps[i].bitsGT(VT: LargestVT)) |
7856 | LargestVT = MemOps[i]; |
7857 | SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl); |
7858 | |
7859 | // Prepare AAInfo for loads/stores after lowering this memset. |
7860 | AAMDNodes NewAAInfo = AAInfo; |
7861 | NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
7862 | |
7863 | for (unsigned i = 0; i < NumMemOps; i++) { |
7864 | EVT VT = MemOps[i]; |
7865 | unsigned VTSize = VT.getSizeInBits() / 8; |
7866 | if (VTSize > Size) { |
7867 | // Issuing an unaligned load / store pair that overlaps with the previous |
7868 | // pair. Adjust the offset accordingly. |
7869 | assert(i == NumMemOps-1 && i != 0); |
7870 | DstOff -= VTSize - Size; |
7871 | } |
7872 | |
7873 | // If this store is smaller than the largest store see whether we can get |
7874 | // the smaller value for free with a truncate or extract vector element and |
7875 | // then store. |
7876 | SDValue Value = MemSetValue; |
7877 | if (VT.bitsLT(VT: LargestVT)) { |
7878 | unsigned Index; |
7879 | unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits(); |
7880 | EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts); |
7881 | if (!LargestVT.isVector() && !VT.isVector() && |
7882 | TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT)) |
7883 | Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue); |
7884 | else if (LargestVT.isVector() && !VT.isVector() && |
7885 | TLI.shallExtractConstSplatVectorElementToStore( |
7886 | VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()), |
7887 | ElemSizeInBits: VT.getSizeInBits(), Index) && |
7888 | TLI.isTypeLegal(VT: SVT) && |
7889 | LargestVT.getSizeInBits() == SVT.getSizeInBits()) { |
7890 | // Target which can combine store(extractelement VectorTy, Idx) can get |
7891 | // the smaller value for free. |
7892 | SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue); |
7893 | Value = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: TailValue, |
7894 | N2: DAG.getVectorIdxConstant(Val: Index, DL: dl)); |
7895 | } else |
7896 | Value = getMemsetValue(Value: Src, VT, DAG, dl); |
7897 | } |
7898 | assert(Value.getValueType() == VT && "Value with wrong type." ); |
7899 | SDValue Store = DAG.getStore( |
7900 | Chain, dl, Val: Value, |
7901 | Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl), |
7902 | PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, |
7903 | MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, |
7904 | AAInfo: NewAAInfo); |
7905 | OutChains.push_back(Elt: Store); |
7906 | DstOff += VT.getSizeInBits() / 8; |
7907 | Size -= VTSize; |
7908 | } |
7909 | |
7910 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
7911 | } |
7912 | |
7913 | static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, |
7914 | unsigned AS) { |
7915 | // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all |
7916 | // pointer operands can be losslessly bitcasted to pointers of address space 0 |
7917 | if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) { |
7918 | report_fatal_error(reason: "cannot lower memory intrinsic in address space " + |
7919 | Twine(AS)); |
7920 | } |
7921 | } |
7922 | |
7923 | SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, |
7924 | SDValue Src, SDValue Size, Align Alignment, |
7925 | bool isVol, bool AlwaysInline, bool isTailCall, |
7926 | MachinePointerInfo DstPtrInfo, |
7927 | MachinePointerInfo SrcPtrInfo, |
7928 | const AAMDNodes &AAInfo, AAResults *AA) { |
7929 | // Check to see if we should lower the memcpy to loads and stores first. |
7930 | // For cases within the target-specified limits, this is the best choice. |
7931 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size); |
7932 | if (ConstantSize) { |
7933 | // Memcpy with size zero? Just return the original chain. |
7934 | if (ConstantSize->isZero()) |
7935 | return Chain; |
7936 | |
7937 | SDValue Result = getMemcpyLoadsAndStores( |
7938 | DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment, |
7939 | isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
7940 | if (Result.getNode()) |
7941 | return Result; |
7942 | } |
7943 | |
7944 | // Then check to see if we should lower the memcpy with target-specific |
7945 | // code. If the target chooses to do this, this is the next best. |
7946 | if (TSI) { |
7947 | SDValue Result = TSI->EmitTargetCodeForMemcpy( |
7948 | DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, |
7949 | DstPtrInfo, SrcPtrInfo); |
7950 | if (Result.getNode()) |
7951 | return Result; |
7952 | } |
7953 | |
7954 | // If we really need inline code and the target declined to provide it, |
7955 | // use a (potentially long) sequence of loads and stores. |
7956 | if (AlwaysInline) { |
7957 | assert(ConstantSize && "AlwaysInline requires a constant size!" ); |
7958 | return getMemcpyLoadsAndStores( |
7959 | DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment, |
7960 | isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
7961 | } |
7962 | |
7963 | checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace()); |
7964 | checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace()); |
7965 | |
7966 | // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc |
7967 | // memcpy is not guaranteed to be safe. libc memcpys aren't required to |
7968 | // respect volatile, so they may do things like read or write memory |
7969 | // beyond the given memory regions. But fixing this isn't easy, and most |
7970 | // people don't care. |
7971 | |
7972 | // Emit a library call. |
7973 | TargetLowering::ArgListTy Args; |
7974 | TargetLowering::ArgListEntry Entry; |
7975 | Entry.Ty = PointerType::getUnqual(C&: *getContext()); |
7976 | Entry.Node = Dst; Args.push_back(x: Entry); |
7977 | Entry.Node = Src; Args.push_back(x: Entry); |
7978 | |
7979 | Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext()); |
7980 | Entry.Node = Size; Args.push_back(x: Entry); |
7981 | // FIXME: pass in SDLoc |
7982 | TargetLowering::CallLoweringInfo CLI(*this); |
7983 | CLI.setDebugLoc(dl) |
7984 | .setChain(Chain) |
7985 | .setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMCPY), |
7986 | ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()), |
7987 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMCPY), |
7988 | VT: TLI->getPointerTy(DL: getDataLayout())), |
7989 | ArgsList: std::move(Args)) |
7990 | .setDiscardResult() |
7991 | .setTailCall(isTailCall); |
7992 | |
7993 | std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); |
7994 | return CallResult.second; |
7995 | } |
7996 | |
7997 | SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, |
7998 | SDValue Dst, SDValue Src, SDValue Size, |
7999 | Type *SizeTy, unsigned ElemSz, |
8000 | bool isTailCall, |
8001 | MachinePointerInfo DstPtrInfo, |
8002 | MachinePointerInfo SrcPtrInfo) { |
8003 | // Emit a library call. |
8004 | TargetLowering::ArgListTy Args; |
8005 | TargetLowering::ArgListEntry Entry; |
8006 | Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext()); |
8007 | Entry.Node = Dst; |
8008 | Args.push_back(x: Entry); |
8009 | |
8010 | Entry.Node = Src; |
8011 | Args.push_back(x: Entry); |
8012 | |
8013 | Entry.Ty = SizeTy; |
8014 | Entry.Node = Size; |
8015 | Args.push_back(x: Entry); |
8016 | |
8017 | RTLIB::Libcall LibraryCall = |
8018 | RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz); |
8019 | if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
8020 | report_fatal_error(reason: "Unsupported element size" ); |
8021 | |
8022 | TargetLowering::CallLoweringInfo CLI(*this); |
8023 | CLI.setDebugLoc(dl) |
8024 | .setChain(Chain) |
8025 | .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall), |
8026 | ResultType: Type::getVoidTy(C&: *getContext()), |
8027 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall), |
8028 | VT: TLI->getPointerTy(DL: getDataLayout())), |
8029 | ArgsList: std::move(Args)) |
8030 | .setDiscardResult() |
8031 | .setTailCall(isTailCall); |
8032 | |
8033 | std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); |
8034 | return CallResult.second; |
8035 | } |
8036 | |
8037 | SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, |
8038 | SDValue Src, SDValue Size, Align Alignment, |
8039 | bool isVol, bool isTailCall, |
8040 | MachinePointerInfo DstPtrInfo, |
8041 | MachinePointerInfo SrcPtrInfo, |
8042 | const AAMDNodes &AAInfo, AAResults *AA) { |
8043 | // Check to see if we should lower the memmove to loads and stores first. |
8044 | // For cases within the target-specified limits, this is the best choice. |
8045 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size); |
8046 | if (ConstantSize) { |
8047 | // Memmove with size zero? Just return the original chain. |
8048 | if (ConstantSize->isZero()) |
8049 | return Chain; |
8050 | |
8051 | SDValue Result = getMemmoveLoadsAndStores( |
8052 | DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment, |
8053 | isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo); |
8054 | if (Result.getNode()) |
8055 | return Result; |
8056 | } |
8057 | |
8058 | // Then check to see if we should lower the memmove with target-specific |
8059 | // code. If the target chooses to do this, this is the next best. |
8060 | if (TSI) { |
8061 | SDValue Result = |
8062 | TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, |
8063 | Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo); |
8064 | if (Result.getNode()) |
8065 | return Result; |
8066 | } |
8067 | |
8068 | checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace()); |
8069 | checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace()); |
8070 | |
8071 | // FIXME: If the memmove is volatile, lowering it to plain libc memmove may |
8072 | // not be safe. See memcpy above for more details. |
8073 | |
8074 | // Emit a library call. |
8075 | TargetLowering::ArgListTy Args; |
8076 | TargetLowering::ArgListEntry Entry; |
8077 | Entry.Ty = PointerType::getUnqual(C&: *getContext()); |
8078 | Entry.Node = Dst; Args.push_back(x: Entry); |
8079 | Entry.Node = Src; Args.push_back(x: Entry); |
8080 | |
8081 | Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext()); |
8082 | Entry.Node = Size; Args.push_back(x: Entry); |
8083 | // FIXME: pass in SDLoc |
8084 | TargetLowering::CallLoweringInfo CLI(*this); |
8085 | CLI.setDebugLoc(dl) |
8086 | .setChain(Chain) |
8087 | .setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMMOVE), |
8088 | ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()), |
8089 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMMOVE), |
8090 | VT: TLI->getPointerTy(DL: getDataLayout())), |
8091 | ArgsList: std::move(Args)) |
8092 | .setDiscardResult() |
8093 | .setTailCall(isTailCall); |
8094 | |
8095 | std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); |
8096 | return CallResult.second; |
8097 | } |
8098 | |
8099 | SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, |
8100 | SDValue Dst, SDValue Src, SDValue Size, |
8101 | Type *SizeTy, unsigned ElemSz, |
8102 | bool isTailCall, |
8103 | MachinePointerInfo DstPtrInfo, |
8104 | MachinePointerInfo SrcPtrInfo) { |
8105 | // Emit a library call. |
8106 | TargetLowering::ArgListTy Args; |
8107 | TargetLowering::ArgListEntry Entry; |
8108 | Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext()); |
8109 | Entry.Node = Dst; |
8110 | Args.push_back(x: Entry); |
8111 | |
8112 | Entry.Node = Src; |
8113 | Args.push_back(x: Entry); |
8114 | |
8115 | Entry.Ty = SizeTy; |
8116 | Entry.Node = Size; |
8117 | Args.push_back(x: Entry); |
8118 | |
8119 | RTLIB::Libcall LibraryCall = |
8120 | RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz); |
8121 | if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
8122 | report_fatal_error(reason: "Unsupported element size" ); |
8123 | |
8124 | TargetLowering::CallLoweringInfo CLI(*this); |
8125 | CLI.setDebugLoc(dl) |
8126 | .setChain(Chain) |
8127 | .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall), |
8128 | ResultType: Type::getVoidTy(C&: *getContext()), |
8129 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall), |
8130 | VT: TLI->getPointerTy(DL: getDataLayout())), |
8131 | ArgsList: std::move(Args)) |
8132 | .setDiscardResult() |
8133 | .setTailCall(isTailCall); |
8134 | |
8135 | std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); |
8136 | return CallResult.second; |
8137 | } |
8138 | |
8139 | SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, |
8140 | SDValue Src, SDValue Size, Align Alignment, |
8141 | bool isVol, bool AlwaysInline, bool isTailCall, |
8142 | MachinePointerInfo DstPtrInfo, |
8143 | const AAMDNodes &AAInfo) { |
8144 | // Check to see if we should lower the memset to stores first. |
8145 | // For cases within the target-specified limits, this is the best choice. |
8146 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size); |
8147 | if (ConstantSize) { |
8148 | // Memset with size zero? Just return the original chain. |
8149 | if (ConstantSize->isZero()) |
8150 | return Chain; |
8151 | |
8152 | SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src, |
8153 | Size: ConstantSize->getZExtValue(), Alignment, |
8154 | isVol, AlwaysInline: false, DstPtrInfo, AAInfo); |
8155 | |
8156 | if (Result.getNode()) |
8157 | return Result; |
8158 | } |
8159 | |
8160 | // Then check to see if we should lower the memset with target-specific |
8161 | // code. If the target chooses to do this, this is the next best. |
8162 | if (TSI) { |
8163 | SDValue Result = TSI->EmitTargetCodeForMemset( |
8164 | DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo); |
8165 | if (Result.getNode()) |
8166 | return Result; |
8167 | } |
8168 | |
8169 | // If we really need inline code and the target declined to provide it, |
8170 | // use a (potentially long) sequence of loads and stores. |
8171 | if (AlwaysInline) { |
8172 | assert(ConstantSize && "AlwaysInline requires a constant size!" ); |
8173 | SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src, |
8174 | Size: ConstantSize->getZExtValue(), Alignment, |
8175 | isVol, AlwaysInline: true, DstPtrInfo, AAInfo); |
8176 | assert(Result && |
8177 | "getMemsetStores must return a valid sequence when AlwaysInline" ); |
8178 | return Result; |
8179 | } |
8180 | |
8181 | checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace()); |
8182 | |
8183 | // Emit a library call. |
8184 | auto &Ctx = *getContext(); |
8185 | const auto& DL = getDataLayout(); |
8186 | |
8187 | TargetLowering::CallLoweringInfo CLI(*this); |
8188 | // FIXME: pass in SDLoc |
8189 | CLI.setDebugLoc(dl).setChain(Chain); |
8190 | |
8191 | const char *BzeroName = getTargetLoweringInfo().getLibcallName(Call: RTLIB::BZERO); |
8192 | |
8193 | // Helper function to create an Entry from Node and Type. |
8194 | const auto CreateEntry = [](SDValue Node, Type *Ty) { |
8195 | TargetLowering::ArgListEntry Entry; |
8196 | Entry.Node = Node; |
8197 | Entry.Ty = Ty; |
8198 | return Entry; |
8199 | }; |
8200 | |
8201 | // If zeroing out and bzero is present, use it. |
8202 | if (isNullConstant(V: Src) && BzeroName) { |
8203 | TargetLowering::ArgListTy Args; |
8204 | Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx))); |
8205 | Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx))); |
8206 | CLI.setLibCallee( |
8207 | CC: TLI->getLibcallCallingConv(Call: RTLIB::BZERO), ResultType: Type::getVoidTy(C&: Ctx), |
8208 | Target: getExternalSymbol(Sym: BzeroName, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args)); |
8209 | } else { |
8210 | TargetLowering::ArgListTy Args; |
8211 | Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx))); |
8212 | Args.push_back(x: CreateEntry(Src, Src.getValueType().getTypeForEVT(Context&: Ctx))); |
8213 | Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx))); |
8214 | CLI.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMSET), |
8215 | ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx), |
8216 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMSET), |
8217 | VT: TLI->getPointerTy(DL)), |
8218 | ArgsList: std::move(Args)); |
8219 | } |
8220 | |
8221 | CLI.setDiscardResult().setTailCall(isTailCall); |
8222 | |
8223 | std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); |
8224 | return CallResult.second; |
8225 | } |
8226 | |
8227 | SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, |
8228 | SDValue Dst, SDValue Value, SDValue Size, |
8229 | Type *SizeTy, unsigned ElemSz, |
8230 | bool isTailCall, |
8231 | MachinePointerInfo DstPtrInfo) { |
8232 | // Emit a library call. |
8233 | TargetLowering::ArgListTy Args; |
8234 | TargetLowering::ArgListEntry Entry; |
8235 | Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext()); |
8236 | Entry.Node = Dst; |
8237 | Args.push_back(x: Entry); |
8238 | |
8239 | Entry.Ty = Type::getInt8Ty(C&: *getContext()); |
8240 | Entry.Node = Value; |
8241 | Args.push_back(x: Entry); |
8242 | |
8243 | Entry.Ty = SizeTy; |
8244 | Entry.Node = Size; |
8245 | Args.push_back(x: Entry); |
8246 | |
8247 | RTLIB::Libcall LibraryCall = |
8248 | RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz); |
8249 | if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
8250 | report_fatal_error(reason: "Unsupported element size" ); |
8251 | |
8252 | TargetLowering::CallLoweringInfo CLI(*this); |
8253 | CLI.setDebugLoc(dl) |
8254 | .setChain(Chain) |
8255 | .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall), |
8256 | ResultType: Type::getVoidTy(C&: *getContext()), |
8257 | Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall), |
8258 | VT: TLI->getPointerTy(DL: getDataLayout())), |
8259 | ArgsList: std::move(Args)) |
8260 | .setDiscardResult() |
8261 | .setTailCall(isTailCall); |
8262 | |
8263 | std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI); |
8264 | return CallResult.second; |
8265 | } |
8266 | |
8267 | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
8268 | SDVTList VTList, ArrayRef<SDValue> Ops, |
8269 | MachineMemOperand *MMO) { |
8270 | FoldingSetNodeID ID; |
8271 | ID.AddInteger(I: MemVT.getRawBits()); |
8272 | AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops); |
8273 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8274 | ID.AddInteger(I: MMO->getFlags()); |
8275 | void* IP = nullptr; |
8276 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8277 | cast<AtomicSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8278 | return SDValue(E, 0); |
8279 | } |
8280 | |
8281 | auto *N = newSDNode<AtomicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
8282 | Args&: VTList, Args&: MemVT, Args&: MMO); |
8283 | createOperands(Node: N, Vals: Ops); |
8284 | |
8285 | CSEMap.InsertNode(N, InsertPos: IP); |
8286 | InsertNode(N); |
8287 | return SDValue(N, 0); |
8288 | } |
8289 | |
8290 | SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, |
8291 | EVT MemVT, SDVTList VTs, SDValue Chain, |
8292 | SDValue Ptr, SDValue Cmp, SDValue Swp, |
8293 | MachineMemOperand *MMO) { |
8294 | assert(Opcode == ISD::ATOMIC_CMP_SWAP || |
8295 | Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
8296 | assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types" ); |
8297 | |
8298 | SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; |
8299 | return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO); |
8300 | } |
8301 | |
8302 | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
8303 | SDValue Chain, SDValue Ptr, SDValue Val, |
8304 | MachineMemOperand *MMO) { |
8305 | assert((Opcode == ISD::ATOMIC_LOAD_ADD || |
8306 | Opcode == ISD::ATOMIC_LOAD_SUB || |
8307 | Opcode == ISD::ATOMIC_LOAD_AND || |
8308 | Opcode == ISD::ATOMIC_LOAD_CLR || |
8309 | Opcode == ISD::ATOMIC_LOAD_OR || |
8310 | Opcode == ISD::ATOMIC_LOAD_XOR || |
8311 | Opcode == ISD::ATOMIC_LOAD_NAND || |
8312 | Opcode == ISD::ATOMIC_LOAD_MIN || |
8313 | Opcode == ISD::ATOMIC_LOAD_MAX || |
8314 | Opcode == ISD::ATOMIC_LOAD_UMIN || |
8315 | Opcode == ISD::ATOMIC_LOAD_UMAX || |
8316 | Opcode == ISD::ATOMIC_LOAD_FADD || |
8317 | Opcode == ISD::ATOMIC_LOAD_FSUB || |
8318 | Opcode == ISD::ATOMIC_LOAD_FMAX || |
8319 | Opcode == ISD::ATOMIC_LOAD_FMIN || |
8320 | Opcode == ISD::ATOMIC_LOAD_UINC_WRAP || |
8321 | Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP || |
8322 | Opcode == ISD::ATOMIC_SWAP || |
8323 | Opcode == ISD::ATOMIC_STORE) && |
8324 | "Invalid Atomic Op" ); |
8325 | |
8326 | EVT VT = Val.getValueType(); |
8327 | |
8328 | SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : |
8329 | getVTList(VT, MVT::Other); |
8330 | SDValue Ops[] = {Chain, Ptr, Val}; |
8331 | return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO); |
8332 | } |
8333 | |
8334 | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
8335 | EVT VT, SDValue Chain, SDValue Ptr, |
8336 | MachineMemOperand *MMO) { |
8337 | assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op" ); |
8338 | |
8339 | SDVTList VTs = getVTList(VT, MVT::Other); |
8340 | SDValue Ops[] = {Chain, Ptr}; |
8341 | return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO); |
8342 | } |
8343 | |
8344 | /// getMergeValues - Create a MERGE_VALUES node from the given operands. |
8345 | SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { |
8346 | if (Ops.size() == 1) |
8347 | return Ops[0]; |
8348 | |
8349 | SmallVector<EVT, 4> VTs; |
8350 | VTs.reserve(N: Ops.size()); |
8351 | for (const SDValue &Op : Ops) |
8352 | VTs.push_back(Elt: Op.getValueType()); |
8353 | return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops); |
8354 | } |
8355 | |
8356 | SDValue SelectionDAG::getMemIntrinsicNode( |
8357 | unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, |
8358 | EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, |
8359 | MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) { |
8360 | if (!Size && MemVT.isScalableVector()) |
8361 | Size = MemoryLocation::UnknownSize; |
8362 | else if (!Size) |
8363 | Size = MemVT.getStoreSize(); |
8364 | |
8365 | MachineFunction &MF = getMachineFunction(); |
8366 | MachineMemOperand *MMO = |
8367 | MF.getMachineMemOperand(PtrInfo, f: Flags, s: Size, base_alignment: Alignment, AAInfo); |
8368 | |
8369 | return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); |
8370 | } |
8371 | |
8372 | SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, |
8373 | SDVTList VTList, |
8374 | ArrayRef<SDValue> Ops, EVT MemVT, |
8375 | MachineMemOperand *MMO) { |
8376 | assert((Opcode == ISD::INTRINSIC_VOID || |
8377 | Opcode == ISD::INTRINSIC_W_CHAIN || |
8378 | Opcode == ISD::PREFETCH || |
8379 | (Opcode <= (unsigned)std::numeric_limits<int>::max() && |
8380 | (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && |
8381 | "Opcode is not a memory-accessing opcode!" ); |
8382 | |
8383 | // Memoize the node unless it returns a glue result. |
8384 | MemIntrinsicSDNode *N; |
8385 | if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
8386 | FoldingSetNodeID ID; |
8387 | AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops); |
8388 | ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>( |
8389 | Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MMO)); |
8390 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8391 | ID.AddInteger(I: MMO->getFlags()); |
8392 | ID.AddInteger(I: MemVT.getRawBits()); |
8393 | void *IP = nullptr; |
8394 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8395 | cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8396 | return SDValue(E, 0); |
8397 | } |
8398 | |
8399 | N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
8400 | Args&: VTList, Args&: MemVT, Args&: MMO); |
8401 | createOperands(Node: N, Vals: Ops); |
8402 | |
8403 | CSEMap.InsertNode(N, InsertPos: IP); |
8404 | } else { |
8405 | N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
8406 | Args&: VTList, Args&: MemVT, Args&: MMO); |
8407 | createOperands(Node: N, Vals: Ops); |
8408 | } |
8409 | InsertNode(N); |
8410 | SDValue V(N, 0); |
8411 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8412 | return V; |
8413 | } |
8414 | |
8415 | SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, |
8416 | SDValue Chain, int FrameIndex, |
8417 | int64_t Size, int64_t Offset) { |
8418 | const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; |
8419 | const auto VTs = getVTList(MVT::Other); |
8420 | SDValue Ops[2] = { |
8421 | Chain, |
8422 | getFrameIndex(FI: FrameIndex, |
8423 | VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()), |
8424 | isTarget: true)}; |
8425 | |
8426 | FoldingSetNodeID ID; |
8427 | AddNodeIDNode(ID, Opcode, VTs, Ops); |
8428 | ID.AddInteger(I: FrameIndex); |
8429 | ID.AddInteger(I: Size); |
8430 | ID.AddInteger(I: Offset); |
8431 | void *IP = nullptr; |
8432 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
8433 | return SDValue(E, 0); |
8434 | |
8435 | LifetimeSDNode *N = newSDNode<LifetimeSDNode>( |
8436 | Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); |
8437 | createOperands(Node: N, Vals: Ops); |
8438 | CSEMap.InsertNode(N, InsertPos: IP); |
8439 | InsertNode(N); |
8440 | SDValue V(N, 0); |
8441 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8442 | return V; |
8443 | } |
8444 | |
8445 | SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, |
8446 | uint64_t Guid, uint64_t Index, |
8447 | uint32_t Attr) { |
8448 | const unsigned Opcode = ISD::PSEUDO_PROBE; |
8449 | const auto VTs = getVTList(MVT::Other); |
8450 | SDValue Ops[] = {Chain}; |
8451 | FoldingSetNodeID ID; |
8452 | AddNodeIDNode(ID, Opcode, VTs, Ops); |
8453 | ID.AddInteger(I: Guid); |
8454 | ID.AddInteger(I: Index); |
8455 | void *IP = nullptr; |
8456 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP)) |
8457 | return SDValue(E, 0); |
8458 | |
8459 | auto *N = newSDNode<PseudoProbeSDNode>( |
8460 | Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); |
8461 | createOperands(Node: N, Vals: Ops); |
8462 | CSEMap.InsertNode(N, IP); |
8463 | InsertNode(N: N); |
8464 | SDValue V(N, 0); |
8465 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8466 | return V; |
8467 | } |
8468 | |
8469 | /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
8470 | /// MachinePointerInfo record from it. This is particularly useful because the |
8471 | /// code generator has many cases where it doesn't bother passing in a |
8472 | /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
8473 | static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
8474 | SelectionDAG &DAG, SDValue Ptr, |
8475 | int64_t Offset = 0) { |
8476 | // If this is FI+Offset, we can model it. |
8477 | if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) |
8478 | return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), |
8479 | FI: FI->getIndex(), Offset); |
8480 | |
8481 | // If this is (FI+Offset1)+Offset2, we can model it. |
8482 | if (Ptr.getOpcode() != ISD::ADD || |
8483 | !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) || |
8484 | !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) |
8485 | return Info; |
8486 | |
8487 | int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex(); |
8488 | return MachinePointerInfo::getFixedStack( |
8489 | MF&: DAG.getMachineFunction(), FI, |
8490 | Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue()); |
8491 | } |
8492 | |
8493 | /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
8494 | /// MachinePointerInfo record from it. This is particularly useful because the |
8495 | /// code generator has many cases where it doesn't bother passing in a |
8496 | /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
8497 | static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
8498 | SelectionDAG &DAG, SDValue Ptr, |
8499 | SDValue OffsetOp) { |
8500 | // If the 'Offset' value isn't a constant, we can't handle this. |
8501 | if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp)) |
8502 | return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue()); |
8503 | if (OffsetOp.isUndef()) |
8504 | return InferPointerInfo(Info, DAG, Ptr); |
8505 | return Info; |
8506 | } |
8507 | |
8508 | SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
8509 | EVT VT, const SDLoc &dl, SDValue Chain, |
8510 | SDValue Ptr, SDValue Offset, |
8511 | MachinePointerInfo PtrInfo, EVT MemVT, |
8512 | Align Alignment, |
8513 | MachineMemOperand::Flags MMOFlags, |
8514 | const AAMDNodes &AAInfo, const MDNode *Ranges) { |
8515 | assert(Chain.getValueType() == MVT::Other && |
8516 | "Invalid chain type" ); |
8517 | |
8518 | MMOFlags |= MachineMemOperand::MOLoad; |
8519 | assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
8520 | // If we don't have a PtrInfo, infer the trivial frame index case to simplify |
8521 | // clients. |
8522 | if (PtrInfo.V.isNull()) |
8523 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset); |
8524 | |
8525 | uint64_t Size = MemoryLocation::getSizeOrUnknown(T: MemVT.getStoreSize()); |
8526 | MachineFunction &MF = getMachineFunction(); |
8527 | MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size, |
8528 | base_alignment: Alignment, AAInfo, Ranges); |
8529 | return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); |
8530 | } |
8531 | |
8532 | SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
8533 | EVT VT, const SDLoc &dl, SDValue Chain, |
8534 | SDValue Ptr, SDValue Offset, EVT MemVT, |
8535 | MachineMemOperand *MMO) { |
8536 | if (VT == MemVT) { |
8537 | ExtType = ISD::NON_EXTLOAD; |
8538 | } else if (ExtType == ISD::NON_EXTLOAD) { |
8539 | assert(VT == MemVT && "Non-extending load from different memory type!" ); |
8540 | } else { |
8541 | // Extending load. |
8542 | assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && |
8543 | "Should only be an extending load, not truncating!" ); |
8544 | assert(VT.isInteger() == MemVT.isInteger() && |
8545 | "Cannot convert from FP to Int or Int -> FP!" ); |
8546 | assert(VT.isVector() == MemVT.isVector() && |
8547 | "Cannot use an ext load to convert to or from a vector!" ); |
8548 | assert((!VT.isVector() || |
8549 | VT.getVectorElementCount() == MemVT.getVectorElementCount()) && |
8550 | "Cannot use an ext load to change the number of vector elements!" ); |
8551 | } |
8552 | |
8553 | bool Indexed = AM != ISD::UNINDEXED; |
8554 | assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!" ); |
8555 | |
8556 | SDVTList VTs = Indexed ? |
8557 | getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); |
8558 | SDValue Ops[] = { Chain, Ptr, Offset }; |
8559 | FoldingSetNodeID ID; |
8560 | AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops); |
8561 | ID.AddInteger(I: MemVT.getRawBits()); |
8562 | ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>( |
8563 | IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO)); |
8564 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8565 | ID.AddInteger(I: MMO->getFlags()); |
8566 | void *IP = nullptr; |
8567 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8568 | cast<LoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8569 | return SDValue(E, 0); |
8570 | } |
8571 | auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, |
8572 | Args&: ExtType, Args&: MemVT, Args&: MMO); |
8573 | createOperands(Node: N, Vals: Ops); |
8574 | |
8575 | CSEMap.InsertNode(N, InsertPos: IP); |
8576 | InsertNode(N); |
8577 | SDValue V(N, 0); |
8578 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8579 | return V; |
8580 | } |
8581 | |
8582 | SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
8583 | SDValue Ptr, MachinePointerInfo PtrInfo, |
8584 | MaybeAlign Alignment, |
8585 | MachineMemOperand::Flags MMOFlags, |
8586 | const AAMDNodes &AAInfo, const MDNode *Ranges) { |
8587 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8588 | return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef, |
8589 | PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges); |
8590 | } |
8591 | |
8592 | SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
8593 | SDValue Ptr, MachineMemOperand *MMO) { |
8594 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8595 | return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef, |
8596 | MemVT: VT, MMO); |
8597 | } |
8598 | |
8599 | SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
8600 | EVT VT, SDValue Chain, SDValue Ptr, |
8601 | MachinePointerInfo PtrInfo, EVT MemVT, |
8602 | MaybeAlign Alignment, |
8603 | MachineMemOperand::Flags MMOFlags, |
8604 | const AAMDNodes &AAInfo) { |
8605 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8606 | return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo, |
8607 | MemVT, Alignment, MMOFlags, AAInfo); |
8608 | } |
8609 | |
8610 | SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
8611 | EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, |
8612 | MachineMemOperand *MMO) { |
8613 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8614 | return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, |
8615 | MemVT, MMO); |
8616 | } |
8617 | |
8618 | SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, |
8619 | SDValue Base, SDValue Offset, |
8620 | ISD::MemIndexedMode AM) { |
8621 | LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad); |
8622 | assert(LD->getOffset().isUndef() && "Load is already a indexed load!" ); |
8623 | // Don't propagate the invariant or dereferenceable flags. |
8624 | auto MMOFlags = |
8625 | LD->getMemOperand()->getFlags() & |
8626 | ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
8627 | return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, |
8628 | Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), |
8629 | MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo()); |
8630 | } |
8631 | |
8632 | SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
8633 | SDValue Ptr, MachinePointerInfo PtrInfo, |
8634 | Align Alignment, |
8635 | MachineMemOperand::Flags MMOFlags, |
8636 | const AAMDNodes &AAInfo) { |
8637 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
8638 | |
8639 | MMOFlags |= MachineMemOperand::MOStore; |
8640 | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
8641 | |
8642 | if (PtrInfo.V.isNull()) |
8643 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr); |
8644 | |
8645 | MachineFunction &MF = getMachineFunction(); |
8646 | uint64_t Size = |
8647 | MemoryLocation::getSizeOrUnknown(T: Val.getValueType().getStoreSize()); |
8648 | MachineMemOperand *MMO = |
8649 | MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size, base_alignment: Alignment, AAInfo); |
8650 | return getStore(Chain, dl, Val, Ptr, MMO); |
8651 | } |
8652 | |
8653 | SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
8654 | SDValue Ptr, MachineMemOperand *MMO) { |
8655 | assert(Chain.getValueType() == MVT::Other && |
8656 | "Invalid chain type" ); |
8657 | EVT VT = Val.getValueType(); |
8658 | SDVTList VTs = getVTList(MVT::Other); |
8659 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8660 | SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
8661 | FoldingSetNodeID ID; |
8662 | AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops); |
8663 | ID.AddInteger(I: VT.getRawBits()); |
8664 | ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>( |
8665 | IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO)); |
8666 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8667 | ID.AddInteger(I: MMO->getFlags()); |
8668 | void *IP = nullptr; |
8669 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8670 | cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8671 | return SDValue(E, 0); |
8672 | } |
8673 | auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
8674 | Args: ISD::UNINDEXED, Args: false, Args&: VT, Args&: MMO); |
8675 | createOperands(Node: N, Vals: Ops); |
8676 | |
8677 | CSEMap.InsertNode(N, InsertPos: IP); |
8678 | InsertNode(N); |
8679 | SDValue V(N, 0); |
8680 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8681 | return V; |
8682 | } |
8683 | |
8684 | SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
8685 | SDValue Ptr, MachinePointerInfo PtrInfo, |
8686 | EVT SVT, Align Alignment, |
8687 | MachineMemOperand::Flags MMOFlags, |
8688 | const AAMDNodes &AAInfo) { |
8689 | assert(Chain.getValueType() == MVT::Other && |
8690 | "Invalid chain type" ); |
8691 | |
8692 | MMOFlags |= MachineMemOperand::MOStore; |
8693 | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
8694 | |
8695 | if (PtrInfo.V.isNull()) |
8696 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr); |
8697 | |
8698 | MachineFunction &MF = getMachineFunction(); |
8699 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
8700 | PtrInfo, f: MMOFlags, s: MemoryLocation::getSizeOrUnknown(T: SVT.getStoreSize()), |
8701 | base_alignment: Alignment, AAInfo); |
8702 | return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); |
8703 | } |
8704 | |
8705 | SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
8706 | SDValue Ptr, EVT SVT, |
8707 | MachineMemOperand *MMO) { |
8708 | EVT VT = Val.getValueType(); |
8709 | |
8710 | assert(Chain.getValueType() == MVT::Other && |
8711 | "Invalid chain type" ); |
8712 | if (VT == SVT) |
8713 | return getStore(Chain, dl, Val, Ptr, MMO); |
8714 | |
8715 | assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
8716 | "Should only be a truncating store, not extending!" ); |
8717 | assert(VT.isInteger() == SVT.isInteger() && |
8718 | "Can't do FP-INT conversion!" ); |
8719 | assert(VT.isVector() == SVT.isVector() && |
8720 | "Cannot use trunc store to convert to or from a vector!" ); |
8721 | assert((!VT.isVector() || |
8722 | VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
8723 | "Cannot use trunc store to change the number of vector elements!" ); |
8724 | |
8725 | SDVTList VTs = getVTList(MVT::Other); |
8726 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8727 | SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
8728 | FoldingSetNodeID ID; |
8729 | AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops); |
8730 | ID.AddInteger(I: SVT.getRawBits()); |
8731 | ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>( |
8732 | IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO)); |
8733 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8734 | ID.AddInteger(I: MMO->getFlags()); |
8735 | void *IP = nullptr; |
8736 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8737 | cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8738 | return SDValue(E, 0); |
8739 | } |
8740 | auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
8741 | Args: ISD::UNINDEXED, Args: true, Args&: SVT, Args&: MMO); |
8742 | createOperands(Node: N, Vals: Ops); |
8743 | |
8744 | CSEMap.InsertNode(N, InsertPos: IP); |
8745 | InsertNode(N); |
8746 | SDValue V(N, 0); |
8747 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8748 | return V; |
8749 | } |
8750 | |
8751 | SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, |
8752 | SDValue Base, SDValue Offset, |
8753 | ISD::MemIndexedMode AM) { |
8754 | StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore); |
8755 | assert(ST->getOffset().isUndef() && "Store is already a indexed store!" ); |
8756 | SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
8757 | SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; |
8758 | FoldingSetNodeID ID; |
8759 | AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops); |
8760 | ID.AddInteger(I: ST->getMemoryVT().getRawBits()); |
8761 | ID.AddInteger(I: ST->getRawSubclassData()); |
8762 | ID.AddInteger(I: ST->getPointerInfo().getAddrSpace()); |
8763 | ID.AddInteger(I: ST->getMemOperand()->getFlags()); |
8764 | void *IP = nullptr; |
8765 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
8766 | return SDValue(E, 0); |
8767 | |
8768 | auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, |
8769 | Args: ST->isTruncatingStore(), Args: ST->getMemoryVT(), |
8770 | Args: ST->getMemOperand()); |
8771 | createOperands(Node: N, Vals: Ops); |
8772 | |
8773 | CSEMap.InsertNode(N, InsertPos: IP); |
8774 | InsertNode(N); |
8775 | SDValue V(N, 0); |
8776 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8777 | return V; |
8778 | } |
8779 | |
8780 | SDValue SelectionDAG::getLoadVP( |
8781 | ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, |
8782 | SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, |
8783 | MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
8784 | MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
8785 | const MDNode *Ranges, bool IsExpanding) { |
8786 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
8787 | |
8788 | MMOFlags |= MachineMemOperand::MOLoad; |
8789 | assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
8790 | // If we don't have a PtrInfo, infer the trivial frame index case to simplify |
8791 | // clients. |
8792 | if (PtrInfo.V.isNull()) |
8793 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset); |
8794 | |
8795 | uint64_t Size = MemoryLocation::getSizeOrUnknown(T: MemVT.getStoreSize()); |
8796 | MachineFunction &MF = getMachineFunction(); |
8797 | MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size, |
8798 | base_alignment: Alignment, AAInfo, Ranges); |
8799 | return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT, |
8800 | MMO, IsExpanding); |
8801 | } |
8802 | |
8803 | SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM, |
8804 | ISD::LoadExtType ExtType, EVT VT, |
8805 | const SDLoc &dl, SDValue Chain, SDValue Ptr, |
8806 | SDValue Offset, SDValue Mask, SDValue EVL, |
8807 | EVT MemVT, MachineMemOperand *MMO, |
8808 | bool IsExpanding) { |
8809 | bool Indexed = AM != ISD::UNINDEXED; |
8810 | assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!" ); |
8811 | |
8812 | SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
8813 | : getVTList(VT, MVT::Other); |
8814 | SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL}; |
8815 | FoldingSetNodeID ID; |
8816 | AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops); |
8817 | ID.AddInteger(I: MemVT.getRawBits()); |
8818 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>( |
8819 | IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO)); |
8820 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8821 | ID.AddInteger(I: MMO->getFlags()); |
8822 | void *IP = nullptr; |
8823 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8824 | cast<VPLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8825 | return SDValue(E, 0); |
8826 | } |
8827 | auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, |
8828 | Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO); |
8829 | createOperands(Node: N, Vals: Ops); |
8830 | |
8831 | CSEMap.InsertNode(N, InsertPos: IP); |
8832 | InsertNode(N); |
8833 | SDValue V(N, 0); |
8834 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8835 | return V; |
8836 | } |
8837 | |
8838 | SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
8839 | SDValue Ptr, SDValue Mask, SDValue EVL, |
8840 | MachinePointerInfo PtrInfo, |
8841 | MaybeAlign Alignment, |
8842 | MachineMemOperand::Flags MMOFlags, |
8843 | const AAMDNodes &AAInfo, const MDNode *Ranges, |
8844 | bool IsExpanding) { |
8845 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8846 | return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef, |
8847 | Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges, |
8848 | IsExpanding); |
8849 | } |
8850 | |
8851 | SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
8852 | SDValue Ptr, SDValue Mask, SDValue EVL, |
8853 | MachineMemOperand *MMO, bool IsExpanding) { |
8854 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8855 | return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef, |
8856 | Mask, EVL, MemVT: VT, MMO, IsExpanding); |
8857 | } |
8858 | |
8859 | SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
8860 | EVT VT, SDValue Chain, SDValue Ptr, |
8861 | SDValue Mask, SDValue EVL, |
8862 | MachinePointerInfo PtrInfo, EVT MemVT, |
8863 | MaybeAlign Alignment, |
8864 | MachineMemOperand::Flags MMOFlags, |
8865 | const AAMDNodes &AAInfo, bool IsExpanding) { |
8866 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8867 | return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask, |
8868 | EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr, |
8869 | IsExpanding); |
8870 | } |
8871 | |
8872 | SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
8873 | EVT VT, SDValue Chain, SDValue Ptr, |
8874 | SDValue Mask, SDValue EVL, EVT MemVT, |
8875 | MachineMemOperand *MMO, bool IsExpanding) { |
8876 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8877 | return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask, |
8878 | EVL, MemVT, MMO, IsExpanding); |
8879 | } |
8880 | |
8881 | SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, |
8882 | SDValue Base, SDValue Offset, |
8883 | ISD::MemIndexedMode AM) { |
8884 | auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad); |
8885 | assert(LD->getOffset().isUndef() && "Load is already a indexed load!" ); |
8886 | // Don't propagate the invariant or dereferenceable flags. |
8887 | auto MMOFlags = |
8888 | LD->getMemOperand()->getFlags() & |
8889 | ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
8890 | return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, |
8891 | Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(), |
8892 | EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(), |
8893 | MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(), |
8894 | Ranges: nullptr, IsExpanding: LD->isExpandingLoad()); |
8895 | } |
8896 | |
8897 | SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, |
8898 | SDValue Ptr, SDValue Offset, SDValue Mask, |
8899 | SDValue EVL, EVT MemVT, MachineMemOperand *MMO, |
8900 | ISD::MemIndexedMode AM, bool IsTruncating, |
8901 | bool IsCompressing) { |
8902 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
8903 | bool Indexed = AM != ISD::UNINDEXED; |
8904 | assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!" ); |
8905 | SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
8906 | : getVTList(MVT::Other); |
8907 | SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL}; |
8908 | FoldingSetNodeID ID; |
8909 | AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops); |
8910 | ID.AddInteger(I: MemVT.getRawBits()); |
8911 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>( |
8912 | IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO)); |
8913 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8914 | ID.AddInteger(I: MMO->getFlags()); |
8915 | void *IP = nullptr; |
8916 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8917 | cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8918 | return SDValue(E, 0); |
8919 | } |
8920 | auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, |
8921 | Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO); |
8922 | createOperands(Node: N, Vals: Ops); |
8923 | |
8924 | CSEMap.InsertNode(N, InsertPos: IP); |
8925 | InsertNode(N); |
8926 | SDValue V(N, 0); |
8927 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
8928 | return V; |
8929 | } |
8930 | |
8931 | SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
8932 | SDValue Val, SDValue Ptr, SDValue Mask, |
8933 | SDValue EVL, MachinePointerInfo PtrInfo, |
8934 | EVT SVT, Align Alignment, |
8935 | MachineMemOperand::Flags MMOFlags, |
8936 | const AAMDNodes &AAInfo, |
8937 | bool IsCompressing) { |
8938 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
8939 | |
8940 | MMOFlags |= MachineMemOperand::MOStore; |
8941 | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
8942 | |
8943 | if (PtrInfo.V.isNull()) |
8944 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr); |
8945 | |
8946 | MachineFunction &MF = getMachineFunction(); |
8947 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
8948 | PtrInfo, f: MMOFlags, s: MemoryLocation::getSizeOrUnknown(T: SVT.getStoreSize()), |
8949 | base_alignment: Alignment, AAInfo); |
8950 | return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, |
8951 | IsCompressing); |
8952 | } |
8953 | |
8954 | SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
8955 | SDValue Val, SDValue Ptr, SDValue Mask, |
8956 | SDValue EVL, EVT SVT, |
8957 | MachineMemOperand *MMO, |
8958 | bool IsCompressing) { |
8959 | EVT VT = Val.getValueType(); |
8960 | |
8961 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
8962 | if (VT == SVT) |
8963 | return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask, |
8964 | EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED, |
8965 | /*IsTruncating*/ false, IsCompressing); |
8966 | |
8967 | assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
8968 | "Should only be a truncating store, not extending!" ); |
8969 | assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!" ); |
8970 | assert(VT.isVector() == SVT.isVector() && |
8971 | "Cannot use trunc store to convert to or from a vector!" ); |
8972 | assert((!VT.isVector() || |
8973 | VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
8974 | "Cannot use trunc store to change the number of vector elements!" ); |
8975 | |
8976 | SDVTList VTs = getVTList(MVT::Other); |
8977 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
8978 | SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL}; |
8979 | FoldingSetNodeID ID; |
8980 | AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops); |
8981 | ID.AddInteger(I: SVT.getRawBits()); |
8982 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>( |
8983 | IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO)); |
8984 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
8985 | ID.AddInteger(I: MMO->getFlags()); |
8986 | void *IP = nullptr; |
8987 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
8988 | cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
8989 | return SDValue(E, 0); |
8990 | } |
8991 | auto *N = |
8992 | newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
8993 | Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO); |
8994 | createOperands(Node: N, Vals: Ops); |
8995 | |
8996 | CSEMap.InsertNode(N, InsertPos: IP); |
8997 | InsertNode(N); |
8998 | SDValue V(N, 0); |
8999 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9000 | return V; |
9001 | } |
9002 | |
9003 | SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, |
9004 | SDValue Base, SDValue Offset, |
9005 | ISD::MemIndexedMode AM) { |
9006 | auto *ST = cast<VPStoreSDNode>(Val&: OrigStore); |
9007 | assert(ST->getOffset().isUndef() && "Store is already an indexed store!" ); |
9008 | SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
9009 | SDValue Ops[] = {ST->getChain(), ST->getValue(), Base, |
9010 | Offset, ST->getMask(), ST->getVectorLength()}; |
9011 | FoldingSetNodeID ID; |
9012 | AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops); |
9013 | ID.AddInteger(I: ST->getMemoryVT().getRawBits()); |
9014 | ID.AddInteger(I: ST->getRawSubclassData()); |
9015 | ID.AddInteger(I: ST->getPointerInfo().getAddrSpace()); |
9016 | ID.AddInteger(I: ST->getMemOperand()->getFlags()); |
9017 | void *IP = nullptr; |
9018 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
9019 | return SDValue(E, 0); |
9020 | |
9021 | auto *N = newSDNode<VPStoreSDNode>( |
9022 | Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(), |
9023 | Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand()); |
9024 | createOperands(Node: N, Vals: Ops); |
9025 | |
9026 | CSEMap.InsertNode(N, InsertPos: IP); |
9027 | InsertNode(N); |
9028 | SDValue V(N, 0); |
9029 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9030 | return V; |
9031 | } |
9032 | |
9033 | SDValue SelectionDAG::getStridedLoadVP( |
9034 | ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
9035 | SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
9036 | SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
9037 | MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
9038 | const MDNode *Ranges, bool IsExpanding) { |
9039 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9040 | |
9041 | MMOFlags |= MachineMemOperand::MOLoad; |
9042 | assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
9043 | // If we don't have a PtrInfo, infer the trivial frame index case to simplify |
9044 | // clients. |
9045 | if (PtrInfo.V.isNull()) |
9046 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset); |
9047 | |
9048 | uint64_t Size = MemoryLocation::UnknownSize; |
9049 | MachineFunction &MF = getMachineFunction(); |
9050 | MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, f: MMOFlags, s: Size, |
9051 | base_alignment: Alignment, AAInfo, Ranges); |
9052 | return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask, |
9053 | EVL, MemVT, MMO, IsExpanding); |
9054 | } |
9055 | |
9056 | SDValue SelectionDAG::getStridedLoadVP( |
9057 | ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
9058 | SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
9059 | SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) { |
9060 | bool Indexed = AM != ISD::UNINDEXED; |
9061 | assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!" ); |
9062 | |
9063 | SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL}; |
9064 | SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
9065 | : getVTList(VT, MVT::Other); |
9066 | FoldingSetNodeID ID; |
9067 | AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops); |
9068 | ID.AddInteger(I: VT.getRawBits()); |
9069 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>( |
9070 | IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO)); |
9071 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9072 | |
9073 | void *IP = nullptr; |
9074 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
9075 | cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9076 | return SDValue(E, 0); |
9077 | } |
9078 | |
9079 | auto *N = |
9080 | newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM, |
9081 | Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO); |
9082 | createOperands(Node: N, Vals: Ops); |
9083 | CSEMap.InsertNode(N, InsertPos: IP); |
9084 | InsertNode(N); |
9085 | SDValue V(N, 0); |
9086 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9087 | return V; |
9088 | } |
9089 | |
9090 | SDValue SelectionDAG::getStridedLoadVP( |
9091 | EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride, |
9092 | SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment, |
9093 | MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
9094 | const MDNode *Ranges, bool IsExpanding) { |
9095 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
9096 | return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
9097 | Offset: Undef, Stride, Mask, EVL, PtrInfo, MemVT: VT, Alignment, |
9098 | MMOFlags, AAInfo, Ranges, IsExpanding); |
9099 | } |
9100 | |
9101 | SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, |
9102 | SDValue Ptr, SDValue Stride, |
9103 | SDValue Mask, SDValue EVL, |
9104 | MachineMemOperand *MMO, |
9105 | bool IsExpanding) { |
9106 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
9107 | return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
9108 | Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding); |
9109 | } |
9110 | |
9111 | SDValue SelectionDAG::getExtStridedLoadVP( |
9112 | ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
9113 | SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, |
9114 | MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, |
9115 | MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
9116 | bool IsExpanding) { |
9117 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
9118 | return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef, |
9119 | Stride, Mask, EVL, PtrInfo, MemVT, Alignment, |
9120 | MMOFlags, AAInfo, Ranges: nullptr, IsExpanding); |
9121 | } |
9122 | |
9123 | SDValue SelectionDAG::getExtStridedLoadVP( |
9124 | ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
9125 | SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, |
9126 | MachineMemOperand *MMO, bool IsExpanding) { |
9127 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
9128 | return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef, |
9129 | Stride, Mask, EVL, MemVT, MMO, IsExpanding); |
9130 | } |
9131 | |
9132 | SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, |
9133 | SDValue Base, SDValue Offset, |
9134 | ISD::MemIndexedMode AM) { |
9135 | auto *SLD = cast<VPStridedLoadSDNode>(Val&: OrigLoad); |
9136 | assert(SLD->getOffset().isUndef() && |
9137 | "Strided load is already a indexed load!" ); |
9138 | // Don't propagate the invariant or dereferenceable flags. |
9139 | auto MMOFlags = |
9140 | SLD->getMemOperand()->getFlags() & |
9141 | ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
9142 | return getStridedLoadVP( |
9143 | AM, ExtType: SLD->getExtensionType(), VT: OrigLoad.getValueType(), DL, Chain: SLD->getChain(), |
9144 | Ptr: Base, Offset, Stride: SLD->getStride(), Mask: SLD->getMask(), EVL: SLD->getVectorLength(), |
9145 | PtrInfo: SLD->getPointerInfo(), MemVT: SLD->getMemoryVT(), Alignment: SLD->getAlign(), MMOFlags, |
9146 | AAInfo: SLD->getAAInfo(), Ranges: nullptr, IsExpanding: SLD->isExpandingLoad()); |
9147 | } |
9148 | |
9149 | SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL, |
9150 | SDValue Val, SDValue Ptr, |
9151 | SDValue Offset, SDValue Stride, |
9152 | SDValue Mask, SDValue EVL, EVT MemVT, |
9153 | MachineMemOperand *MMO, |
9154 | ISD::MemIndexedMode AM, |
9155 | bool IsTruncating, bool IsCompressing) { |
9156 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9157 | bool Indexed = AM != ISD::UNINDEXED; |
9158 | assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!" ); |
9159 | SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
9160 | : getVTList(MVT::Other); |
9161 | SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL}; |
9162 | FoldingSetNodeID ID; |
9163 | AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops); |
9164 | ID.AddInteger(I: MemVT.getRawBits()); |
9165 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>( |
9166 | IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO)); |
9167 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9168 | void *IP = nullptr; |
9169 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
9170 | cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9171 | return SDValue(E, 0); |
9172 | } |
9173 | auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), |
9174 | Args&: VTs, Args&: AM, Args&: IsTruncating, |
9175 | Args&: IsCompressing, Args&: MemVT, Args&: MMO); |
9176 | createOperands(Node: N, Vals: Ops); |
9177 | |
9178 | CSEMap.InsertNode(N, InsertPos: IP); |
9179 | InsertNode(N); |
9180 | SDValue V(N, 0); |
9181 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9182 | return V; |
9183 | } |
9184 | |
9185 | SDValue SelectionDAG::getTruncStridedStoreVP( |
9186 | SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, |
9187 | SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, |
9188 | Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
9189 | bool IsCompressing) { |
9190 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9191 | |
9192 | MMOFlags |= MachineMemOperand::MOStore; |
9193 | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
9194 | |
9195 | if (PtrInfo.V.isNull()) |
9196 | PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr); |
9197 | |
9198 | MachineFunction &MF = getMachineFunction(); |
9199 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
9200 | PtrInfo, f: MMOFlags, s: MemoryLocation::UnknownSize, base_alignment: Alignment, AAInfo); |
9201 | return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT, |
9202 | MMO, IsCompressing); |
9203 | } |
9204 | |
9205 | SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, |
9206 | SDValue Val, SDValue Ptr, |
9207 | SDValue Stride, SDValue Mask, |
9208 | SDValue EVL, EVT SVT, |
9209 | MachineMemOperand *MMO, |
9210 | bool IsCompressing) { |
9211 | EVT VT = Val.getValueType(); |
9212 | |
9213 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9214 | if (VT == SVT) |
9215 | return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), |
9216 | Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED, |
9217 | /*IsTruncating*/ false, IsCompressing); |
9218 | |
9219 | assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
9220 | "Should only be a truncating store, not extending!" ); |
9221 | assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!" ); |
9222 | assert(VT.isVector() == SVT.isVector() && |
9223 | "Cannot use trunc store to convert to or from a vector!" ); |
9224 | assert((!VT.isVector() || |
9225 | VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
9226 | "Cannot use trunc store to change the number of vector elements!" ); |
9227 | |
9228 | SDVTList VTs = getVTList(MVT::Other); |
9229 | SDValue Undef = getUNDEF(VT: Ptr.getValueType()); |
9230 | SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL}; |
9231 | FoldingSetNodeID ID; |
9232 | AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops); |
9233 | ID.AddInteger(I: SVT.getRawBits()); |
9234 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>( |
9235 | IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO)); |
9236 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9237 | void *IP = nullptr; |
9238 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
9239 | cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9240 | return SDValue(E, 0); |
9241 | } |
9242 | auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), |
9243 | Args&: VTs, Args: ISD::UNINDEXED, Args: true, |
9244 | Args&: IsCompressing, Args&: SVT, Args&: MMO); |
9245 | createOperands(Node: N, Vals: Ops); |
9246 | |
9247 | CSEMap.InsertNode(N, InsertPos: IP); |
9248 | InsertNode(N); |
9249 | SDValue V(N, 0); |
9250 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9251 | return V; |
9252 | } |
9253 | |
9254 | SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore, |
9255 | const SDLoc &DL, SDValue Base, |
9256 | SDValue Offset, |
9257 | ISD::MemIndexedMode AM) { |
9258 | auto *SST = cast<VPStridedStoreSDNode>(Val&: OrigStore); |
9259 | assert(SST->getOffset().isUndef() && |
9260 | "Strided store is already an indexed store!" ); |
9261 | SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
9262 | SDValue Ops[] = { |
9263 | SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(), |
9264 | SST->getMask(), SST->getVectorLength()}; |
9265 | FoldingSetNodeID ID; |
9266 | AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops); |
9267 | ID.AddInteger(I: SST->getMemoryVT().getRawBits()); |
9268 | ID.AddInteger(I: SST->getRawSubclassData()); |
9269 | ID.AddInteger(I: SST->getPointerInfo().getAddrSpace()); |
9270 | void *IP = nullptr; |
9271 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
9272 | return SDValue(E, 0); |
9273 | |
9274 | auto *N = newSDNode<VPStridedStoreSDNode>( |
9275 | Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM, Args: SST->isTruncatingStore(), |
9276 | Args: SST->isCompressingStore(), Args: SST->getMemoryVT(), Args: SST->getMemOperand()); |
9277 | createOperands(Node: N, Vals: Ops); |
9278 | |
9279 | CSEMap.InsertNode(N, InsertPos: IP); |
9280 | InsertNode(N); |
9281 | SDValue V(N, 0); |
9282 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9283 | return V; |
9284 | } |
9285 | |
9286 | SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
9287 | ArrayRef<SDValue> Ops, MachineMemOperand *MMO, |
9288 | ISD::MemIndexType IndexType) { |
9289 | assert(Ops.size() == 6 && "Incompatible number of operands" ); |
9290 | |
9291 | FoldingSetNodeID ID; |
9292 | AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops); |
9293 | ID.AddInteger(I: VT.getRawBits()); |
9294 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>( |
9295 | IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType)); |
9296 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9297 | ID.AddInteger(I: MMO->getFlags()); |
9298 | void *IP = nullptr; |
9299 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9300 | cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9301 | return SDValue(E, 0); |
9302 | } |
9303 | |
9304 | auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
9305 | Args&: VT, Args&: MMO, Args&: IndexType); |
9306 | createOperands(Node: N, Vals: Ops); |
9307 | |
9308 | assert(N->getMask().getValueType().getVectorElementCount() == |
9309 | N->getValueType(0).getVectorElementCount() && |
9310 | "Vector width mismatch between mask and data" ); |
9311 | assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
9312 | N->getValueType(0).getVectorElementCount().isScalable() && |
9313 | "Scalable flags of index and data do not match" ); |
9314 | assert(ElementCount::isKnownGE( |
9315 | N->getIndex().getValueType().getVectorElementCount(), |
9316 | N->getValueType(0).getVectorElementCount()) && |
9317 | "Vector width mismatch between index and data" ); |
9318 | assert(isa<ConstantSDNode>(N->getScale()) && |
9319 | N->getScale()->getAsAPIntVal().isPowerOf2() && |
9320 | "Scale should be a constant power of 2" ); |
9321 | |
9322 | CSEMap.InsertNode(N, InsertPos: IP); |
9323 | InsertNode(N); |
9324 | SDValue V(N, 0); |
9325 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9326 | return V; |
9327 | } |
9328 | |
9329 | SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
9330 | ArrayRef<SDValue> Ops, |
9331 | MachineMemOperand *MMO, |
9332 | ISD::MemIndexType IndexType) { |
9333 | assert(Ops.size() == 7 && "Incompatible number of operands" ); |
9334 | |
9335 | FoldingSetNodeID ID; |
9336 | AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops); |
9337 | ID.AddInteger(I: VT.getRawBits()); |
9338 | ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>( |
9339 | IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType)); |
9340 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9341 | ID.AddInteger(I: MMO->getFlags()); |
9342 | void *IP = nullptr; |
9343 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9344 | cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9345 | return SDValue(E, 0); |
9346 | } |
9347 | auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
9348 | Args&: VT, Args&: MMO, Args&: IndexType); |
9349 | createOperands(Node: N, Vals: Ops); |
9350 | |
9351 | assert(N->getMask().getValueType().getVectorElementCount() == |
9352 | N->getValue().getValueType().getVectorElementCount() && |
9353 | "Vector width mismatch between mask and data" ); |
9354 | assert( |
9355 | N->getIndex().getValueType().getVectorElementCount().isScalable() == |
9356 | N->getValue().getValueType().getVectorElementCount().isScalable() && |
9357 | "Scalable flags of index and data do not match" ); |
9358 | assert(ElementCount::isKnownGE( |
9359 | N->getIndex().getValueType().getVectorElementCount(), |
9360 | N->getValue().getValueType().getVectorElementCount()) && |
9361 | "Vector width mismatch between index and data" ); |
9362 | assert(isa<ConstantSDNode>(N->getScale()) && |
9363 | N->getScale()->getAsAPIntVal().isPowerOf2() && |
9364 | "Scale should be a constant power of 2" ); |
9365 | |
9366 | CSEMap.InsertNode(N, InsertPos: IP); |
9367 | InsertNode(N); |
9368 | SDValue V(N, 0); |
9369 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9370 | return V; |
9371 | } |
9372 | |
9373 | SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
9374 | SDValue Base, SDValue Offset, SDValue Mask, |
9375 | SDValue PassThru, EVT MemVT, |
9376 | MachineMemOperand *MMO, |
9377 | ISD::MemIndexedMode AM, |
9378 | ISD::LoadExtType ExtTy, bool isExpanding) { |
9379 | bool Indexed = AM != ISD::UNINDEXED; |
9380 | assert((Indexed || Offset.isUndef()) && |
9381 | "Unindexed masked load with an offset!" ); |
9382 | SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) |
9383 | : getVTList(VT, MVT::Other); |
9384 | SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; |
9385 | FoldingSetNodeID ID; |
9386 | AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops); |
9387 | ID.AddInteger(I: MemVT.getRawBits()); |
9388 | ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>( |
9389 | IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO)); |
9390 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9391 | ID.AddInteger(I: MMO->getFlags()); |
9392 | void *IP = nullptr; |
9393 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9394 | cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9395 | return SDValue(E, 0); |
9396 | } |
9397 | auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, |
9398 | Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO); |
9399 | createOperands(Node: N, Vals: Ops); |
9400 | |
9401 | CSEMap.InsertNode(N, InsertPos: IP); |
9402 | InsertNode(N); |
9403 | SDValue V(N, 0); |
9404 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9405 | return V; |
9406 | } |
9407 | |
9408 | SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, |
9409 | SDValue Base, SDValue Offset, |
9410 | ISD::MemIndexedMode AM) { |
9411 | MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad); |
9412 | assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!" ); |
9413 | return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base, |
9414 | Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(), |
9415 | MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM, |
9416 | ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad()); |
9417 | } |
9418 | |
9419 | SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, |
9420 | SDValue Val, SDValue Base, SDValue Offset, |
9421 | SDValue Mask, EVT MemVT, |
9422 | MachineMemOperand *MMO, |
9423 | ISD::MemIndexedMode AM, bool IsTruncating, |
9424 | bool IsCompressing) { |
9425 | assert(Chain.getValueType() == MVT::Other && |
9426 | "Invalid chain type" ); |
9427 | bool Indexed = AM != ISD::UNINDEXED; |
9428 | assert((Indexed || Offset.isUndef()) && |
9429 | "Unindexed masked store with an offset!" ); |
9430 | SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) |
9431 | : getVTList(MVT::Other); |
9432 | SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; |
9433 | FoldingSetNodeID ID; |
9434 | AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops); |
9435 | ID.AddInteger(I: MemVT.getRawBits()); |
9436 | ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>( |
9437 | IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO)); |
9438 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9439 | ID.AddInteger(I: MMO->getFlags()); |
9440 | void *IP = nullptr; |
9441 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9442 | cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9443 | return SDValue(E, 0); |
9444 | } |
9445 | auto *N = |
9446 | newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, |
9447 | Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO); |
9448 | createOperands(Node: N, Vals: Ops); |
9449 | |
9450 | CSEMap.InsertNode(N, InsertPos: IP); |
9451 | InsertNode(N); |
9452 | SDValue V(N, 0); |
9453 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9454 | return V; |
9455 | } |
9456 | |
9457 | SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, |
9458 | SDValue Base, SDValue Offset, |
9459 | ISD::MemIndexedMode AM) { |
9460 | MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore); |
9461 | assert(ST->getOffset().isUndef() && |
9462 | "Masked store is already a indexed store!" ); |
9463 | return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset, |
9464 | Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), |
9465 | AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore()); |
9466 | } |
9467 | |
9468 | SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
9469 | ArrayRef<SDValue> Ops, |
9470 | MachineMemOperand *MMO, |
9471 | ISD::MemIndexType IndexType, |
9472 | ISD::LoadExtType ExtTy) { |
9473 | assert(Ops.size() == 6 && "Incompatible number of operands" ); |
9474 | |
9475 | FoldingSetNodeID ID; |
9476 | AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops); |
9477 | ID.AddInteger(I: MemVT.getRawBits()); |
9478 | ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>( |
9479 | IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy)); |
9480 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9481 | ID.AddInteger(I: MMO->getFlags()); |
9482 | void *IP = nullptr; |
9483 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9484 | cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9485 | return SDValue(E, 0); |
9486 | } |
9487 | |
9488 | auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
9489 | Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy); |
9490 | createOperands(Node: N, Vals: Ops); |
9491 | |
9492 | assert(N->getPassThru().getValueType() == N->getValueType(0) && |
9493 | "Incompatible type of the PassThru value in MaskedGatherSDNode" ); |
9494 | assert(N->getMask().getValueType().getVectorElementCount() == |
9495 | N->getValueType(0).getVectorElementCount() && |
9496 | "Vector width mismatch between mask and data" ); |
9497 | assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
9498 | N->getValueType(0).getVectorElementCount().isScalable() && |
9499 | "Scalable flags of index and data do not match" ); |
9500 | assert(ElementCount::isKnownGE( |
9501 | N->getIndex().getValueType().getVectorElementCount(), |
9502 | N->getValueType(0).getVectorElementCount()) && |
9503 | "Vector width mismatch between index and data" ); |
9504 | assert(isa<ConstantSDNode>(N->getScale()) && |
9505 | N->getScale()->getAsAPIntVal().isPowerOf2() && |
9506 | "Scale should be a constant power of 2" ); |
9507 | |
9508 | CSEMap.InsertNode(N, InsertPos: IP); |
9509 | InsertNode(N); |
9510 | SDValue V(N, 0); |
9511 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9512 | return V; |
9513 | } |
9514 | |
9515 | SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
9516 | ArrayRef<SDValue> Ops, |
9517 | MachineMemOperand *MMO, |
9518 | ISD::MemIndexType IndexType, |
9519 | bool IsTrunc) { |
9520 | assert(Ops.size() == 6 && "Incompatible number of operands" ); |
9521 | |
9522 | FoldingSetNodeID ID; |
9523 | AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops); |
9524 | ID.AddInteger(I: MemVT.getRawBits()); |
9525 | ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>( |
9526 | IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc)); |
9527 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9528 | ID.AddInteger(I: MMO->getFlags()); |
9529 | void *IP = nullptr; |
9530 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) { |
9531 | cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO); |
9532 | return SDValue(E, 0); |
9533 | } |
9534 | |
9535 | auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), |
9536 | Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc); |
9537 | createOperands(Node: N, Vals: Ops); |
9538 | |
9539 | assert(N->getMask().getValueType().getVectorElementCount() == |
9540 | N->getValue().getValueType().getVectorElementCount() && |
9541 | "Vector width mismatch between mask and data" ); |
9542 | assert( |
9543 | N->getIndex().getValueType().getVectorElementCount().isScalable() == |
9544 | N->getValue().getValueType().getVectorElementCount().isScalable() && |
9545 | "Scalable flags of index and data do not match" ); |
9546 | assert(ElementCount::isKnownGE( |
9547 | N->getIndex().getValueType().getVectorElementCount(), |
9548 | N->getValue().getValueType().getVectorElementCount()) && |
9549 | "Vector width mismatch between index and data" ); |
9550 | assert(isa<ConstantSDNode>(N->getScale()) && |
9551 | N->getScale()->getAsAPIntVal().isPowerOf2() && |
9552 | "Scale should be a constant power of 2" ); |
9553 | |
9554 | CSEMap.InsertNode(N, InsertPos: IP); |
9555 | InsertNode(N); |
9556 | SDValue V(N, 0); |
9557 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9558 | return V; |
9559 | } |
9560 | |
9561 | SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
9562 | EVT MemVT, MachineMemOperand *MMO) { |
9563 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9564 | SDVTList VTs = getVTList(MVT::Other); |
9565 | SDValue Ops[] = {Chain, Ptr}; |
9566 | FoldingSetNodeID ID; |
9567 | AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops); |
9568 | ID.AddInteger(I: MemVT.getRawBits()); |
9569 | ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>( |
9570 | Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO)); |
9571 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9572 | ID.AddInteger(I: MMO->getFlags()); |
9573 | void *IP = nullptr; |
9574 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
9575 | return SDValue(E, 0); |
9576 | |
9577 | auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(), |
9578 | Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO); |
9579 | createOperands(Node: N, Vals: Ops); |
9580 | |
9581 | CSEMap.InsertNode(N, InsertPos: IP); |
9582 | InsertNode(N); |
9583 | SDValue V(N, 0); |
9584 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9585 | return V; |
9586 | } |
9587 | |
9588 | SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
9589 | EVT MemVT, MachineMemOperand *MMO) { |
9590 | assert(Chain.getValueType() == MVT::Other && "Invalid chain type" ); |
9591 | SDVTList VTs = getVTList(MVT::Other); |
9592 | SDValue Ops[] = {Chain, Ptr}; |
9593 | FoldingSetNodeID ID; |
9594 | AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops); |
9595 | ID.AddInteger(I: MemVT.getRawBits()); |
9596 | ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>( |
9597 | Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO)); |
9598 | ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace()); |
9599 | ID.AddInteger(I: MMO->getFlags()); |
9600 | void *IP = nullptr; |
9601 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) |
9602 | return SDValue(E, 0); |
9603 | |
9604 | auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(), |
9605 | Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO); |
9606 | createOperands(Node: N, Vals: Ops); |
9607 | |
9608 | CSEMap.InsertNode(N, InsertPos: IP); |
9609 | InsertNode(N); |
9610 | SDValue V(N, 0); |
9611 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9612 | return V; |
9613 | } |
9614 | |
9615 | SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { |
9616 | // select undef, T, F --> T (if T is a constant), otherwise F |
9617 | // select, ?, undef, F --> F |
9618 | // select, ?, T, undef --> T |
9619 | if (Cond.isUndef()) |
9620 | return isConstantValueOfAnyType(N: T) ? T : F; |
9621 | if (T.isUndef()) |
9622 | return F; |
9623 | if (F.isUndef()) |
9624 | return T; |
9625 | |
9626 | // select true, T, F --> T |
9627 | // select false, T, F --> F |
9628 | if (auto *CondC = dyn_cast<ConstantSDNode>(Val&: Cond)) |
9629 | return CondC->isZero() ? F : T; |
9630 | |
9631 | // TODO: This should simplify VSELECT with non-zero constant condition using |
9632 | // something like this (but check boolean contents to be complete?): |
9633 | if (ConstantSDNode *CondC = isConstOrConstSplat(N: Cond, /*AllowUndefs*/ false, |
9634 | /*AllowTruncation*/ true)) |
9635 | if (CondC->isZero()) |
9636 | return F; |
9637 | |
9638 | // select ?, T, T --> T |
9639 | if (T == F) |
9640 | return T; |
9641 | |
9642 | return SDValue(); |
9643 | } |
9644 | |
9645 | SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { |
9646 | // shift undef, Y --> 0 (can always assume that the undef value is 0) |
9647 | if (X.isUndef()) |
9648 | return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType()); |
9649 | // shift X, undef --> undef (because it may shift by the bitwidth) |
9650 | if (Y.isUndef()) |
9651 | return getUNDEF(VT: X.getValueType()); |
9652 | |
9653 | // shift 0, Y --> 0 |
9654 | // shift X, 0 --> X |
9655 | if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y)) |
9656 | return X; |
9657 | |
9658 | // shift X, C >= bitwidth(X) --> undef |
9659 | // All vector elements must be too big (or undef) to avoid partial undefs. |
9660 | auto isShiftTooBig = [X](ConstantSDNode *Val) { |
9661 | return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits()); |
9662 | }; |
9663 | if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true)) |
9664 | return getUNDEF(VT: X.getValueType()); |
9665 | |
9666 | return SDValue(); |
9667 | } |
9668 | |
9669 | SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, |
9670 | SDNodeFlags Flags) { |
9671 | // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand |
9672 | // (an undef operand can be chosen to be Nan/Inf), then the result of this |
9673 | // operation is poison. That result can be relaxed to undef. |
9674 | ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true); |
9675 | ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true); |
9676 | bool HasNan = (XC && XC->getValueAPF().isNaN()) || |
9677 | (YC && YC->getValueAPF().isNaN()); |
9678 | bool HasInf = (XC && XC->getValueAPF().isInfinity()) || |
9679 | (YC && YC->getValueAPF().isInfinity()); |
9680 | |
9681 | if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef())) |
9682 | return getUNDEF(VT: X.getValueType()); |
9683 | |
9684 | if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef())) |
9685 | return getUNDEF(VT: X.getValueType()); |
9686 | |
9687 | if (!YC) |
9688 | return SDValue(); |
9689 | |
9690 | // X + -0.0 --> X |
9691 | if (Opcode == ISD::FADD) |
9692 | if (YC->getValueAPF().isNegZero()) |
9693 | return X; |
9694 | |
9695 | // X - +0.0 --> X |
9696 | if (Opcode == ISD::FSUB) |
9697 | if (YC->getValueAPF().isPosZero()) |
9698 | return X; |
9699 | |
9700 | // X * 1.0 --> X |
9701 | // X / 1.0 --> X |
9702 | if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) |
9703 | if (YC->getValueAPF().isExactlyValue(V: 1.0)) |
9704 | return X; |
9705 | |
9706 | // X * 0.0 --> 0.0 |
9707 | if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) |
9708 | if (YC->getValueAPF().isZero()) |
9709 | return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType()); |
9710 | |
9711 | return SDValue(); |
9712 | } |
9713 | |
9714 | SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, |
9715 | SDValue Ptr, SDValue SV, unsigned Align) { |
9716 | SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; |
9717 | return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); |
9718 | } |
9719 | |
9720 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
9721 | ArrayRef<SDUse> Ops) { |
9722 | switch (Ops.size()) { |
9723 | case 0: return getNode(Opcode, DL, VT); |
9724 | case 1: return getNode(Opcode, DL, VT, N1: static_cast<const SDValue>(Ops[0])); |
9725 | case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]); |
9726 | case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]); |
9727 | default: break; |
9728 | } |
9729 | |
9730 | // Copy from an SDUse array into an SDValue array for use with |
9731 | // the regular getNode logic. |
9732 | SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end()); |
9733 | return getNode(Opcode, DL, VT, Ops: NewOps); |
9734 | } |
9735 | |
9736 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
9737 | ArrayRef<SDValue> Ops) { |
9738 | SDNodeFlags Flags; |
9739 | if (Inserter) |
9740 | Flags = Inserter->getFlags(); |
9741 | return getNode(Opcode, DL, VT, Ops, Flags); |
9742 | } |
9743 | |
9744 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
9745 | ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { |
9746 | unsigned NumOps = Ops.size(); |
9747 | switch (NumOps) { |
9748 | case 0: return getNode(Opcode, DL, VT); |
9749 | case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags); |
9750 | case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags); |
9751 | case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags); |
9752 | default: break; |
9753 | } |
9754 | |
9755 | #ifndef NDEBUG |
9756 | for (const auto &Op : Ops) |
9757 | assert(Op.getOpcode() != ISD::DELETED_NODE && |
9758 | "Operand is DELETED_NODE!" ); |
9759 | #endif |
9760 | |
9761 | switch (Opcode) { |
9762 | default: break; |
9763 | case ISD::BUILD_VECTOR: |
9764 | // Attempt to simplify BUILD_VECTOR. |
9765 | if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this)) |
9766 | return V; |
9767 | break; |
9768 | case ISD::CONCAT_VECTORS: |
9769 | if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this)) |
9770 | return V; |
9771 | break; |
9772 | case ISD::SELECT_CC: |
9773 | assert(NumOps == 5 && "SELECT_CC takes 5 operands!" ); |
9774 | assert(Ops[0].getValueType() == Ops[1].getValueType() && |
9775 | "LHS and RHS of condition must have same type!" ); |
9776 | assert(Ops[2].getValueType() == Ops[3].getValueType() && |
9777 | "True and False arms of SelectCC must have same type!" ); |
9778 | assert(Ops[2].getValueType() == VT && |
9779 | "select_cc node must be of same type as true and false value!" ); |
9780 | assert((!Ops[0].getValueType().isVector() || |
9781 | Ops[0].getValueType().getVectorElementCount() == |
9782 | VT.getVectorElementCount()) && |
9783 | "Expected select_cc with vector result to have the same sized " |
9784 | "comparison type!" ); |
9785 | break; |
9786 | case ISD::BR_CC: |
9787 | assert(NumOps == 5 && "BR_CC takes 5 operands!" ); |
9788 | assert(Ops[2].getValueType() == Ops[3].getValueType() && |
9789 | "LHS/RHS of comparison should match types!" ); |
9790 | break; |
9791 | case ISD::VP_ADD: |
9792 | case ISD::VP_SUB: |
9793 | // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR |
9794 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
9795 | Opcode = ISD::VP_XOR; |
9796 | break; |
9797 | case ISD::VP_MUL: |
9798 | // If it is VP_MUL mask operation then turn it to VP_AND |
9799 | if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
9800 | Opcode = ISD::VP_AND; |
9801 | break; |
9802 | case ISD::VP_REDUCE_MUL: |
9803 | // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND |
9804 | if (VT == MVT::i1) |
9805 | Opcode = ISD::VP_REDUCE_AND; |
9806 | break; |
9807 | case ISD::VP_REDUCE_ADD: |
9808 | // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR |
9809 | if (VT == MVT::i1) |
9810 | Opcode = ISD::VP_REDUCE_XOR; |
9811 | break; |
9812 | case ISD::VP_REDUCE_SMAX: |
9813 | case ISD::VP_REDUCE_UMIN: |
9814 | // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to |
9815 | // VP_REDUCE_AND. |
9816 | if (VT == MVT::i1) |
9817 | Opcode = ISD::VP_REDUCE_AND; |
9818 | break; |
9819 | case ISD::VP_REDUCE_SMIN: |
9820 | case ISD::VP_REDUCE_UMAX: |
9821 | // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to |
9822 | // VP_REDUCE_OR. |
9823 | if (VT == MVT::i1) |
9824 | Opcode = ISD::VP_REDUCE_OR; |
9825 | break; |
9826 | } |
9827 | |
9828 | // Memoize nodes. |
9829 | SDNode *N; |
9830 | SDVTList VTs = getVTList(VT); |
9831 | |
9832 | if (VT != MVT::Glue) { |
9833 | FoldingSetNodeID ID; |
9834 | AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops); |
9835 | void *IP = nullptr; |
9836 | |
9837 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
9838 | return SDValue(E, 0); |
9839 | |
9840 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
9841 | createOperands(Node: N, Vals: Ops); |
9842 | |
9843 | CSEMap.InsertNode(N, InsertPos: IP); |
9844 | } else { |
9845 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
9846 | createOperands(Node: N, Vals: Ops); |
9847 | } |
9848 | |
9849 | N->setFlags(Flags); |
9850 | InsertNode(N); |
9851 | SDValue V(N, 0); |
9852 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
9853 | return V; |
9854 | } |
9855 | |
9856 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
9857 | ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { |
9858 | return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops); |
9859 | } |
9860 | |
9861 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
9862 | ArrayRef<SDValue> Ops) { |
9863 | SDNodeFlags Flags; |
9864 | if (Inserter) |
9865 | Flags = Inserter->getFlags(); |
9866 | return getNode(Opcode, DL, VTList, Ops, Flags); |
9867 | } |
9868 | |
9869 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
9870 | ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { |
9871 | if (VTList.NumVTs == 1) |
9872 | return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags); |
9873 | |
9874 | #ifndef NDEBUG |
9875 | for (const auto &Op : Ops) |
9876 | assert(Op.getOpcode() != ISD::DELETED_NODE && |
9877 | "Operand is DELETED_NODE!" ); |
9878 | #endif |
9879 | |
9880 | switch (Opcode) { |
9881 | case ISD::SADDO: |
9882 | case ISD::UADDO: |
9883 | case ISD::SSUBO: |
9884 | case ISD::USUBO: { |
9885 | assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
9886 | "Invalid add/sub overflow op!" ); |
9887 | assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() && |
9888 | Ops[0].getValueType() == Ops[1].getValueType() && |
9889 | Ops[0].getValueType() == VTList.VTs[0] && |
9890 | "Binary operator types must match!" ); |
9891 | SDValue N1 = Ops[0], N2 = Ops[1]; |
9892 | canonicalizeCommutativeBinop(Opcode, N1, N2); |
9893 | |
9894 | // (X +- 0) -> X with zero-overflow. |
9895 | ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, |
9896 | /*AllowTruncation*/ true); |
9897 | if (N2CV && N2CV->isZero()) { |
9898 | SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]); |
9899 | return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags); |
9900 | } |
9901 | |
9902 | if (VTList.VTs[0].isVector() && |
9903 | VTList.VTs[0].getVectorElementType() == MVT::i1 && |
9904 | VTList.VTs[1].getVectorElementType() == MVT::i1) { |
9905 | SDValue F1 = getFreeze(V: N1); |
9906 | SDValue F2 = getFreeze(V: N2); |
9907 | // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)} |
9908 | if (Opcode == ISD::UADDO || Opcode == ISD::SADDO) |
9909 | return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, |
9910 | Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2), |
9911 | getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)}, |
9912 | Flags); |
9913 | // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)} |
9914 | if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) { |
9915 | SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]); |
9916 | return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, |
9917 | Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2), |
9918 | getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)}, |
9919 | Flags); |
9920 | } |
9921 | } |
9922 | break; |
9923 | } |
9924 | case ISD::SMUL_LOHI: |
9925 | case ISD::UMUL_LOHI: { |
9926 | assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!" ); |
9927 | assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] && |
9928 | VTList.VTs[0] == Ops[0].getValueType() && |
9929 | VTList.VTs[0] == Ops[1].getValueType() && |
9930 | "Binary operator types must match!" ); |
9931 | // Constant fold. |
9932 | ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]); |
9933 | ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]); |
9934 | if (LHS && RHS) { |
9935 | unsigned Width = VTList.VTs[0].getScalarSizeInBits(); |
9936 | unsigned OutWidth = Width * 2; |
9937 | APInt Val = LHS->getAPIntValue(); |
9938 | APInt Mul = RHS->getAPIntValue(); |
9939 | if (Opcode == ISD::SMUL_LOHI) { |
9940 | Val = Val.sext(width: OutWidth); |
9941 | Mul = Mul.sext(width: OutWidth); |
9942 | } else { |
9943 | Val = Val.zext(width: OutWidth); |
9944 | Mul = Mul.zext(width: OutWidth); |
9945 | } |
9946 | Val *= Mul; |
9947 | |
9948 | SDValue Hi = |
9949 | getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]); |
9950 | SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]); |
9951 | return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags); |
9952 | } |
9953 | break; |
9954 | } |
9955 | case ISD::FFREXP: { |
9956 | assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!" ); |
9957 | assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() && |
9958 | VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch" ); |
9959 | |
9960 | if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) { |
9961 | int FrexpExp; |
9962 | APFloat FrexpMant = |
9963 | frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven); |
9964 | SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]); |
9965 | SDValue Result1 = |
9966 | getConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0, DL, VT: VTList.VTs[1]); |
9967 | return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags); |
9968 | } |
9969 | |
9970 | break; |
9971 | } |
9972 | case ISD::STRICT_FP_EXTEND: |
9973 | assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
9974 | "Invalid STRICT_FP_EXTEND!" ); |
9975 | assert(VTList.VTs[0].isFloatingPoint() && |
9976 | Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!" ); |
9977 | assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
9978 | "STRICT_FP_EXTEND result type should be vector iff the operand " |
9979 | "type is vector!" ); |
9980 | assert((!VTList.VTs[0].isVector() || |
9981 | VTList.VTs[0].getVectorElementCount() == |
9982 | Ops[1].getValueType().getVectorElementCount()) && |
9983 | "Vector element count mismatch!" ); |
9984 | assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && |
9985 | "Invalid fpext node, dst <= src!" ); |
9986 | break; |
9987 | case ISD::STRICT_FP_ROUND: |
9988 | assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!" ); |
9989 | assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
9990 | "STRICT_FP_ROUND result type should be vector iff the operand " |
9991 | "type is vector!" ); |
9992 | assert((!VTList.VTs[0].isVector() || |
9993 | VTList.VTs[0].getVectorElementCount() == |
9994 | Ops[1].getValueType().getVectorElementCount()) && |
9995 | "Vector element count mismatch!" ); |
9996 | assert(VTList.VTs[0].isFloatingPoint() && |
9997 | Ops[1].getValueType().isFloatingPoint() && |
9998 | VTList.VTs[0].bitsLT(Ops[1].getValueType()) && |
9999 | isa<ConstantSDNode>(Ops[2]) && |
10000 | (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) && |
10001 | "Invalid STRICT_FP_ROUND!" ); |
10002 | break; |
10003 | #if 0 |
10004 | // FIXME: figure out how to safely handle things like |
10005 | // int foo(int x) { return 1 << (x & 255); } |
10006 | // int bar() { return foo(256); } |
10007 | case ISD::SRA_PARTS: |
10008 | case ISD::SRL_PARTS: |
10009 | case ISD::SHL_PARTS: |
10010 | if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && |
10011 | cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) |
10012 | return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
10013 | else if (N3.getOpcode() == ISD::AND) |
10014 | if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { |
10015 | // If the and is only masking out bits that cannot effect the shift, |
10016 | // eliminate the and. |
10017 | unsigned NumBits = VT.getScalarSizeInBits()*2; |
10018 | if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) |
10019 | return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
10020 | } |
10021 | break; |
10022 | #endif |
10023 | } |
10024 | |
10025 | // Memoize the node unless it returns a glue result. |
10026 | SDNode *N; |
10027 | if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
10028 | FoldingSetNodeID ID; |
10029 | AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops); |
10030 | void *IP = nullptr; |
10031 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) |
10032 | return SDValue(E, 0); |
10033 | |
10034 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList); |
10035 | createOperands(Node: N, Vals: Ops); |
10036 | CSEMap.InsertNode(N, InsertPos: IP); |
10037 | } else { |
10038 | N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList); |
10039 | createOperands(Node: N, Vals: Ops); |
10040 | } |
10041 | |
10042 | N->setFlags(Flags); |
10043 | InsertNode(N); |
10044 | SDValue V(N, 0); |
10045 | NewSDValueDbgMsg(V, Msg: "Creating new node: " , G: this); |
10046 | return V; |
10047 | } |
10048 | |
10049 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
10050 | SDVTList VTList) { |
10051 | return getNode(Opcode, DL, VTList, Ops: std::nullopt); |
10052 | } |
10053 | |
10054 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
10055 | SDValue N1) { |
10056 | SDValue Ops[] = { N1 }; |
10057 | return getNode(Opcode, DL, VTList, Ops); |
10058 | } |
10059 | |
10060 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
10061 | SDValue N1, SDValue N2) { |
10062 | SDValue Ops[] = { N1, N2 }; |
10063 | return getNode(Opcode, DL, VTList, Ops); |
10064 | } |
10065 | |
10066 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
10067 | SDValue N1, SDValue N2, SDValue N3) { |
10068 | SDValue Ops[] = { N1, N2, N3 }; |
10069 | return getNode(Opcode, DL, VTList, Ops); |
10070 | } |
10071 | |
10072 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
10073 | SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
10074 | SDValue Ops[] = { N1, N2, N3, N4 }; |
10075 | return getNode(Opcode, DL, VTList, Ops); |
10076 | } |
10077 | |
10078 | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
10079 | SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
10080 | SDValue N5) { |
10081 | SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
10082 | return getNode(Opcode, DL, VTList, Ops); |
10083 | } |
10084 | |
10085 | SDVTList SelectionDAG::getVTList(EVT VT) { |
10086 | return makeVTList(VTs: SDNode::getValueTypeList(VT), NumVTs: 1); |
10087 | } |
10088 | |
10089 | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { |
10090 | FoldingSetNodeID ID; |
10091 | ID.AddInteger(I: 2U); |
10092 | ID.AddInteger(I: VT1.getRawBits()); |
10093 | ID.AddInteger(I: VT2.getRawBits()); |
10094 | |
10095 | void *IP = nullptr; |
10096 | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP); |
10097 | if (!Result) { |
10098 | EVT *Array = Allocator.Allocate<EVT>(Num: 2); |
10099 | Array[0] = VT1; |
10100 | Array[1] = VT2; |
10101 | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); |
10102 | VTListMap.InsertNode(N: Result, InsertPos: IP); |
10103 | } |
10104 | return Result->getSDVTList(); |
10105 | } |
10106 | |
10107 | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { |
10108 | FoldingSetNodeID ID; |
10109 | ID.AddInteger(I: 3U); |
10110 | ID.AddInteger(I: VT1.getRawBits()); |
10111 | ID.AddInteger(I: VT2.getRawBits()); |
10112 | ID.AddInteger(I: VT3.getRawBits()); |
10113 | |
10114 | void *IP = nullptr; |
10115 | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP); |
10116 | if (!Result) { |
10117 | EVT *Array = Allocator.Allocate<EVT>(Num: 3); |
10118 | Array[0] = VT1; |
10119 | Array[1] = VT2; |
10120 | Array[2] = VT3; |
10121 | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); |
10122 | VTListMap.InsertNode(N: Result, InsertPos: IP); |
10123 | } |
10124 | return Result->getSDVTList(); |
10125 | } |
10126 | |
10127 | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { |
10128 | FoldingSetNodeID ID; |
10129 | ID.AddInteger(I: 4U); |
10130 | ID.AddInteger(I: VT1.getRawBits()); |
10131 | ID.AddInteger(I: VT2.getRawBits()); |
10132 | ID.AddInteger(I: VT3.getRawBits()); |
10133 | ID.AddInteger(I: VT4.getRawBits()); |
10134 | |
10135 | void *IP = nullptr; |
10136 | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP); |
10137 | if (!Result) { |
10138 | EVT *Array = Allocator.Allocate<EVT>(Num: 4); |
10139 | Array[0] = VT1; |
10140 | Array[1] = VT2; |
10141 | Array[2] = VT3; |
10142 | Array[3] = VT4; |
10143 | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); |
10144 | VTListMap.InsertNode(N: Result, InsertPos: IP); |
10145 | } |
10146 | return Result->getSDVTList(); |
10147 | } |
10148 | |
10149 | SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { |
10150 | unsigned NumVTs = VTs.size(); |
10151 | FoldingSetNodeID ID; |
10152 | ID.AddInteger(I: NumVTs); |
10153 | for (unsigned index = 0; index < NumVTs; index++) { |
10154 | ID.AddInteger(I: VTs[index].getRawBits()); |
10155 | } |
10156 | |
10157 | void *IP = nullptr; |
10158 | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP); |
10159 | if (!Result) { |
10160 | EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs); |
10161 | llvm::copy(Range&: VTs, Out: Array); |
10162 | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); |
10163 | VTListMap.InsertNode(N: Result, InsertPos: IP); |
10164 | } |
10165 | return Result->getSDVTList(); |
10166 | } |
10167 | |
10168 | |
10169 | /// UpdateNodeOperands - *Mutate* the specified node in-place to have the |
10170 | /// specified operands. If the resultant node already exists in the DAG, |
10171 | /// this does not modify the specified node, instead it returns the node that |
10172 | /// already exists. If the resultant node does not exist in the DAG, the |
10173 | /// input node is returned. As a degenerate case, if you specify the same |
10174 | /// input operands as the node already has, the input node is returned. |
10175 | SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { |
10176 | assert(N->getNumOperands() == 1 && "Update with wrong number of operands" ); |
10177 | |
10178 | // Check to see if there is no change. |
10179 | if (Op == N->getOperand(Num: 0)) return N; |
10180 | |
10181 | // See if the modified node already exists. |
10182 | void *InsertPos = nullptr; |
10183 | if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) |
10184 | return Existing; |
10185 | |
10186 | // Nope it doesn't. Remove the node from its current place in the maps. |
10187 | if (InsertPos) |
10188 | if (!RemoveNodeFromCSEMaps(N)) |
10189 | InsertPos = nullptr; |
10190 | |
10191 | // Now we update the operands. |
10192 | N->OperandList[0].set(Op); |
10193 | |
10194 | updateDivergence(N); |
10195 | // If this gets put into a CSE map, add it. |
10196 | if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
10197 | return N; |
10198 | } |
10199 | |
10200 | SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { |
10201 | assert(N->getNumOperands() == 2 && "Update with wrong number of operands" ); |
10202 | |
10203 | // Check to see if there is no change. |
10204 | if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1)) |
10205 | return N; // No operands changed, just return the input node. |
10206 | |
10207 | // See if the modified node already exists. |
10208 | void *InsertPos = nullptr; |
10209 | if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) |
10210 | return Existing; |
10211 | |
10212 | // Nope it doesn't. Remove the node from its current place in the maps. |
10213 | if (InsertPos) |
10214 | if (!RemoveNodeFromCSEMaps(N)) |
10215 | InsertPos = nullptr; |
10216 | |
10217 | // Now we update the operands. |
10218 | if (N->OperandList[0] != Op1) |
10219 | N->OperandList[0].set(Op1); |
10220 | if (N->OperandList[1] != Op2) |
10221 | N->OperandList[1].set(Op2); |
10222 | |
10223 | updateDivergence(N); |
10224 | // If this gets put into a CSE map, add it. |
10225 | if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
10226 | return N; |
10227 | } |
10228 | |
10229 | SDNode *SelectionDAG:: |
10230 | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { |
10231 | SDValue Ops[] = { Op1, Op2, Op3 }; |
10232 | return UpdateNodeOperands(N, Ops); |
10233 | } |
10234 | |
10235 | SDNode *SelectionDAG:: |
10236 | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
10237 | SDValue Op3, SDValue Op4) { |
10238 | SDValue Ops[] = { Op1, Op2, Op3, Op4 }; |
10239 | return UpdateNodeOperands(N, Ops); |
10240 | } |
10241 | |
10242 | SDNode *SelectionDAG:: |
10243 | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
10244 | SDValue Op3, SDValue Op4, SDValue Op5) { |
10245 | SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; |
10246 | return UpdateNodeOperands(N, Ops); |
10247 | } |
10248 | |
10249 | SDNode *SelectionDAG:: |
10250 | UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { |
10251 | unsigned NumOps = Ops.size(); |
10252 | assert(N->getNumOperands() == NumOps && |
10253 | "Update with wrong number of operands" ); |
10254 | |
10255 | // If no operands changed just return the input node. |
10256 | if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin())) |
10257 | return N; |
10258 | |
10259 | // See if the modified node already exists. |
10260 | void *InsertPos = nullptr; |
10261 | if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) |
10262 | return Existing; |
10263 | |
10264 | // Nope it doesn't. Remove the node from its current place in the maps. |
10265 | if (InsertPos) |
10266 | if (!RemoveNodeFromCSEMaps(N)) |
10267 | InsertPos = nullptr; |
10268 | |
10269 | // Now we update the operands. |
10270 | for (unsigned i = 0; i != NumOps; ++i) |
10271 | if (N->OperandList[i] != Ops[i]) |
10272 | N->OperandList[i].set(Ops[i]); |
10273 | |
10274 | updateDivergence(N); |
10275 | // If this gets put into a CSE map, add it. |
10276 | if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
10277 | return N; |
10278 | } |
10279 | |
10280 | /// DropOperands - Release the operands and set this node to have |
10281 | /// zero operands. |
10282 | void SDNode::DropOperands() { |
10283 | // Unlike the code in MorphNodeTo that does this, we don't need to |
10284 | // watch for dead nodes here. |
10285 | for (op_iterator I = op_begin(), E = op_end(); I != E; ) { |
10286 | SDUse &Use = *I++; |
10287 | Use.set(SDValue()); |
10288 | } |
10289 | } |
10290 | |
10291 | void SelectionDAG::setNodeMemRefs(MachineSDNode *N, |
10292 | ArrayRef<MachineMemOperand *> NewMemRefs) { |
10293 | if (NewMemRefs.empty()) { |
10294 | N->clearMemRefs(); |
10295 | return; |
10296 | } |
10297 | |
10298 | // Check if we can avoid allocating by storing a single reference directly. |
10299 | if (NewMemRefs.size() == 1) { |
10300 | N->MemRefs = NewMemRefs[0]; |
10301 | N->NumMemRefs = 1; |
10302 | return; |
10303 | } |
10304 | |
10305 | MachineMemOperand **MemRefsBuffer = |
10306 | Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size()); |
10307 | llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer); |
10308 | N->MemRefs = MemRefsBuffer; |
10309 | N->NumMemRefs = static_cast<int>(NewMemRefs.size()); |
10310 | } |
10311 | |
10312 | /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a |
10313 | /// machine opcode. |
10314 | /// |
10315 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10316 | EVT VT) { |
10317 | SDVTList VTs = getVTList(VT); |
10318 | return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt); |
10319 | } |
10320 | |
10321 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10322 | EVT VT, SDValue Op1) { |
10323 | SDVTList VTs = getVTList(VT); |
10324 | SDValue Ops[] = { Op1 }; |
10325 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10326 | } |
10327 | |
10328 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10329 | EVT VT, SDValue Op1, |
10330 | SDValue Op2) { |
10331 | SDVTList VTs = getVTList(VT); |
10332 | SDValue Ops[] = { Op1, Op2 }; |
10333 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10334 | } |
10335 | |
10336 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10337 | EVT VT, SDValue Op1, |
10338 | SDValue Op2, SDValue Op3) { |
10339 | SDVTList VTs = getVTList(VT); |
10340 | SDValue Ops[] = { Op1, Op2, Op3 }; |
10341 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10342 | } |
10343 | |
10344 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10345 | EVT VT, ArrayRef<SDValue> Ops) { |
10346 | SDVTList VTs = getVTList(VT); |
10347 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10348 | } |
10349 | |
10350 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10351 | EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { |
10352 | SDVTList VTs = getVTList(VT1, VT2); |
10353 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10354 | } |
10355 | |
10356 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10357 | EVT VT1, EVT VT2) { |
10358 | SDVTList VTs = getVTList(VT1, VT2); |
10359 | return SelectNodeTo(N, MachineOpc, VTs, Ops: std::nullopt); |
10360 | } |
10361 | |
10362 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10363 | EVT VT1, EVT VT2, EVT VT3, |
10364 | ArrayRef<SDValue> Ops) { |
10365 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
10366 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10367 | } |
10368 | |
10369 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10370 | EVT VT1, EVT VT2, |
10371 | SDValue Op1, SDValue Op2) { |
10372 | SDVTList VTs = getVTList(VT1, VT2); |
10373 | SDValue Ops[] = { Op1, Op2 }; |
10374 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
10375 | } |
10376 | |
10377 | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
10378 | SDVTList VTs,ArrayRef<SDValue> Ops) { |
10379 | SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops); |
10380 | // Reset the NodeID to -1. |
10381 | New->setNodeId(-1); |
10382 | if (New != N) { |
10383 | ReplaceAllUsesWith(From: N, To: New); |
10384 | RemoveDeadNode(N); |
10385 | } |
10386 | return New; |
10387 | } |
10388 | |
10389 | /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away |
10390 | /// the line number information on the merged node since it is not possible to |
10391 | /// preserve the information that operation is associated with multiple lines. |
10392 | /// This will make the debugger working better at -O0, were there is a higher |
10393 | /// probability having other instructions associated with that line. |
10394 | /// |
10395 | /// For IROrder, we keep the smaller of the two |
10396 | SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { |
10397 | DebugLoc NLoc = N->getDebugLoc(); |
10398 | if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) { |
10399 | N->setDebugLoc(DebugLoc()); |
10400 | } |
10401 | unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder()); |
10402 | N->setIROrder(Order); |
10403 | return N; |
10404 | } |
10405 | |
10406 | /// MorphNodeTo - This *mutates* the specified node to have the specified |
10407 | /// return type, opcode, and operands. |
10408 | /// |
10409 | /// Note that MorphNodeTo returns the resultant node. If there is already a |
10410 | /// node of the specified opcode and operands, it returns that node instead of |
10411 | /// the current one. Note that the SDLoc need not be the same. |
10412 | /// |
10413 | /// Using MorphNodeTo is faster than creating a new node and swapping it in |
10414 | /// with ReplaceAllUsesWith both because it often avoids allocating a new |
10415 | /// node, and because it doesn't require CSE recalculation for any of |
10416 | /// the node's users. |
10417 | /// |
10418 | /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. |
10419 | /// As a consequence it isn't appropriate to use from within the DAG combiner or |
10420 | /// the legalizer which maintain worklists that would need to be updated when |
10421 | /// deleting things. |
10422 | SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, |
10423 | SDVTList VTs, ArrayRef<SDValue> Ops) { |
10424 | // If an identical node already exists, use it. |
10425 | void *IP = nullptr; |
10426 | if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { |
10427 | FoldingSetNodeID ID; |
10428 | AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops); |
10429 | if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP)) |
10430 | return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N)); |
10431 | } |
10432 | |
10433 | if (!RemoveNodeFromCSEMaps(N)) |
10434 | IP = nullptr; |
10435 | |
10436 | // Start the morphing. |
10437 | N->NodeType = Opc; |
10438 | N->ValueList = VTs.VTs; |
10439 | N->NumValues = VTs.NumVTs; |
10440 | |
10441 | // Clear the operands list, updating used nodes to remove this from their |
10442 | // use list. Keep track of any operands that become dead as a result. |
10443 | SmallPtrSet<SDNode*, 16> DeadNodeSet; |
10444 | for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
10445 | SDUse &Use = *I++; |
10446 | SDNode *Used = Use.getNode(); |
10447 | Use.set(SDValue()); |
10448 | if (Used->use_empty()) |
10449 | DeadNodeSet.insert(Ptr: Used); |
10450 | } |
10451 | |
10452 | // For MachineNode, initialize the memory references information. |
10453 | if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N)) |
10454 | MN->clearMemRefs(); |
10455 | |
10456 | // Swap for an appropriately sized array from the recycler. |
10457 | removeOperands(Node: N); |
10458 | createOperands(Node: N, Vals: Ops); |
10459 | |
10460 | // Delete any nodes that are still dead after adding the uses for the |
10461 | // new operands. |
10462 | if (!DeadNodeSet.empty()) { |
10463 | SmallVector<SDNode *, 16> DeadNodes; |
10464 | for (SDNode *N : DeadNodeSet) |
10465 | if (N->use_empty()) |
10466 | DeadNodes.push_back(Elt: N); |
10467 | RemoveDeadNodes(DeadNodes); |
10468 | } |
10469 | |
10470 | if (IP) |
10471 | CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node. |
10472 | return N; |
10473 | } |
10474 | |
10475 | SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { |
10476 | unsigned OrigOpc = Node->getOpcode(); |
10477 | unsigned NewOpc; |
10478 | switch (OrigOpc) { |
10479 | default: |
10480 | llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!" ); |
10481 | #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
10482 | case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; |
10483 | #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
10484 | case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; |
10485 | #include "llvm/IR/ConstrainedOps.def" |
10486 | } |
10487 | |
10488 | assert(Node->getNumValues() == 2 && "Unexpected number of results!" ); |
10489 | |
10490 | // We're taking this node out of the chain, so we need to re-link things. |
10491 | SDValue InputChain = Node->getOperand(Num: 0); |
10492 | SDValue OutputChain = SDValue(Node, 1); |
10493 | ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain); |
10494 | |
10495 | SmallVector<SDValue, 3> Ops; |
10496 | for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) |
10497 | Ops.push_back(Elt: Node->getOperand(Num: i)); |
10498 | |
10499 | SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0)); |
10500 | SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops); |
10501 | |
10502 | // MorphNodeTo can operate in two ways: if an existing node with the |
10503 | // specified operands exists, it can just return it. Otherwise, it |
10504 | // updates the node in place to have the requested operands. |
10505 | if (Res == Node) { |
10506 | // If we updated the node in place, reset the node ID. To the isel, |
10507 | // this should be just like a newly allocated machine node. |
10508 | Res->setNodeId(-1); |
10509 | } else { |
10510 | ReplaceAllUsesWith(From: Node, To: Res); |
10511 | RemoveDeadNode(N: Node); |
10512 | } |
10513 | |
10514 | return Res; |
10515 | } |
10516 | |
10517 | /// getMachineNode - These are used for target selectors to create a new node |
10518 | /// with specified return type(s), MachineInstr opcode, and operands. |
10519 | /// |
10520 | /// Note that getMachineNode returns the resultant node. If there is already a |
10521 | /// node of the specified opcode and operands, it returns that node instead of |
10522 | /// the current one. |
10523 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10524 | EVT VT) { |
10525 | SDVTList VTs = getVTList(VT); |
10526 | return getMachineNode(Opcode, dl, VTs, Ops: std::nullopt); |
10527 | } |
10528 | |
10529 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10530 | EVT VT, SDValue Op1) { |
10531 | SDVTList VTs = getVTList(VT); |
10532 | SDValue Ops[] = { Op1 }; |
10533 | return getMachineNode(Opcode, dl, VTs, Ops); |
10534 | } |
10535 | |
10536 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10537 | EVT VT, SDValue Op1, SDValue Op2) { |
10538 | SDVTList VTs = getVTList(VT); |
10539 | SDValue Ops[] = { Op1, Op2 }; |
10540 | return getMachineNode(Opcode, dl, VTs, Ops); |
10541 | } |
10542 | |
10543 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10544 | EVT VT, SDValue Op1, SDValue Op2, |
10545 | SDValue Op3) { |
10546 | SDVTList VTs = getVTList(VT); |
10547 | SDValue Ops[] = { Op1, Op2, Op3 }; |
10548 | return getMachineNode(Opcode, dl, VTs, Ops); |
10549 | } |
10550 | |
10551 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10552 | EVT VT, ArrayRef<SDValue> Ops) { |
10553 | SDVTList VTs = getVTList(VT); |
10554 | return getMachineNode(Opcode, dl, VTs, Ops); |
10555 | } |
10556 | |
10557 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10558 | EVT VT1, EVT VT2, SDValue Op1, |
10559 | SDValue Op2) { |
10560 | SDVTList VTs = getVTList(VT1, VT2); |
10561 | SDValue Ops[] = { Op1, Op2 }; |
10562 | return getMachineNode(Opcode, dl, VTs, Ops); |
10563 | } |
10564 | |
10565 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10566 | EVT VT1, EVT VT2, SDValue Op1, |
10567 | SDValue Op2, SDValue Op3) { |
10568 | SDVTList VTs = getVTList(VT1, VT2); |
10569 | SDValue Ops[] = { Op1, Op2, Op3 }; |
10570 | return getMachineNode(Opcode, dl, VTs, Ops); |
10571 | } |
10572 | |
10573 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10574 | EVT VT1, EVT VT2, |
10575 | ArrayRef<SDValue> Ops) { |
10576 | SDVTList VTs = getVTList(VT1, VT2); |
10577 | return getMachineNode(Opcode, dl, VTs, Ops); |
10578 | } |
10579 | |
10580 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10581 | EVT VT1, EVT VT2, EVT VT3, |
10582 | SDValue Op1, SDValue Op2) { |
10583 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
10584 | SDValue Ops[] = { Op1, Op2 }; |
10585 | return getMachineNode(Opcode, dl, VTs, Ops); |
10586 | } |
10587 | |
10588 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10589 | EVT VT1, EVT VT2, EVT VT3, |
10590 | SDValue Op1, SDValue Op2, |
10591 | SDValue Op3) { |
10592 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
10593 | SDValue Ops[] = { Op1, Op2, Op3 }; |
10594 | return getMachineNode(Opcode, dl, VTs, Ops); |
10595 | } |
10596 | |
10597 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10598 | EVT VT1, EVT VT2, EVT VT3, |
10599 | ArrayRef<SDValue> Ops) { |
10600 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
10601 | return getMachineNode(Opcode, dl, VTs, Ops); |
10602 | } |
10603 | |
10604 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
10605 | ArrayRef<EVT> ResultTys, |
10606 | ArrayRef<SDValue> Ops) { |
10607 | SDVTList VTs = getVTList(VTs: ResultTys); |
10608 | return getMachineNode(Opcode, dl, VTs, Ops); |
10609 | } |
10610 | |
10611 | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, |
10612 | SDVTList VTs, |
10613 | ArrayRef<SDValue> Ops) { |
10614 | bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; |
10615 | MachineSDNode *N; |
10616 | void *IP = nullptr; |
10617 | |
10618 | if (DoCSE) { |
10619 | FoldingSetNodeID ID; |
10620 | AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops); |
10621 | IP = nullptr; |
10622 | if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) { |
10623 | return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL)); |
10624 | } |
10625 | } |
10626 | |
10627 | // Allocate a new MachineSDNode. |
10628 | N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs); |
10629 | createOperands(Node: N, Vals: Ops); |
10630 | |
10631 | if (DoCSE) |
10632 | CSEMap.InsertNode(N, InsertPos: IP); |
10633 | |
10634 | InsertNode(N); |
10635 | NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: " , G: this); |
10636 | return N; |
10637 | } |
10638 | |
10639 | /// getTargetExtractSubreg - A convenience function for creating |
10640 | /// TargetOpcode::EXTRACT_SUBREG nodes. |
10641 | SDValue SelectionDAG::(int SRIdx, const SDLoc &DL, EVT VT, |
10642 | SDValue Operand) { |
10643 | SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
10644 | SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, |
10645 | VT, Op1: Operand, Op2: SRIdxVal); |
10646 | return SDValue(Subreg, 0); |
10647 | } |
10648 | |
10649 | /// getTargetInsertSubreg - A convenience function for creating |
10650 | /// TargetOpcode::INSERT_SUBREG nodes. |
10651 | SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
10652 | SDValue Operand, SDValue Subreg) { |
10653 | SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
10654 | SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL, |
10655 | VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal); |
10656 | return SDValue(Result, 0); |
10657 | } |
10658 | |
10659 | /// getNodeIfExists - Get the specified node if it's already available, or |
10660 | /// else return NULL. |
10661 | SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
10662 | ArrayRef<SDValue> Ops) { |
10663 | SDNodeFlags Flags; |
10664 | if (Inserter) |
10665 | Flags = Inserter->getFlags(); |
10666 | return getNodeIfExists(Opcode, VTList, Ops, Flags); |
10667 | } |
10668 | |
10669 | SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
10670 | ArrayRef<SDValue> Ops, |
10671 | const SDNodeFlags Flags) { |
10672 | if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
10673 | FoldingSetNodeID ID; |
10674 | AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops); |
10675 | void *IP = nullptr; |
10676 | if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP)) { |
10677 | E->intersectFlagsWith(Flags); |
10678 | return E; |
10679 | } |
10680 | } |
10681 | return nullptr; |
10682 | } |
10683 | |
10684 | /// doesNodeExist - Check if a node exists without modifying its flags. |
10685 | bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, |
10686 | ArrayRef<SDValue> Ops) { |
10687 | if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
10688 | FoldingSetNodeID ID; |
10689 | AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops); |
10690 | void *IP = nullptr; |
10691 | if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP)) |
10692 | return true; |
10693 | } |
10694 | return false; |
10695 | } |
10696 | |
10697 | /// getDbgValue - Creates a SDDbgValue node. |
10698 | /// |
10699 | /// SDNode |
10700 | SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, |
10701 | SDNode *N, unsigned R, bool IsIndirect, |
10702 | const DebugLoc &DL, unsigned O) { |
10703 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10704 | "Expected inlined-at fields to agree" ); |
10705 | return new (DbgInfo->getAlloc()) |
10706 | SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R), |
10707 | {}, IsIndirect, DL, O, |
10708 | /*IsVariadic=*/false); |
10709 | } |
10710 | |
10711 | /// Constant |
10712 | SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, |
10713 | DIExpression *Expr, |
10714 | const Value *C, |
10715 | const DebugLoc &DL, unsigned O) { |
10716 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10717 | "Expected inlined-at fields to agree" ); |
10718 | return new (DbgInfo->getAlloc()) |
10719 | SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {}, |
10720 | /*IsIndirect=*/false, DL, O, |
10721 | /*IsVariadic=*/false); |
10722 | } |
10723 | |
10724 | /// FrameIndex |
10725 | SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
10726 | DIExpression *Expr, unsigned FI, |
10727 | bool IsIndirect, |
10728 | const DebugLoc &DL, |
10729 | unsigned O) { |
10730 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10731 | "Expected inlined-at fields to agree" ); |
10732 | return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O); |
10733 | } |
10734 | |
10735 | /// FrameIndex with dependencies |
10736 | SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
10737 | DIExpression *Expr, unsigned FI, |
10738 | ArrayRef<SDNode *> Dependencies, |
10739 | bool IsIndirect, |
10740 | const DebugLoc &DL, |
10741 | unsigned O) { |
10742 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10743 | "Expected inlined-at fields to agree" ); |
10744 | return new (DbgInfo->getAlloc()) |
10745 | SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI), |
10746 | Dependencies, IsIndirect, DL, O, |
10747 | /*IsVariadic=*/false); |
10748 | } |
10749 | |
10750 | /// VReg |
10751 | SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr, |
10752 | unsigned VReg, bool IsIndirect, |
10753 | const DebugLoc &DL, unsigned O) { |
10754 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10755 | "Expected inlined-at fields to agree" ); |
10756 | return new (DbgInfo->getAlloc()) |
10757 | SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg), |
10758 | {}, IsIndirect, DL, O, |
10759 | /*IsVariadic=*/false); |
10760 | } |
10761 | |
10762 | SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr, |
10763 | ArrayRef<SDDbgOperand> Locs, |
10764 | ArrayRef<SDNode *> Dependencies, |
10765 | bool IsIndirect, const DebugLoc &DL, |
10766 | unsigned O, bool IsVariadic) { |
10767 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
10768 | "Expected inlined-at fields to agree" ); |
10769 | return new (DbgInfo->getAlloc()) |
10770 | SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect, |
10771 | DL, O, IsVariadic); |
10772 | } |
10773 | |
10774 | void SelectionDAG::transferDbgValues(SDValue From, SDValue To, |
10775 | unsigned OffsetInBits, unsigned SizeInBits, |
10776 | bool InvalidateDbg) { |
10777 | SDNode *FromNode = From.getNode(); |
10778 | SDNode *ToNode = To.getNode(); |
10779 | assert(FromNode && ToNode && "Can't modify dbg values" ); |
10780 | |
10781 | // PR35338 |
10782 | // TODO: assert(From != To && "Redundant dbg value transfer"); |
10783 | // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); |
10784 | if (From == To || FromNode == ToNode) |
10785 | return; |
10786 | |
10787 | if (!FromNode->getHasDebugValue()) |
10788 | return; |
10789 | |
10790 | SDDbgOperand FromLocOp = |
10791 | SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo()); |
10792 | SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo()); |
10793 | |
10794 | SmallVector<SDDbgValue *, 2> ClonedDVs; |
10795 | for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) { |
10796 | if (Dbg->isInvalidated()) |
10797 | continue; |
10798 | |
10799 | // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); |
10800 | |
10801 | // Create a new location ops vector that is equal to the old vector, but |
10802 | // with each instance of FromLocOp replaced with ToLocOp. |
10803 | bool Changed = false; |
10804 | auto NewLocOps = Dbg->copyLocationOps(); |
10805 | std::replace_if( |
10806 | first: NewLocOps.begin(), last: NewLocOps.end(), |
10807 | pred: [&Changed, FromLocOp](const SDDbgOperand &Op) { |
10808 | bool Match = Op == FromLocOp; |
10809 | Changed |= Match; |
10810 | return Match; |
10811 | }, |
10812 | new_value: ToLocOp); |
10813 | // Ignore this SDDbgValue if we didn't find a matching location. |
10814 | if (!Changed) |
10815 | continue; |
10816 | |
10817 | DIVariable *Var = Dbg->getVariable(); |
10818 | auto *Expr = Dbg->getExpression(); |
10819 | // If a fragment is requested, update the expression. |
10820 | if (SizeInBits) { |
10821 | // When splitting a larger (e.g., sign-extended) value whose |
10822 | // lower bits are described with an SDDbgValue, do not attempt |
10823 | // to transfer the SDDbgValue to the upper bits. |
10824 | if (auto FI = Expr->getFragmentInfo()) |
10825 | if (OffsetInBits + SizeInBits > FI->SizeInBits) |
10826 | continue; |
10827 | auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, |
10828 | SizeInBits); |
10829 | if (!Fragment) |
10830 | continue; |
10831 | Expr = *Fragment; |
10832 | } |
10833 | |
10834 | auto AdditionalDependencies = Dbg->getAdditionalDependencies(); |
10835 | // Clone the SDDbgValue and move it to To. |
10836 | SDDbgValue *Clone = getDbgValueList( |
10837 | Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(), |
10838 | DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()), |
10839 | IsVariadic: Dbg->isVariadic()); |
10840 | ClonedDVs.push_back(Elt: Clone); |
10841 | |
10842 | if (InvalidateDbg) { |
10843 | // Invalidate value and indicate the SDDbgValue should not be emitted. |
10844 | Dbg->setIsInvalidated(); |
10845 | Dbg->setIsEmitted(); |
10846 | } |
10847 | } |
10848 | |
10849 | for (SDDbgValue *Dbg : ClonedDVs) { |
10850 | assert(is_contained(Dbg->getSDNodes(), ToNode) && |
10851 | "Transferred DbgValues should depend on the new SDNode" ); |
10852 | AddDbgValue(DB: Dbg, isParameter: false); |
10853 | } |
10854 | } |
10855 | |
10856 | void SelectionDAG::salvageDebugInfo(SDNode &N) { |
10857 | if (!N.getHasDebugValue()) |
10858 | return; |
10859 | |
10860 | SmallVector<SDDbgValue *, 2> ClonedDVs; |
10861 | for (auto *DV : GetDbgValues(SD: &N)) { |
10862 | if (DV->isInvalidated()) |
10863 | continue; |
10864 | switch (N.getOpcode()) { |
10865 | default: |
10866 | break; |
10867 | case ISD::ADD: { |
10868 | SDValue N0 = N.getOperand(Num: 0); |
10869 | SDValue N1 = N.getOperand(Num: 1); |
10870 | if (!isa<ConstantSDNode>(Val: N0)) { |
10871 | bool RHSConstant = isa<ConstantSDNode>(Val: N1); |
10872 | uint64_t Offset; |
10873 | if (RHSConstant) |
10874 | Offset = N.getConstantOperandVal(Num: 1); |
10875 | // We are not allowed to turn indirect debug values variadic, so |
10876 | // don't salvage those. |
10877 | if (!RHSConstant && DV->isIndirect()) |
10878 | continue; |
10879 | |
10880 | // Rewrite an ADD constant node into a DIExpression. Since we are |
10881 | // performing arithmetic to compute the variable's *value* in the |
10882 | // DIExpression, we need to mark the expression with a |
10883 | // DW_OP_stack_value. |
10884 | auto *DIExpr = DV->getExpression(); |
10885 | auto NewLocOps = DV->copyLocationOps(); |
10886 | bool Changed = false; |
10887 | size_t OrigLocOpsSize = NewLocOps.size(); |
10888 | for (size_t i = 0; i < OrigLocOpsSize; ++i) { |
10889 | // We're not given a ResNo to compare against because the whole |
10890 | // node is going away. We know that any ISD::ADD only has one |
10891 | // result, so we can assume any node match is using the result. |
10892 | if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE || |
10893 | NewLocOps[i].getSDNode() != &N) |
10894 | continue; |
10895 | NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo()); |
10896 | if (RHSConstant) { |
10897 | SmallVector<uint64_t, 3> ExprOps; |
10898 | DIExpression::appendOffset(Ops&: ExprOps, Offset); |
10899 | DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true); |
10900 | } else { |
10901 | // Convert to a variadic expression (if not already). |
10902 | // convertToVariadicExpression() returns a const pointer, so we use |
10903 | // a temporary const variable here. |
10904 | const auto *TmpDIExpr = |
10905 | DIExpression::convertToVariadicExpression(Expr: DIExpr); |
10906 | SmallVector<uint64_t, 3> ExprOps; |
10907 | ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg); |
10908 | ExprOps.push_back(Elt: NewLocOps.size()); |
10909 | ExprOps.push_back(Elt: dwarf::DW_OP_plus); |
10910 | SDDbgOperand RHS = |
10911 | SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo()); |
10912 | NewLocOps.push_back(Elt: RHS); |
10913 | DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true); |
10914 | } |
10915 | Changed = true; |
10916 | } |
10917 | (void)Changed; |
10918 | assert(Changed && "Salvage target doesn't use N" ); |
10919 | |
10920 | bool IsVariadic = |
10921 | DV->isVariadic() || OrigLocOpsSize != NewLocOps.size(); |
10922 | |
10923 | auto AdditionalDependencies = DV->getAdditionalDependencies(); |
10924 | SDDbgValue *Clone = getDbgValueList( |
10925 | Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies, |
10926 | IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic); |
10927 | ClonedDVs.push_back(Elt: Clone); |
10928 | DV->setIsInvalidated(); |
10929 | DV->setIsEmitted(); |
10930 | LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting" ; |
10931 | N0.getNode()->dumprFull(this); |
10932 | dbgs() << " into " << *DIExpr << '\n'); |
10933 | } |
10934 | break; |
10935 | } |
10936 | case ISD::TRUNCATE: { |
10937 | SDValue N0 = N.getOperand(Num: 0); |
10938 | TypeSize FromSize = N0.getValueSizeInBits(); |
10939 | TypeSize ToSize = N.getValueSizeInBits(ResNo: 0); |
10940 | |
10941 | DIExpression *DbgExpression = DV->getExpression(); |
10942 | auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false); |
10943 | auto NewLocOps = DV->copyLocationOps(); |
10944 | bool Changed = false; |
10945 | for (size_t i = 0; i < NewLocOps.size(); ++i) { |
10946 | if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE || |
10947 | NewLocOps[i].getSDNode() != &N) |
10948 | continue; |
10949 | |
10950 | NewLocOps[i] = SDDbgOperand::fromNode(Node: N0.getNode(), ResNo: N0.getResNo()); |
10951 | DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i); |
10952 | Changed = true; |
10953 | } |
10954 | assert(Changed && "Salvage target doesn't use N" ); |
10955 | (void)Changed; |
10956 | |
10957 | SDDbgValue *Clone = |
10958 | getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps, |
10959 | Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(), |
10960 | DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic()); |
10961 | |
10962 | ClonedDVs.push_back(Elt: Clone); |
10963 | DV->setIsInvalidated(); |
10964 | DV->setIsEmitted(); |
10965 | LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting" ; N0.getNode()->dumprFull(this); |
10966 | dbgs() << " into " << *DbgExpression << '\n'); |
10967 | break; |
10968 | } |
10969 | } |
10970 | } |
10971 | |
10972 | for (SDDbgValue *Dbg : ClonedDVs) { |
10973 | assert(!Dbg->getSDNodes().empty() && |
10974 | "Salvaged DbgValue should depend on a new SDNode" ); |
10975 | AddDbgValue(DB: Dbg, isParameter: false); |
10976 | } |
10977 | } |
10978 | |
10979 | /// Creates a SDDbgLabel node. |
10980 | SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, |
10981 | const DebugLoc &DL, unsigned O) { |
10982 | assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) && |
10983 | "Expected inlined-at fields to agree" ); |
10984 | return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); |
10985 | } |
10986 | |
10987 | namespace { |
10988 | |
10989 | /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node |
10990 | /// pointed to by a use iterator is deleted, increment the use iterator |
10991 | /// so that it doesn't dangle. |
10992 | /// |
10993 | class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { |
10994 | SDNode::use_iterator &UI; |
10995 | SDNode::use_iterator &UE; |
10996 | |
10997 | void NodeDeleted(SDNode *N, SDNode *E) override { |
10998 | // Increment the iterator as needed. |
10999 | while (UI != UE && N == *UI) |
11000 | ++UI; |
11001 | } |
11002 | |
11003 | public: |
11004 | RAUWUpdateListener(SelectionDAG &d, |
11005 | SDNode::use_iterator &ui, |
11006 | SDNode::use_iterator &ue) |
11007 | : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} |
11008 | }; |
11009 | |
11010 | } // end anonymous namespace |
11011 | |
11012 | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
11013 | /// This can cause recursive merging of nodes in the DAG. |
11014 | /// |
11015 | /// This version assumes From has a single result value. |
11016 | /// |
11017 | void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { |
11018 | SDNode *From = FromN.getNode(); |
11019 | assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && |
11020 | "Cannot replace with this method!" ); |
11021 | assert(From != To.getNode() && "Cannot replace uses of with self" ); |
11022 | |
11023 | // Preserve Debug Values |
11024 | transferDbgValues(From: FromN, To); |
11025 | // Preserve extra info. |
11026 | copyExtraInfo(From, To: To.getNode()); |
11027 | |
11028 | // Iterate over all the existing uses of From. New uses will be added |
11029 | // to the beginning of the use list, which we avoid visiting. |
11030 | // This specifically avoids visiting uses of From that arise while the |
11031 | // replacement is happening, because any such uses would be the result |
11032 | // of CSE: If an existing node looks like From after one of its operands |
11033 | // is replaced by To, we don't want to replace of all its users with To |
11034 | // too. See PR3018 for more info. |
11035 | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
11036 | RAUWUpdateListener Listener(*this, UI, UE); |
11037 | while (UI != UE) { |
11038 | SDNode *User = *UI; |
11039 | |
11040 | // This node is about to morph, remove its old self from the CSE maps. |
11041 | RemoveNodeFromCSEMaps(N: User); |
11042 | |
11043 | // A user can appear in a use list multiple times, and when this |
11044 | // happens the uses are usually next to each other in the list. |
11045 | // To help reduce the number of CSE recomputations, process all |
11046 | // the uses of this user that we can find this way. |
11047 | do { |
11048 | SDUse &Use = UI.getUse(); |
11049 | ++UI; |
11050 | Use.set(To); |
11051 | if (To->isDivergent() != From->isDivergent()) |
11052 | updateDivergence(N: User); |
11053 | } while (UI != UE && *UI == User); |
11054 | // Now that we have modified User, add it back to the CSE maps. If it |
11055 | // already exists there, recursively merge the results together. |
11056 | AddModifiedNodeToCSEMaps(N: User); |
11057 | } |
11058 | |
11059 | // If we just RAUW'd the root, take note. |
11060 | if (FromN == getRoot()) |
11061 | setRoot(To); |
11062 | } |
11063 | |
11064 | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
11065 | /// This can cause recursive merging of nodes in the DAG. |
11066 | /// |
11067 | /// This version assumes that for each value of From, there is a |
11068 | /// corresponding value in To in the same position with the same type. |
11069 | /// |
11070 | void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { |
11071 | #ifndef NDEBUG |
11072 | for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
11073 | assert((!From->hasAnyUseOfValue(i) || |
11074 | From->getValueType(i) == To->getValueType(i)) && |
11075 | "Cannot use this version of ReplaceAllUsesWith!" ); |
11076 | #endif |
11077 | |
11078 | // Handle the trivial case. |
11079 | if (From == To) |
11080 | return; |
11081 | |
11082 | // Preserve Debug Info. Only do this if there's a use. |
11083 | for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
11084 | if (From->hasAnyUseOfValue(Value: i)) { |
11085 | assert((i < To->getNumValues()) && "Invalid To location" ); |
11086 | transferDbgValues(From: SDValue(From, i), To: SDValue(To, i)); |
11087 | } |
11088 | // Preserve extra info. |
11089 | copyExtraInfo(From, To); |
11090 | |
11091 | // Iterate over just the existing users of From. See the comments in |
11092 | // the ReplaceAllUsesWith above. |
11093 | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
11094 | RAUWUpdateListener Listener(*this, UI, UE); |
11095 | while (UI != UE) { |
11096 | SDNode *User = *UI; |
11097 | |
11098 | // This node is about to morph, remove its old self from the CSE maps. |
11099 | RemoveNodeFromCSEMaps(N: User); |
11100 | |
11101 | // A user can appear in a use list multiple times, and when this |
11102 | // happens the uses are usually next to each other in the list. |
11103 | // To help reduce the number of CSE recomputations, process all |
11104 | // the uses of this user that we can find this way. |
11105 | do { |
11106 | SDUse &Use = UI.getUse(); |
11107 | ++UI; |
11108 | Use.setNode(To); |
11109 | if (To->isDivergent() != From->isDivergent()) |
11110 | updateDivergence(N: User); |
11111 | } while (UI != UE && *UI == User); |
11112 | |
11113 | // Now that we have modified User, add it back to the CSE maps. If it |
11114 | // already exists there, recursively merge the results together. |
11115 | AddModifiedNodeToCSEMaps(N: User); |
11116 | } |
11117 | |
11118 | // If we just RAUW'd the root, take note. |
11119 | if (From == getRoot().getNode()) |
11120 | setRoot(SDValue(To, getRoot().getResNo())); |
11121 | } |
11122 | |
11123 | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
11124 | /// This can cause recursive merging of nodes in the DAG. |
11125 | /// |
11126 | /// This version can replace From with any result values. To must match the |
11127 | /// number and types of values returned by From. |
11128 | void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { |
11129 | if (From->getNumValues() == 1) // Handle the simple case efficiently. |
11130 | return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]); |
11131 | |
11132 | for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) { |
11133 | // Preserve Debug Info. |
11134 | transferDbgValues(From: SDValue(From, i), To: To[i]); |
11135 | // Preserve extra info. |
11136 | copyExtraInfo(From, To: To[i].getNode()); |
11137 | } |
11138 | |
11139 | // Iterate over just the existing users of From. See the comments in |
11140 | // the ReplaceAllUsesWith above. |
11141 | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
11142 | RAUWUpdateListener Listener(*this, UI, UE); |
11143 | while (UI != UE) { |
11144 | SDNode *User = *UI; |
11145 | |
11146 | // This node is about to morph, remove its old self from the CSE maps. |
11147 | RemoveNodeFromCSEMaps(N: User); |
11148 | |
11149 | // A user can appear in a use list multiple times, and when this happens the |
11150 | // uses are usually next to each other in the list. To help reduce the |
11151 | // number of CSE and divergence recomputations, process all the uses of this |
11152 | // user that we can find this way. |
11153 | bool To_IsDivergent = false; |
11154 | do { |
11155 | SDUse &Use = UI.getUse(); |
11156 | const SDValue &ToOp = To[Use.getResNo()]; |
11157 | ++UI; |
11158 | Use.set(ToOp); |
11159 | To_IsDivergent |= ToOp->isDivergent(); |
11160 | } while (UI != UE && *UI == User); |
11161 | |
11162 | if (To_IsDivergent != From->isDivergent()) |
11163 | updateDivergence(N: User); |
11164 | |
11165 | // Now that we have modified User, add it back to the CSE maps. If it |
11166 | // already exists there, recursively merge the results together. |
11167 | AddModifiedNodeToCSEMaps(N: User); |
11168 | } |
11169 | |
11170 | // If we just RAUW'd the root, take note. |
11171 | if (From == getRoot().getNode()) |
11172 | setRoot(SDValue(To[getRoot().getResNo()])); |
11173 | } |
11174 | |
11175 | /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving |
11176 | /// uses of other values produced by From.getNode() alone. The Deleted |
11177 | /// vector is handled the same way as for ReplaceAllUsesWith. |
11178 | void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ |
11179 | // Handle the really simple, really trivial case efficiently. |
11180 | if (From == To) return; |
11181 | |
11182 | // Handle the simple, trivial, case efficiently. |
11183 | if (From.getNode()->getNumValues() == 1) { |
11184 | ReplaceAllUsesWith(FromN: From, To); |
11185 | return; |
11186 | } |
11187 | |
11188 | // Preserve Debug Info. |
11189 | transferDbgValues(From, To); |
11190 | copyExtraInfo(From: From.getNode(), To: To.getNode()); |
11191 | |
11192 | // Iterate over just the existing users of From. See the comments in |
11193 | // the ReplaceAllUsesWith above. |
11194 | SDNode::use_iterator UI = From.getNode()->use_begin(), |
11195 | UE = From.getNode()->use_end(); |
11196 | RAUWUpdateListener Listener(*this, UI, UE); |
11197 | while (UI != UE) { |
11198 | SDNode *User = *UI; |
11199 | bool UserRemovedFromCSEMaps = false; |
11200 | |
11201 | // A user can appear in a use list multiple times, and when this |
11202 | // happens the uses are usually next to each other in the list. |
11203 | // To help reduce the number of CSE recomputations, process all |
11204 | // the uses of this user that we can find this way. |
11205 | do { |
11206 | SDUse &Use = UI.getUse(); |
11207 | |
11208 | // Skip uses of different values from the same node. |
11209 | if (Use.getResNo() != From.getResNo()) { |
11210 | ++UI; |
11211 | continue; |
11212 | } |
11213 | |
11214 | // If this node hasn't been modified yet, it's still in the CSE maps, |
11215 | // so remove its old self from the CSE maps. |
11216 | if (!UserRemovedFromCSEMaps) { |
11217 | RemoveNodeFromCSEMaps(N: User); |
11218 | UserRemovedFromCSEMaps = true; |
11219 | } |
11220 | |
11221 | ++UI; |
11222 | Use.set(To); |
11223 | if (To->isDivergent() != From->isDivergent()) |
11224 | updateDivergence(N: User); |
11225 | } while (UI != UE && *UI == User); |
11226 | // We are iterating over all uses of the From node, so if a use |
11227 | // doesn't use the specific value, no changes are made. |
11228 | if (!UserRemovedFromCSEMaps) |
11229 | continue; |
11230 | |
11231 | // Now that we have modified User, add it back to the CSE maps. If it |
11232 | // already exists there, recursively merge the results together. |
11233 | AddModifiedNodeToCSEMaps(N: User); |
11234 | } |
11235 | |
11236 | // If we just RAUW'd the root, take note. |
11237 | if (From == getRoot()) |
11238 | setRoot(To); |
11239 | } |
11240 | |
11241 | namespace { |
11242 | |
11243 | /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith |
11244 | /// to record information about a use. |
11245 | struct UseMemo { |
11246 | SDNode *User; |
11247 | unsigned Index; |
11248 | SDUse *Use; |
11249 | }; |
11250 | |
11251 | /// operator< - Sort Memos by User. |
11252 | bool operator<(const UseMemo &L, const UseMemo &R) { |
11253 | return (intptr_t)L.User < (intptr_t)R.User; |
11254 | } |
11255 | |
11256 | /// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node |
11257 | /// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that |
11258 | /// the node already has been taken care of recursively. |
11259 | class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener { |
11260 | SmallVector<UseMemo, 4> &Uses; |
11261 | |
11262 | void NodeDeleted(SDNode *N, SDNode *E) override { |
11263 | for (UseMemo &Memo : Uses) |
11264 | if (Memo.User == N) |
11265 | Memo.User = nullptr; |
11266 | } |
11267 | |
11268 | public: |
11269 | RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses) |
11270 | : SelectionDAG::DAGUpdateListener(d), Uses(uses) {} |
11271 | }; |
11272 | |
11273 | } // end anonymous namespace |
11274 | |
11275 | bool SelectionDAG::calculateDivergence(SDNode *N) { |
11276 | if (TLI->isSDNodeAlwaysUniform(N)) { |
11277 | assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) && |
11278 | "Conflicting divergence information!" ); |
11279 | return false; |
11280 | } |
11281 | if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA)) |
11282 | return true; |
11283 | for (const auto &Op : N->ops()) { |
11284 | if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) |
11285 | return true; |
11286 | } |
11287 | return false; |
11288 | } |
11289 | |
11290 | void SelectionDAG::updateDivergence(SDNode *N) { |
11291 | SmallVector<SDNode *, 16> Worklist(1, N); |
11292 | do { |
11293 | N = Worklist.pop_back_val(); |
11294 | bool IsDivergent = calculateDivergence(N); |
11295 | if (N->SDNodeBits.IsDivergent != IsDivergent) { |
11296 | N->SDNodeBits.IsDivergent = IsDivergent; |
11297 | llvm::append_range(C&: Worklist, R: N->uses()); |
11298 | } |
11299 | } while (!Worklist.empty()); |
11300 | } |
11301 | |
11302 | void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { |
11303 | DenseMap<SDNode *, unsigned> Degree; |
11304 | Order.reserve(n: AllNodes.size()); |
11305 | for (auto &N : allnodes()) { |
11306 | unsigned NOps = N.getNumOperands(); |
11307 | Degree[&N] = NOps; |
11308 | if (0 == NOps) |
11309 | Order.push_back(x: &N); |
11310 | } |
11311 | for (size_t I = 0; I != Order.size(); ++I) { |
11312 | SDNode *N = Order[I]; |
11313 | for (auto *U : N->uses()) { |
11314 | unsigned &UnsortedOps = Degree[U]; |
11315 | if (0 == --UnsortedOps) |
11316 | Order.push_back(x: U); |
11317 | } |
11318 | } |
11319 | } |
11320 | |
11321 | #ifndef NDEBUG |
11322 | void SelectionDAG::VerifyDAGDivergence() { |
11323 | std::vector<SDNode *> TopoOrder; |
11324 | CreateTopologicalOrder(Order&: TopoOrder); |
11325 | for (auto *N : TopoOrder) { |
11326 | assert(calculateDivergence(N) == N->isDivergent() && |
11327 | "Divergence bit inconsistency detected" ); |
11328 | } |
11329 | } |
11330 | #endif |
11331 | |
11332 | /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving |
11333 | /// uses of other values produced by From.getNode() alone. The same value |
11334 | /// may appear in both the From and To list. The Deleted vector is |
11335 | /// handled the same way as for ReplaceAllUsesWith. |
11336 | void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, |
11337 | const SDValue *To, |
11338 | unsigned Num){ |
11339 | // Handle the simple, trivial case efficiently. |
11340 | if (Num == 1) |
11341 | return ReplaceAllUsesOfValueWith(From: *From, To: *To); |
11342 | |
11343 | transferDbgValues(From: *From, To: *To); |
11344 | copyExtraInfo(From: From->getNode(), To: To->getNode()); |
11345 | |
11346 | // Read up all the uses and make records of them. This helps |
11347 | // processing new uses that are introduced during the |
11348 | // replacement process. |
11349 | SmallVector<UseMemo, 4> Uses; |
11350 | for (unsigned i = 0; i != Num; ++i) { |
11351 | unsigned FromResNo = From[i].getResNo(); |
11352 | SDNode *FromNode = From[i].getNode(); |
11353 | for (SDNode::use_iterator UI = FromNode->use_begin(), |
11354 | E = FromNode->use_end(); UI != E; ++UI) { |
11355 | SDUse &Use = UI.getUse(); |
11356 | if (Use.getResNo() == FromResNo) { |
11357 | UseMemo Memo = { .User: *UI, .Index: i, .Use: &Use }; |
11358 | Uses.push_back(Elt: Memo); |
11359 | } |
11360 | } |
11361 | } |
11362 | |
11363 | // Sort the uses, so that all the uses from a given User are together. |
11364 | llvm::sort(C&: Uses); |
11365 | RAUOVWUpdateListener Listener(*this, Uses); |
11366 | |
11367 | for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); |
11368 | UseIndex != UseIndexEnd; ) { |
11369 | // We know that this user uses some value of From. If it is the right |
11370 | // value, update it. |
11371 | SDNode *User = Uses[UseIndex].User; |
11372 | // If the node has been deleted by recursive CSE updates when updating |
11373 | // another node, then just skip this entry. |
11374 | if (User == nullptr) { |
11375 | ++UseIndex; |
11376 | continue; |
11377 | } |
11378 | |
11379 | // This node is about to morph, remove its old self from the CSE maps. |
11380 | RemoveNodeFromCSEMaps(N: User); |
11381 | |
11382 | // The Uses array is sorted, so all the uses for a given User |
11383 | // are next to each other in the list. |
11384 | // To help reduce the number of CSE recomputations, process all |
11385 | // the uses of this user that we can find this way. |
11386 | do { |
11387 | unsigned i = Uses[UseIndex].Index; |
11388 | SDUse &Use = *Uses[UseIndex].Use; |
11389 | ++UseIndex; |
11390 | |
11391 | Use.set(To[i]); |
11392 | } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); |
11393 | |
11394 | // Now that we have modified User, add it back to the CSE maps. If it |
11395 | // already exists there, recursively merge the results together. |
11396 | AddModifiedNodeToCSEMaps(N: User); |
11397 | } |
11398 | } |
11399 | |
11400 | /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG |
11401 | /// based on their topological order. It returns the maximum id and a vector |
11402 | /// of the SDNodes* in assigned order by reference. |
11403 | unsigned SelectionDAG::AssignTopologicalOrder() { |
11404 | unsigned DAGSize = 0; |
11405 | |
11406 | // SortedPos tracks the progress of the algorithm. Nodes before it are |
11407 | // sorted, nodes after it are unsorted. When the algorithm completes |
11408 | // it is at the end of the list. |
11409 | allnodes_iterator SortedPos = allnodes_begin(); |
11410 | |
11411 | // Visit all the nodes. Move nodes with no operands to the front of |
11412 | // the list immediately. Annotate nodes that do have operands with their |
11413 | // operand count. Before we do this, the Node Id fields of the nodes |
11414 | // may contain arbitrary values. After, the Node Id fields for nodes |
11415 | // before SortedPos will contain the topological sort index, and the |
11416 | // Node Id fields for nodes At SortedPos and after will contain the |
11417 | // count of outstanding operands. |
11418 | for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) { |
11419 | checkForCycles(N: &N, DAG: this); |
11420 | unsigned Degree = N.getNumOperands(); |
11421 | if (Degree == 0) { |
11422 | // A node with no uses, add it to the result array immediately. |
11423 | N.setNodeId(DAGSize++); |
11424 | allnodes_iterator Q(&N); |
11425 | if (Q != SortedPos) |
11426 | SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q)); |
11427 | assert(SortedPos != AllNodes.end() && "Overran node list" ); |
11428 | ++SortedPos; |
11429 | } else { |
11430 | // Temporarily use the Node Id as scratch space for the degree count. |
11431 | N.setNodeId(Degree); |
11432 | } |
11433 | } |
11434 | |
11435 | // Visit all the nodes. As we iterate, move nodes into sorted order, |
11436 | // such that by the time the end is reached all nodes will be sorted. |
11437 | for (SDNode &Node : allnodes()) { |
11438 | SDNode *N = &Node; |
11439 | checkForCycles(N, DAG: this); |
11440 | // N is in sorted position, so all its uses have one less operand |
11441 | // that needs to be sorted. |
11442 | for (SDNode *P : N->uses()) { |
11443 | unsigned Degree = P->getNodeId(); |
11444 | assert(Degree != 0 && "Invalid node degree" ); |
11445 | --Degree; |
11446 | if (Degree == 0) { |
11447 | // All of P's operands are sorted, so P may sorted now. |
11448 | P->setNodeId(DAGSize++); |
11449 | if (P->getIterator() != SortedPos) |
11450 | SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P)); |
11451 | assert(SortedPos != AllNodes.end() && "Overran node list" ); |
11452 | ++SortedPos; |
11453 | } else { |
11454 | // Update P's outstanding operand count. |
11455 | P->setNodeId(Degree); |
11456 | } |
11457 | } |
11458 | if (Node.getIterator() == SortedPos) { |
11459 | #ifndef NDEBUG |
11460 | allnodes_iterator I(N); |
11461 | SDNode *S = &*++I; |
11462 | dbgs() << "Overran sorted position:\n" ; |
11463 | S->dumprFull(G: this); dbgs() << "\n" ; |
11464 | dbgs() << "Checking if this is due to cycles\n" ; |
11465 | checkForCycles(DAG: this, force: true); |
11466 | #endif |
11467 | llvm_unreachable(nullptr); |
11468 | } |
11469 | } |
11470 | |
11471 | assert(SortedPos == AllNodes.end() && |
11472 | "Topological sort incomplete!" ); |
11473 | assert(AllNodes.front().getOpcode() == ISD::EntryToken && |
11474 | "First node in topological sort is not the entry token!" ); |
11475 | assert(AllNodes.front().getNodeId() == 0 && |
11476 | "First node in topological sort has non-zero id!" ); |
11477 | assert(AllNodes.front().getNumOperands() == 0 && |
11478 | "First node in topological sort has operands!" ); |
11479 | assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && |
11480 | "Last node in topologic sort has unexpected id!" ); |
11481 | assert(AllNodes.back().use_empty() && |
11482 | "Last node in topologic sort has users!" ); |
11483 | assert(DAGSize == allnodes_size() && "Node count mismatch!" ); |
11484 | return DAGSize; |
11485 | } |
11486 | |
11487 | /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the |
11488 | /// value is produced by SD. |
11489 | void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) { |
11490 | for (SDNode *SD : DB->getSDNodes()) { |
11491 | if (!SD) |
11492 | continue; |
11493 | assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); |
11494 | SD->setHasDebugValue(true); |
11495 | } |
11496 | DbgInfo->add(V: DB, isParameter); |
11497 | } |
11498 | |
11499 | void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); } |
11500 | |
11501 | SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, |
11502 | SDValue NewMemOpChain) { |
11503 | assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node" ); |
11504 | assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT" ); |
11505 | // The new memory operation must have the same position as the old load in |
11506 | // terms of memory dependency. Create a TokenFactor for the old load and new |
11507 | // memory operation and update uses of the old load's output chain to use that |
11508 | // TokenFactor. |
11509 | if (OldChain == NewMemOpChain || OldChain.use_empty()) |
11510 | return NewMemOpChain; |
11511 | |
11512 | SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, |
11513 | OldChain, NewMemOpChain); |
11514 | ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor); |
11515 | UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain); |
11516 | return TokenFactor; |
11517 | } |
11518 | |
11519 | SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, |
11520 | SDValue NewMemOp) { |
11521 | assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node" ); |
11522 | SDValue OldChain = SDValue(OldLoad, 1); |
11523 | SDValue NewMemOpChain = NewMemOp.getValue(R: 1); |
11524 | return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); |
11525 | } |
11526 | |
11527 | SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, |
11528 | Function **OutFunction) { |
11529 | assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol" ); |
11530 | |
11531 | auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol(); |
11532 | auto *Module = MF->getFunction().getParent(); |
11533 | auto *Function = Module->getFunction(Name: Symbol); |
11534 | |
11535 | if (OutFunction != nullptr) |
11536 | *OutFunction = Function; |
11537 | |
11538 | if (Function != nullptr) { |
11539 | auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace()); |
11540 | return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy); |
11541 | } |
11542 | |
11543 | std::string ErrorStr; |
11544 | raw_string_ostream ErrorFormatter(ErrorStr); |
11545 | ErrorFormatter << "Undefined external symbol " ; |
11546 | ErrorFormatter << '"' << Symbol << '"'; |
11547 | report_fatal_error(reason: Twine(ErrorFormatter.str())); |
11548 | } |
11549 | |
11550 | //===----------------------------------------------------------------------===// |
11551 | // SDNode Class |
11552 | //===----------------------------------------------------------------------===// |
11553 | |
11554 | bool llvm::isNullConstant(SDValue V) { |
11555 | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V); |
11556 | return Const != nullptr && Const->isZero(); |
11557 | } |
11558 | |
11559 | bool llvm::isNullFPConstant(SDValue V) { |
11560 | ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V); |
11561 | return Const != nullptr && Const->isZero() && !Const->isNegative(); |
11562 | } |
11563 | |
11564 | bool llvm::isAllOnesConstant(SDValue V) { |
11565 | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V); |
11566 | return Const != nullptr && Const->isAllOnes(); |
11567 | } |
11568 | |
11569 | bool llvm::isOneConstant(SDValue V) { |
11570 | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V); |
11571 | return Const != nullptr && Const->isOne(); |
11572 | } |
11573 | |
11574 | bool llvm::isMinSignedConstant(SDValue V) { |
11575 | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V); |
11576 | return Const != nullptr && Const->isMinSignedValue(); |
11577 | } |
11578 | |
11579 | bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V, |
11580 | unsigned OperandNo) { |
11581 | // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity(). |
11582 | // TODO: Target-specific opcodes could be added. |
11583 | if (auto *Const = isConstOrConstSplat(N: V)) { |
11584 | switch (Opcode) { |
11585 | case ISD::ADD: |
11586 | case ISD::OR: |
11587 | case ISD::XOR: |
11588 | case ISD::UMAX: |
11589 | return Const->isZero(); |
11590 | case ISD::MUL: |
11591 | return Const->isOne(); |
11592 | case ISD::AND: |
11593 | case ISD::UMIN: |
11594 | return Const->isAllOnes(); |
11595 | case ISD::SMAX: |
11596 | return Const->isMinSignedValue(); |
11597 | case ISD::SMIN: |
11598 | return Const->isMaxSignedValue(); |
11599 | case ISD::SUB: |
11600 | case ISD::SHL: |
11601 | case ISD::SRA: |
11602 | case ISD::SRL: |
11603 | return OperandNo == 1 && Const->isZero(); |
11604 | case ISD::UDIV: |
11605 | case ISD::SDIV: |
11606 | return OperandNo == 1 && Const->isOne(); |
11607 | } |
11608 | } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) { |
11609 | switch (Opcode) { |
11610 | case ISD::FADD: |
11611 | return ConstFP->isZero() && |
11612 | (Flags.hasNoSignedZeros() || ConstFP->isNegative()); |
11613 | case ISD::FSUB: |
11614 | return OperandNo == 1 && ConstFP->isZero() && |
11615 | (Flags.hasNoSignedZeros() || !ConstFP->isNegative()); |
11616 | case ISD::FMUL: |
11617 | return ConstFP->isExactlyValue(V: 1.0); |
11618 | case ISD::FDIV: |
11619 | return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0); |
11620 | case ISD::FMINNUM: |
11621 | case ISD::FMAXNUM: { |
11622 | // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
11623 | EVT VT = V.getValueType(); |
11624 | const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT); |
11625 | APFloat NeutralAF = !Flags.hasNoNaNs() |
11626 | ? APFloat::getQNaN(Sem: Semantics) |
11627 | : !Flags.hasNoInfs() |
11628 | ? APFloat::getInf(Sem: Semantics) |
11629 | : APFloat::getLargest(Sem: Semantics); |
11630 | if (Opcode == ISD::FMAXNUM) |
11631 | NeutralAF.changeSign(); |
11632 | |
11633 | return ConstFP->isExactlyValue(V: NeutralAF); |
11634 | } |
11635 | } |
11636 | } |
11637 | return false; |
11638 | } |
11639 | |
11640 | SDValue llvm::peekThroughBitcasts(SDValue V) { |
11641 | while (V.getOpcode() == ISD::BITCAST) |
11642 | V = V.getOperand(i: 0); |
11643 | return V; |
11644 | } |
11645 | |
11646 | SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { |
11647 | while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse()) |
11648 | V = V.getOperand(i: 0); |
11649 | return V; |
11650 | } |
11651 | |
11652 | SDValue llvm::(SDValue V) { |
11653 | while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) |
11654 | V = V.getOperand(i: 0); |
11655 | return V; |
11656 | } |
11657 | |
11658 | SDValue llvm::peekThroughTruncates(SDValue V) { |
11659 | while (V.getOpcode() == ISD::TRUNCATE) |
11660 | V = V.getOperand(i: 0); |
11661 | return V; |
11662 | } |
11663 | |
11664 | bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { |
11665 | if (V.getOpcode() != ISD::XOR) |
11666 | return false; |
11667 | V = peekThroughBitcasts(V: V.getOperand(i: 1)); |
11668 | unsigned NumBits = V.getScalarValueSizeInBits(); |
11669 | ConstantSDNode *C = |
11670 | isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true); |
11671 | return C && (C->getAPIntValue().countr_one() >= NumBits); |
11672 | } |
11673 | |
11674 | ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, |
11675 | bool AllowTruncation) { |
11676 | EVT VT = N.getValueType(); |
11677 | APInt DemandedElts = VT.isFixedLengthVector() |
11678 | ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements()) |
11679 | : APInt(1, 1); |
11680 | return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation); |
11681 | } |
11682 | |
11683 | ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, |
11684 | bool AllowUndefs, |
11685 | bool AllowTruncation) { |
11686 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) |
11687 | return CN; |
11688 | |
11689 | // SplatVectors can truncate their operands. Ignore that case here unless |
11690 | // AllowTruncation is set. |
11691 | if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
11692 | EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType(); |
11693 | if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) { |
11694 | EVT CVT = CN->getValueType(ResNo: 0); |
11695 | assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension" ); |
11696 | if (AllowTruncation || CVT == VecEltVT) |
11697 | return CN; |
11698 | } |
11699 | } |
11700 | |
11701 | if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) { |
11702 | BitVector UndefElements; |
11703 | ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements); |
11704 | |
11705 | // BuildVectors can truncate their operands. Ignore that case here unless |
11706 | // AllowTruncation is set. |
11707 | // TODO: Look into whether we should allow UndefElements in non-DemandedElts |
11708 | if (CN && (UndefElements.none() || AllowUndefs)) { |
11709 | EVT CVT = CN->getValueType(ResNo: 0); |
11710 | EVT NSVT = N.getValueType().getScalarType(); |
11711 | assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension" ); |
11712 | if (AllowTruncation || (CVT == NSVT)) |
11713 | return CN; |
11714 | } |
11715 | } |
11716 | |
11717 | return nullptr; |
11718 | } |
11719 | |
11720 | ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { |
11721 | EVT VT = N.getValueType(); |
11722 | APInt DemandedElts = VT.isFixedLengthVector() |
11723 | ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements()) |
11724 | : APInt(1, 1); |
11725 | return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs); |
11726 | } |
11727 | |
11728 | ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, |
11729 | const APInt &DemandedElts, |
11730 | bool AllowUndefs) { |
11731 | if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N)) |
11732 | return CN; |
11733 | |
11734 | if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) { |
11735 | BitVector UndefElements; |
11736 | ConstantFPSDNode *CN = |
11737 | BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements); |
11738 | // TODO: Look into whether we should allow UndefElements in non-DemandedElts |
11739 | if (CN && (UndefElements.none() || AllowUndefs)) |
11740 | return CN; |
11741 | } |
11742 | |
11743 | if (N.getOpcode() == ISD::SPLAT_VECTOR) |
11744 | if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0))) |
11745 | return CN; |
11746 | |
11747 | return nullptr; |
11748 | } |
11749 | |
11750 | bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { |
11751 | // TODO: may want to use peekThroughBitcast() here. |
11752 | ConstantSDNode *C = |
11753 | isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true); |
11754 | return C && C->isZero(); |
11755 | } |
11756 | |
11757 | bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) { |
11758 | ConstantSDNode *C = |
11759 | isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true); |
11760 | return C && C->isOne(); |
11761 | } |
11762 | |
11763 | bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) { |
11764 | N = peekThroughBitcasts(V: N); |
11765 | unsigned BitWidth = N.getScalarValueSizeInBits(); |
11766 | ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); |
11767 | return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth; |
11768 | } |
11769 | |
11770 | HandleSDNode::~HandleSDNode() { |
11771 | DropOperands(); |
11772 | } |
11773 | |
11774 | GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, |
11775 | const DebugLoc &DL, |
11776 | const GlobalValue *GA, EVT VT, |
11777 | int64_t o, unsigned TF) |
11778 | : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { |
11779 | TheGlobal = GA; |
11780 | } |
11781 | |
11782 | AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, |
11783 | EVT VT, unsigned SrcAS, |
11784 | unsigned DestAS) |
11785 | : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), |
11786 | SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} |
11787 | |
11788 | MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
11789 | SDVTList VTs, EVT memvt, MachineMemOperand *mmo) |
11790 | : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { |
11791 | MemSDNodeBits.IsVolatile = MMO->isVolatile(); |
11792 | MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); |
11793 | MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); |
11794 | MemSDNodeBits.IsInvariant = MMO->isInvariant(); |
11795 | |
11796 | // We check here that the size of the memory operand fits within the size of |
11797 | // the MMO. This is because the MMO might indicate only a possible address |
11798 | // range instead of specifying the affected memory addresses precisely. |
11799 | // TODO: Make MachineMemOperands aware of scalable vectors. |
11800 | assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() && |
11801 | "Size mismatch!" ); |
11802 | } |
11803 | |
11804 | /// Profile - Gather unique data for the node. |
11805 | /// |
11806 | void SDNode::Profile(FoldingSetNodeID &ID) const { |
11807 | AddNodeIDNode(ID, N: this); |
11808 | } |
11809 | |
11810 | namespace { |
11811 | |
11812 | struct EVTArray { |
11813 | std::vector<EVT> VTs; |
11814 | |
11815 | EVTArray() { |
11816 | VTs.reserve(n: MVT::VALUETYPE_SIZE); |
11817 | for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i) |
11818 | VTs.push_back(x: MVT((MVT::SimpleValueType)i)); |
11819 | } |
11820 | }; |
11821 | |
11822 | } // end anonymous namespace |
11823 | |
11824 | /// getValueTypeList - Return a pointer to the specified value type. |
11825 | /// |
11826 | const EVT *SDNode::getValueTypeList(EVT VT) { |
11827 | static std::set<EVT, EVT::compareRawBits> EVTs; |
11828 | static EVTArray SimpleVTArray; |
11829 | static sys::SmartMutex<true> VTMutex; |
11830 | |
11831 | if (VT.isExtended()) { |
11832 | sys::SmartScopedLock<true> Lock(VTMutex); |
11833 | return &(*EVTs.insert(x: VT).first); |
11834 | } |
11835 | assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!" ); |
11836 | return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy]; |
11837 | } |
11838 | |
11839 | /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the |
11840 | /// indicated value. This method ignores uses of other values defined by this |
11841 | /// operation. |
11842 | bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { |
11843 | assert(Value < getNumValues() && "Bad value!" ); |
11844 | |
11845 | // TODO: Only iterate over uses of a given value of the node |
11846 | for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { |
11847 | if (UI.getUse().getResNo() == Value) { |
11848 | if (NUses == 0) |
11849 | return false; |
11850 | --NUses; |
11851 | } |
11852 | } |
11853 | |
11854 | // Found exactly the right number of uses? |
11855 | return NUses == 0; |
11856 | } |
11857 | |
11858 | /// hasAnyUseOfValue - Return true if there are any use of the indicated |
11859 | /// value. This method ignores uses of other values defined by this operation. |
11860 | bool SDNode::hasAnyUseOfValue(unsigned Value) const { |
11861 | assert(Value < getNumValues() && "Bad value!" ); |
11862 | |
11863 | for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) |
11864 | if (UI.getUse().getResNo() == Value) |
11865 | return true; |
11866 | |
11867 | return false; |
11868 | } |
11869 | |
11870 | /// isOnlyUserOf - Return true if this node is the only use of N. |
11871 | bool SDNode::isOnlyUserOf(const SDNode *N) const { |
11872 | bool Seen = false; |
11873 | for (const SDNode *User : N->uses()) { |
11874 | if (User == this) |
11875 | Seen = true; |
11876 | else |
11877 | return false; |
11878 | } |
11879 | |
11880 | return Seen; |
11881 | } |
11882 | |
11883 | /// Return true if the only users of N are contained in Nodes. |
11884 | bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { |
11885 | bool Seen = false; |
11886 | for (const SDNode *User : N->uses()) { |
11887 | if (llvm::is_contained(Range&: Nodes, Element: User)) |
11888 | Seen = true; |
11889 | else |
11890 | return false; |
11891 | } |
11892 | |
11893 | return Seen; |
11894 | } |
11895 | |
11896 | /// isOperand - Return true if this node is an operand of N. |
11897 | bool SDValue::isOperandOf(const SDNode *N) const { |
11898 | return is_contained(Range: N->op_values(), Element: *this); |
11899 | } |
11900 | |
11901 | bool SDNode::isOperandOf(const SDNode *N) const { |
11902 | return any_of(Range: N->op_values(), |
11903 | P: [this](SDValue Op) { return this == Op.getNode(); }); |
11904 | } |
11905 | |
11906 | /// reachesChainWithoutSideEffects - Return true if this operand (which must |
11907 | /// be a chain) reaches the specified operand without crossing any |
11908 | /// side-effecting instructions on any chain path. In practice, this looks |
11909 | /// through token factors and non-volatile loads. In order to remain efficient, |
11910 | /// this only looks a couple of nodes in, it does not do an exhaustive search. |
11911 | /// |
11912 | /// Note that we only need to examine chains when we're searching for |
11913 | /// side-effects; SelectionDAG requires that all side-effects are represented |
11914 | /// by chains, even if another operand would force a specific ordering. This |
11915 | /// constraint is necessary to allow transformations like splitting loads. |
11916 | bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, |
11917 | unsigned Depth) const { |
11918 | if (*this == Dest) return true; |
11919 | |
11920 | // Don't search too deeply, we just want to be able to see through |
11921 | // TokenFactor's etc. |
11922 | if (Depth == 0) return false; |
11923 | |
11924 | // If this is a token factor, all inputs to the TF happen in parallel. |
11925 | if (getOpcode() == ISD::TokenFactor) { |
11926 | // First, try a shallow search. |
11927 | if (is_contained(Range: (*this)->ops(), Element: Dest)) { |
11928 | // We found the chain we want as an operand of this TokenFactor. |
11929 | // Essentially, we reach the chain without side-effects if we could |
11930 | // serialize the TokenFactor into a simple chain of operations with |
11931 | // Dest as the last operation. This is automatically true if the |
11932 | // chain has one use: there are no other ordering constraints. |
11933 | // If the chain has more than one use, we give up: some other |
11934 | // use of Dest might force a side-effect between Dest and the current |
11935 | // node. |
11936 | if (Dest.hasOneUse()) |
11937 | return true; |
11938 | } |
11939 | // Next, try a deep search: check whether every operand of the TokenFactor |
11940 | // reaches Dest. |
11941 | return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) { |
11942 | return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1); |
11943 | }); |
11944 | } |
11945 | |
11946 | // Loads don't have side effects, look through them. |
11947 | if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) { |
11948 | if (Ld->isUnordered()) |
11949 | return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1); |
11950 | } |
11951 | return false; |
11952 | } |
11953 | |
11954 | bool SDNode::hasPredecessor(const SDNode *N) const { |
11955 | SmallPtrSet<const SDNode *, 32> Visited; |
11956 | SmallVector<const SDNode *, 16> Worklist; |
11957 | Worklist.push_back(Elt: this); |
11958 | return hasPredecessorHelper(N, Visited, Worklist); |
11959 | } |
11960 | |
11961 | void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { |
11962 | this->Flags.intersectWith(Flags); |
11963 | } |
11964 | |
11965 | SDValue |
11966 | SelectionDAG::matchBinOpReduction(SDNode *, ISD::NodeType &BinOp, |
11967 | ArrayRef<ISD::NodeType> CandidateBinOps, |
11968 | bool AllowPartials) { |
11969 | // The pattern must end in an extract from index 0. |
11970 | if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
11971 | !isNullConstant(V: Extract->getOperand(Num: 1))) |
11972 | return SDValue(); |
11973 | |
11974 | // Match against one of the candidate binary ops. |
11975 | SDValue Op = Extract->getOperand(Num: 0); |
11976 | if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) { |
11977 | return Op.getOpcode() == unsigned(BinOp); |
11978 | })) |
11979 | return SDValue(); |
11980 | |
11981 | // Floating-point reductions may require relaxed constraints on the final step |
11982 | // of the reduction because they may reorder intermediate operations. |
11983 | unsigned CandidateBinOp = Op.getOpcode(); |
11984 | if (Op.getValueType().isFloatingPoint()) { |
11985 | SDNodeFlags Flags = Op->getFlags(); |
11986 | switch (CandidateBinOp) { |
11987 | case ISD::FADD: |
11988 | if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation()) |
11989 | return SDValue(); |
11990 | break; |
11991 | default: |
11992 | llvm_unreachable("Unhandled FP opcode for binop reduction" ); |
11993 | } |
11994 | } |
11995 | |
11996 | // Matching failed - attempt to see if we did enough stages that a partial |
11997 | // reduction from a subvector is possible. |
11998 | auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) { |
11999 | if (!AllowPartials || !Op) |
12000 | return SDValue(); |
12001 | EVT OpVT = Op.getValueType(); |
12002 | EVT OpSVT = OpVT.getScalarType(); |
12003 | EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts); |
12004 | if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0)) |
12005 | return SDValue(); |
12006 | BinOp = (ISD::NodeType)CandidateBinOp; |
12007 | return getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(Op), VT: SubVT, N1: Op, |
12008 | N2: getVectorIdxConstant(Val: 0, DL: SDLoc(Op))); |
12009 | }; |
12010 | |
12011 | // At each stage, we're looking for something that looks like: |
12012 | // %s = shufflevector <8 x i32> %op, <8 x i32> undef, |
12013 | // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, |
12014 | // i32 undef, i32 undef, i32 undef, i32 undef> |
12015 | // %a = binop <8 x i32> %op, %s |
12016 | // Where the mask changes according to the stage. E.g. for a 3-stage pyramid, |
12017 | // we expect something like: |
12018 | // <4,5,6,7,u,u,u,u> |
12019 | // <2,3,u,u,u,u,u,u> |
12020 | // <1,u,u,u,u,u,u,u> |
12021 | // While a partial reduction match would be: |
12022 | // <2,3,u,u,u,u,u,u> |
12023 | // <1,u,u,u,u,u,u,u> |
12024 | unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements()); |
12025 | SDValue PrevOp; |
12026 | for (unsigned i = 0; i < Stages; ++i) { |
12027 | unsigned MaskEnd = (1 << i); |
12028 | |
12029 | if (Op.getOpcode() != CandidateBinOp) |
12030 | return PartialReduction(PrevOp, MaskEnd); |
12031 | |
12032 | SDValue Op0 = Op.getOperand(i: 0); |
12033 | SDValue Op1 = Op.getOperand(i: 1); |
12034 | |
12035 | ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0); |
12036 | if (Shuffle) { |
12037 | Op = Op1; |
12038 | } else { |
12039 | Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1); |
12040 | Op = Op0; |
12041 | } |
12042 | |
12043 | // The first operand of the shuffle should be the same as the other operand |
12044 | // of the binop. |
12045 | if (!Shuffle || Shuffle->getOperand(Num: 0) != Op) |
12046 | return PartialReduction(PrevOp, MaskEnd); |
12047 | |
12048 | // Verify the shuffle has the expected (at this stage of the pyramid) mask. |
12049 | for (int Index = 0; Index < (int)MaskEnd; ++Index) |
12050 | if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index)) |
12051 | return PartialReduction(PrevOp, MaskEnd); |
12052 | |
12053 | PrevOp = Op; |
12054 | } |
12055 | |
12056 | // Handle subvector reductions, which tend to appear after the shuffle |
12057 | // reduction stages. |
12058 | while (Op.getOpcode() == CandidateBinOp) { |
12059 | unsigned NumElts = Op.getValueType().getVectorNumElements(); |
12060 | SDValue Op0 = Op.getOperand(i: 0); |
12061 | SDValue Op1 = Op.getOperand(i: 1); |
12062 | if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
12063 | Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
12064 | Op0.getOperand(i: 0) != Op1.getOperand(i: 0)) |
12065 | break; |
12066 | SDValue Src = Op0.getOperand(i: 0); |
12067 | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
12068 | if (NumSrcElts != (2 * NumElts)) |
12069 | break; |
12070 | if (!(Op0.getConstantOperandAPInt(i: 1) == 0 && |
12071 | Op1.getConstantOperandAPInt(i: 1) == NumElts) && |
12072 | !(Op1.getConstantOperandAPInt(i: 1) == 0 && |
12073 | Op0.getConstantOperandAPInt(i: 1) == NumElts)) |
12074 | break; |
12075 | Op = Src; |
12076 | } |
12077 | |
12078 | BinOp = (ISD::NodeType)CandidateBinOp; |
12079 | return Op; |
12080 | } |
12081 | |
12082 | SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { |
12083 | EVT VT = N->getValueType(ResNo: 0); |
12084 | EVT EltVT = VT.getVectorElementType(); |
12085 | unsigned NE = VT.getVectorNumElements(); |
12086 | |
12087 | SDLoc dl(N); |
12088 | |
12089 | // If ResNE is 0, fully unroll the vector op. |
12090 | if (ResNE == 0) |
12091 | ResNE = NE; |
12092 | else if (NE > ResNE) |
12093 | NE = ResNE; |
12094 | |
12095 | if (N->getNumValues() == 2) { |
12096 | SmallVector<SDValue, 8> Scalars0, Scalars1; |
12097 | SmallVector<SDValue, 4> Operands(N->getNumOperands()); |
12098 | EVT VT1 = N->getValueType(ResNo: 1); |
12099 | EVT EltVT1 = VT1.getVectorElementType(); |
12100 | |
12101 | unsigned i; |
12102 | for (i = 0; i != NE; ++i) { |
12103 | for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
12104 | SDValue Operand = N->getOperand(Num: j); |
12105 | EVT OperandVT = Operand.getValueType(); |
12106 | |
12107 | // A vector operand; extract a single element. |
12108 | EVT OperandEltVT = OperandVT.getVectorElementType(); |
12109 | Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT, |
12110 | N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl)); |
12111 | } |
12112 | |
12113 | SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands); |
12114 | Scalars0.push_back(Elt: EltOp); |
12115 | Scalars1.push_back(Elt: EltOp.getValue(R: 1)); |
12116 | } |
12117 | |
12118 | SDValue Vec0 = getBuildVector(VT, DL: dl, Ops: Scalars0); |
12119 | SDValue Vec1 = getBuildVector(VT: VT1, DL: dl, Ops: Scalars1); |
12120 | return getMergeValues(Ops: {Vec0, Vec1}, dl); |
12121 | } |
12122 | |
12123 | assert(N->getNumValues() == 1 && |
12124 | "Can't unroll a vector with multiple results!" ); |
12125 | |
12126 | SmallVector<SDValue, 8> Scalars; |
12127 | SmallVector<SDValue, 4> Operands(N->getNumOperands()); |
12128 | |
12129 | unsigned i; |
12130 | for (i= 0; i != NE; ++i) { |
12131 | for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
12132 | SDValue Operand = N->getOperand(Num: j); |
12133 | EVT OperandVT = Operand.getValueType(); |
12134 | if (OperandVT.isVector()) { |
12135 | // A vector operand; extract a single element. |
12136 | EVT OperandEltVT = OperandVT.getVectorElementType(); |
12137 | Operands[j] = getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OperandEltVT, |
12138 | N1: Operand, N2: getVectorIdxConstant(Val: i, DL: dl)); |
12139 | } else { |
12140 | // A scalar operand; just use it as is. |
12141 | Operands[j] = Operand; |
12142 | } |
12143 | } |
12144 | |
12145 | switch (N->getOpcode()) { |
12146 | default: { |
12147 | Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands, |
12148 | Flags: N->getFlags())); |
12149 | break; |
12150 | } |
12151 | case ISD::VSELECT: |
12152 | Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands)); |
12153 | break; |
12154 | case ISD::SHL: |
12155 | case ISD::SRA: |
12156 | case ISD::SRL: |
12157 | case ISD::ROTL: |
12158 | case ISD::ROTR: |
12159 | Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0], |
12160 | N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(), |
12161 | Op: Operands[1]))); |
12162 | break; |
12163 | case ISD::SIGN_EXTEND_INREG: { |
12164 | EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType(); |
12165 | Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, |
12166 | N1: Operands[0], |
12167 | N2: getValueType(VT: ExtVT))); |
12168 | } |
12169 | } |
12170 | } |
12171 | |
12172 | for (; i < ResNE; ++i) |
12173 | Scalars.push_back(Elt: getUNDEF(VT: EltVT)); |
12174 | |
12175 | EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE); |
12176 | return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars); |
12177 | } |
12178 | |
12179 | std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp( |
12180 | SDNode *N, unsigned ResNE) { |
12181 | unsigned Opcode = N->getOpcode(); |
12182 | assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || |
12183 | Opcode == ISD::USUBO || Opcode == ISD::SSUBO || |
12184 | Opcode == ISD::UMULO || Opcode == ISD::SMULO) && |
12185 | "Expected an overflow opcode" ); |
12186 | |
12187 | EVT ResVT = N->getValueType(ResNo: 0); |
12188 | EVT OvVT = N->getValueType(ResNo: 1); |
12189 | EVT ResEltVT = ResVT.getVectorElementType(); |
12190 | EVT OvEltVT = OvVT.getVectorElementType(); |
12191 | SDLoc dl(N); |
12192 | |
12193 | // If ResNE is 0, fully unroll the vector op. |
12194 | unsigned NE = ResVT.getVectorNumElements(); |
12195 | if (ResNE == 0) |
12196 | ResNE = NE; |
12197 | else if (NE > ResNE) |
12198 | NE = ResNE; |
12199 | |
12200 | SmallVector<SDValue, 8> LHSScalars; |
12201 | SmallVector<SDValue, 8> RHSScalars; |
12202 | ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE); |
12203 | ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE); |
12204 | |
12205 | EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT); |
12206 | SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT); |
12207 | SmallVector<SDValue, 8> ResScalars; |
12208 | SmallVector<SDValue, 8> OvScalars; |
12209 | for (unsigned i = 0; i < NE; ++i) { |
12210 | SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]); |
12211 | SDValue Ov = |
12212 | getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1), |
12213 | LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT), |
12214 | RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT)); |
12215 | |
12216 | ResScalars.push_back(Elt: Res); |
12217 | OvScalars.push_back(Elt: Ov); |
12218 | } |
12219 | |
12220 | ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT)); |
12221 | OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT)); |
12222 | |
12223 | EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE); |
12224 | EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE); |
12225 | return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars), |
12226 | y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars)); |
12227 | } |
12228 | |
12229 | bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, |
12230 | LoadSDNode *Base, |
12231 | unsigned Bytes, |
12232 | int Dist) const { |
12233 | if (LD->isVolatile() || Base->isVolatile()) |
12234 | return false; |
12235 | // TODO: probably too restrictive for atomics, revisit |
12236 | if (!LD->isSimple()) |
12237 | return false; |
12238 | if (LD->isIndexed() || Base->isIndexed()) |
12239 | return false; |
12240 | if (LD->getChain() != Base->getChain()) |
12241 | return false; |
12242 | EVT VT = LD->getMemoryVT(); |
12243 | if (VT.getSizeInBits() / 8 != Bytes) |
12244 | return false; |
12245 | |
12246 | auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this); |
12247 | auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this); |
12248 | |
12249 | int64_t Offset = 0; |
12250 | if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset)) |
12251 | return (Dist * (int64_t)Bytes == Offset); |
12252 | return false; |
12253 | } |
12254 | |
12255 | /// InferPtrAlignment - Infer alignment of a load / store address. Return |
12256 | /// std::nullopt if it cannot be inferred. |
12257 | MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { |
12258 | // If this is a GlobalAddress + cst, return the alignment. |
12259 | const GlobalValue *GV = nullptr; |
12260 | int64_t GVOffset = 0; |
12261 | if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) { |
12262 | unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
12263 | KnownBits Known(PtrWidth); |
12264 | llvm::computeKnownBits(V: GV, Known, DL: getDataLayout()); |
12265 | unsigned AlignBits = Known.countMinTrailingZeros(); |
12266 | if (AlignBits) |
12267 | return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset); |
12268 | } |
12269 | |
12270 | // If this is a direct reference to a stack slot, use information about the |
12271 | // stack slot's alignment. |
12272 | int FrameIdx = INT_MIN; |
12273 | int64_t FrameOffset = 0; |
12274 | if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) { |
12275 | FrameIdx = FI->getIndex(); |
12276 | } else if (isBaseWithConstantOffset(Op: Ptr) && |
12277 | isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) { |
12278 | // Handle FI+Cst |
12279 | FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex(); |
12280 | FrameOffset = Ptr.getConstantOperandVal(i: 1); |
12281 | } |
12282 | |
12283 | if (FrameIdx != INT_MIN) { |
12284 | const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
12285 | return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset); |
12286 | } |
12287 | |
12288 | return std::nullopt; |
12289 | } |
12290 | |
12291 | /// Split the scalar node with EXTRACT_ELEMENT using the provided |
12292 | /// VTs and return the low/high part. |
12293 | std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N, |
12294 | const SDLoc &DL, |
12295 | const EVT &LoVT, |
12296 | const EVT &HiVT) { |
12297 | assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() && |
12298 | "Split node must be a scalar type" ); |
12299 | SDValue Lo = |
12300 | getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL)); |
12301 | SDValue Hi = |
12302 | getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL)); |
12303 | return std::make_pair(x&: Lo, y&: Hi); |
12304 | } |
12305 | |
12306 | /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type |
12307 | /// which is split (or expanded) into two not necessarily identical pieces. |
12308 | std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { |
12309 | // Currently all types are split in half. |
12310 | EVT LoVT, HiVT; |
12311 | if (!VT.isVector()) |
12312 | LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT); |
12313 | else |
12314 | LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext()); |
12315 | |
12316 | return std::make_pair(x&: LoVT, y&: HiVT); |
12317 | } |
12318 | |
12319 | /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a |
12320 | /// type, dependent on an enveloping VT that has been split into two identical |
12321 | /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. |
12322 | std::pair<EVT, EVT> |
12323 | SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, |
12324 | bool *HiIsEmpty) const { |
12325 | EVT EltTp = VT.getVectorElementType(); |
12326 | // Examples: |
12327 | // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) |
12328 | // custom VL=9 with enveloping VL=8/8 yields 8/1 |
12329 | // custom VL=10 with enveloping VL=8/8 yields 8/2 |
12330 | // etc. |
12331 | ElementCount VTNumElts = VT.getVectorElementCount(); |
12332 | ElementCount EnvNumElts = EnvVT.getVectorElementCount(); |
12333 | assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && |
12334 | "Mixing fixed width and scalable vectors when enveloping a type" ); |
12335 | EVT LoVT, HiVT; |
12336 | if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { |
12337 | LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts); |
12338 | HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts); |
12339 | *HiIsEmpty = false; |
12340 | } else { |
12341 | // Flag that hi type has zero storage size, but return split envelop type |
12342 | // (this would be easier if vector types with zero elements were allowed). |
12343 | LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts); |
12344 | HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts); |
12345 | *HiIsEmpty = true; |
12346 | } |
12347 | return std::make_pair(x&: LoVT, y&: HiVT); |
12348 | } |
12349 | |
12350 | /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the |
12351 | /// low/high part. |
12352 | std::pair<SDValue, SDValue> |
12353 | SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, |
12354 | const EVT &HiVT) { |
12355 | assert(LoVT.isScalableVector() == HiVT.isScalableVector() && |
12356 | LoVT.isScalableVector() == N.getValueType().isScalableVector() && |
12357 | "Splitting vector with an invalid mixture of fixed and scalable " |
12358 | "vector types" ); |
12359 | assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= |
12360 | N.getValueType().getVectorMinNumElements() && |
12361 | "More vector elements requested than available!" ); |
12362 | SDValue Lo, Hi; |
12363 | Lo = |
12364 | getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N, N2: getVectorIdxConstant(Val: 0, DL)); |
12365 | // For scalable vectors it is safe to use LoVT.getVectorMinNumElements() |
12366 | // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales |
12367 | // IDX with the runtime scaling factor of the result vector type. For |
12368 | // fixed-width result vectors, that runtime scaling factor is 1. |
12369 | Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N, |
12370 | N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL)); |
12371 | return std::make_pair(x&: Lo, y&: Hi); |
12372 | } |
12373 | |
12374 | std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT, |
12375 | const SDLoc &DL) { |
12376 | // Split the vector length parameter. |
12377 | // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts). |
12378 | EVT VT = N.getValueType(); |
12379 | assert(VecVT.getVectorElementCount().isKnownEven() && |
12380 | "Expecting the mask to be an evenly-sized vector" ); |
12381 | unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2; |
12382 | SDValue HalfNumElts = |
12383 | VecVT.isFixedLengthVector() |
12384 | ? getConstant(Val: HalfMinNumElts, DL, VT) |
12385 | : getVScale(DL, VT, MulImm: APInt(VT.getScalarSizeInBits(), HalfMinNumElts)); |
12386 | SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts); |
12387 | SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts); |
12388 | return std::make_pair(x&: Lo, y&: Hi); |
12389 | } |
12390 | |
12391 | /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. |
12392 | SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { |
12393 | EVT VT = N.getValueType(); |
12394 | EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(), |
12395 | NumElements: NextPowerOf2(A: VT.getVectorNumElements())); |
12396 | return getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: getUNDEF(VT: WideVT), N2: N, |
12397 | N3: getVectorIdxConstant(Val: 0, DL)); |
12398 | } |
12399 | |
12400 | void SelectionDAG::(SDValue Op, |
12401 | SmallVectorImpl<SDValue> &Args, |
12402 | unsigned Start, unsigned Count, |
12403 | EVT EltVT) { |
12404 | EVT VT = Op.getValueType(); |
12405 | if (Count == 0) |
12406 | Count = VT.getVectorNumElements(); |
12407 | if (EltVT == EVT()) |
12408 | EltVT = VT.getVectorElementType(); |
12409 | SDLoc SL(Op); |
12410 | for (unsigned i = Start, e = Start + Count; i != e; ++i) { |
12411 | Args.push_back(Elt: getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Op, |
12412 | N2: getVectorIdxConstant(Val: i, DL: SL))); |
12413 | } |
12414 | } |
12415 | |
12416 | // getAddressSpace - Return the address space this GlobalAddress belongs to. |
12417 | unsigned GlobalAddressSDNode::getAddressSpace() const { |
12418 | return getGlobal()->getType()->getAddressSpace(); |
12419 | } |
12420 | |
12421 | Type *ConstantPoolSDNode::getType() const { |
12422 | if (isMachineConstantPoolEntry()) |
12423 | return Val.MachineCPVal->getType(); |
12424 | return Val.ConstVal->getType(); |
12425 | } |
12426 | |
12427 | bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
12428 | unsigned &SplatBitSize, |
12429 | bool &HasAnyUndefs, |
12430 | unsigned MinSplatBits, |
12431 | bool IsBigEndian) const { |
12432 | EVT VT = getValueType(ResNo: 0); |
12433 | assert(VT.isVector() && "Expected a vector type" ); |
12434 | unsigned VecWidth = VT.getSizeInBits(); |
12435 | if (MinSplatBits > VecWidth) |
12436 | return false; |
12437 | |
12438 | // FIXME: The widths are based on this node's type, but build vectors can |
12439 | // truncate their operands. |
12440 | SplatValue = APInt(VecWidth, 0); |
12441 | SplatUndef = APInt(VecWidth, 0); |
12442 | |
12443 | // Get the bits. Bits with undefined values (when the corresponding element |
12444 | // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared |
12445 | // in SplatValue. If any of the values are not constant, give up and return |
12446 | // false. |
12447 | unsigned int NumOps = getNumOperands(); |
12448 | assert(NumOps > 0 && "isConstantSplat has 0-size build vector" ); |
12449 | unsigned EltWidth = VT.getScalarSizeInBits(); |
12450 | |
12451 | for (unsigned j = 0; j < NumOps; ++j) { |
12452 | unsigned i = IsBigEndian ? NumOps - 1 - j : j; |
12453 | SDValue OpVal = getOperand(Num: i); |
12454 | unsigned BitPos = j * EltWidth; |
12455 | |
12456 | if (OpVal.isUndef()) |
12457 | SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth); |
12458 | else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal)) |
12459 | SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos); |
12460 | else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal)) |
12461 | SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos); |
12462 | else |
12463 | return false; |
12464 | } |
12465 | |
12466 | // The build_vector is all constants or undefs. Find the smallest element |
12467 | // size that splats the vector. |
12468 | HasAnyUndefs = (SplatUndef != 0); |
12469 | |
12470 | // FIXME: This does not work for vectors with elements less than 8 bits. |
12471 | while (VecWidth > 8) { |
12472 | // If we can't split in half, stop here. |
12473 | if (VecWidth & 1) |
12474 | break; |
12475 | |
12476 | unsigned HalfSize = VecWidth / 2; |
12477 | APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize); |
12478 | APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0); |
12479 | APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize); |
12480 | APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0); |
12481 | |
12482 | // If the two halves do not match (ignoring undef bits), stop here. |
12483 | if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || |
12484 | MinSplatBits > HalfSize) |
12485 | break; |
12486 | |
12487 | SplatValue = HighValue | LowValue; |
12488 | SplatUndef = HighUndef & LowUndef; |
12489 | |
12490 | VecWidth = HalfSize; |
12491 | } |
12492 | |
12493 | // FIXME: The loop above only tries to split in halves. But if the input |
12494 | // vector for example is <3 x i16> it wouldn't be able to detect a |
12495 | // SplatBitSize of 16. No idea if that is a design flaw currently limiting |
12496 | // optimizations. I guess that back in the days when this helper was created |
12497 | // vectors normally was power-of-2 sized. |
12498 | |
12499 | SplatBitSize = VecWidth; |
12500 | return true; |
12501 | } |
12502 | |
12503 | SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, |
12504 | BitVector *UndefElements) const { |
12505 | unsigned NumOps = getNumOperands(); |
12506 | if (UndefElements) { |
12507 | UndefElements->clear(); |
12508 | UndefElements->resize(N: NumOps); |
12509 | } |
12510 | assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size" ); |
12511 | if (!DemandedElts) |
12512 | return SDValue(); |
12513 | SDValue Splatted; |
12514 | for (unsigned i = 0; i != NumOps; ++i) { |
12515 | if (!DemandedElts[i]) |
12516 | continue; |
12517 | SDValue Op = getOperand(Num: i); |
12518 | if (Op.isUndef()) { |
12519 | if (UndefElements) |
12520 | (*UndefElements)[i] = true; |
12521 | } else if (!Splatted) { |
12522 | Splatted = Op; |
12523 | } else if (Splatted != Op) { |
12524 | return SDValue(); |
12525 | } |
12526 | } |
12527 | |
12528 | if (!Splatted) { |
12529 | unsigned FirstDemandedIdx = DemandedElts.countr_zero(); |
12530 | assert(getOperand(FirstDemandedIdx).isUndef() && |
12531 | "Can only have a splat without a constant for all undefs." ); |
12532 | return getOperand(Num: FirstDemandedIdx); |
12533 | } |
12534 | |
12535 | return Splatted; |
12536 | } |
12537 | |
12538 | SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { |
12539 | APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands()); |
12540 | return getSplatValue(DemandedElts, UndefElements); |
12541 | } |
12542 | |
12543 | bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, |
12544 | SmallVectorImpl<SDValue> &Sequence, |
12545 | BitVector *UndefElements) const { |
12546 | unsigned NumOps = getNumOperands(); |
12547 | Sequence.clear(); |
12548 | if (UndefElements) { |
12549 | UndefElements->clear(); |
12550 | UndefElements->resize(N: NumOps); |
12551 | } |
12552 | assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size" ); |
12553 | if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps)) |
12554 | return false; |
12555 | |
12556 | // Set the undefs even if we don't find a sequence (like getSplatValue). |
12557 | if (UndefElements) |
12558 | for (unsigned I = 0; I != NumOps; ++I) |
12559 | if (DemandedElts[I] && getOperand(Num: I).isUndef()) |
12560 | (*UndefElements)[I] = true; |
12561 | |
12562 | // Iteratively widen the sequence length looking for repetitions. |
12563 | for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { |
12564 | Sequence.append(NumInputs: SeqLen, Elt: SDValue()); |
12565 | for (unsigned I = 0; I != NumOps; ++I) { |
12566 | if (!DemandedElts[I]) |
12567 | continue; |
12568 | SDValue &SeqOp = Sequence[I % SeqLen]; |
12569 | SDValue Op = getOperand(Num: I); |
12570 | if (Op.isUndef()) { |
12571 | if (!SeqOp) |
12572 | SeqOp = Op; |
12573 | continue; |
12574 | } |
12575 | if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { |
12576 | Sequence.clear(); |
12577 | break; |
12578 | } |
12579 | SeqOp = Op; |
12580 | } |
12581 | if (!Sequence.empty()) |
12582 | return true; |
12583 | } |
12584 | |
12585 | assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern" ); |
12586 | return false; |
12587 | } |
12588 | |
12589 | bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, |
12590 | BitVector *UndefElements) const { |
12591 | APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands()); |
12592 | return getRepeatedSequence(DemandedElts, Sequence, UndefElements); |
12593 | } |
12594 | |
12595 | ConstantSDNode * |
12596 | BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, |
12597 | BitVector *UndefElements) const { |
12598 | return dyn_cast_or_null<ConstantSDNode>( |
12599 | Val: getSplatValue(DemandedElts, UndefElements)); |
12600 | } |
12601 | |
12602 | ConstantSDNode * |
12603 | BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { |
12604 | return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements)); |
12605 | } |
12606 | |
12607 | ConstantFPSDNode * |
12608 | BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, |
12609 | BitVector *UndefElements) const { |
12610 | return dyn_cast_or_null<ConstantFPSDNode>( |
12611 | Val: getSplatValue(DemandedElts, UndefElements)); |
12612 | } |
12613 | |
12614 | ConstantFPSDNode * |
12615 | BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { |
12616 | return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements)); |
12617 | } |
12618 | |
12619 | int32_t |
12620 | BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
12621 | uint32_t BitWidth) const { |
12622 | if (ConstantFPSDNode *CN = |
12623 | dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) { |
12624 | bool IsExact; |
12625 | APSInt IntVal(BitWidth); |
12626 | const APFloat &APF = CN->getValueAPF(); |
12627 | if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) != |
12628 | APFloat::opOK || |
12629 | !IsExact) |
12630 | return -1; |
12631 | |
12632 | return IntVal.exactLogBase2(); |
12633 | } |
12634 | return -1; |
12635 | } |
12636 | |
12637 | bool BuildVectorSDNode::getConstantRawBits( |
12638 | bool IsLittleEndian, unsigned DstEltSizeInBits, |
12639 | SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const { |
12640 | // Early-out if this contains anything but Undef/Constant/ConstantFP. |
12641 | if (!isConstant()) |
12642 | return false; |
12643 | |
12644 | unsigned NumSrcOps = getNumOperands(); |
12645 | unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits(); |
12646 | assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
12647 | "Invalid bitcast scale" ); |
12648 | |
12649 | // Extract raw src bits. |
12650 | SmallVector<APInt> SrcBitElements(NumSrcOps, |
12651 | APInt::getZero(numBits: SrcEltSizeInBits)); |
12652 | BitVector SrcUndeElements(NumSrcOps, false); |
12653 | |
12654 | for (unsigned I = 0; I != NumSrcOps; ++I) { |
12655 | SDValue Op = getOperand(Num: I); |
12656 | if (Op.isUndef()) { |
12657 | SrcUndeElements.set(I); |
12658 | continue; |
12659 | } |
12660 | auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op); |
12661 | auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op); |
12662 | assert((CInt || CFP) && "Unknown constant" ); |
12663 | SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits) |
12664 | : CFP->getValueAPF().bitcastToAPInt(); |
12665 | } |
12666 | |
12667 | // Recast to dst width. |
12668 | recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements, |
12669 | SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements); |
12670 | return true; |
12671 | } |
12672 | |
12673 | void BuildVectorSDNode::recastRawBits(bool IsLittleEndian, |
12674 | unsigned DstEltSizeInBits, |
12675 | SmallVectorImpl<APInt> &DstBitElements, |
12676 | ArrayRef<APInt> SrcBitElements, |
12677 | BitVector &DstUndefElements, |
12678 | const BitVector &SrcUndefElements) { |
12679 | unsigned NumSrcOps = SrcBitElements.size(); |
12680 | unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth(); |
12681 | assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
12682 | "Invalid bitcast scale" ); |
12683 | assert(NumSrcOps == SrcUndefElements.size() && |
12684 | "Vector size mismatch" ); |
12685 | |
12686 | unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits; |
12687 | DstUndefElements.clear(); |
12688 | DstUndefElements.resize(N: NumDstOps, t: false); |
12689 | DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits)); |
12690 | |
12691 | // Concatenate src elements constant bits together into dst element. |
12692 | if (SrcEltSizeInBits <= DstEltSizeInBits) { |
12693 | unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits; |
12694 | for (unsigned I = 0; I != NumDstOps; ++I) { |
12695 | DstUndefElements.set(I); |
12696 | APInt &DstBits = DstBitElements[I]; |
12697 | for (unsigned J = 0; J != Scale; ++J) { |
12698 | unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
12699 | if (SrcUndefElements[Idx]) |
12700 | continue; |
12701 | DstUndefElements.reset(Idx: I); |
12702 | const APInt &SrcBits = SrcBitElements[Idx]; |
12703 | assert(SrcBits.getBitWidth() == SrcEltSizeInBits && |
12704 | "Illegal constant bitwidths" ); |
12705 | DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits); |
12706 | } |
12707 | } |
12708 | return; |
12709 | } |
12710 | |
12711 | // Split src element constant bits into dst elements. |
12712 | unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits; |
12713 | for (unsigned I = 0; I != NumSrcOps; ++I) { |
12714 | if (SrcUndefElements[I]) { |
12715 | DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale); |
12716 | continue; |
12717 | } |
12718 | const APInt &SrcBits = SrcBitElements[I]; |
12719 | for (unsigned J = 0; J != Scale; ++J) { |
12720 | unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
12721 | APInt &DstBits = DstBitElements[Idx]; |
12722 | DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits); |
12723 | } |
12724 | } |
12725 | } |
12726 | |
12727 | bool BuildVectorSDNode::isConstant() const { |
12728 | for (const SDValue &Op : op_values()) { |
12729 | unsigned Opc = Op.getOpcode(); |
12730 | if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) |
12731 | return false; |
12732 | } |
12733 | return true; |
12734 | } |
12735 | |
12736 | std::optional<std::pair<APInt, APInt>> |
12737 | BuildVectorSDNode::isConstantSequence() const { |
12738 | unsigned NumOps = getNumOperands(); |
12739 | if (NumOps < 2) |
12740 | return std::nullopt; |
12741 | |
12742 | if (!isa<ConstantSDNode>(Val: getOperand(Num: 0)) || |
12743 | !isa<ConstantSDNode>(Val: getOperand(Num: 1))) |
12744 | return std::nullopt; |
12745 | |
12746 | unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits(); |
12747 | APInt Start = getConstantOperandAPInt(Num: 0).trunc(width: EltSize); |
12748 | APInt Stride = getConstantOperandAPInt(Num: 1).trunc(width: EltSize) - Start; |
12749 | |
12750 | if (Stride.isZero()) |
12751 | return std::nullopt; |
12752 | |
12753 | for (unsigned i = 2; i < NumOps; ++i) { |
12754 | if (!isa<ConstantSDNode>(Val: getOperand(Num: i))) |
12755 | return std::nullopt; |
12756 | |
12757 | APInt Val = getConstantOperandAPInt(Num: i).trunc(width: EltSize); |
12758 | if (Val != (Start + (Stride * i))) |
12759 | return std::nullopt; |
12760 | } |
12761 | |
12762 | return std::make_pair(x&: Start, y&: Stride); |
12763 | } |
12764 | |
12765 | bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { |
12766 | // Find the first non-undef value in the shuffle mask. |
12767 | unsigned i, e; |
12768 | for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) |
12769 | /* search */; |
12770 | |
12771 | // If all elements are undefined, this shuffle can be considered a splat |
12772 | // (although it should eventually get simplified away completely). |
12773 | if (i == e) |
12774 | return true; |
12775 | |
12776 | // Make sure all remaining elements are either undef or the same as the first |
12777 | // non-undef value. |
12778 | for (int Idx = Mask[i]; i != e; ++i) |
12779 | if (Mask[i] >= 0 && Mask[i] != Idx) |
12780 | return false; |
12781 | return true; |
12782 | } |
12783 | |
12784 | // Returns the SDNode if it is a constant integer BuildVector |
12785 | // or constant integer. |
12786 | SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { |
12787 | if (isa<ConstantSDNode>(Val: N)) |
12788 | return N.getNode(); |
12789 | if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())) |
12790 | return N.getNode(); |
12791 | // Treat a GlobalAddress supporting constant offset folding as a |
12792 | // constant integer. |
12793 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: N)) |
12794 | if (GA->getOpcode() == ISD::GlobalAddress && |
12795 | TLI->isOffsetFoldingLegal(GA)) |
12796 | return GA; |
12797 | if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
12798 | isa<ConstantSDNode>(Val: N.getOperand(i: 0))) |
12799 | return N.getNode(); |
12800 | return nullptr; |
12801 | } |
12802 | |
12803 | // Returns the SDNode if it is a constant float BuildVector |
12804 | // or constant float. |
12805 | SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { |
12806 | if (isa<ConstantFPSDNode>(Val: N)) |
12807 | return N.getNode(); |
12808 | |
12809 | if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode())) |
12810 | return N.getNode(); |
12811 | |
12812 | if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
12813 | isa<ConstantFPSDNode>(Val: N.getOperand(i: 0))) |
12814 | return N.getNode(); |
12815 | |
12816 | return nullptr; |
12817 | } |
12818 | |
12819 | void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { |
12820 | assert(!Node->OperandList && "Node already has operands" ); |
12821 | assert(SDNode::getMaxNumOperands() >= Vals.size() && |
12822 | "too many operands to fit into SDNode" ); |
12823 | SDUse *Ops = OperandRecycler.allocate( |
12824 | Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator); |
12825 | |
12826 | bool IsDivergent = false; |
12827 | for (unsigned I = 0; I != Vals.size(); ++I) { |
12828 | Ops[I].setUser(Node); |
12829 | Ops[I].setInitial(Vals[I]); |
12830 | if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. |
12831 | IsDivergent |= Ops[I].getNode()->isDivergent(); |
12832 | } |
12833 | Node->NumOperands = Vals.size(); |
12834 | Node->OperandList = Ops; |
12835 | if (!TLI->isSDNodeAlwaysUniform(N: Node)) { |
12836 | IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA); |
12837 | Node->SDNodeBits.IsDivergent = IsDivergent; |
12838 | } |
12839 | checkForCycles(N: Node); |
12840 | } |
12841 | |
12842 | SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, |
12843 | SmallVectorImpl<SDValue> &Vals) { |
12844 | size_t Limit = SDNode::getMaxNumOperands(); |
12845 | while (Vals.size() > Limit) { |
12846 | unsigned SliceIdx = Vals.size() - Limit; |
12847 | auto = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit); |
12848 | SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); |
12849 | Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end()); |
12850 | Vals.emplace_back(Args&: NewTF); |
12851 | } |
12852 | return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); |
12853 | } |
12854 | |
12855 | SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, |
12856 | EVT VT, SDNodeFlags Flags) { |
12857 | switch (Opcode) { |
12858 | default: |
12859 | return SDValue(); |
12860 | case ISD::ADD: |
12861 | case ISD::OR: |
12862 | case ISD::XOR: |
12863 | case ISD::UMAX: |
12864 | return getConstant(Val: 0, DL, VT); |
12865 | case ISD::MUL: |
12866 | return getConstant(Val: 1, DL, VT); |
12867 | case ISD::AND: |
12868 | case ISD::UMIN: |
12869 | return getAllOnesConstant(DL, VT); |
12870 | case ISD::SMAX: |
12871 | return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT); |
12872 | case ISD::SMIN: |
12873 | return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT); |
12874 | case ISD::FADD: |
12875 | return getConstantFP(Val: -0.0, DL, VT); |
12876 | case ISD::FMUL: |
12877 | return getConstantFP(Val: 1.0, DL, VT); |
12878 | case ISD::FMINNUM: |
12879 | case ISD::FMAXNUM: { |
12880 | // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
12881 | const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
12882 | APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) : |
12883 | !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) : |
12884 | APFloat::getLargest(Sem: Semantics); |
12885 | if (Opcode == ISD::FMAXNUM) |
12886 | NeutralAF.changeSign(); |
12887 | |
12888 | return getConstantFP(V: NeutralAF, DL, VT); |
12889 | } |
12890 | case ISD::FMINIMUM: |
12891 | case ISD::FMAXIMUM: { |
12892 | // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF. |
12893 | const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
12894 | APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) |
12895 | : APFloat::getLargest(Sem: Semantics); |
12896 | if (Opcode == ISD::FMAXIMUM) |
12897 | NeutralAF.changeSign(); |
12898 | |
12899 | return getConstantFP(V: NeutralAF, DL, VT); |
12900 | } |
12901 | |
12902 | } |
12903 | } |
12904 | |
12905 | /// Helper used to make a call to a library function that has one argument of |
12906 | /// pointer type. |
12907 | /// |
12908 | /// Such functions include 'fegetmode', 'fesetenv' and some others, which are |
12909 | /// used to get or set floating-point state. They have one argument of pointer |
12910 | /// type, which points to the memory region containing bits of the |
12911 | /// floating-point state. The value returned by such function is ignored in the |
12912 | /// created call. |
12913 | /// |
12914 | /// \param LibFunc Reference to library function (value of RTLIB::Libcall). |
12915 | /// \param Ptr Pointer used to save/load state. |
12916 | /// \param InChain Ingoing token chain. |
12917 | /// \returns Outgoing chain token. |
12918 | SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, |
12919 | SDValue InChain, |
12920 | const SDLoc &DLoc) { |
12921 | assert(InChain.getValueType() == MVT::Other && "Expected token chain" ); |
12922 | TargetLowering::ArgListTy Args; |
12923 | TargetLowering::ArgListEntry Entry; |
12924 | Entry.Node = Ptr; |
12925 | Entry.Ty = Ptr.getValueType().getTypeForEVT(Context&: *getContext()); |
12926 | Args.push_back(x: Entry); |
12927 | RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc); |
12928 | SDValue Callee = getExternalSymbol(Sym: TLI->getLibcallName(Call: LC), |
12929 | VT: TLI->getPointerTy(DL: getDataLayout())); |
12930 | TargetLowering::CallLoweringInfo CLI(*this); |
12931 | CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee( |
12932 | CC: TLI->getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, |
12933 | ArgsList: std::move(Args)); |
12934 | return TLI->LowerCallTo(CLI).second; |
12935 | } |
12936 | |
12937 | void SelectionDAG::(SDNode *From, SDNode *To) { |
12938 | assert(From && To && "Invalid SDNode; empty source SDValue?" ); |
12939 | auto I = SDEI.find(Val: From); |
12940 | if (I == SDEI.end()) |
12941 | return; |
12942 | |
12943 | // Use of operator[] on the DenseMap may cause an insertion, which invalidates |
12944 | // the iterator, hence the need to make a copy to prevent a use-after-free. |
12945 | NodeExtraInfo NEI = I->second; |
12946 | if (LLVM_LIKELY(!NEI.PCSections)) { |
12947 | // No deep copy required for the types of extra info set. |
12948 | // |
12949 | // FIXME: Investigate if other types of extra info also need deep copy. This |
12950 | // depends on the types of nodes they can be attached to: if some extra info |
12951 | // is only ever attached to nodes where a replacement To node is always the |
12952 | // node where later use and propagation of the extra info has the intended |
12953 | // semantics, no deep copy is required. |
12954 | SDEI[To] = std::move(NEI); |
12955 | return; |
12956 | } |
12957 | |
12958 | // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced |
12959 | // through the replacement of From with To. Otherwise, replacements of a node |
12960 | // (From) with more complex nodes (To and its operands) may result in lost |
12961 | // extra info where the root node (To) is insignificant in further propagating |
12962 | // and using extra info when further lowering to MIR. |
12963 | // |
12964 | // In the first step pre-populate the visited set with the nodes reachable |
12965 | // from the old From node. This avoids copying NodeExtraInfo to parts of the |
12966 | // DAG that is not new and should be left untouched. |
12967 | SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom. |
12968 | DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From. |
12969 | auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) { |
12970 | if (MaxDepth == 0) { |
12971 | // Remember this node in case we need to increase MaxDepth and continue |
12972 | // populating FromReach from this node. |
12973 | Leafs.emplace_back(Args&: N); |
12974 | return; |
12975 | } |
12976 | if (!FromReach.insert(V: N).second) |
12977 | return; |
12978 | for (const SDValue &Op : N->op_values()) |
12979 | Self(Self, Op.getNode(), MaxDepth - 1); |
12980 | }; |
12981 | |
12982 | // Copy extra info to To and all its transitive operands (that are new). |
12983 | SmallPtrSet<const SDNode *, 8> Visited; |
12984 | auto DeepCopyTo = [&](auto &&Self, const SDNode *N) { |
12985 | if (FromReach.contains(V: N)) |
12986 | return true; |
12987 | if (!Visited.insert(Ptr: N).second) |
12988 | return true; |
12989 | if (getEntryNode().getNode() == N) |
12990 | return false; |
12991 | for (const SDValue &Op : N->op_values()) { |
12992 | if (!Self(Self, Op.getNode())) |
12993 | return false; |
12994 | } |
12995 | // Copy only if entry node was not reached. |
12996 | SDEI[N] = NEI; |
12997 | return true; |
12998 | }; |
12999 | |
13000 | // We first try with a lower MaxDepth, assuming that the path to common |
13001 | // operands between From and To is relatively short. This significantly |
13002 | // improves performance in the common case. The initial MaxDepth is big |
13003 | // enough to avoid retry in the common case; the last MaxDepth is large |
13004 | // enough to avoid having to use the fallback below (and protects from |
13005 | // potential stack exhaustion from recursion). |
13006 | for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024; |
13007 | PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) { |
13008 | // StartFrom is the previous (or initial) set of leafs reachable at the |
13009 | // previous maximum depth. |
13010 | SmallVector<const SDNode *> StartFrom; |
13011 | std::swap(LHS&: StartFrom, RHS&: Leafs); |
13012 | for (const SDNode *N : StartFrom) |
13013 | VisitFrom(VisitFrom, N, MaxDepth - PrevDepth); |
13014 | if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To))) |
13015 | return; |
13016 | // This should happen very rarely (reached the entry node). |
13017 | LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n" ); |
13018 | assert(!Leafs.empty()); |
13019 | } |
13020 | |
13021 | // This should not happen - but if it did, that means the subgraph reachable |
13022 | // from From has depth greater or equal to maximum MaxDepth, and VisitFrom() |
13023 | // could not visit all reachable common operands. Consequently, we were able |
13024 | // to reach the entry node. |
13025 | errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n" ; |
13026 | assert(false && "From subgraph too complex - increase max. MaxDepth?" ); |
13027 | // Best-effort fallback if assertions disabled. |
13028 | SDEI[To] = std::move(NEI); |
13029 | } |
13030 | |
13031 | #ifndef NDEBUG |
13032 | static void checkForCyclesHelper(const SDNode *N, |
13033 | SmallPtrSetImpl<const SDNode*> &Visited, |
13034 | SmallPtrSetImpl<const SDNode*> &Checked, |
13035 | const llvm::SelectionDAG *DAG) { |
13036 | // If this node has already been checked, don't check it again. |
13037 | if (Checked.count(Ptr: N)) |
13038 | return; |
13039 | |
13040 | // If a node has already been visited on this depth-first walk, reject it as |
13041 | // a cycle. |
13042 | if (!Visited.insert(Ptr: N).second) { |
13043 | errs() << "Detected cycle in SelectionDAG\n" ; |
13044 | dbgs() << "Offending node:\n" ; |
13045 | N->dumprFull(G: DAG); dbgs() << "\n" ; |
13046 | abort(); |
13047 | } |
13048 | |
13049 | for (const SDValue &Op : N->op_values()) |
13050 | checkForCyclesHelper(N: Op.getNode(), Visited, Checked, DAG); |
13051 | |
13052 | Checked.insert(Ptr: N); |
13053 | Visited.erase(Ptr: N); |
13054 | } |
13055 | #endif |
13056 | |
13057 | void llvm::checkForCycles(const llvm::SDNode *N, |
13058 | const llvm::SelectionDAG *DAG, |
13059 | bool force) { |
13060 | #ifndef NDEBUG |
13061 | bool check = force; |
13062 | #ifdef EXPENSIVE_CHECKS |
13063 | check = true; |
13064 | #endif // EXPENSIVE_CHECKS |
13065 | if (check) { |
13066 | assert(N && "Checking nonexistent SDNode" ); |
13067 | SmallPtrSet<const SDNode*, 32> visited; |
13068 | SmallPtrSet<const SDNode*, 32> checked; |
13069 | checkForCyclesHelper(N, Visited&: visited, Checked&: checked, DAG); |
13070 | } |
13071 | #endif // !NDEBUG |
13072 | } |
13073 | |
13074 | void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { |
13075 | checkForCycles(N: DAG->getRoot().getNode(), DAG, force); |
13076 | } |
13077 | |