1 | //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements integer type expansion and promotion for LegalizeTypes. |
10 | // Promotion is the act of changing a computation in an illegal type into a |
11 | // computation in a larger type. For example, implementing i8 arithmetic in an |
12 | // i32 register (often needed on powerpc). |
13 | // Expansion is the act of changing a computation in an illegal type into a |
14 | // computation in two identical registers of a smaller type. For example, |
15 | // implementing i64 arithmetic in two i32 registers (often needed on 32-bit |
16 | // targets). |
17 | // |
18 | //===----------------------------------------------------------------------===// |
19 | |
20 | #include "LegalizeTypes.h" |
21 | #include "llvm/Analysis/TargetLibraryInfo.h" |
22 | #include "llvm/CodeGen/StackMaps.h" |
23 | #include "llvm/CodeGen/TargetLowering.h" |
24 | #include "llvm/IR/DerivedTypes.h" |
25 | #include "llvm/Support/ErrorHandling.h" |
26 | #include "llvm/Support/KnownBits.h" |
27 | #include "llvm/Support/raw_ostream.h" |
28 | #include <algorithm> |
29 | using namespace llvm; |
30 | |
31 | #define DEBUG_TYPE "legalize-types" |
32 | |
33 | //===----------------------------------------------------------------------===// |
34 | // Integer Result Promotion |
35 | //===----------------------------------------------------------------------===// |
36 | |
37 | /// PromoteIntegerResult - This method is called when a result of a node is |
38 | /// found to be in need of promotion to a larger type. At this point, the node |
39 | /// may also have invalid operands or may have other results that need |
40 | /// expansion, we just know that (at least) one result needs promotion. |
41 | void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
42 | LLVM_DEBUG(dbgs() << "Promote integer result: " ; N->dump(&DAG)); |
43 | SDValue Res = SDValue(); |
44 | |
45 | // See if the target wants to custom expand this node. |
46 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) { |
47 | LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n" ); |
48 | return; |
49 | } |
50 | |
51 | switch (N->getOpcode()) { |
52 | default: |
53 | #ifndef NDEBUG |
54 | dbgs() << "PromoteIntegerResult #" << ResNo << ": " ; |
55 | N->dump(G: &DAG); dbgs() << "\n" ; |
56 | #endif |
57 | report_fatal_error(reason: "Do not know how to promote this operator!" ); |
58 | case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
59 | case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
60 | case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
61 | case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
62 | case ISD::VP_BITREVERSE: |
63 | case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
64 | case ISD::VP_BSWAP: |
65 | case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
66 | case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
67 | case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
68 | case ISD::VP_CTLZ_ZERO_UNDEF: |
69 | case ISD::VP_CTLZ: |
70 | case ISD::CTLZ_ZERO_UNDEF: |
71 | case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
72 | case ISD::PARITY: |
73 | case ISD::VP_CTPOP: |
74 | case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; |
75 | case ISD::VP_CTTZ_ZERO_UNDEF: |
76 | case ISD::VP_CTTZ: |
77 | case ISD::CTTZ_ZERO_UNDEF: |
78 | case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
79 | case ISD::EXTRACT_VECTOR_ELT: |
80 | Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
81 | case ISD::LOAD: Res = PromoteIntRes_LOAD(N: cast<LoadSDNode>(Val: N)); break; |
82 | case ISD::MLOAD: Res = PromoteIntRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N)); |
83 | break; |
84 | case ISD::MGATHER: Res = PromoteIntRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N)); |
85 | break; |
86 | case ISD::SELECT: |
87 | case ISD::VSELECT: |
88 | case ISD::VP_SELECT: |
89 | case ISD::VP_MERGE: |
90 | Res = PromoteIntRes_Select(N); |
91 | break; |
92 | case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
93 | case ISD::STRICT_FSETCC: |
94 | case ISD::STRICT_FSETCCS: |
95 | case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
96 | case ISD::SMIN: |
97 | case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
98 | case ISD::UMIN: |
99 | case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; |
100 | |
101 | case ISD::SHL: |
102 | case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; |
103 | case ISD::SIGN_EXTEND_INREG: |
104 | Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
105 | case ISD::SRA: |
106 | case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; |
107 | case ISD::SRL: |
108 | case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; |
109 | case ISD::VP_TRUNCATE: |
110 | case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
111 | case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
112 | case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
113 | case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; |
114 | |
115 | case ISD::EXTRACT_SUBVECTOR: |
116 | Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
117 | case ISD::INSERT_SUBVECTOR: |
118 | Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; |
119 | case ISD::VECTOR_REVERSE: |
120 | Res = PromoteIntRes_VECTOR_REVERSE(N); break; |
121 | case ISD::VECTOR_SHUFFLE: |
122 | Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
123 | case ISD::VECTOR_SPLICE: |
124 | Res = PromoteIntRes_VECTOR_SPLICE(N); break; |
125 | case ISD::VECTOR_INTERLEAVE: |
126 | case ISD::VECTOR_DEINTERLEAVE: |
127 | Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); |
128 | return; |
129 | case ISD::INSERT_VECTOR_ELT: |
130 | Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
131 | case ISD::BUILD_VECTOR: |
132 | Res = PromoteIntRes_BUILD_VECTOR(N); |
133 | break; |
134 | case ISD::SPLAT_VECTOR: |
135 | case ISD::SCALAR_TO_VECTOR: |
136 | Res = PromoteIntRes_ScalarOp(N); |
137 | break; |
138 | case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; |
139 | case ISD::CONCAT_VECTORS: |
140 | Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
141 | |
142 | case ISD::ANY_EXTEND_VECTOR_INREG: |
143 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
144 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
145 | Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
146 | |
147 | case ISD::SIGN_EXTEND: |
148 | case ISD::VP_SIGN_EXTEND: |
149 | case ISD::ZERO_EXTEND: |
150 | case ISD::VP_ZERO_EXTEND: |
151 | case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
152 | |
153 | case ISD::VP_FP_TO_SINT: |
154 | case ISD::VP_FP_TO_UINT: |
155 | case ISD::STRICT_FP_TO_SINT: |
156 | case ISD::STRICT_FP_TO_UINT: |
157 | case ISD::FP_TO_SINT: |
158 | case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
159 | |
160 | case ISD::FP_TO_SINT_SAT: |
161 | case ISD::FP_TO_UINT_SAT: |
162 | Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; |
163 | |
164 | case ISD::FP_TO_BF16: |
165 | case ISD::FP_TO_FP16: |
166 | Res = PromoteIntRes_FP_TO_FP16_BF16(N); |
167 | break; |
168 | case ISD::STRICT_FP_TO_FP16: |
169 | Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N); |
170 | break; |
171 | case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; |
172 | |
173 | case ISD::AND: |
174 | case ISD::OR: |
175 | case ISD::XOR: |
176 | case ISD::ADD: |
177 | case ISD::SUB: |
178 | case ISD::MUL: |
179 | case ISD::VP_AND: |
180 | case ISD::VP_OR: |
181 | case ISD::VP_XOR: |
182 | case ISD::VP_ADD: |
183 | case ISD::VP_SUB: |
184 | case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
185 | |
186 | case ISD::VP_SMIN: |
187 | case ISD::VP_SMAX: |
188 | case ISD::SDIV: |
189 | case ISD::SREM: |
190 | case ISD::VP_SDIV: |
191 | case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
192 | |
193 | case ISD::VP_UMIN: |
194 | case ISD::VP_UMAX: |
195 | case ISD::UDIV: |
196 | case ISD::UREM: |
197 | case ISD::VP_UDIV: |
198 | case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
199 | |
200 | case ISD::SADDO: |
201 | case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
202 | case ISD::UADDO: |
203 | case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
204 | case ISD::SMULO: |
205 | case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
206 | |
207 | case ISD::ADDE: |
208 | case ISD::SUBE: |
209 | case ISD::UADDO_CARRY: |
210 | case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; |
211 | |
212 | case ISD::SADDO_CARRY: |
213 | case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; |
214 | |
215 | case ISD::SADDSAT: |
216 | case ISD::UADDSAT: |
217 | case ISD::SSUBSAT: |
218 | case ISD::USUBSAT: |
219 | case ISD::SSHLSAT: |
220 | case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; |
221 | |
222 | case ISD::SMULFIX: |
223 | case ISD::SMULFIXSAT: |
224 | case ISD::UMULFIX: |
225 | case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; |
226 | |
227 | case ISD::SDIVFIX: |
228 | case ISD::SDIVFIXSAT: |
229 | case ISD::UDIVFIX: |
230 | case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; |
231 | |
232 | case ISD::ABS: Res = PromoteIntRes_ABS(N); break; |
233 | |
234 | case ISD::ATOMIC_LOAD: |
235 | Res = PromoteIntRes_Atomic0(N: cast<AtomicSDNode>(Val: N)); break; |
236 | |
237 | case ISD::ATOMIC_LOAD_ADD: |
238 | case ISD::ATOMIC_LOAD_SUB: |
239 | case ISD::ATOMIC_LOAD_AND: |
240 | case ISD::ATOMIC_LOAD_CLR: |
241 | case ISD::ATOMIC_LOAD_OR: |
242 | case ISD::ATOMIC_LOAD_XOR: |
243 | case ISD::ATOMIC_LOAD_NAND: |
244 | case ISD::ATOMIC_LOAD_MIN: |
245 | case ISD::ATOMIC_LOAD_MAX: |
246 | case ISD::ATOMIC_LOAD_UMIN: |
247 | case ISD::ATOMIC_LOAD_UMAX: |
248 | case ISD::ATOMIC_SWAP: |
249 | Res = PromoteIntRes_Atomic1(N: cast<AtomicSDNode>(Val: N)); break; |
250 | |
251 | case ISD::ATOMIC_CMP_SWAP: |
252 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
253 | Res = PromoteIntRes_AtomicCmpSwap(N: cast<AtomicSDNode>(Val: N), ResNo); |
254 | break; |
255 | |
256 | case ISD::VECREDUCE_ADD: |
257 | case ISD::VECREDUCE_MUL: |
258 | case ISD::VECREDUCE_AND: |
259 | case ISD::VECREDUCE_OR: |
260 | case ISD::VECREDUCE_XOR: |
261 | case ISD::VECREDUCE_SMAX: |
262 | case ISD::VECREDUCE_SMIN: |
263 | case ISD::VECREDUCE_UMAX: |
264 | case ISD::VECREDUCE_UMIN: |
265 | Res = PromoteIntRes_VECREDUCE(N); |
266 | break; |
267 | |
268 | case ISD::VP_REDUCE_ADD: |
269 | case ISD::VP_REDUCE_MUL: |
270 | case ISD::VP_REDUCE_AND: |
271 | case ISD::VP_REDUCE_OR: |
272 | case ISD::VP_REDUCE_XOR: |
273 | case ISD::VP_REDUCE_SMAX: |
274 | case ISD::VP_REDUCE_SMIN: |
275 | case ISD::VP_REDUCE_UMAX: |
276 | case ISD::VP_REDUCE_UMIN: |
277 | Res = PromoteIntRes_VP_REDUCE(N); |
278 | break; |
279 | |
280 | case ISD::FREEZE: |
281 | Res = PromoteIntRes_FREEZE(N); |
282 | break; |
283 | |
284 | case ISD::ROTL: |
285 | case ISD::ROTR: |
286 | Res = PromoteIntRes_Rotate(N); |
287 | break; |
288 | |
289 | case ISD::FSHL: |
290 | case ISD::FSHR: |
291 | Res = PromoteIntRes_FunnelShift(N); |
292 | break; |
293 | |
294 | case ISD::VP_FSHL: |
295 | case ISD::VP_FSHR: |
296 | Res = PromoteIntRes_VPFunnelShift(N); |
297 | break; |
298 | |
299 | case ISD::IS_FPCLASS: |
300 | Res = PromoteIntRes_IS_FPCLASS(N); |
301 | break; |
302 | case ISD::FFREXP: |
303 | Res = PromoteIntRes_FFREXP(N); |
304 | break; |
305 | |
306 | case ISD::LRINT: |
307 | case ISD::LLRINT: |
308 | Res = PromoteIntRes_XRINT(N); |
309 | break; |
310 | } |
311 | |
312 | // If the result is null then the sub-method took care of registering it. |
313 | if (Res.getNode()) |
314 | SetPromotedInteger(Op: SDValue(N, ResNo), Result: Res); |
315 | } |
316 | |
317 | SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
318 | unsigned ResNo) { |
319 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
320 | return GetPromotedInteger(Op); |
321 | } |
322 | |
323 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
324 | // Sign-extend the new bits, and continue the assertion. |
325 | SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
326 | return DAG.getNode(Opcode: ISD::AssertSext, DL: SDLoc(N), |
327 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
328 | } |
329 | |
330 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
331 | // Zero the new bits, and continue the assertion. |
332 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
333 | return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), |
334 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
335 | } |
336 | |
337 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
338 | EVT ResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
339 | SDValue Res = DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), |
340 | MemVT: N->getMemoryVT(), VT: ResVT, |
341 | Chain: N->getChain(), Ptr: N->getBasePtr(), |
342 | MMO: N->getMemOperand()); |
343 | // Legalize the chain result - switch anything that used the old chain to |
344 | // use the new one. |
345 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
346 | return Res; |
347 | } |
348 | |
349 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
350 | SDValue Op2 = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
351 | SDValue Res = DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), |
352 | MemVT: N->getMemoryVT(), |
353 | Chain: N->getChain(), Ptr: N->getBasePtr(), |
354 | Val: Op2, MMO: N->getMemOperand()); |
355 | // Legalize the chain result - switch anything that used the old chain to |
356 | // use the new one. |
357 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
358 | return Res; |
359 | } |
360 | |
361 | SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
362 | unsigned ResNo) { |
363 | if (ResNo == 1) { |
364 | assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
365 | EVT SVT = getSetCCResultType(VT: N->getOperand(Num: 2).getValueType()); |
366 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
367 | |
368 | // Only use the result of getSetCCResultType if it is legal, |
369 | // otherwise just use the promoted result type (NVT). |
370 | if (!TLI.isTypeLegal(VT: SVT)) |
371 | SVT = NVT; |
372 | |
373 | SDVTList VTs = DAG.getVTList(N->getValueType(ResNo: 0), SVT, MVT::Other); |
374 | SDValue Res = DAG.getAtomicCmpSwap( |
375 | Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs, |
376 | Chain: N->getChain(), Ptr: N->getBasePtr(), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3), |
377 | MMO: N->getMemOperand()); |
378 | ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0)); |
379 | ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2)); |
380 | return DAG.getSExtOrTrunc(Op: Res.getValue(R: 1), DL: SDLoc(N), VT: NVT); |
381 | } |
382 | |
383 | // Op2 is used for the comparison and thus must be extended according to the |
384 | // target's atomic operations. Op3 is merely stored and so can be left alone. |
385 | SDValue Op2 = N->getOperand(Num: 2); |
386 | SDValue Op3 = GetPromotedInteger(Op: N->getOperand(Num: 3)); |
387 | switch (TLI.getExtendForAtomicCmpSwapArg()) { |
388 | case ISD::SIGN_EXTEND: |
389 | Op2 = SExtPromotedInteger(Op: Op2); |
390 | break; |
391 | case ISD::ZERO_EXTEND: |
392 | Op2 = ZExtPromotedInteger(Op: Op2); |
393 | break; |
394 | case ISD::ANY_EXTEND: |
395 | Op2 = GetPromotedInteger(Op: Op2); |
396 | break; |
397 | default: |
398 | llvm_unreachable("Invalid atomic op extension" ); |
399 | } |
400 | |
401 | SDVTList VTs = |
402 | DAG.getVTList(Op2.getValueType(), N->getValueType(ResNo: 1), MVT::Other); |
403 | SDValue Res = DAG.getAtomicCmpSwap( |
404 | Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs, Chain: N->getChain(), |
405 | Ptr: N->getBasePtr(), Cmp: Op2, Swp: Op3, MMO: N->getMemOperand()); |
406 | // Update the use to N with the newly created Res. |
407 | for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) |
408 | ReplaceValueWith(From: SDValue(N, i), To: Res.getValue(R: i)); |
409 | return Res; |
410 | } |
411 | |
412 | SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
413 | SDValue InOp = N->getOperand(Num: 0); |
414 | EVT InVT = InOp.getValueType(); |
415 | EVT NInVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT); |
416 | EVT OutVT = N->getValueType(ResNo: 0); |
417 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
418 | SDLoc dl(N); |
419 | |
420 | switch (getTypeAction(VT: InVT)) { |
421 | case TargetLowering::TypeLegal: |
422 | break; |
423 | case TargetLowering::TypePromoteInteger: |
424 | if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector() && !NInVT.isVector()) |
425 | // The input promotes to the same size. Convert the promoted value. |
426 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetPromotedInteger(Op: InOp)); |
427 | break; |
428 | case TargetLowering::TypeSoftenFloat: |
429 | // Promote the integer operand by hand. |
430 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftenedFloat(Op: InOp)); |
431 | case TargetLowering::TypeSoftPromoteHalf: |
432 | // Promote the integer operand by hand. |
433 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftPromotedHalf(Op: InOp)); |
434 | case TargetLowering::TypePromoteFloat: { |
435 | // Convert the promoted float by hand. |
436 | if (!NOutVT.isVector()) |
437 | return DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: NOutVT, Operand: GetPromotedFloat(Op: InOp)); |
438 | break; |
439 | } |
440 | case TargetLowering::TypeExpandInteger: |
441 | case TargetLowering::TypeExpandFloat: |
442 | break; |
443 | case TargetLowering::TypeScalarizeVector: |
444 | // Convert the element to an integer and promote it by hand. |
445 | if (!NOutVT.isVector()) |
446 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, |
447 | Operand: BitConvertToInteger(Op: GetScalarizedVector(Op: InOp))); |
448 | break; |
449 | case TargetLowering::TypeScalarizeScalableVector: |
450 | report_fatal_error(reason: "Scalarization of scalable vectors is not supported." ); |
451 | case TargetLowering::TypeSplitVector: { |
452 | if (!NOutVT.isVector()) { |
453 | // For example, i32 = BITCAST v2i16 on alpha. Convert the split |
454 | // pieces of the input into integers and reassemble in the final type. |
455 | SDValue Lo, Hi; |
456 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
457 | Lo = BitConvertToInteger(Op: Lo); |
458 | Hi = BitConvertToInteger(Op: Hi); |
459 | |
460 | if (DAG.getDataLayout().isBigEndian()) |
461 | std::swap(a&: Lo, b&: Hi); |
462 | |
463 | InOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, |
464 | VT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
465 | BitWidth: NOutVT.getSizeInBits()), |
466 | Operand: JoinIntegers(Lo, Hi)); |
467 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: InOp); |
468 | } |
469 | break; |
470 | } |
471 | case TargetLowering::TypeWidenVector: |
472 | // The input is widened to the same size. Convert to the widened value. |
473 | // Make sure that the outgoing value is not a vector, because this would |
474 | // make us bitcast between two vectors which are legalized in different ways. |
475 | if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector()) { |
476 | SDValue Res = |
477 | DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetWidenedVector(Op: InOp)); |
478 | |
479 | // For big endian targets we need to shift the casted value or the |
480 | // interesting bits will end up at the wrong place. |
481 | if (DAG.getDataLayout().isBigEndian()) { |
482 | unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
483 | assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!" ); |
484 | Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NOutVT, N1: Res, |
485 | N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NOutVT, DL: dl)); |
486 | } |
487 | return Res; |
488 | } |
489 | // If the output type is also a vector and widening it to the same size |
490 | // as the widened input type would be a legal type, we can widen the bitcast |
491 | // and handle the promotion after. |
492 | if (NOutVT.isVector()) { |
493 | TypeSize WidenInSize = NInVT.getSizeInBits(); |
494 | TypeSize OutSize = OutVT.getSizeInBits(); |
495 | if (WidenInSize.hasKnownScalarFactor(RHS: OutSize)) { |
496 | unsigned Scale = WidenInSize.getKnownScalarFactor(RHS: OutSize); |
497 | EVT WideOutVT = |
498 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: OutVT.getVectorElementType(), |
499 | EC: OutVT.getVectorElementCount() * Scale); |
500 | if (isTypeLegal(VT: WideOutVT)) { |
501 | InOp = DAG.getBitcast(VT: WideOutVT, V: GetWidenedVector(Op: InOp)); |
502 | InOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: InOp, |
503 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
504 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: InOp); |
505 | } |
506 | } |
507 | } |
508 | } |
509 | |
510 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, |
511 | Operand: CreateStackStoreLoad(Op: InOp, DestVT: OutVT)); |
512 | } |
513 | |
514 | SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { |
515 | SDValue V = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
516 | return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), |
517 | VT: V.getValueType(), Operand: V); |
518 | } |
519 | |
520 | SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
521 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
522 | EVT OVT = N->getValueType(ResNo: 0); |
523 | EVT NVT = Op.getValueType(); |
524 | SDLoc dl(N); |
525 | |
526 | // If the larger BSWAP isn't supported by the target, try to expand now. |
527 | // If we expand later we'll end up with more operations since we lost the |
528 | // original type. We only do this for scalars since we have a shuffle |
529 | // based lowering for vectors in LegalizeVectorOps. |
530 | if (!OVT.isVector() && |
531 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BSWAP, VT: NVT)) { |
532 | if (SDValue Res = TLI.expandBSWAP(N, DAG)) |
533 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res); |
534 | } |
535 | |
536 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
537 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl); |
538 | if (N->getOpcode() == ISD::BSWAP) |
539 | return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: NVT, Operand: Op), |
540 | N2: ShAmt); |
541 | SDValue Mask = N->getOperand(Num: 1); |
542 | SDValue EVL = N->getOperand(Num: 2); |
543 | return DAG.getNode(Opcode: ISD::VP_LSHR, DL: dl, VT: NVT, |
544 | N1: DAG.getNode(Opcode: ISD::VP_BSWAP, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), N2: ShAmt, |
545 | N3: Mask, N4: EVL); |
546 | } |
547 | |
548 | SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
549 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
550 | EVT OVT = N->getValueType(ResNo: 0); |
551 | EVT NVT = Op.getValueType(); |
552 | SDLoc dl(N); |
553 | |
554 | // If the larger BITREVERSE isn't supported by the target, try to expand now. |
555 | // If we expand later we'll end up with more operations since we lost the |
556 | // original type. We only do this for scalars since we have a shuffle |
557 | // based lowering for vectors in LegalizeVectorOps. |
558 | if (!OVT.isVector() && OVT.isSimple() && |
559 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BITREVERSE, VT: NVT)) { |
560 | if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) |
561 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res); |
562 | } |
563 | |
564 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
565 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl); |
566 | if (N->getOpcode() == ISD::BITREVERSE) |
567 | return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, |
568 | N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: NVT, Operand: Op), N2: ShAmt); |
569 | SDValue Mask = N->getOperand(Num: 1); |
570 | SDValue EVL = N->getOperand(Num: 2); |
571 | return DAG.getNode(Opcode: ISD::VP_LSHR, DL: dl, VT: NVT, |
572 | N1: DAG.getNode(Opcode: ISD::VP_BITREVERSE, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), |
573 | N2: ShAmt, N3: Mask, N4: EVL); |
574 | } |
575 | |
576 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
577 | // The pair element type may be legal, or may not promote to the same type as |
578 | // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. |
579 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), |
580 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
581 | VT: N->getValueType(ResNo: 0)), Operand: JoinIntegers(Lo: N->getOperand(Num: 0), |
582 | Hi: N->getOperand(Num: 1))); |
583 | } |
584 | |
585 | SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
586 | EVT VT = N->getValueType(ResNo: 0); |
587 | // FIXME there is no actual debug info here |
588 | SDLoc dl(N); |
589 | // Zero extend things like i1, sign extend everything else. It shouldn't |
590 | // matter in theory which one we pick, but this tends to give better code? |
591 | unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
592 | SDValue Result = DAG.getNode(Opcode: Opc, DL: dl, |
593 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT), |
594 | Operand: SDValue(N, 0)); |
595 | assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?" ); |
596 | return Result; |
597 | } |
598 | |
599 | SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
600 | EVT OVT = N->getValueType(ResNo: 0); |
601 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
602 | SDLoc dl(N); |
603 | |
604 | // If the larger CTLZ isn't supported by the target, try to expand now. |
605 | // If we expand later we'll end up with more operations since we lost the |
606 | // original type. |
607 | if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
608 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ, VT: NVT) && |
609 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ_ZERO_UNDEF, VT: NVT)) { |
610 | if (SDValue Result = TLI.expandCTLZ(N, DAG)) { |
611 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result); |
612 | return Result; |
613 | } |
614 | } |
615 | |
616 | // Zero extend to the promoted type and do the count there. |
617 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
618 | |
619 | // Subtract off the extra leading bits in the bigger type. |
620 | SDValue = DAG.getConstant( |
621 | Val: NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), DL: dl, VT: NVT); |
622 | if (!N->isVPOpcode()) |
623 | return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, |
624 | N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op), |
625 | N2: ExtractLeadingBits); |
626 | SDValue Mask = N->getOperand(Num: 1); |
627 | SDValue EVL = N->getOperand(Num: 2); |
628 | return DAG.getNode(Opcode: ISD::VP_SUB, DL: dl, VT: NVT, |
629 | N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), |
630 | N2: ExtractLeadingBits, N3: Mask, N4: EVL); |
631 | } |
632 | |
633 | SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { |
634 | EVT OVT = N->getValueType(ResNo: 0); |
635 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
636 | |
637 | // If the larger CTPOP isn't supported by the target, try to expand now. |
638 | // If we expand later we'll end up with more operations since we lost the |
639 | // original type. |
640 | // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to |
641 | // TargetLowering. |
642 | if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
643 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT: NVT)) { |
644 | if (SDValue Result = TLI.expandCTPOP(N, DAG)) { |
645 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Result); |
646 | return Result; |
647 | } |
648 | } |
649 | |
650 | // Zero extend to the promoted type and do the count or parity there. |
651 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
652 | if (!N->isVPOpcode()) |
653 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), Operand: Op); |
654 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op, |
655 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
656 | } |
657 | |
658 | SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
659 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
660 | EVT OVT = N->getValueType(ResNo: 0); |
661 | EVT NVT = Op.getValueType(); |
662 | SDLoc dl(N); |
663 | |
664 | // If the larger CTTZ isn't supported by the target, try to expand now. |
665 | // If we expand later we'll end up with more operations since we lost the |
666 | // original type. Don't expand if we can use CTPOP or CTLZ expansion on the |
667 | // larger type. |
668 | if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
669 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ, VT: NVT) && |
670 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ_ZERO_UNDEF, VT: NVT) && |
671 | !TLI.isOperationLegal(Op: ISD::CTPOP, VT: NVT) && |
672 | !TLI.isOperationLegal(Op: ISD::CTLZ, VT: NVT)) { |
673 | if (SDValue Result = TLI.expandCTTZ(N, DAG)) { |
674 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result); |
675 | return Result; |
676 | } |
677 | } |
678 | |
679 | if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) { |
680 | // The count is the same in the promoted type except if the original |
681 | // value was zero. This can be handled by setting the bit just off |
682 | // the top of the original type. |
683 | auto TopBit = APInt::getOneBitSet(numBits: NVT.getScalarSizeInBits(), |
684 | BitNo: OVT.getScalarSizeInBits()); |
685 | if (N->getOpcode() == ISD::CTTZ) |
686 | Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT)); |
687 | else |
688 | Op = |
689 | DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT), |
690 | N3: N->getOperand(Num: 1), N4: N->getOperand(Num: 2)); |
691 | } |
692 | if (!N->isVPOpcode()) |
693 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op); |
694 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op, N2: N->getOperand(Num: 1), |
695 | N3: N->getOperand(Num: 2)); |
696 | } |
697 | |
698 | SDValue DAGTypeLegalizer::(SDNode *N) { |
699 | SDLoc dl(N); |
700 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
701 | |
702 | SDValue Op0 = N->getOperand(Num: 0); |
703 | SDValue Op1 = N->getOperand(Num: 1); |
704 | |
705 | // If the input also needs to be promoted, do that first so we can get a |
706 | // get a good idea for the output type. |
707 | if (TLI.getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) |
708 | == TargetLowering::TypePromoteInteger) { |
709 | SDValue In = GetPromotedInteger(Op: Op0); |
710 | |
711 | // If the new type is larger than NVT, use it. We probably won't need to |
712 | // promote it again. |
713 | EVT SVT = In.getValueType().getScalarType(); |
714 | if (SVT.bitsGE(VT: NVT)) { |
715 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT, N1: In, N2: Op1); |
716 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NVT); |
717 | } |
718 | } |
719 | |
720 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NVT, N1: Op0, N2: Op1); |
721 | } |
722 | |
723 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
724 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
725 | unsigned NewOpc = N->getOpcode(); |
726 | SDLoc dl(N); |
727 | |
728 | // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is |
729 | // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT |
730 | // and SINT conversions are Custom, there is no way to tell which is |
731 | // preferable. We choose SINT because that's the right thing on PPC.) |
732 | if (N->getOpcode() == ISD::FP_TO_UINT && |
733 | !TLI.isOperationLegal(Op: ISD::FP_TO_UINT, VT: NVT) && |
734 | TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT)) |
735 | NewOpc = ISD::FP_TO_SINT; |
736 | |
737 | if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && |
738 | !TLI.isOperationLegal(Op: ISD::STRICT_FP_TO_UINT, VT: NVT) && |
739 | TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT)) |
740 | NewOpc = ISD::STRICT_FP_TO_SINT; |
741 | |
742 | if (N->getOpcode() == ISD::VP_FP_TO_UINT && |
743 | !TLI.isOperationLegal(Op: ISD::VP_FP_TO_UINT, VT: NVT) && |
744 | TLI.isOperationLegalOrCustom(Op: ISD::VP_FP_TO_SINT, VT: NVT)) |
745 | NewOpc = ISD::VP_FP_TO_SINT; |
746 | |
747 | SDValue Res; |
748 | if (N->isStrictFPOpcode()) { |
749 | Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, |
750 | {N->getOperand(0), N->getOperand(1)}); |
751 | // Legalize the chain result - switch anything that used the old chain to |
752 | // use the new one. |
753 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
754 | } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { |
755 | Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1), |
756 | N->getOperand(Num: 2)}); |
757 | } else { |
758 | Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
759 | } |
760 | |
761 | // Assert that the converted value fits in the original type. If it doesn't |
762 | // (eg: because the value being converted is too big), then the result of the |
763 | // original operation was undefined anyway, so the assert is still correct. |
764 | // |
765 | // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: |
766 | // before legalization: fp-to-uint16, 65534. -> 0xfffe |
767 | // after legalization: fp-to-sint32, 65534. -> 0x0000fffe |
768 | return DAG.getNode(Opcode: (N->getOpcode() == ISD::FP_TO_UINT || |
769 | N->getOpcode() == ISD::STRICT_FP_TO_UINT || |
770 | N->getOpcode() == ISD::VP_FP_TO_UINT) |
771 | ? ISD::AssertZext |
772 | : ISD::AssertSext, |
773 | DL: dl, VT: NVT, N1: Res, |
774 | N2: DAG.getValueType(N->getValueType(ResNo: 0).getScalarType())); |
775 | } |
776 | |
777 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { |
778 | // Promote the result type, while keeping the original width in Op1. |
779 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
780 | SDLoc dl(N); |
781 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0), |
782 | N2: N->getOperand(Num: 1)); |
783 | } |
784 | |
785 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { |
786 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
787 | SDLoc dl(N); |
788 | |
789 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
790 | } |
791 | |
792 | SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) { |
793 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
794 | SDLoc dl(N); |
795 | |
796 | SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), |
797 | N->getOperand(Num: 0), N->getOperand(Num: 1)); |
798 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
799 | return Res; |
800 | } |
801 | |
802 | SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) { |
803 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
804 | SDLoc dl(N); |
805 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
806 | } |
807 | |
808 | SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { |
809 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
810 | SDLoc dl(N); |
811 | |
812 | SDValue Res = |
813 | DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); |
814 | |
815 | // Legalize the chain result - switch anything that used the old chain to |
816 | // use the new one. |
817 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
818 | return Res; |
819 | } |
820 | |
821 | SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
822 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
823 | SDLoc dl(N); |
824 | |
825 | if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) |
826 | == TargetLowering::TypePromoteInteger) { |
827 | SDValue Res = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
828 | assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!" ); |
829 | |
830 | // If the result and operand types are the same after promotion, simplify |
831 | // to an in-register extension. Unless this is a VP_*_EXTEND. |
832 | if (NVT == Res.getValueType() && N->getNumOperands() == 1) { |
833 | // The high bits are not guaranteed to be anything. Insert an extend. |
834 | if (N->getOpcode() == ISD::SIGN_EXTEND) |
835 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res, |
836 | N2: DAG.getValueType(N->getOperand(Num: 0).getValueType())); |
837 | if (N->getOpcode() == ISD::ZERO_EXTEND) |
838 | return DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: N->getOperand(Num: 0).getValueType()); |
839 | assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!" ); |
840 | return Res; |
841 | } |
842 | } |
843 | |
844 | // Otherwise, just extend the original operand all the way to the larger type. |
845 | if (N->getNumOperands() != 1) { |
846 | assert(N->getNumOperands() == 3 && "Unexpected number of operands!" ); |
847 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
848 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0), |
849 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
850 | } |
851 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
852 | } |
853 | |
854 | SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
855 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!" ); |
856 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
857 | ISD::LoadExtType ExtType = |
858 | ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); |
859 | SDLoc dl(N); |
860 | SDValue Res = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: N->getChain(), Ptr: N->getBasePtr(), |
861 | MemVT: N->getMemoryVT(), MMO: N->getMemOperand()); |
862 | |
863 | // Legalize the chain result - switch anything that used the old chain to |
864 | // use the new one. |
865 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
866 | return Res; |
867 | } |
868 | |
869 | SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
870 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
871 | SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru()); |
872 | |
873 | ISD::LoadExtType ExtType = N->getExtensionType(); |
874 | if (ExtType == ISD::NON_EXTLOAD) |
875 | ExtType = ISD::EXTLOAD; |
876 | |
877 | SDLoc dl(N); |
878 | SDValue Res = DAG.getMaskedLoad(VT: NVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), |
879 | Offset: N->getOffset(), Mask: N->getMask(), Src0: ExtPassThru, |
880 | MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), |
881 | AM: N->getAddressingMode(), ExtType, |
882 | IsExpanding: N->isExpandingLoad()); |
883 | // Legalize the chain result - switch anything that used the old chain to |
884 | // use the new one. |
885 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
886 | return Res; |
887 | } |
888 | |
889 | SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
890 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
891 | SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru()); |
892 | assert(NVT == ExtPassThru.getValueType() && |
893 | "Gather result type and the passThru argument type should be the same" ); |
894 | |
895 | ISD::LoadExtType ExtType = N->getExtensionType(); |
896 | if (ExtType == ISD::NON_EXTLOAD) |
897 | ExtType = ISD::EXTLOAD; |
898 | |
899 | SDLoc dl(N); |
900 | SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), |
901 | N->getIndex(), N->getScale() }; |
902 | SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(NVT, MVT::Other), |
903 | MemVT: N->getMemoryVT(), dl, Ops, |
904 | MMO: N->getMemOperand(), IndexType: N->getIndexType(), |
905 | ExtTy: ExtType); |
906 | // Legalize the chain result - switch anything that used the old chain to |
907 | // use the new one. |
908 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
909 | return Res; |
910 | } |
911 | |
912 | /// Promote the overflow flag of an overflowing arithmetic node. |
913 | SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
914 | // Change the return type of the boolean result while obeying |
915 | // getSetCCResultType. |
916 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
917 | EVT VT = N->getValueType(ResNo: 0); |
918 | EVT SVT = getSetCCResultType(VT); |
919 | SDValue Ops[3] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
920 | unsigned NumOps = N->getNumOperands(); |
921 | assert(NumOps <= 3 && "Too many operands" ); |
922 | if (NumOps == 3) |
923 | Ops[2] = N->getOperand(Num: 2); |
924 | |
925 | SDLoc dl(N); |
926 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: SVT), |
927 | Ops: ArrayRef(Ops, NumOps)); |
928 | |
929 | // Modified the sum result - switch anything that used the old sum to use |
930 | // the new one. |
931 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
932 | |
933 | // Convert to the expected type. |
934 | return DAG.getBoolExtOrTrunc(Op: Res.getValue(R: 1), SL: dl, VT: NVT, OpVT: VT); |
935 | } |
936 | |
937 | SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { |
938 | // If the promoted type is legal, we can convert this to: |
939 | // 1. ANY_EXTEND iN to iM |
940 | // 2. SHL by M-N |
941 | // 3. [US][ADD|SUB|SHL]SAT |
942 | // 4. L/ASHR by M-N |
943 | // Else it is more efficient to convert this to a min and a max |
944 | // operation in the higher precision arithmetic. |
945 | SDLoc dl(N); |
946 | SDValue Op1 = N->getOperand(Num: 0); |
947 | SDValue Op2 = N->getOperand(Num: 1); |
948 | unsigned OldBits = Op1.getScalarValueSizeInBits(); |
949 | |
950 | unsigned Opcode = N->getOpcode(); |
951 | bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; |
952 | |
953 | SDValue Op1Promoted, Op2Promoted; |
954 | if (IsShift) { |
955 | Op1Promoted = GetPromotedInteger(Op: Op1); |
956 | Op2Promoted = ZExtPromotedInteger(Op: Op2); |
957 | } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { |
958 | Op1Promoted = ZExtPromotedInteger(Op: Op1); |
959 | Op2Promoted = ZExtPromotedInteger(Op: Op2); |
960 | } else { |
961 | Op1Promoted = SExtPromotedInteger(Op: Op1); |
962 | Op2Promoted = SExtPromotedInteger(Op: Op2); |
963 | } |
964 | EVT PromotedType = Op1Promoted.getValueType(); |
965 | unsigned NewBits = PromotedType.getScalarSizeInBits(); |
966 | |
967 | if (Opcode == ISD::UADDSAT) { |
968 | APInt MaxVal = APInt::getAllOnes(numBits: OldBits).zext(width: NewBits); |
969 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: PromotedType); |
970 | SDValue Add = |
971 | DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted); |
972 | return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: PromotedType, N1: Add, N2: SatMax); |
973 | } |
974 | |
975 | // USUBSAT can always be promoted as long as we have zero-extended the args. |
976 | if (Opcode == ISD::USUBSAT) |
977 | return DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT: PromotedType, N1: Op1Promoted, |
978 | N2: Op2Promoted); |
979 | |
980 | // Shift cannot use a min/max expansion, we can't detect overflow if all of |
981 | // the bits have been shifted out. |
982 | if (IsShift || TLI.isOperationLegal(Op: Opcode, VT: PromotedType)) { |
983 | unsigned ShiftOp; |
984 | switch (Opcode) { |
985 | case ISD::SADDSAT: |
986 | case ISD::SSUBSAT: |
987 | case ISD::SSHLSAT: |
988 | ShiftOp = ISD::SRA; |
989 | break; |
990 | case ISD::USHLSAT: |
991 | ShiftOp = ISD::SRL; |
992 | break; |
993 | default: |
994 | llvm_unreachable("Expected opcode to be signed or unsigned saturation " |
995 | "addition, subtraction or left shift" ); |
996 | } |
997 | |
998 | unsigned SHLAmount = NewBits - OldBits; |
999 | SDValue ShiftAmount = |
1000 | DAG.getShiftAmountConstant(Val: SHLAmount, VT: PromotedType, DL: dl); |
1001 | Op1Promoted = |
1002 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, N2: ShiftAmount); |
1003 | if (!IsShift) |
1004 | Op2Promoted = |
1005 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op2Promoted, N2: ShiftAmount); |
1006 | |
1007 | SDValue Result = |
1008 | DAG.getNode(Opcode, DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted); |
1009 | return DAG.getNode(Opcode: ShiftOp, DL: dl, VT: PromotedType, N1: Result, N2: ShiftAmount); |
1010 | } |
1011 | |
1012 | unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; |
1013 | APInt MinVal = APInt::getSignedMinValue(numBits: OldBits).sext(width: NewBits); |
1014 | APInt MaxVal = APInt::getSignedMaxValue(numBits: OldBits).sext(width: NewBits); |
1015 | SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT: PromotedType); |
1016 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: PromotedType); |
1017 | SDValue Result = |
1018 | DAG.getNode(Opcode: AddOp, DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted); |
1019 | Result = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: PromotedType, N1: Result, N2: SatMax); |
1020 | Result = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: PromotedType, N1: Result, N2: SatMin); |
1021 | return Result; |
1022 | } |
1023 | |
1024 | SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { |
1025 | // Can just promote the operands then continue with operation. |
1026 | SDLoc dl(N); |
1027 | SDValue Op1Promoted, Op2Promoted; |
1028 | bool Signed = |
1029 | N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; |
1030 | bool Saturating = |
1031 | N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; |
1032 | if (Signed) { |
1033 | Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1034 | Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1035 | } else { |
1036 | Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1037 | Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1038 | } |
1039 | EVT OldType = N->getOperand(Num: 0).getValueType(); |
1040 | EVT PromotedType = Op1Promoted.getValueType(); |
1041 | unsigned DiffSize = |
1042 | PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); |
1043 | |
1044 | if (Saturating) { |
1045 | // Promoting the operand and result values changes the saturation width, |
1046 | // which is extends the values that we clamp to on saturation. This could be |
1047 | // resolved by shifting one of the operands the same amount, which would |
1048 | // also shift the result we compare against, then shifting back. |
1049 | Op1Promoted = |
1050 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, |
1051 | N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl)); |
1052 | SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, |
1053 | N2: Op2Promoted, N3: N->getOperand(Num: 2)); |
1054 | unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; |
1055 | return DAG.getNode(Opcode: ShiftOp, DL: dl, VT: PromotedType, N1: Result, |
1056 | N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl)); |
1057 | } |
1058 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted, |
1059 | N3: N->getOperand(Num: 2)); |
1060 | } |
1061 | |
1062 | static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, |
1063 | unsigned SatW, bool Signed, |
1064 | const TargetLowering &TLI, |
1065 | SelectionDAG &DAG) { |
1066 | EVT VT = V.getValueType(); |
1067 | unsigned VTW = VT.getScalarSizeInBits(); |
1068 | |
1069 | if (!Signed) { |
1070 | // Saturate to the unsigned maximum by getting the minimum of V and the |
1071 | // maximum. |
1072 | return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: V, |
1073 | N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW), |
1074 | DL: dl, VT)); |
1075 | } |
1076 | |
1077 | // Saturate to the signed maximum (the low SatW - 1 bits) by taking the |
1078 | // signed minimum of it and V. |
1079 | V = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: V, |
1080 | N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW - 1), |
1081 | DL: dl, VT)); |
1082 | // Saturate to the signed minimum (the high SatW + 1 bits) by taking the |
1083 | // signed maximum of it and V. |
1084 | V = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: V, |
1085 | N2: DAG.getConstant(Val: APInt::getHighBitsSet(numBits: VTW, hiBitsSet: VTW - SatW + 1), |
1086 | DL: dl, VT)); |
1087 | return V; |
1088 | } |
1089 | |
1090 | static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, |
1091 | unsigned Scale, const TargetLowering &TLI, |
1092 | SelectionDAG &DAG, unsigned SatW = 0) { |
1093 | EVT VT = LHS.getValueType(); |
1094 | unsigned VTSize = VT.getScalarSizeInBits(); |
1095 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
1096 | N->getOpcode() == ISD::SDIVFIXSAT; |
1097 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
1098 | N->getOpcode() == ISD::UDIVFIXSAT; |
1099 | |
1100 | SDLoc dl(N); |
1101 | // Widen the types by a factor of two. This is guaranteed to expand, since it |
1102 | // will always have enough high bits in the LHS to shift into. |
1103 | EVT WideVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VTSize * 2); |
1104 | if (VT.isVector()) |
1105 | WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT, |
1106 | EC: VT.getVectorElementCount()); |
1107 | LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL: dl, VT: WideVT); |
1108 | RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL: dl, VT: WideVT); |
1109 | SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS, RHS, Scale, |
1110 | DAG); |
1111 | assert(Res && "Expanding DIVFIX with wide type failed?" ); |
1112 | if (Saturating) { |
1113 | // If the caller has told us to saturate at something less, use that width |
1114 | // instead of the type before doubling. However, it cannot be more than |
1115 | // what we just widened! |
1116 | assert(SatW <= VTSize && |
1117 | "Tried to saturate to more than the original type?" ); |
1118 | Res = SaturateWidenedDIVFIX(V: Res, dl, SatW: SatW == 0 ? VTSize : SatW, Signed, |
1119 | TLI, DAG); |
1120 | } |
1121 | return DAG.getZExtOrTrunc(Op: Res, DL: dl, VT); |
1122 | } |
1123 | |
1124 | SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { |
1125 | SDLoc dl(N); |
1126 | SDValue Op1Promoted, Op2Promoted; |
1127 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
1128 | N->getOpcode() == ISD::SDIVFIXSAT; |
1129 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
1130 | N->getOpcode() == ISD::UDIVFIXSAT; |
1131 | if (Signed) { |
1132 | Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1133 | Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1134 | } else { |
1135 | Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1136 | Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1137 | } |
1138 | EVT PromotedType = Op1Promoted.getValueType(); |
1139 | unsigned Scale = N->getConstantOperandVal(Num: 2); |
1140 | |
1141 | // If the type is already legal and the operation is legal in that type, we |
1142 | // should not early expand. |
1143 | if (TLI.isTypeLegal(VT: PromotedType)) { |
1144 | TargetLowering::LegalizeAction Action = |
1145 | TLI.getFixedPointOperationAction(Op: N->getOpcode(), VT: PromotedType, Scale); |
1146 | if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { |
1147 | unsigned Diff = PromotedType.getScalarSizeInBits() - |
1148 | N->getValueType(ResNo: 0).getScalarSizeInBits(); |
1149 | if (Saturating) |
1150 | Op1Promoted = |
1151 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, |
1152 | N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl)); |
1153 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, |
1154 | N2: Op2Promoted, N3: N->getOperand(Num: 2)); |
1155 | if (Saturating) |
1156 | Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT: PromotedType, N1: Res, |
1157 | N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl)); |
1158 | return Res; |
1159 | } |
1160 | } |
1161 | |
1162 | // See if we can perform the division in this type without expanding. |
1163 | if (SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: Op1Promoted, |
1164 | RHS: Op2Promoted, Scale, DAG)) { |
1165 | if (Saturating) |
1166 | Res = SaturateWidenedDIVFIX(V: Res, dl, |
1167 | SatW: N->getValueType(ResNo: 0).getScalarSizeInBits(), |
1168 | Signed, TLI, DAG); |
1169 | return Res; |
1170 | } |
1171 | // If we cannot, expand it to twice the type width. If we are saturating, give |
1172 | // it the original width as a saturating width so we don't need to emit |
1173 | // two saturations. |
1174 | return earlyExpandDIVFIX(N, LHS: Op1Promoted, RHS: Op2Promoted, Scale, TLI, DAG, |
1175 | SatW: N->getValueType(ResNo: 0).getScalarSizeInBits()); |
1176 | } |
1177 | |
1178 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
1179 | if (ResNo == 1) |
1180 | return PromoteIntRes_Overflow(N); |
1181 | |
1182 | // The operation overflowed iff the result in the larger type is not the |
1183 | // sign extension of its truncation to the original type. |
1184 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1185 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1186 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
1187 | EVT NVT = LHS.getValueType(); |
1188 | SDLoc dl(N); |
1189 | |
1190 | // Do the arithmetic in the larger type. |
1191 | unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; |
1192 | SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS); |
1193 | |
1194 | // Calculate the overflow flag: sign extend the arithmetic result from |
1195 | // the original type. |
1196 | SDValue Ofl = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res, |
1197 | N2: DAG.getValueType(OVT)); |
1198 | // Overflowed if and only if this is not equal to Res. |
1199 | Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE); |
1200 | |
1201 | // Use the calculated overflow everywhere. |
1202 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
1203 | |
1204 | return Res; |
1205 | } |
1206 | |
1207 | SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { |
1208 | SDValue Mask = N->getOperand(Num: 0); |
1209 | |
1210 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1211 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
1212 | |
1213 | unsigned Opcode = N->getOpcode(); |
1214 | if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) |
1215 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS, |
1216 | N4: N->getOperand(Num: 3)); |
1217 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS); |
1218 | } |
1219 | |
1220 | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
1221 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
1222 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 3)); |
1223 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), |
1224 | VT: LHS.getValueType(), N1: N->getOperand(Num: 0), |
1225 | N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4)); |
1226 | } |
1227 | |
1228 | SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
1229 | unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
1230 | EVT InVT = N->getOperand(Num: OpNo).getValueType(); |
1231 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1232 | |
1233 | EVT SVT = getSetCCResultType(VT: InVT); |
1234 | |
1235 | // If we got back a type that needs to be promoted, this likely means the |
1236 | // the input type also needs to be promoted. So get the promoted type for |
1237 | // the input and try the query again. |
1238 | if (getTypeAction(VT: SVT) == TargetLowering::TypePromoteInteger) { |
1239 | if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) { |
1240 | InVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT); |
1241 | SVT = getSetCCResultType(VT: InVT); |
1242 | } else { |
1243 | // Input type isn't promoted, just use the default promoted type. |
1244 | SVT = NVT; |
1245 | } |
1246 | } |
1247 | |
1248 | SDLoc dl(N); |
1249 | assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && |
1250 | "Vector compare must return a vector result!" ); |
1251 | |
1252 | // Get the SETCC result using the canonical SETCC type. |
1253 | SDValue SetCC; |
1254 | if (N->isStrictFPOpcode()) { |
1255 | SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); |
1256 | SDValue Opers[] = {N->getOperand(Num: 0), N->getOperand(Num: 1), |
1257 | N->getOperand(Num: 2), N->getOperand(Num: 3)}; |
1258 | SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, Ops: Opers, Flags: N->getFlags()); |
1259 | // Legalize the chain result - switch anything that used the old chain to |
1260 | // use the new one. |
1261 | ReplaceValueWith(From: SDValue(N, 1), To: SetCC.getValue(R: 1)); |
1262 | } else |
1263 | SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SVT, N1: N->getOperand(Num: 0), |
1264 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), Flags: N->getFlags()); |
1265 | |
1266 | // Convert to the expected type. |
1267 | return DAG.getSExtOrTrunc(Op: SetCC, DL: dl, VT: NVT); |
1268 | } |
1269 | |
1270 | SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { |
1271 | SDLoc DL(N); |
1272 | SDValue Arg = N->getOperand(Num: 0); |
1273 | SDValue Test = N->getOperand(Num: 1); |
1274 | EVT NResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1275 | return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: NResVT, N1: Arg, N2: Test); |
1276 | } |
1277 | |
1278 | SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { |
1279 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
1280 | EVT VT = N->getValueType(ResNo: 0); |
1281 | |
1282 | SDLoc dl(N); |
1283 | SDValue Res = |
1284 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: NVT), N: N->getOperand(Num: 0)); |
1285 | |
1286 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1287 | return Res.getValue(R: 1); |
1288 | } |
1289 | |
1290 | SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
1291 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1292 | SDValue RHS = N->getOperand(Num: 1); |
1293 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1294 | RHS = ZExtPromotedInteger(Op: RHS); |
1295 | if (N->getOpcode() != ISD::VP_SHL) |
1296 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1297 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1298 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1299 | } |
1300 | |
1301 | SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
1302 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1303 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), |
1304 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
1305 | } |
1306 | |
1307 | SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
1308 | // The input may have strange things in the top bits of the registers, but |
1309 | // these operations don't care. They may have weird bits going out, but |
1310 | // that too is okay if they are integer operations. |
1311 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1312 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1313 | if (N->getNumOperands() == 2) |
1314 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1315 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1316 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1317 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1318 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1319 | } |
1320 | |
1321 | SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
1322 | // Sign extend the input. |
1323 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1324 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1325 | if (N->getNumOperands() == 2) |
1326 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1327 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1328 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1329 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1330 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1331 | } |
1332 | |
1333 | SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
1334 | // Zero extend the input. |
1335 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1336 | SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1337 | if (N->getNumOperands() == 2) |
1338 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1339 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1340 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1341 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1342 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1343 | } |
1344 | |
1345 | SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { |
1346 | // It doesn't matter if we sign extend or zero extend in the inputs. So do |
1347 | // whatever is best for the target. |
1348 | SDValue LHS = SExtOrZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1349 | SDValue RHS = SExtOrZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1350 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), |
1351 | VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1352 | } |
1353 | |
1354 | SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
1355 | // The input value must be properly sign extended. |
1356 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1357 | SDValue RHS = N->getOperand(Num: 1); |
1358 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1359 | RHS = ZExtPromotedInteger(Op: RHS); |
1360 | if (N->getOpcode() != ISD::VP_ASHR) |
1361 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1362 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1363 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1364 | } |
1365 | |
1366 | SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
1367 | // The input value must be properly zero extended. |
1368 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1369 | SDValue RHS = N->getOperand(Num: 1); |
1370 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1371 | RHS = ZExtPromotedInteger(Op: RHS); |
1372 | if (N->getOpcode() != ISD::VP_LSHR) |
1373 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1374 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1375 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1376 | } |
1377 | |
1378 | SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { |
1379 | // Lower the rotate to shifts and ORs which can be promoted. |
1380 | SDValue Res = TLI.expandROT(N, AllowVectorOps: true /*AllowVectorOps*/, DAG); |
1381 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1382 | return SDValue(); |
1383 | } |
1384 | |
1385 | SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { |
1386 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1387 | SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1388 | SDValue Amt = N->getOperand(Num: 2); |
1389 | if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
1390 | Amt = ZExtPromotedInteger(Op: Amt); |
1391 | EVT AmtVT = Amt.getValueType(); |
1392 | |
1393 | SDLoc DL(N); |
1394 | EVT OldVT = N->getOperand(Num: 0).getValueType(); |
1395 | EVT VT = Lo.getValueType(); |
1396 | unsigned Opcode = N->getOpcode(); |
1397 | bool IsFSHR = Opcode == ISD::FSHR; |
1398 | unsigned OldBits = OldVT.getScalarSizeInBits(); |
1399 | unsigned NewBits = VT.getScalarSizeInBits(); |
1400 | |
1401 | // Amount has to be interpreted modulo the old bit width. |
1402 | Amt = DAG.getNode(Opcode: ISD::UREM, DL, VT: AmtVT, N1: Amt, |
1403 | N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT)); |
1404 | |
1405 | // If the promoted type is twice the size (or more), then we use the |
1406 | // traditional funnel 'double' shift codegen. This isn't necessary if the |
1407 | // shift amount is constant. |
1408 | // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
1409 | // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
1410 | if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) && |
1411 | !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) { |
1412 | SDValue HiShift = DAG.getConstant(Val: OldBits, DL, VT); |
1413 | Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: HiShift); |
1414 | Lo = DAG.getZeroExtendInReg(Op: Lo, DL, VT: OldVT); |
1415 | SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Hi, N2: Lo); |
1416 | Res = DAG.getNode(Opcode: IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, N1: Res, N2: Amt); |
1417 | if (!IsFSHR) |
1418 | Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res, N2: HiShift); |
1419 | return Res; |
1420 | } |
1421 | |
1422 | // Shift Lo up to occupy the upper bits of the promoted type. |
1423 | SDValue ShiftOffset = DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT); |
1424 | Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShiftOffset); |
1425 | |
1426 | // Increase Amount to shift the result into the lower bits of the promoted |
1427 | // type. |
1428 | if (IsFSHR) |
1429 | Amt = DAG.getNode(Opcode: ISD::ADD, DL, VT: AmtVT, N1: Amt, N2: ShiftOffset); |
1430 | |
1431 | return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt); |
1432 | } |
1433 | |
1434 | // A vp version of PromoteIntRes_FunnelShift. |
1435 | SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) { |
1436 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1437 | SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1438 | SDValue Amt = N->getOperand(Num: 2); |
1439 | SDValue Mask = N->getOperand(Num: 3); |
1440 | SDValue EVL = N->getOperand(Num: 4); |
1441 | if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
1442 | Amt = ZExtPromotedInteger(Op: Amt); |
1443 | EVT AmtVT = Amt.getValueType(); |
1444 | |
1445 | SDLoc DL(N); |
1446 | EVT OldVT = N->getOperand(Num: 0).getValueType(); |
1447 | EVT VT = Lo.getValueType(); |
1448 | unsigned Opcode = N->getOpcode(); |
1449 | bool IsFSHR = Opcode == ISD::VP_FSHR; |
1450 | unsigned OldBits = OldVT.getScalarSizeInBits(); |
1451 | unsigned NewBits = VT.getScalarSizeInBits(); |
1452 | |
1453 | // Amount has to be interpreted modulo the old bit width. |
1454 | Amt = DAG.getNode(Opcode: ISD::VP_UREM, DL, VT: AmtVT, N1: Amt, |
1455 | N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT), N3: Mask, N4: EVL); |
1456 | |
1457 | // If the promoted type is twice the size (or more), then we use the |
1458 | // traditional funnel 'double' shift codegen. This isn't necessary if the |
1459 | // shift amount is constant. |
1460 | // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
1461 | // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
1462 | if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) && |
1463 | !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) { |
1464 | SDValue HiShift = DAG.getConstant(Val: OldBits, DL, VT); |
1465 | Hi = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Hi, N2: HiShift, N3: Mask, N4: EVL); |
1466 | // FIXME: Replace it by vp operations. |
1467 | Lo = DAG.getZeroExtendInReg(Op: Lo, DL, VT: OldVT); |
1468 | SDValue Res = DAG.getNode(Opcode: ISD::VP_OR, DL, VT, N1: Hi, N2: Lo, N3: Mask, N4: EVL); |
1469 | Res = DAG.getNode(Opcode: IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, N1: Res, N2: Amt, |
1470 | N3: Mask, N4: EVL); |
1471 | if (!IsFSHR) |
1472 | Res = DAG.getNode(Opcode: ISD::VP_LSHR, DL, VT, N1: Res, N2: HiShift, N3: Mask, N4: EVL); |
1473 | return Res; |
1474 | } |
1475 | |
1476 | // Shift Lo up to occupy the upper bits of the promoted type. |
1477 | SDValue ShiftOffset = DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT); |
1478 | Lo = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Lo, N2: ShiftOffset, N3: Mask, N4: EVL); |
1479 | |
1480 | // Increase Amount to shift the result into the lower bits of the promoted |
1481 | // type. |
1482 | if (IsFSHR) |
1483 | Amt = DAG.getNode(Opcode: ISD::VP_ADD, DL, VT: AmtVT, N1: Amt, N2: ShiftOffset, N3: Mask, N4: EVL); |
1484 | |
1485 | return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt, N4: Mask, N5: EVL); |
1486 | } |
1487 | |
1488 | SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
1489 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1490 | SDValue Res; |
1491 | SDValue InOp = N->getOperand(Num: 0); |
1492 | SDLoc dl(N); |
1493 | |
1494 | switch (getTypeAction(VT: InOp.getValueType())) { |
1495 | default: llvm_unreachable("Unknown type action!" ); |
1496 | case TargetLowering::TypeLegal: |
1497 | case TargetLowering::TypeExpandInteger: |
1498 | Res = InOp; |
1499 | break; |
1500 | case TargetLowering::TypePromoteInteger: |
1501 | Res = GetPromotedInteger(Op: InOp); |
1502 | break; |
1503 | case TargetLowering::TypeSplitVector: { |
1504 | EVT InVT = InOp.getValueType(); |
1505 | assert(InVT.isVector() && "Cannot split scalar types" ); |
1506 | ElementCount NumElts = InVT.getVectorElementCount(); |
1507 | assert(NumElts == NVT.getVectorElementCount() && |
1508 | "Dst and Src must have the same number of elements" ); |
1509 | assert(isPowerOf2_32(NumElts.getKnownMinValue()) && |
1510 | "Promoted vector type must be a power of two" ); |
1511 | |
1512 | SDValue EOp1, EOp2; |
1513 | GetSplitVector(Op: InOp, Lo&: EOp1, Hi&: EOp2); |
1514 | |
1515 | EVT HalfNVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getScalarType(), |
1516 | EC: NumElts.divideCoefficientBy(RHS: 2)); |
1517 | if (N->getOpcode() == ISD::TRUNCATE) { |
1518 | EOp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp1); |
1519 | EOp2 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp2); |
1520 | } else { |
1521 | assert(N->getOpcode() == ISD::VP_TRUNCATE && |
1522 | "Expected VP_TRUNCATE opcode" ); |
1523 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
1524 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
1525 | std::tie(args&: EVLLo, args&: EVLHi) = |
1526 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl); |
1527 | EOp1 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp1, N2: MaskLo, N3: EVLLo); |
1528 | EOp2 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp2, N2: MaskHi, N3: EVLHi); |
1529 | } |
1530 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, N1: EOp1, N2: EOp2); |
1531 | } |
1532 | // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some |
1533 | // targets. |
1534 | case TargetLowering::TypeWidenVector: { |
1535 | SDValue WideInOp = GetWidenedVector(Op: InOp); |
1536 | |
1537 | // Truncate widened InOp. |
1538 | unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
1539 | EVT TruncVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
1540 | VT: N->getValueType(ResNo: 0).getScalarType(), NumElements: NumElem); |
1541 | SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncVT, Operand: WideInOp); |
1542 | |
1543 | // Zero extend so that the elements are of same type as those of NVT |
1544 | EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(), |
1545 | NumElements: NumElem); |
1546 | SDValue WideExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: ExtVT, Operand: WideTrunc); |
1547 | |
1548 | // Extract the low NVT subvector. |
1549 | SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: dl); |
1550 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: WideExt, N2: ZeroIdx); |
1551 | } |
1552 | } |
1553 | |
1554 | // Truncate to NVT instead of VT |
1555 | if (N->getOpcode() == ISD::VP_TRUNCATE) |
1556 | return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: NVT, N1: Res, N2: N->getOperand(Num: 1), |
1557 | N3: N->getOperand(Num: 2)); |
1558 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Res); |
1559 | } |
1560 | |
1561 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
1562 | if (ResNo == 1) |
1563 | return PromoteIntRes_Overflow(N); |
1564 | |
1565 | // The operation overflowed iff the result in the larger type is not the |
1566 | // zero extension of its truncation to the original type. |
1567 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1568 | SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1569 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
1570 | EVT NVT = LHS.getValueType(); |
1571 | SDLoc dl(N); |
1572 | |
1573 | // Do the arithmetic in the larger type. |
1574 | unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; |
1575 | SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS); |
1576 | |
1577 | // Calculate the overflow flag: zero extend the arithmetic result from |
1578 | // the original type. |
1579 | SDValue Ofl = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: OVT); |
1580 | // Overflowed if and only if this is not equal to Res. |
1581 | Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE); |
1582 | |
1583 | // Use the calculated overflow everywhere. |
1584 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
1585 | |
1586 | return Res; |
1587 | } |
1588 | |
1589 | // Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that |
1590 | // the third operand of ADDE/SUBE nodes is carry flag, which differs from |
1591 | // the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. |
1592 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, |
1593 | unsigned ResNo) { |
1594 | if (ResNo == 1) |
1595 | return PromoteIntRes_Overflow(N); |
1596 | |
1597 | // We need to sign-extend the operands so the carry value computed by the |
1598 | // wide operation will be equivalent to the carry value computed by the |
1599 | // narrow operation. |
1600 | // An UADDO_CARRY can generate carry only if any of the operands has its |
1601 | // most significant bit set. Sign extension propagates the most significant |
1602 | // bit into the higher bits which means the extra bit that the narrow |
1603 | // addition would need (i.e. the carry) will be propagated through the higher |
1604 | // bits of the wide addition. |
1605 | // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will |
1606 | // be preserved by sign extension. |
1607 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1608 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1609 | |
1610 | EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(ResNo: 1)}; |
1611 | |
1612 | // Do the arithmetic in the wide type. |
1613 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: DAG.getVTList(VTs: ValueVTs), |
1614 | N1: LHS, N2: RHS, N3: N->getOperand(Num: 2)); |
1615 | |
1616 | // Update the users of the original carry/borrow value. |
1617 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
1618 | |
1619 | return SDValue(Res.getNode(), 0); |
1620 | } |
1621 | |
1622 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, |
1623 | unsigned ResNo) { |
1624 | assert(ResNo == 1 && "Don't know how to promote other results yet." ); |
1625 | return PromoteIntRes_Overflow(N); |
1626 | } |
1627 | |
1628 | SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { |
1629 | EVT OVT = N->getValueType(ResNo: 0); |
1630 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
1631 | |
1632 | // If a larger ABS or SMAX isn't supported by the target, try to expand now. |
1633 | // If we expand later we'll end up sign extending more than just the sra input |
1634 | // in sra+xor+sub expansion. |
1635 | if (!OVT.isVector() && |
1636 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::ABS, VT: NVT) && |
1637 | !TLI.isOperationLegal(Op: ISD::SMAX, VT: NVT)) { |
1638 | if (SDValue Res = TLI.expandABS(N, DAG)) |
1639 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Res); |
1640 | } |
1641 | |
1642 | SDValue Op0 = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1643 | return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(N), VT: Op0.getValueType(), Operand: Op0); |
1644 | } |
1645 | |
1646 | SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
1647 | // Promote the overflow bit trivially. |
1648 | if (ResNo == 1) |
1649 | return PromoteIntRes_Overflow(N); |
1650 | |
1651 | SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1); |
1652 | SDLoc DL(N); |
1653 | EVT SmallVT = LHS.getValueType(); |
1654 | |
1655 | // To determine if the result overflowed in a larger type, we extend the |
1656 | // input to the larger type, do the multiply (checking if it overflows), |
1657 | // then also check the high bits of the result to see if overflow happened |
1658 | // there. |
1659 | if (N->getOpcode() == ISD::SMULO) { |
1660 | LHS = SExtPromotedInteger(Op: LHS); |
1661 | RHS = SExtPromotedInteger(Op: RHS); |
1662 | } else { |
1663 | LHS = ZExtPromotedInteger(Op: LHS); |
1664 | RHS = ZExtPromotedInteger(Op: RHS); |
1665 | } |
1666 | SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: N->getValueType(ResNo: 1)); |
1667 | SDValue Mul = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N1: LHS, N2: RHS); |
1668 | |
1669 | // Overflow occurred if it occurred in the larger type, or if the high part |
1670 | // of the result does not zero/sign-extend the low part. Check this second |
1671 | // possibility first. |
1672 | SDValue Overflow; |
1673 | if (N->getOpcode() == ISD::UMULO) { |
1674 | // Unsigned overflow occurred if the high part is non-zero. |
1675 | unsigned Shift = SmallVT.getScalarSizeInBits(); |
1676 | SDValue Hi = |
1677 | DAG.getNode(Opcode: ISD::SRL, DL, VT: Mul.getValueType(), N1: Mul, |
1678 | N2: DAG.getShiftAmountConstant(Val: Shift, VT: Mul.getValueType(), DL)); |
1679 | Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Hi, |
1680 | RHS: DAG.getConstant(Val: 0, DL, VT: Hi.getValueType()), |
1681 | Cond: ISD::SETNE); |
1682 | } else { |
1683 | // Signed overflow occurred if the high part does not sign extend the low. |
1684 | SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: Mul.getValueType(), |
1685 | N1: Mul, N2: DAG.getValueType(SmallVT)); |
1686 | Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: SExt, RHS: Mul, Cond: ISD::SETNE); |
1687 | } |
1688 | |
1689 | // The only other way for overflow to occur is if the multiplication in the |
1690 | // larger type itself overflowed. |
1691 | Overflow = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 1), N1: Overflow, |
1692 | N2: SDValue(Mul.getNode(), 1)); |
1693 | |
1694 | // Use the calculated overflow everywhere. |
1695 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
1696 | return Mul; |
1697 | } |
1698 | |
1699 | SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
1700 | return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
1701 | VT: N->getValueType(ResNo: 0))); |
1702 | } |
1703 | |
1704 | SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { |
1705 | EVT VT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1706 | |
1707 | const APInt &MulImm = N->getConstantOperandAPInt(Num: 0); |
1708 | return DAG.getVScale(DL: SDLoc(N), VT, MulImm: MulImm.sext(width: VT.getSizeInBits())); |
1709 | } |
1710 | |
1711 | SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
1712 | SDValue Chain = N->getOperand(Num: 0); // Get the chain. |
1713 | SDValue Ptr = N->getOperand(Num: 1); // Get the pointer. |
1714 | EVT VT = N->getValueType(ResNo: 0); |
1715 | SDLoc dl(N); |
1716 | |
1717 | MVT RegVT = TLI.getRegisterType(Context&: *DAG.getContext(), VT); |
1718 | unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT); |
1719 | // The argument is passed as NumRegs registers of type RegVT. |
1720 | |
1721 | SmallVector<SDValue, 8> Parts(NumRegs); |
1722 | for (unsigned i = 0; i < NumRegs; ++i) { |
1723 | Parts[i] = DAG.getVAArg(VT: RegVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2), |
1724 | Align: N->getConstantOperandVal(Num: 3)); |
1725 | Chain = Parts[i].getValue(R: 1); |
1726 | } |
1727 | |
1728 | // Handle endianness of the load. |
1729 | if (DAG.getDataLayout().isBigEndian()) |
1730 | std::reverse(first: Parts.begin(), last: Parts.end()); |
1731 | |
1732 | // Assemble the parts in the promoted type. |
1733 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1734 | SDValue Res = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[0]); |
1735 | for (unsigned i = 1; i < NumRegs; ++i) { |
1736 | SDValue Part = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[i]); |
1737 | // Shift it to the right position and "or" it in. |
1738 | Part = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Part, |
1739 | N2: DAG.getConstant(Val: i * RegVT.getSizeInBits(), DL: dl, |
1740 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
1741 | Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Res, N2: Part); |
1742 | } |
1743 | |
1744 | // Modified the chain result - switch anything that used the old chain to |
1745 | // use the new one. |
1746 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1747 | |
1748 | return Res; |
1749 | } |
1750 | |
1751 | //===----------------------------------------------------------------------===// |
1752 | // Integer Operand Promotion |
1753 | //===----------------------------------------------------------------------===// |
1754 | |
1755 | /// PromoteIntegerOperand - This method is called when the specified operand of |
1756 | /// the specified node is found to need promotion. At this point, all of the |
1757 | /// result types of the node are known to be legal, but other operands of the |
1758 | /// node may need promotion or expansion as well as the specified one. |
1759 | bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
1760 | LLVM_DEBUG(dbgs() << "Promote integer operand: " ; N->dump(&DAG)); |
1761 | SDValue Res = SDValue(); |
1762 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) { |
1763 | LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n" ); |
1764 | return false; |
1765 | } |
1766 | |
1767 | switch (N->getOpcode()) { |
1768 | default: |
1769 | #ifndef NDEBUG |
1770 | dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": " ; |
1771 | N->dump(G: &DAG); dbgs() << "\n" ; |
1772 | #endif |
1773 | report_fatal_error(reason: "Do not know how to promote this operator's operand!" ); |
1774 | |
1775 | case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
1776 | case ISD::ATOMIC_STORE: |
1777 | Res = PromoteIntOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N)); |
1778 | break; |
1779 | case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
1780 | case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
1781 | case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
1782 | case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
1783 | case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
1784 | case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
1785 | case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
1786 | case ISD::INSERT_VECTOR_ELT: |
1787 | Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); |
1788 | break; |
1789 | case ISD::SPLAT_VECTOR: |
1790 | case ISD::SCALAR_TO_VECTOR: |
1791 | Res = PromoteIntOp_ScalarOp(N); |
1792 | break; |
1793 | case ISD::VSELECT: |
1794 | case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
1795 | case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
1796 | case ISD::VP_SETCC: |
1797 | case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
1798 | case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
1799 | case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; |
1800 | case ISD::VP_SINT_TO_FP: |
1801 | case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
1802 | case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; |
1803 | case ISD::STORE: Res = PromoteIntOp_STORE(N: cast<StoreSDNode>(Val: N), |
1804 | OpNo); break; |
1805 | case ISD::MSTORE: Res = PromoteIntOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), |
1806 | OpNo); break; |
1807 | case ISD::MLOAD: Res = PromoteIntOp_MLOAD(N: cast<MaskedLoadSDNode>(Val: N), |
1808 | OpNo); break; |
1809 | case ISD::MGATHER: Res = PromoteIntOp_MGATHER(N: cast<MaskedGatherSDNode>(Val: N), |
1810 | OpNo); break; |
1811 | case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(N: cast<MaskedScatterSDNode>(Val: N), |
1812 | OpNo); break; |
1813 | case ISD::VP_TRUNCATE: |
1814 | case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
1815 | case ISD::BF16_TO_FP: |
1816 | case ISD::FP16_TO_FP: |
1817 | case ISD::VP_UINT_TO_FP: |
1818 | case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
1819 | case ISD::STRICT_FP16_TO_FP: |
1820 | case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; |
1821 | case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
1822 | case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; |
1823 | case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
1824 | case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; |
1825 | |
1826 | case ISD::SHL: |
1827 | case ISD::SRA: |
1828 | case ISD::SRL: |
1829 | case ISD::ROTL: |
1830 | case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
1831 | |
1832 | case ISD::FSHL: |
1833 | case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; |
1834 | |
1835 | case ISD::SADDO_CARRY: |
1836 | case ISD::SSUBO_CARRY: |
1837 | case ISD::UADDO_CARRY: |
1838 | case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break; |
1839 | |
1840 | case ISD::FRAMEADDR: |
1841 | case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; |
1842 | |
1843 | case ISD::SMULFIX: |
1844 | case ISD::SMULFIXSAT: |
1845 | case ISD::UMULFIX: |
1846 | case ISD::UMULFIXSAT: |
1847 | case ISD::SDIVFIX: |
1848 | case ISD::SDIVFIXSAT: |
1849 | case ISD::UDIVFIX: |
1850 | case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; |
1851 | case ISD::FPOWI: |
1852 | case ISD::STRICT_FPOWI: |
1853 | case ISD::FLDEXP: |
1854 | case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; |
1855 | case ISD::VECREDUCE_ADD: |
1856 | case ISD::VECREDUCE_MUL: |
1857 | case ISD::VECREDUCE_AND: |
1858 | case ISD::VECREDUCE_OR: |
1859 | case ISD::VECREDUCE_XOR: |
1860 | case ISD::VECREDUCE_SMAX: |
1861 | case ISD::VECREDUCE_SMIN: |
1862 | case ISD::VECREDUCE_UMAX: |
1863 | case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; |
1864 | case ISD::VP_REDUCE_ADD: |
1865 | case ISD::VP_REDUCE_MUL: |
1866 | case ISD::VP_REDUCE_AND: |
1867 | case ISD::VP_REDUCE_OR: |
1868 | case ISD::VP_REDUCE_XOR: |
1869 | case ISD::VP_REDUCE_SMAX: |
1870 | case ISD::VP_REDUCE_SMIN: |
1871 | case ISD::VP_REDUCE_UMAX: |
1872 | case ISD::VP_REDUCE_UMIN: |
1873 | Res = PromoteIntOp_VP_REDUCE(N, OpNo); |
1874 | break; |
1875 | |
1876 | case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; |
1877 | case ISD::STACKMAP: |
1878 | Res = PromoteIntOp_STACKMAP(N, OpNo); |
1879 | break; |
1880 | case ISD::PATCHPOINT: |
1881 | Res = PromoteIntOp_PATCHPOINT(N, OpNo); |
1882 | break; |
1883 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
1884 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
1885 | Res = PromoteIntOp_VP_STRIDED(N, OpNo); |
1886 | break; |
1887 | case ISD::EXPERIMENTAL_VP_SPLICE: |
1888 | Res = PromoteIntOp_VP_SPLICE(N, OpNo); |
1889 | break; |
1890 | } |
1891 | |
1892 | // If the result is null, the sub-method took care of registering results etc. |
1893 | if (!Res.getNode()) return false; |
1894 | |
1895 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
1896 | // core about this. |
1897 | if (Res.getNode() == N) |
1898 | return true; |
1899 | |
1900 | const bool IsStrictFp = N->isStrictFPOpcode(); |
1901 | assert(Res.getValueType() == N->getValueType(0) && |
1902 | N->getNumValues() == (IsStrictFp ? 2 : 1) && |
1903 | "Invalid operand expansion" ); |
1904 | LLVM_DEBUG(dbgs() << "Replacing: " ; N->dump(&DAG); dbgs() << " with: " ; |
1905 | Res.dump()); |
1906 | |
1907 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1908 | if (IsStrictFp) |
1909 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1)); |
1910 | |
1911 | return false; |
1912 | } |
1913 | |
1914 | /// PromoteSetCCOperands - Promote the operands of a comparison. This code is |
1915 | /// shared among BR_CC, SELECT_CC, and SETCC handlers. |
1916 | void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, |
1917 | ISD::CondCode CCCode) { |
1918 | // We have to insert explicit sign or zero extends. Note that we could |
1919 | // insert sign extends for ALL conditions. For those operations where either |
1920 | // zero or sign extension would be valid, we ask the target which extension |
1921 | // it would prefer. |
1922 | |
1923 | // Signed comparisons always require sign extension. |
1924 | if (ISD::isSignedIntSetCC(Code: CCCode)) { |
1925 | LHS = SExtPromotedInteger(Op: LHS); |
1926 | RHS = SExtPromotedInteger(Op: RHS); |
1927 | return; |
1928 | } |
1929 | |
1930 | assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && |
1931 | "Unknown integer comparison!" ); |
1932 | |
1933 | SDValue OpL = GetPromotedInteger(Op: LHS); |
1934 | SDValue OpR = GetPromotedInteger(Op: RHS); |
1935 | |
1936 | if (TLI.isSExtCheaperThanZExt(FromTy: LHS.getValueType(), ToTy: OpL.getValueType())) { |
1937 | // The target would prefer to promote the comparison operand with sign |
1938 | // extension. Honor that unless the promoted values are already zero |
1939 | // extended. |
1940 | unsigned OpLEffectiveBits = |
1941 | DAG.computeKnownBits(Op: OpL).countMaxActiveBits(); |
1942 | unsigned OpREffectiveBits = |
1943 | DAG.computeKnownBits(Op: OpR).countMaxActiveBits(); |
1944 | if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
1945 | OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
1946 | LHS = OpL; |
1947 | RHS = OpR; |
1948 | return; |
1949 | } |
1950 | |
1951 | // The promoted values aren't zero extended, use a sext_inreg. |
1952 | LHS = SExtPromotedInteger(Op: LHS); |
1953 | RHS = SExtPromotedInteger(Op: RHS); |
1954 | return; |
1955 | } |
1956 | |
1957 | // Prefer to promote the comparison operand with zero extension. |
1958 | |
1959 | // If the width of OpL/OpR excluding the duplicated sign bits is no greater |
1960 | // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation |
1961 | // that we might not be able to remove. |
1962 | unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpL); |
1963 | unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpR); |
1964 | if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
1965 | OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
1966 | LHS = OpL; |
1967 | RHS = OpR; |
1968 | return; |
1969 | } |
1970 | |
1971 | // Otherwise, use zext_inreg. |
1972 | LHS = ZExtPromotedInteger(Op: LHS); |
1973 | RHS = ZExtPromotedInteger(Op: RHS); |
1974 | } |
1975 | |
1976 | SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
1977 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1978 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
1979 | } |
1980 | |
1981 | SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
1982 | SDValue Op1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1983 | return DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(), |
1984 | Chain: N->getChain(), Ptr: Op1, Val: N->getBasePtr(), MMO: N->getMemOperand()); |
1985 | } |
1986 | |
1987 | SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
1988 | // This should only occur in unusual situations like bitcasting to an |
1989 | // x86_fp80, so just turn it into a store+load |
1990 | return CreateStackStoreLoad(Op: N->getOperand(Num: 0), DestVT: N->getValueType(ResNo: 0)); |
1991 | } |
1992 | |
1993 | SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
1994 | assert(OpNo == 2 && "Don't know how to promote this operand!" ); |
1995 | |
1996 | SDValue LHS = N->getOperand(Num: 2); |
1997 | SDValue RHS = N->getOperand(Num: 3); |
1998 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get()); |
1999 | |
2000 | // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always |
2001 | // legal types. |
2002 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2003 | Op2: N->getOperand(Num: 1), Op3: LHS, Op4: RHS, Op5: N->getOperand(Num: 4)), |
2004 | 0); |
2005 | } |
2006 | |
2007 | SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
2008 | assert(OpNo == 1 && "only know how to promote condition" ); |
2009 | |
2010 | // Promote all the way up to the canonical SetCC type. |
2011 | SDValue Cond = PromoteTargetBoolean(Bool: N->getOperand(Num: 1), MVT::ValVT: Other); |
2012 | |
2013 | // The chain (Op#0) and basic block destination (Op#2) are always legal types. |
2014 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Cond, |
2015 | Op3: N->getOperand(Num: 2)), 0); |
2016 | } |
2017 | |
2018 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
2019 | // Since the result type is legal, the operands must promote to it. |
2020 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
2021 | SDValue Lo = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2022 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
2023 | assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?" ); |
2024 | SDLoc dl(N); |
2025 | |
2026 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: Hi, |
2027 | N2: DAG.getConstant(Val: OVT.getSizeInBits(), DL: dl, |
2028 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
2029 | return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
2030 | } |
2031 | |
2032 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
2033 | // The vector type is legal but the element type is not. This implies |
2034 | // that the vector is a power-of-two in length and that the element |
2035 | // type does not have a strange size (eg: it is not i1). |
2036 | EVT VecVT = N->getValueType(ResNo: 0); |
2037 | unsigned NumElts = VecVT.getVectorNumElements(); |
2038 | assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
2039 | "Legal vector of one illegal element?" ); |
2040 | |
2041 | // Promote the inserted value. The type does not need to match the |
2042 | // vector element type. Check that any extra bits introduced will be |
2043 | // truncated away. |
2044 | assert(N->getOperand(0).getValueSizeInBits() >= |
2045 | N->getValueType(0).getScalarSizeInBits() && |
2046 | "Type of inserted value narrower than vector element type!" ); |
2047 | |
2048 | SmallVector<SDValue, 16> NewOps; |
2049 | for (unsigned i = 0; i < NumElts; ++i) |
2050 | NewOps.push_back(Elt: GetPromotedInteger(Op: N->getOperand(Num: i))); |
2051 | |
2052 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2053 | } |
2054 | |
2055 | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
2056 | unsigned OpNo) { |
2057 | if (OpNo == 1) { |
2058 | // Promote the inserted value. This is valid because the type does not |
2059 | // have to match the vector element type. |
2060 | |
2061 | // Check that any extra bits introduced will be truncated away. |
2062 | assert(N->getOperand(1).getValueSizeInBits() >= |
2063 | N->getValueType(0).getScalarSizeInBits() && |
2064 | "Type of inserted value narrower than vector element type!" ); |
2065 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2066 | Op2: GetPromotedInteger(Op: N->getOperand(Num: 1)), |
2067 | Op3: N->getOperand(Num: 2)), |
2068 | 0); |
2069 | } |
2070 | |
2071 | assert(OpNo == 2 && "Different operand and result vector types?" ); |
2072 | |
2073 | // Promote the index. |
2074 | SDValue Idx = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 2), DL: SDLoc(N), |
2075 | VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())); |
2076 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2077 | Op2: N->getOperand(Num: 1), Op3: Idx), 0); |
2078 | } |
2079 | |
2080 | SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { |
2081 | // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, |
2082 | // so just promote the operand in place. |
2083 | return SDValue(DAG.UpdateNodeOperands(N, |
2084 | Op: GetPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2085 | } |
2086 | |
2087 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
2088 | assert(OpNo == 0 && "Only know how to promote the condition!" ); |
2089 | SDValue Cond = N->getOperand(Num: 0); |
2090 | EVT OpTy = N->getOperand(Num: 1).getValueType(); |
2091 | |
2092 | if (N->getOpcode() == ISD::VSELECT) |
2093 | if (SDValue Res = WidenVSELECTMask(N)) |
2094 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), |
2095 | N1: Res, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
2096 | |
2097 | // Promote all the way up to the canonical SetCC type. |
2098 | EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; |
2099 | Cond = PromoteTargetBoolean(Bool: Cond, ValVT: OpVT); |
2100 | |
2101 | return SDValue(DAG.UpdateNodeOperands(N, Op1: Cond, Op2: N->getOperand(Num: 1), |
2102 | Op3: N->getOperand(Num: 2)), 0); |
2103 | } |
2104 | |
2105 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
2106 | assert(OpNo == 0 && "Don't know how to promote this operand!" ); |
2107 | |
2108 | SDValue LHS = N->getOperand(Num: 0); |
2109 | SDValue RHS = N->getOperand(Num: 1); |
2110 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get()); |
2111 | |
2112 | // The CC (#4) and the possible return values (#2 and #3) have legal types. |
2113 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2), |
2114 | Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)), 0); |
2115 | } |
2116 | |
2117 | SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
2118 | assert(OpNo == 0 && "Don't know how to promote this operand!" ); |
2119 | |
2120 | SDValue LHS = N->getOperand(Num: 0); |
2121 | SDValue RHS = N->getOperand(Num: 1); |
2122 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get()); |
2123 | |
2124 | // The CC (#2) is always legal. |
2125 | if (N->getOpcode() == ISD::SETCC) |
2126 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2)), 0); |
2127 | |
2128 | assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode" ); |
2129 | |
2130 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2), |
2131 | Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)), |
2132 | 0); |
2133 | } |
2134 | |
2135 | SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
2136 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2137 | Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2138 | } |
2139 | |
2140 | SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { |
2141 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1), |
2142 | Op3: ZExtPromotedInteger(Op: N->getOperand(Num: 2))), 0); |
2143 | } |
2144 | |
2145 | SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
2146 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2147 | SDLoc dl(N); |
2148 | Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op); |
2149 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Op.getValueType(), |
2150 | N1: Op, N2: DAG.getValueType(N->getOperand(Num: 0).getValueType())); |
2151 | } |
2152 | |
2153 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { |
2154 | SDLoc dl(N); |
2155 | EVT VT = N->getValueType(ResNo: 0); |
2156 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2157 | // FIXME: There is no VP_ANY_EXTEND yet. |
2158 | Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), |
2159 | N3: N->getOperand(Num: 2)); |
2160 | unsigned Diff = |
2161 | VT.getScalarSizeInBits() - N->getOperand(Num: 0).getScalarValueSizeInBits(); |
2162 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: Diff, VT, DL: dl); |
2163 | // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. |
2164 | SDValue Shl = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: ShAmt, N3: N->getOperand(Num: 1), |
2165 | N4: N->getOperand(Num: 2)); |
2166 | return DAG.getNode(Opcode: ISD::VP_ASHR, DL: dl, VT, N1: Shl, N2: ShAmt, N3: N->getOperand(Num: 1), |
2167 | N4: N->getOperand(Num: 2)); |
2168 | } |
2169 | |
2170 | SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
2171 | if (N->getOpcode() == ISD::VP_SINT_TO_FP) |
2172 | return SDValue(DAG.UpdateNodeOperands(N, |
2173 | Op1: SExtPromotedInteger(Op: N->getOperand(Num: 0)), |
2174 | Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)), |
2175 | 0); |
2176 | return SDValue(DAG.UpdateNodeOperands(N, |
2177 | Op: SExtPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2178 | } |
2179 | |
2180 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { |
2181 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2182 | Op2: SExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2183 | } |
2184 | |
2185 | SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
2186 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
2187 | SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
2188 | SDLoc dl(N); |
2189 | |
2190 | SDValue Val = GetPromotedInteger(Op: N->getValue()); // Get promoted value. |
2191 | |
2192 | // Truncate the value and store the result. |
2193 | return DAG.getTruncStore(Chain: Ch, dl, Val, Ptr, |
2194 | SVT: N->getMemoryVT(), MMO: N->getMemOperand()); |
2195 | } |
2196 | |
2197 | SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
2198 | unsigned OpNo) { |
2199 | SDValue DataOp = N->getValue(); |
2200 | SDValue Mask = N->getMask(); |
2201 | |
2202 | if (OpNo == 4) { |
2203 | // The Mask. Update in place. |
2204 | EVT DataVT = DataOp.getValueType(); |
2205 | Mask = PromoteTargetBoolean(Bool: Mask, ValVT: DataVT); |
2206 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2207 | NewOps[4] = Mask; |
2208 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2209 | } |
2210 | |
2211 | assert(OpNo == 1 && "Unexpected operand for promotion" ); |
2212 | DataOp = GetPromotedInteger(Op: DataOp); |
2213 | |
2214 | return DAG.getMaskedStore(Chain: N->getChain(), dl: SDLoc(N), Val: DataOp, Base: N->getBasePtr(), |
2215 | Offset: N->getOffset(), Mask, MemVT: N->getMemoryVT(), |
2216 | MMO: N->getMemOperand(), AM: N->getAddressingMode(), |
2217 | /*IsTruncating*/ true, IsCompressing: N->isCompressingStore()); |
2218 | } |
2219 | |
2220 | SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
2221 | unsigned OpNo) { |
2222 | assert(OpNo == 3 && "Only know how to promote the mask!" ); |
2223 | EVT DataVT = N->getValueType(ResNo: 0); |
2224 | SDValue Mask = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2225 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2226 | NewOps[OpNo] = Mask; |
2227 | SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps); |
2228 | if (Res == N) |
2229 | return SDValue(Res, 0); |
2230 | |
2231 | // Update triggered CSE, do our own replacement since caller can't. |
2232 | ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0)); |
2233 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1)); |
2234 | return SDValue(); |
2235 | } |
2236 | |
2237 | SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
2238 | unsigned OpNo) { |
2239 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
2240 | |
2241 | if (OpNo == 2) { |
2242 | // The Mask |
2243 | EVT DataVT = N->getValueType(ResNo: 0); |
2244 | NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2245 | } else if (OpNo == 4) { |
2246 | // The Index |
2247 | if (N->isIndexSigned()) |
2248 | // Need to sign extend the index since the bits will likely be used. |
2249 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2250 | else |
2251 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2252 | } else |
2253 | NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2254 | |
2255 | SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps); |
2256 | if (Res == N) |
2257 | return SDValue(Res, 0); |
2258 | |
2259 | // Update triggered CSE, do our own replacement since caller can't. |
2260 | ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0)); |
2261 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1)); |
2262 | return SDValue(); |
2263 | } |
2264 | |
2265 | SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
2266 | unsigned OpNo) { |
2267 | bool TruncateStore = N->isTruncatingStore(); |
2268 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
2269 | |
2270 | if (OpNo == 2) { |
2271 | // The Mask |
2272 | EVT DataVT = N->getValue().getValueType(); |
2273 | NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2274 | } else if (OpNo == 4) { |
2275 | // The Index |
2276 | if (N->isIndexSigned()) |
2277 | // Need to sign extend the index since the bits will likely be used. |
2278 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2279 | else |
2280 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2281 | } else { |
2282 | NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2283 | TruncateStore = true; |
2284 | } |
2285 | |
2286 | return DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: N->getMemoryVT(), |
2287 | dl: SDLoc(N), Ops: NewOps, MMO: N->getMemOperand(), |
2288 | IndexType: N->getIndexType(), IsTruncating: TruncateStore); |
2289 | } |
2290 | |
2291 | SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
2292 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2293 | if (N->getOpcode() == ISD::VP_TRUNCATE) |
2294 | return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op, |
2295 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
2296 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
2297 | } |
2298 | |
2299 | SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
2300 | if (N->getOpcode() == ISD::VP_UINT_TO_FP) |
2301 | return SDValue(DAG.UpdateNodeOperands(N, |
2302 | Op1: ZExtPromotedInteger(Op: N->getOperand(Num: 0)), |
2303 | Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)), |
2304 | 0); |
2305 | return SDValue(DAG.UpdateNodeOperands(N, |
2306 | Op: ZExtPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2307 | } |
2308 | |
2309 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { |
2310 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2311 | Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2312 | } |
2313 | |
2314 | SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
2315 | SDLoc dl(N); |
2316 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2317 | Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op); |
2318 | return DAG.getZeroExtendInReg(Op, DL: dl, VT: N->getOperand(Num: 0).getValueType()); |
2319 | } |
2320 | |
2321 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { |
2322 | SDLoc dl(N); |
2323 | EVT VT = N->getValueType(ResNo: 0); |
2324 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2325 | // FIXME: There is no VP_ANY_EXTEND yet. |
2326 | Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), |
2327 | N3: N->getOperand(Num: 2)); |
2328 | APInt Imm = APInt::getLowBitsSet(numBits: VT.getScalarSizeInBits(), |
2329 | loBitsSet: N->getOperand(Num: 0).getScalarValueSizeInBits()); |
2330 | return DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: Imm, DL: dl, VT), |
2331 | N3: N->getOperand(Num: 1), N4: N->getOperand(Num: 2)); |
2332 | } |
2333 | |
2334 | SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) { |
2335 | assert(OpNo == 2 && "Don't know how to promote this operand!" ); |
2336 | |
2337 | SDValue LHS = N->getOperand(Num: 0); |
2338 | SDValue RHS = N->getOperand(Num: 1); |
2339 | SDValue Carry = N->getOperand(Num: 2); |
2340 | SDLoc DL(N); |
2341 | |
2342 | Carry = PromoteTargetBoolean(Bool: Carry, ValVT: LHS.getValueType()); |
2343 | |
2344 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: Carry), 0); |
2345 | } |
2346 | |
2347 | SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { |
2348 | SDValue Op2 = ZExtPromotedInteger(Op: N->getOperand(Num: 2)); |
2349 | return SDValue( |
2350 | DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1), Op3: Op2), 0); |
2351 | } |
2352 | |
2353 | SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { |
2354 | // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. |
2355 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2356 | return SDValue(DAG.UpdateNodeOperands(N, Op), 0); |
2357 | } |
2358 | |
2359 | SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { |
2360 | bool IsStrict = N->isStrictFPOpcode(); |
2361 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
2362 | |
2363 | bool IsPowI = |
2364 | N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; |
2365 | |
2366 | // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result |
2367 | // and floating point operand is already type legalized). |
2368 | RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)) |
2369 | : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)); |
2370 | |
2371 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC)) { |
2372 | SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
2373 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Op), 0); |
2374 | } |
2375 | |
2376 | // We can't just promote the exponent type in FPOWI, since we want to lower |
2377 | // the node to a libcall and we if we promote to a type larger than |
2378 | // sizeof(int) the libcall might not be according to the targets ABI. Instead |
2379 | // we rewrite to a libcall here directly, letting makeLibCall handle promotion |
2380 | // if the target accepts it according to shouldSignExtendTypeInLibCall. |
2381 | |
2382 | unsigned OpOffset = IsStrict ? 1 : 0; |
2383 | // The exponent should fit in a sizeof(int) type for the libcall to be valid. |
2384 | assert(DAG.getLibInfo().getIntSize() == |
2385 | N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && |
2386 | "POWI exponent should match with sizeof(int) when doing the libcall." ); |
2387 | TargetLowering::MakeLibCallOptions CallOptions; |
2388 | CallOptions.setSExt(true); |
2389 | SDValue Ops[2] = {N->getOperand(Num: 0 + OpOffset), N->getOperand(Num: 1 + OpOffset)}; |
2390 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( |
2391 | DAG, LC, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain); |
2392 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
2393 | if (IsStrict) |
2394 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
2395 | return SDValue(); |
2396 | } |
2397 | |
2398 | static unsigned getExtendForIntVecReduction(SDNode *N) { |
2399 | switch (N->getOpcode()) { |
2400 | default: |
2401 | llvm_unreachable("Expected integer vector reduction" ); |
2402 | case ISD::VECREDUCE_ADD: |
2403 | case ISD::VECREDUCE_MUL: |
2404 | case ISD::VECREDUCE_AND: |
2405 | case ISD::VECREDUCE_OR: |
2406 | case ISD::VECREDUCE_XOR: |
2407 | case ISD::VP_REDUCE_ADD: |
2408 | case ISD::VP_REDUCE_MUL: |
2409 | case ISD::VP_REDUCE_AND: |
2410 | case ISD::VP_REDUCE_OR: |
2411 | case ISD::VP_REDUCE_XOR: |
2412 | return ISD::ANY_EXTEND; |
2413 | case ISD::VECREDUCE_SMAX: |
2414 | case ISD::VECREDUCE_SMIN: |
2415 | case ISD::VP_REDUCE_SMAX: |
2416 | case ISD::VP_REDUCE_SMIN: |
2417 | return ISD::SIGN_EXTEND; |
2418 | case ISD::VECREDUCE_UMAX: |
2419 | case ISD::VECREDUCE_UMIN: |
2420 | case ISD::VP_REDUCE_UMAX: |
2421 | case ISD::VP_REDUCE_UMIN: |
2422 | return ISD::ZERO_EXTEND; |
2423 | } |
2424 | } |
2425 | |
2426 | SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { |
2427 | switch (getExtendForIntVecReduction(N)) { |
2428 | default: |
2429 | llvm_unreachable("Impossible extension kind for integer reduction" ); |
2430 | case ISD::ANY_EXTEND: |
2431 | return GetPromotedInteger(Op: V); |
2432 | case ISD::SIGN_EXTEND: |
2433 | return SExtPromotedInteger(Op: V); |
2434 | case ISD::ZERO_EXTEND: |
2435 | return ZExtPromotedInteger(Op: V); |
2436 | } |
2437 | } |
2438 | |
2439 | SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { |
2440 | SDLoc dl(N); |
2441 | SDValue Op = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0)); |
2442 | |
2443 | EVT OrigEltVT = N->getOperand(Num: 0).getValueType().getVectorElementType(); |
2444 | EVT InVT = Op.getValueType(); |
2445 | EVT EltVT = InVT.getVectorElementType(); |
2446 | EVT ResVT = N->getValueType(ResNo: 0); |
2447 | unsigned Opcode = N->getOpcode(); |
2448 | |
2449 | // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if |
2450 | // vecreduce_xor is not legal |
2451 | if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && |
2452 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_XOR, VT: InVT) && |
2453 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_ADD, VT: InVT)) |
2454 | Opcode = ISD::VECREDUCE_ADD; |
2455 | |
2456 | // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if |
2457 | // vecreduce_or is not legal |
2458 | else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && |
2459 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_OR, VT: InVT) && |
2460 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMAX, VT: InVT)) { |
2461 | Opcode = ISD::VECREDUCE_UMAX; |
2462 | // Can't use promoteTargetBoolean here because we still need |
2463 | // to either sign_ext or zero_ext in the undefined case. |
2464 | switch (TLI.getBooleanContents(Type: InVT)) { |
2465 | case TargetLoweringBase::UndefinedBooleanContent: |
2466 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
2467 | Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2468 | break; |
2469 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
2470 | Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2471 | break; |
2472 | } |
2473 | } |
2474 | |
2475 | // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if |
2476 | // vecreduce_and is not legal |
2477 | else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && |
2478 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_AND, VT: InVT) && |
2479 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMIN, VT: InVT)) { |
2480 | Opcode = ISD::VECREDUCE_UMIN; |
2481 | // Can't use promoteTargetBoolean here because we still need |
2482 | // to either sign_ext or zero_ext in the undefined case. |
2483 | switch (TLI.getBooleanContents(Type: InVT)) { |
2484 | case TargetLoweringBase::UndefinedBooleanContent: |
2485 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
2486 | Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2487 | break; |
2488 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
2489 | Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2490 | break; |
2491 | } |
2492 | } |
2493 | |
2494 | if (ResVT.bitsGE(VT: EltVT)) |
2495 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: ResVT, Operand: Op); |
2496 | |
2497 | // Result size must be >= element size. If this is not the case after |
2498 | // promotion, also promote the result type and then truncate. |
2499 | SDValue Reduce = DAG.getNode(Opcode, DL: dl, VT: EltVT, Operand: Op); |
2500 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Reduce); |
2501 | } |
2502 | |
2503 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { |
2504 | SDLoc DL(N); |
2505 | SDValue Op = N->getOperand(Num: OpNo); |
2506 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2507 | |
2508 | if (OpNo == 2) { // Mask |
2509 | // Update in place. |
2510 | NewOps[2] = PromoteTargetBoolean(Bool: Op, ValVT: N->getOperand(Num: 1).getValueType()); |
2511 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2512 | } |
2513 | |
2514 | assert(OpNo == 1 && "Unexpected operand for promotion" ); |
2515 | |
2516 | Op = PromoteIntOpVectorReduction(N, V: Op); |
2517 | |
2518 | NewOps[OpNo] = Op; |
2519 | |
2520 | EVT VT = N->getValueType(ResNo: 0); |
2521 | EVT EltVT = Op.getValueType().getScalarType(); |
2522 | |
2523 | if (VT.bitsGE(VT: EltVT)) |
2524 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, Ops: NewOps); |
2525 | |
2526 | // Result size must be >= element/start-value size. If this is not the case |
2527 | // after promotion, also promote both the start value and result type and |
2528 | // then truncate. |
2529 | NewOps[0] = |
2530 | DAG.getNode(Opcode: getExtendForIntVecReduction(N), DL, VT: EltVT, Operand: N->getOperand(Num: 0)); |
2531 | SDValue Reduce = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, Ops: NewOps); |
2532 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Reduce); |
2533 | } |
2534 | |
2535 | SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { |
2536 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
2537 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Op), 0); |
2538 | } |
2539 | |
2540 | SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
2541 | assert(OpNo > 1); // Because the first two arguments are guaranteed legal. |
2542 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
2543 | SDValue Operand = N->getOperand(Num: OpNo); |
2544 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Operand.getValueType()); |
2545 | NewOps[OpNo] = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand); |
2546 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2547 | } |
2548 | |
2549 | SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
2550 | assert(OpNo >= 7); |
2551 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
2552 | SDValue Operand = N->getOperand(Num: OpNo); |
2553 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Operand.getValueType()); |
2554 | NewOps[OpNo] = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand); |
2555 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2556 | } |
2557 | |
2558 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
2559 | assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
2560 | (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
2561 | |
2562 | SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); |
2563 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2564 | |
2565 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2566 | } |
2567 | |
2568 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) { |
2569 | SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end()); |
2570 | |
2571 | if (OpNo == 2) { // Offset operand |
2572 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2573 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2574 | } |
2575 | |
2576 | assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion" ); |
2577 | |
2578 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2579 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2580 | } |
2581 | |
2582 | //===----------------------------------------------------------------------===// |
2583 | // Integer Result Expansion |
2584 | //===----------------------------------------------------------------------===// |
2585 | |
2586 | /// ExpandIntegerResult - This method is called when the specified result of the |
2587 | /// specified node is found to need expansion. At this point, the node may also |
2588 | /// have invalid operands or may have other results that need promotion, we just |
2589 | /// know that (at least) one result needs expansion. |
2590 | void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
2591 | LLVM_DEBUG(dbgs() << "Expand integer result: " ; N->dump(&DAG)); |
2592 | SDValue Lo, Hi; |
2593 | Lo = Hi = SDValue(); |
2594 | |
2595 | // See if the target wants to custom expand this node. |
2596 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) |
2597 | return; |
2598 | |
2599 | switch (N->getOpcode()) { |
2600 | default: |
2601 | #ifndef NDEBUG |
2602 | dbgs() << "ExpandIntegerResult #" << ResNo << ": " ; |
2603 | N->dump(G: &DAG); dbgs() << "\n" ; |
2604 | #endif |
2605 | report_fatal_error(reason: "Do not know how to expand the result of this " |
2606 | "operator!" ); |
2607 | |
2608 | case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; |
2609 | case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
2610 | case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; |
2611 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
2612 | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
2613 | case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; |
2614 | |
2615 | case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
2616 | case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
2617 | case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
2618 | case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
2619 | case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
2620 | |
2621 | case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
2622 | case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
2623 | case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
2624 | case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
2625 | case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
2626 | case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; |
2627 | case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
2628 | case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; |
2629 | case ISD::CTLZ_ZERO_UNDEF: |
2630 | case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
2631 | case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
2632 | case ISD::CTTZ_ZERO_UNDEF: |
2633 | case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
2634 | case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; |
2635 | case ISD::STRICT_FP_TO_SINT: |
2636 | case ISD::FP_TO_SINT: |
2637 | case ISD::STRICT_FP_TO_UINT: |
2638 | case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break; |
2639 | case ISD::FP_TO_SINT_SAT: |
2640 | case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; |
2641 | case ISD::STRICT_LROUND: |
2642 | case ISD::STRICT_LRINT: |
2643 | case ISD::LROUND: |
2644 | case ISD::LRINT: |
2645 | case ISD::STRICT_LLROUND: |
2646 | case ISD::STRICT_LLRINT: |
2647 | case ISD::LLROUND: |
2648 | case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; |
2649 | case ISD::LOAD: ExpandIntRes_LOAD(N: cast<LoadSDNode>(Val: N), Lo, Hi); break; |
2650 | case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
2651 | case ISD::READCYCLECOUNTER: |
2652 | case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break; |
2653 | case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
2654 | case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
2655 | case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
2656 | case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
2657 | case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
2658 | case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
2659 | case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
2660 | case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
2661 | case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
2662 | |
2663 | case ISD::ATOMIC_LOAD_ADD: |
2664 | case ISD::ATOMIC_LOAD_SUB: |
2665 | case ISD::ATOMIC_LOAD_AND: |
2666 | case ISD::ATOMIC_LOAD_CLR: |
2667 | case ISD::ATOMIC_LOAD_OR: |
2668 | case ISD::ATOMIC_LOAD_XOR: |
2669 | case ISD::ATOMIC_LOAD_NAND: |
2670 | case ISD::ATOMIC_LOAD_MIN: |
2671 | case ISD::ATOMIC_LOAD_MAX: |
2672 | case ISD::ATOMIC_LOAD_UMIN: |
2673 | case ISD::ATOMIC_LOAD_UMAX: |
2674 | case ISD::ATOMIC_SWAP: |
2675 | case ISD::ATOMIC_CMP_SWAP: { |
2676 | std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node: N); |
2677 | SplitInteger(Op: Tmp.first, Lo, Hi); |
2678 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
2679 | break; |
2680 | } |
2681 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2682 | AtomicSDNode *AN = cast<AtomicSDNode>(Val: N); |
2683 | SDVTList VTs = DAG.getVTList(N->getValueType(ResNo: 0), MVT::Other); |
2684 | SDValue Tmp = DAG.getAtomicCmpSwap( |
2685 | Opcode: ISD::ATOMIC_CMP_SWAP, dl: SDLoc(N), MemVT: AN->getMemoryVT(), VTs, |
2686 | Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 1), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3), |
2687 | MMO: AN->getMemOperand()); |
2688 | |
2689 | // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine |
2690 | // success simply by comparing the loaded value against the ingoing |
2691 | // comparison. |
2692 | SDValue Success = DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 1), LHS: Tmp, |
2693 | RHS: N->getOperand(Num: 2), Cond: ISD::SETEQ); |
2694 | |
2695 | SplitInteger(Op: Tmp, Lo, Hi); |
2696 | ReplaceValueWith(From: SDValue(N, 1), To: Success); |
2697 | ReplaceValueWith(From: SDValue(N, 2), To: Tmp.getValue(R: 1)); |
2698 | break; |
2699 | } |
2700 | |
2701 | case ISD::AND: |
2702 | case ISD::OR: |
2703 | case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
2704 | |
2705 | case ISD::UMAX: |
2706 | case ISD::SMAX: |
2707 | case ISD::UMIN: |
2708 | case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
2709 | |
2710 | case ISD::ADD: |
2711 | case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
2712 | |
2713 | case ISD::ADDC: |
2714 | case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
2715 | |
2716 | case ISD::ADDE: |
2717 | case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
2718 | |
2719 | case ISD::UADDO_CARRY: |
2720 | case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; |
2721 | |
2722 | case ISD::SADDO_CARRY: |
2723 | case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; |
2724 | |
2725 | case ISD::SHL: |
2726 | case ISD::SRA: |
2727 | case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
2728 | |
2729 | case ISD::SADDO: |
2730 | case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
2731 | case ISD::UADDO: |
2732 | case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
2733 | case ISD::UMULO: |
2734 | case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
2735 | |
2736 | case ISD::SADDSAT: |
2737 | case ISD::UADDSAT: |
2738 | case ISD::SSUBSAT: |
2739 | case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; |
2740 | |
2741 | case ISD::SSHLSAT: |
2742 | case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; |
2743 | |
2744 | case ISD::SMULFIX: |
2745 | case ISD::SMULFIXSAT: |
2746 | case ISD::UMULFIX: |
2747 | case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; |
2748 | |
2749 | case ISD::SDIVFIX: |
2750 | case ISD::SDIVFIXSAT: |
2751 | case ISD::UDIVFIX: |
2752 | case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; |
2753 | |
2754 | case ISD::VECREDUCE_ADD: |
2755 | case ISD::VECREDUCE_MUL: |
2756 | case ISD::VECREDUCE_AND: |
2757 | case ISD::VECREDUCE_OR: |
2758 | case ISD::VECREDUCE_XOR: |
2759 | case ISD::VECREDUCE_SMAX: |
2760 | case ISD::VECREDUCE_SMIN: |
2761 | case ISD::VECREDUCE_UMAX: |
2762 | case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; |
2763 | |
2764 | case ISD::ROTL: |
2765 | case ISD::ROTR: |
2766 | ExpandIntRes_Rotate(N, Lo, Hi); |
2767 | break; |
2768 | |
2769 | case ISD::FSHL: |
2770 | case ISD::FSHR: |
2771 | ExpandIntRes_FunnelShift(N, Lo, Hi); |
2772 | break; |
2773 | |
2774 | case ISD::VSCALE: |
2775 | ExpandIntRes_VSCALE(N, Lo, Hi); |
2776 | break; |
2777 | } |
2778 | |
2779 | // If Lo/Hi is null, the sub-method took care of registering results etc. |
2780 | if (Lo.getNode()) |
2781 | SetExpandedInteger(Op: SDValue(N, ResNo), Lo, Hi); |
2782 | } |
2783 | |
2784 | /// Lower an atomic node to the appropriate builtin call. |
2785 | std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
2786 | unsigned Opc = Node->getOpcode(); |
2787 | MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT(); |
2788 | AtomicOrdering order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering(); |
2789 | // Lower to outline atomic libcall if outline atomics enabled, |
2790 | // or to sync libcall otherwise |
2791 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order: order, VT); |
2792 | EVT RetVT = Node->getValueType(ResNo: 0); |
2793 | TargetLowering::MakeLibCallOptions CallOptions; |
2794 | SmallVector<SDValue, 4> Ops; |
2795 | if (TLI.getLibcallName(Call: LC)) { |
2796 | Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end()); |
2797 | Ops.push_back(Elt: Node->getOperand(Num: 1)); |
2798 | } else { |
2799 | LC = RTLIB::getSYNC(Opc, VT); |
2800 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
2801 | "Unexpected atomic op or value type!" ); |
2802 | Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end()); |
2803 | } |
2804 | return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, dl: SDLoc(Node), |
2805 | Chain: Node->getOperand(Num: 0)); |
2806 | } |
2807 | |
2808 | /// N is a shift by a value that needs to be expanded, |
2809 | /// and the shift amount is a constant 'Amt'. Expand the operation. |
2810 | void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
2811 | SDValue &Lo, SDValue &Hi) { |
2812 | SDLoc DL(N); |
2813 | // Expand the incoming operand to be shifted, so that we have its parts |
2814 | SDValue InL, InH; |
2815 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
2816 | |
2817 | // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization |
2818 | // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. |
2819 | if (!Amt) { |
2820 | Lo = InL; |
2821 | Hi = InH; |
2822 | return; |
2823 | } |
2824 | |
2825 | EVT NVT = InL.getValueType(); |
2826 | unsigned VTBits = N->getValueType(ResNo: 0).getSizeInBits(); |
2827 | unsigned NVTBits = NVT.getSizeInBits(); |
2828 | |
2829 | if (N->getOpcode() == ISD::SHL) { |
2830 | if (Amt.uge(RHS: VTBits)) { |
2831 | Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2832 | } else if (Amt.ugt(RHS: NVTBits)) { |
2833 | Lo = DAG.getConstant(Val: 0, DL, VT: NVT); |
2834 | Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL, |
2835 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2836 | } else if (Amt == NVTBits) { |
2837 | Lo = DAG.getConstant(Val: 0, DL, VT: NVT); |
2838 | Hi = InL; |
2839 | } else { |
2840 | Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL, |
2841 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2842 | Hi = DAG.getNode( |
2843 | Opcode: ISD::OR, DL, VT: NVT, |
2844 | N1: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2845 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2846 | N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2847 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2848 | } |
2849 | return; |
2850 | } |
2851 | |
2852 | if (N->getOpcode() == ISD::SRL) { |
2853 | if (Amt.uge(RHS: VTBits)) { |
2854 | Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2855 | } else if (Amt.ugt(RHS: NVTBits)) { |
2856 | Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH, |
2857 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2858 | Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2859 | } else if (Amt == NVTBits) { |
2860 | Lo = InH; |
2861 | Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2862 | } else { |
2863 | Lo = DAG.getNode( |
2864 | Opcode: ISD::OR, DL, VT: NVT, |
2865 | N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2866 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2867 | N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2868 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2869 | Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH, |
2870 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2871 | } |
2872 | return; |
2873 | } |
2874 | |
2875 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!" ); |
2876 | if (Amt.uge(RHS: VTBits)) { |
2877 | Hi = Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2878 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2879 | } else if (Amt.ugt(RHS: NVTBits)) { |
2880 | Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2881 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2882 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2883 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2884 | } else if (Amt == NVTBits) { |
2885 | Lo = InH; |
2886 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2887 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2888 | } else { |
2889 | Lo = DAG.getNode( |
2890 | Opcode: ISD::OR, DL, VT: NVT, |
2891 | N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2892 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2893 | N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2894 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2895 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2896 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2897 | } |
2898 | } |
2899 | |
2900 | /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify |
2901 | /// this shift based on knowledge of the high bit of the shift amount. If we |
2902 | /// can tell this, we know that it is >= 32 or < 32, without knowing the actual |
2903 | /// shift amount. |
2904 | bool DAGTypeLegalizer:: |
2905 | ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2906 | unsigned Opc = N->getOpcode(); |
2907 | SDValue In = N->getOperand(Num: 0); |
2908 | SDValue Amt = N->getOperand(Num: 1); |
2909 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
2910 | EVT ShTy = Amt.getValueType(); |
2911 | unsigned ShBits = ShTy.getScalarSizeInBits(); |
2912 | unsigned NVTBits = NVT.getScalarSizeInBits(); |
2913 | assert(isPowerOf2_32(NVTBits) && |
2914 | "Expanded integer type size not a power of two!" ); |
2915 | SDLoc dl(N); |
2916 | |
2917 | APInt HighBitMask = APInt::getHighBitsSet(numBits: ShBits, hiBitsSet: ShBits - Log2_32(Value: NVTBits)); |
2918 | KnownBits Known = DAG.computeKnownBits(Op: Amt); |
2919 | |
2920 | // If we don't know anything about the high bits, exit. |
2921 | if (((Known.Zero | Known.One) & HighBitMask) == 0) |
2922 | return false; |
2923 | |
2924 | // Get the incoming operand to be shifted. |
2925 | SDValue InL, InH; |
2926 | GetExpandedInteger(Op: In, Lo&: InL, Hi&: InH); |
2927 | |
2928 | // If we know that any of the high bits of the shift amount are one, then we |
2929 | // can do this as a couple of simple shifts. |
2930 | if (Known.One.intersects(RHS: HighBitMask)) { |
2931 | // Mask out the high bit, which we know is set. |
2932 | Amt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShTy, N1: Amt, |
2933 | N2: DAG.getConstant(Val: ~HighBitMask, DL: dl, VT: ShTy)); |
2934 | |
2935 | switch (Opc) { |
2936 | default: llvm_unreachable("Unknown shift" ); |
2937 | case ISD::SHL: |
2938 | Lo = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Low part is zero. |
2939 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt); // High part from Lo part. |
2940 | return true; |
2941 | case ISD::SRL: |
2942 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero. |
2943 | Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part. |
2944 | return true; |
2945 | case ISD::SRA: |
2946 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign extend high part. |
2947 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
2948 | Lo = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part. |
2949 | return true; |
2950 | } |
2951 | } |
2952 | |
2953 | // If we know that all of the high bits of the shift amount are zero, then we |
2954 | // can do this as a couple of simple shifts. |
2955 | if (HighBitMask.isSubsetOf(RHS: Known.Zero)) { |
2956 | // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined |
2957 | // shift if x is zero. We can use XOR here because x is known to be smaller |
2958 | // than 32. |
2959 | SDValue Amt2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ShTy, N1: Amt, |
2960 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
2961 | |
2962 | unsigned Op1, Op2; |
2963 | switch (Opc) { |
2964 | default: llvm_unreachable("Unknown shift" ); |
2965 | case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
2966 | case ISD::SRL: |
2967 | case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
2968 | } |
2969 | |
2970 | // When shifting right the arithmetic for Lo and Hi is swapped. |
2971 | if (Opc != ISD::SHL) |
2972 | std::swap(a&: InL, b&: InH); |
2973 | |
2974 | // Use a little trick to get the bits that move from Lo to Hi. First |
2975 | // shift by one bit. |
2976 | SDValue Sh1 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: InL, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShTy)); |
2977 | // Then compute the remaining shift with amount-1. |
2978 | SDValue Sh2 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: Sh1, N2: Amt2); |
2979 | |
2980 | Lo = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, N1: InL, N2: Amt); |
2981 | Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: Op1, DL: dl, VT: NVT, N1: InH, N2: Amt),N2: Sh2); |
2982 | |
2983 | if (Opc != ISD::SHL) |
2984 | std::swap(a&: Hi, b&: Lo); |
2985 | return true; |
2986 | } |
2987 | |
2988 | return false; |
2989 | } |
2990 | |
2991 | /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift |
2992 | /// of any size. |
2993 | bool DAGTypeLegalizer:: |
2994 | ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2995 | SDValue Amt = N->getOperand(Num: 1); |
2996 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
2997 | EVT ShTy = Amt.getValueType(); |
2998 | unsigned NVTBits = NVT.getSizeInBits(); |
2999 | assert(isPowerOf2_32(NVTBits) && |
3000 | "Expanded integer type size not a power of two!" ); |
3001 | SDLoc dl(N); |
3002 | |
3003 | // Get the incoming operand to be shifted. |
3004 | SDValue InL, InH; |
3005 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
3006 | |
3007 | SDValue NVBitsNode = DAG.getConstant(Val: NVTBits, DL: dl, VT: ShTy); |
3008 | SDValue AmtExcess = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: Amt, N2: NVBitsNode); |
3009 | SDValue AmtLack = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: NVBitsNode, N2: Amt); |
3010 | SDValue isShort = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy), |
3011 | LHS: Amt, RHS: NVBitsNode, Cond: ISD::SETULT); |
3012 | SDValue isZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy), |
3013 | LHS: Amt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShTy), |
3014 | Cond: ISD::SETEQ); |
3015 | |
3016 | SDValue LoS, HiS, LoL, HiL; |
3017 | switch (N->getOpcode()) { |
3018 | default: llvm_unreachable("Unknown shift" ); |
3019 | case ISD::SHL: |
3020 | // Short: ShAmt < NVTBits |
3021 | LoS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt); |
3022 | HiS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3023 | N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: Amt), |
3024 | N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: AmtLack)); |
3025 | |
3026 | // Long: ShAmt >= NVTBits |
3027 | LoL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Lo part is zero. |
3028 | HiL = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: AmtExcess); // Hi from Lo part. |
3029 | |
3030 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL); |
3031 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InH, |
3032 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL)); |
3033 | return true; |
3034 | case ISD::SRL: |
3035 | // Short: ShAmt < NVTBits |
3036 | HiS = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt); |
3037 | LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3038 | N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt), |
3039 | // FIXME: If Amt is zero, the following shift generates an undefined result |
3040 | // on some architectures. |
3041 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack)); |
3042 | |
3043 | // Long: ShAmt >= NVTBits |
3044 | HiL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero. |
3045 | LoL = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part. |
3046 | |
3047 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL, |
3048 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL)); |
3049 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL); |
3050 | return true; |
3051 | case ISD::SRA: |
3052 | // Short: ShAmt < NVTBits |
3053 | HiS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt); |
3054 | LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3055 | N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt), |
3056 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack)); |
3057 | |
3058 | // Long: ShAmt >= NVTBits |
3059 | HiL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign of Hi part. |
3060 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
3061 | LoL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part. |
3062 | |
3063 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL, |
3064 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL)); |
3065 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL); |
3066 | return true; |
3067 | } |
3068 | } |
3069 | |
3070 | static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { |
3071 | |
3072 | switch (Op) { |
3073 | default: llvm_unreachable("invalid min/max opcode" ); |
3074 | case ISD::SMAX: |
3075 | return std::make_pair(x: ISD::SETGT, y: ISD::UMAX); |
3076 | case ISD::UMAX: |
3077 | return std::make_pair(x: ISD::SETUGT, y: ISD::UMAX); |
3078 | case ISD::SMIN: |
3079 | return std::make_pair(x: ISD::SETLT, y: ISD::UMIN); |
3080 | case ISD::UMIN: |
3081 | return std::make_pair(x: ISD::SETULT, y: ISD::UMIN); |
3082 | } |
3083 | } |
3084 | |
3085 | void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
3086 | SDValue &Lo, SDValue &Hi) { |
3087 | SDLoc DL(N); |
3088 | |
3089 | SDValue LHS = N->getOperand(Num: 0); |
3090 | SDValue RHS = N->getOperand(Num: 1); |
3091 | |
3092 | // If the upper halves are all sign bits, then we can perform the MINMAX on |
3093 | // the lower half and sign-extend the result to the upper half. |
3094 | unsigned NumBits = N->getValueType(ResNo: 0).getScalarSizeInBits(); |
3095 | unsigned NumHalfBits = NumBits / 2; |
3096 | if (DAG.ComputeNumSignBits(Op: LHS) > NumHalfBits && |
3097 | DAG.ComputeNumSignBits(Op: RHS) > NumHalfBits) { |
3098 | SDValue LHSL, LHSH, RHSL, RHSH; |
3099 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3100 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3101 | EVT NVT = LHSL.getValueType(); |
3102 | |
3103 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: LHSL, N2: RHSL); |
3104 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: Lo, |
3105 | N2: DAG.getShiftAmountConstant(Val: NumHalfBits - 1, VT: NVT, DL)); |
3106 | return; |
3107 | } |
3108 | |
3109 | // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. |
3110 | // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. |
3111 | if ((N->getOpcode() == ISD::SMAX && isNullConstant(V: RHS)) || |
3112 | (N->getOpcode() == ISD::SMIN && isAllOnesConstant(V: RHS))) { |
3113 | SDValue LHSL, LHSH, RHSL, RHSH; |
3114 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3115 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3116 | EVT NVT = LHSL.getValueType(); |
3117 | EVT CCT = getSetCCResultType(VT: NVT); |
3118 | |
3119 | SDValue HiNeg = |
3120 | DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: DAG.getConstant(Val: 0, DL, VT: NVT), Cond: ISD::SETLT); |
3121 | if (N->getOpcode() == ISD::SMIN) { |
3122 | Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: LHSL, RHS: DAG.getConstant(Val: -1, DL, VT: NVT)); |
3123 | } else { |
3124 | Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: DAG.getConstant(Val: 0, DL, VT: NVT), RHS: LHSL); |
3125 | } |
3126 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH}); |
3127 | return; |
3128 | } |
3129 | |
3130 | const APInt *RHSVal = nullptr; |
3131 | if (auto *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS)) |
3132 | RHSVal = &RHSConst->getAPIntValue(); |
3133 | |
3134 | // The high half of MIN/MAX is always just the the MIN/MAX of the |
3135 | // high halves of the operands. Expand this way if it appears profitable. |
3136 | if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && |
3137 | (RHSVal->countLeadingOnes() >= NumHalfBits || |
3138 | RHSVal->countLeadingZeros() >= NumHalfBits)) { |
3139 | SDValue LHSL, LHSH, RHSL, RHSH; |
3140 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3141 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3142 | EVT NVT = LHSL.getValueType(); |
3143 | EVT CCT = getSetCCResultType(VT: NVT); |
3144 | |
3145 | ISD::NodeType LoOpc; |
3146 | ISD::CondCode CondC; |
3147 | std::tie(args&: CondC, args&: LoOpc) = getExpandedMinMaxOps(Op: N->getOpcode()); |
3148 | |
3149 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH}); |
3150 | // We need to know whether to select Lo part that corresponds to 'winning' |
3151 | // Hi part or if Hi parts are equal. |
3152 | SDValue IsHiLeft = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: CondC); |
3153 | SDValue IsHiEq = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: ISD::SETEQ); |
3154 | |
3155 | // Lo part corresponding to the 'winning' Hi part |
3156 | SDValue LoCmp = DAG.getSelect(DL, VT: NVT, Cond: IsHiLeft, LHS: LHSL, RHS: RHSL); |
3157 | |
3158 | // Recursed Lo part if Hi parts are equal, this uses unsigned version |
3159 | SDValue LoMinMax = DAG.getNode(Opcode: LoOpc, DL, VT: NVT, Ops: {LHSL, RHSL}); |
3160 | |
3161 | Lo = DAG.getSelect(DL, VT: NVT, Cond: IsHiEq, LHS: LoMinMax, RHS: LoCmp); |
3162 | return; |
3163 | } |
3164 | |
3165 | // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies |
3166 | // the compare. |
3167 | ISD::CondCode Pred; |
3168 | switch (N->getOpcode()) { |
3169 | default: llvm_unreachable("How did we get here?" ); |
3170 | case ISD::SMAX: |
3171 | if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
3172 | Pred = ISD::SETGE; |
3173 | else |
3174 | Pred = ISD::SETGT; |
3175 | break; |
3176 | case ISD::SMIN: |
3177 | if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
3178 | Pred = ISD::SETLE; |
3179 | else |
3180 | Pred = ISD::SETLT; |
3181 | break; |
3182 | case ISD::UMAX: |
3183 | if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
3184 | Pred = ISD::SETUGE; |
3185 | else |
3186 | Pred = ISD::SETUGT; |
3187 | break; |
3188 | case ISD::UMIN: |
3189 | if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
3190 | Pred = ISD::SETULE; |
3191 | else |
3192 | Pred = ISD::SETULT; |
3193 | break; |
3194 | } |
3195 | EVT VT = N->getValueType(ResNo: 0); |
3196 | EVT CCT = getSetCCResultType(VT); |
3197 | SDValue Cond = DAG.getSetCC(DL, VT: CCT, LHS, RHS, Cond: Pred); |
3198 | SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); |
3199 | SplitInteger(Op: Result, Lo, Hi); |
3200 | } |
3201 | |
3202 | void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
3203 | SDValue &Lo, SDValue &Hi) { |
3204 | SDLoc dl(N); |
3205 | // Expand the subcomponents. |
3206 | SDValue LHSL, LHSH, RHSL, RHSH; |
3207 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3208 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3209 | |
3210 | EVT NVT = LHSL.getValueType(); |
3211 | SDValue LoOps[2] = { LHSL, RHSL }; |
3212 | SDValue HiOps[3] = { LHSH, RHSH }; |
3213 | |
3214 | bool HasOpCarry = TLI.isOperationLegalOrCustom( |
3215 | Op: N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, |
3216 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3217 | if (HasOpCarry) { |
3218 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT)); |
3219 | if (N->getOpcode() == ISD::ADD) { |
3220 | Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps); |
3221 | HiOps[2] = Lo.getValue(R: 1); |
3222 | Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero() |
3223 | ? DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: ArrayRef(HiOps, 2)) |
3224 | : DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, Ops: HiOps); |
3225 | } else { |
3226 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps); |
3227 | HiOps[2] = Lo.getValue(R: 1); |
3228 | Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero() |
3229 | ? DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: ArrayRef(HiOps, 2)) |
3230 | : DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, Ops: HiOps); |
3231 | } |
3232 | return; |
3233 | } |
3234 | |
3235 | // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support |
3236 | // them. TODO: Teach operation legalization how to expand unsupported |
3237 | // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate |
3238 | // a carry of type MVT::Glue, but there doesn't seem to be any way to |
3239 | // generate a value of this type in the expanded code sequence. |
3240 | bool hasCarry = |
3241 | TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ? |
3242 | ISD::ADDC : ISD::SUBC, |
3243 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3244 | |
3245 | if (hasCarry) { |
3246 | SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
3247 | if (N->getOpcode() == ISD::ADD) { |
3248 | Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps); |
3249 | HiOps[2] = Lo.getValue(R: 1); |
3250 | Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps); |
3251 | } else { |
3252 | Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps); |
3253 | HiOps[2] = Lo.getValue(R: 1); |
3254 | Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps); |
3255 | } |
3256 | return; |
3257 | } |
3258 | |
3259 | bool hasOVF = |
3260 | TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ? |
3261 | ISD::UADDO : ISD::USUBO, |
3262 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3263 | TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(Type: NVT); |
3264 | |
3265 | if (hasOVF) { |
3266 | EVT OvfVT = getSetCCResultType(VT: NVT); |
3267 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: OvfVT); |
3268 | int RevOpc; |
3269 | if (N->getOpcode() == ISD::ADD) { |
3270 | RevOpc = ISD::SUB; |
3271 | Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps); |
3272 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3273 | } else { |
3274 | RevOpc = ISD::ADD; |
3275 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps); |
3276 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3277 | } |
3278 | SDValue OVF = Lo.getValue(R: 1); |
3279 | |
3280 | switch (BoolType) { |
3281 | case TargetLoweringBase::UndefinedBooleanContent: |
3282 | OVF = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OvfVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: OvfVT), N2: OVF); |
3283 | [[fallthrough]]; |
3284 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
3285 | OVF = DAG.getZExtOrTrunc(Op: OVF, DL: dl, VT: NVT); |
3286 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Hi, N2: OVF); |
3287 | break; |
3288 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
3289 | OVF = DAG.getSExtOrTrunc(Op: OVF, DL: dl, VT: NVT); |
3290 | Hi = DAG.getNode(Opcode: RevOpc, DL: dl, VT: NVT, N1: Hi, N2: OVF); |
3291 | } |
3292 | return; |
3293 | } |
3294 | |
3295 | if (N->getOpcode() == ISD::ADD) { |
3296 | Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: LoOps); |
3297 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3298 | SDValue Cmp; |
3299 | // Special case: X+1 has a carry out if X+1==0. This may reduce the live |
3300 | // range of X. We assume comparing with 0 is cheap. |
3301 | if (isOneConstant(V: LoOps[1])) |
3302 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, |
3303 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ); |
3304 | else if (isAllOnesConstant(V: LoOps[1])) { |
3305 | if (isAllOnesConstant(V: HiOps[1])) |
3306 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0], |
3307 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ); |
3308 | else |
3309 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0], |
3310 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3311 | } else |
3312 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, RHS: LoOps[0], |
3313 | Cond: ISD::SETULT); |
3314 | |
3315 | SDValue Carry; |
3316 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
3317 | Carry = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT); |
3318 | else |
3319 | Carry = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT), |
3320 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT)); |
3321 | |
3322 | if (isAllOnesConstant(V: LoOps[1]) && isAllOnesConstant(V: HiOps[1])) |
3323 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: HiOps[0], N2: Carry); |
3324 | else |
3325 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: Hi, N2: Carry); |
3326 | } else { |
3327 | Lo = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: LoOps); |
3328 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3329 | SDValue Cmp = |
3330 | DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LoOps[0].getValueType()), |
3331 | LHS: LoOps[0], RHS: LoOps[1], Cond: ISD::SETULT); |
3332 | |
3333 | SDValue Borrow; |
3334 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
3335 | Borrow = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT); |
3336 | else |
3337 | Borrow = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT), |
3338 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT)); |
3339 | |
3340 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Hi, N2: Borrow); |
3341 | } |
3342 | } |
3343 | |
3344 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
3345 | SDValue &Lo, SDValue &Hi) { |
3346 | // Expand the subcomponents. |
3347 | SDValue LHSL, LHSH, RHSL, RHSH; |
3348 | SDLoc dl(N); |
3349 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3350 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3351 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
3352 | SDValue LoOps[2] = { LHSL, RHSL }; |
3353 | SDValue HiOps[3] = { LHSH, RHSH }; |
3354 | |
3355 | if (N->getOpcode() == ISD::ADDC) { |
3356 | Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps); |
3357 | HiOps[2] = Lo.getValue(R: 1); |
3358 | Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps); |
3359 | } else { |
3360 | Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps); |
3361 | HiOps[2] = Lo.getValue(R: 1); |
3362 | Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps); |
3363 | } |
3364 | |
3365 | // Legalized the flag result - switch anything that used the old flag to |
3366 | // use the new one. |
3367 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3368 | } |
3369 | |
3370 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
3371 | SDValue &Lo, SDValue &Hi) { |
3372 | // Expand the subcomponents. |
3373 | SDValue LHSL, LHSH, RHSL, RHSH; |
3374 | SDLoc dl(N); |
3375 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3376 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3377 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
3378 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) }; |
3379 | SDValue HiOps[3] = { LHSH, RHSH }; |
3380 | |
3381 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3382 | HiOps[2] = Lo.getValue(R: 1); |
3383 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps); |
3384 | |
3385 | // Legalized the flag result - switch anything that used the old flag to |
3386 | // use the new one. |
3387 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3388 | } |
3389 | |
3390 | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
3391 | SDValue &Lo, SDValue &Hi) { |
3392 | SDValue LHS = N->getOperand(Num: 0); |
3393 | SDValue RHS = N->getOperand(Num: 1); |
3394 | SDLoc dl(N); |
3395 | |
3396 | SDValue Ovf; |
3397 | |
3398 | unsigned CarryOp, NoCarryOp; |
3399 | ISD::CondCode Cond; |
3400 | switch(N->getOpcode()) { |
3401 | case ISD::UADDO: |
3402 | CarryOp = ISD::UADDO_CARRY; |
3403 | NoCarryOp = ISD::ADD; |
3404 | Cond = ISD::SETULT; |
3405 | break; |
3406 | case ISD::USUBO: |
3407 | CarryOp = ISD::USUBO_CARRY; |
3408 | NoCarryOp = ISD::SUB; |
3409 | Cond = ISD::SETUGT; |
3410 | break; |
3411 | default: |
3412 | llvm_unreachable("Node has unexpected Opcode" ); |
3413 | } |
3414 | |
3415 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
3416 | Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType())); |
3417 | |
3418 | if (HasCarryOp) { |
3419 | // Expand the subcomponents. |
3420 | SDValue LHSL, LHSH, RHSL, RHSH; |
3421 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3422 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3423 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3424 | SDValue LoOps[2] = { LHSL, RHSL }; |
3425 | SDValue HiOps[3] = { LHSH, RHSH }; |
3426 | |
3427 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3428 | HiOps[2] = Lo.getValue(R: 1); |
3429 | Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: HiOps); |
3430 | |
3431 | Ovf = Hi.getValue(R: 1); |
3432 | } else { |
3433 | // Expand the result by simply replacing it with the equivalent |
3434 | // non-overflow-checking operation. |
3435 | SDValue Sum = DAG.getNode(Opcode: NoCarryOp, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS); |
3436 | SplitInteger(Op: Sum, Lo, Hi); |
3437 | |
3438 | if (N->getOpcode() == ISD::UADDO && isOneConstant(V: RHS)) { |
3439 | // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this |
3440 | // with (Lo | Hi) == 0. |
3441 | SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Lo.getValueType(), N1: Lo, N2: Hi); |
3442 | Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Or, |
3443 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: Lo.getValueType()), Cond: ISD::SETEQ); |
3444 | } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(V: RHS)) { |
3445 | // Special case: uaddo X, -1 overflows if X == 0. |
3446 | Ovf = |
3447 | DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS, |
3448 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType()), Cond: ISD::SETNE); |
3449 | } else { |
3450 | // Calculate the overflow: addition overflows iff a + b < a, and |
3451 | // subtraction overflows iff a - b > a. |
3452 | Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Sum, RHS: LHS, Cond); |
3453 | } |
3454 | } |
3455 | |
3456 | // Legalized the flag result - switch anything that used the old flag to |
3457 | // use the new one. |
3458 | ReplaceValueWith(From: SDValue(N, 1), To: Ovf); |
3459 | } |
3460 | |
3461 | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, |
3462 | SDValue &Hi) { |
3463 | // Expand the subcomponents. |
3464 | SDValue LHSL, LHSH, RHSL, RHSH; |
3465 | SDLoc dl(N); |
3466 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3467 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3468 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3469 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) }; |
3470 | SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
3471 | |
3472 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3473 | HiOps[2] = Lo.getValue(R: 1); |
3474 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps); |
3475 | |
3476 | // Legalized the flag result - switch anything that used the old flag to |
3477 | // use the new one. |
3478 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3479 | } |
3480 | |
3481 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, |
3482 | SDValue &Lo, SDValue &Hi) { |
3483 | // Expand the subcomponents. |
3484 | SDValue LHSL, LHSH, RHSL, RHSH; |
3485 | SDLoc dl(N); |
3486 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3487 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3488 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3489 | |
3490 | // We need to use an unsigned carry op for the lo part. |
3491 | unsigned CarryOp = |
3492 | N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; |
3493 | Lo = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSL, RHSL, N->getOperand(Num: 2) }); |
3494 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) }); |
3495 | |
3496 | // Legalized the flag result - switch anything that used the old flag to |
3497 | // use the new one. |
3498 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3499 | } |
3500 | |
3501 | void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
3502 | SDValue &Lo, SDValue &Hi) { |
3503 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3504 | SDLoc dl(N); |
3505 | SDValue Op = N->getOperand(Num: 0); |
3506 | if (Op.getValueType().bitsLE(VT: NVT)) { |
3507 | // The low part is any extension of the input (which degenerates to a copy). |
3508 | Lo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Op); |
3509 | Hi = DAG.getUNDEF(VT: NVT); // The high part is undefined. |
3510 | } else { |
3511 | // For example, extension of an i48 to an i64. The operand type necessarily |
3512 | // promotes to the result type, so will end up being expanded too. |
3513 | assert(getTypeAction(Op.getValueType()) == |
3514 | TargetLowering::TypePromoteInteger && |
3515 | "Only know how to promote this result!" ); |
3516 | SDValue Res = GetPromotedInteger(Op); |
3517 | assert(Res.getValueType() == N->getValueType(0) && |
3518 | "Operand over promoted?" ); |
3519 | // Split the promoted operand. This will simplify when it is expanded. |
3520 | SplitInteger(Op: Res, Lo, Hi); |
3521 | } |
3522 | } |
3523 | |
3524 | void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
3525 | SDValue &Lo, SDValue &Hi) { |
3526 | SDLoc dl(N); |
3527 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3528 | EVT NVT = Lo.getValueType(); |
3529 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
3530 | unsigned NVTBits = NVT.getSizeInBits(); |
3531 | unsigned EVTBits = EVT.getSizeInBits(); |
3532 | |
3533 | if (NVTBits < EVTBits) { |
3534 | Hi = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Hi, |
3535 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
3536 | BitWidth: EVTBits - NVTBits))); |
3537 | } else { |
3538 | Lo = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT)); |
3539 | // The high part replicates the sign bit of Lo, make it explicit. |
3540 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3541 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, |
3542 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3543 | } |
3544 | } |
3545 | |
3546 | void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
3547 | SDValue &Lo, SDValue &Hi) { |
3548 | SDLoc dl(N); |
3549 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3550 | EVT NVT = Lo.getValueType(); |
3551 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
3552 | unsigned NVTBits = NVT.getSizeInBits(); |
3553 | unsigned EVTBits = EVT.getSizeInBits(); |
3554 | |
3555 | if (NVTBits < EVTBits) { |
3556 | Hi = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Hi, |
3557 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
3558 | BitWidth: EVTBits - NVTBits))); |
3559 | } else { |
3560 | Lo = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT)); |
3561 | // The high part must be zero, make it explicit. |
3562 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3563 | } |
3564 | } |
3565 | |
3566 | void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
3567 | SDValue &Lo, SDValue &Hi) { |
3568 | SDLoc dl(N); |
3569 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands. |
3570 | Lo = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
3571 | Hi = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
3572 | } |
3573 | |
3574 | void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
3575 | SDValue &Lo, SDValue &Hi) { |
3576 | SDLoc dl(N); |
3577 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands. |
3578 | Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
3579 | Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
3580 | } |
3581 | |
3582 | void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, |
3583 | SDValue &Hi) { |
3584 | SDLoc dl(N); |
3585 | // parity(HiLo) -> parity(Lo^Hi) |
3586 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3587 | EVT NVT = Lo.getValueType(); |
3588 | Lo = |
3589 | DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT: NVT, Operand: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Hi)); |
3590 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3591 | } |
3592 | |
3593 | void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
3594 | SDValue &Lo, SDValue &Hi) { |
3595 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3596 | unsigned NBitWidth = NVT.getSizeInBits(); |
3597 | auto Constant = cast<ConstantSDNode>(Val: N); |
3598 | const APInt &Cst = Constant->getAPIntValue(); |
3599 | bool IsTarget = Constant->isTargetOpcode(); |
3600 | bool IsOpaque = Constant->isOpaque(); |
3601 | SDLoc dl(N); |
3602 | Lo = DAG.getConstant(Val: Cst.trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget, isOpaque: IsOpaque); |
3603 | Hi = DAG.getConstant(Val: Cst.lshr(shiftAmt: NBitWidth).trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget, |
3604 | isOpaque: IsOpaque); |
3605 | } |
3606 | |
3607 | void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { |
3608 | SDLoc dl(N); |
3609 | |
3610 | SDValue N0 = N->getOperand(Num: 0); |
3611 | GetExpandedInteger(Op: N0, Lo, Hi); |
3612 | EVT NVT = Lo.getValueType(); |
3613 | |
3614 | // If the upper half is all sign bits, then we can perform the ABS on the |
3615 | // lower half and zero-extend. |
3616 | if (DAG.ComputeNumSignBits(Op: N0) > NVT.getScalarSizeInBits()) { |
3617 | Lo = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: NVT, Operand: Lo); |
3618 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3619 | return; |
3620 | } |
3621 | |
3622 | // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence |
3623 | // we use in LegalizeDAG. The SUB part of the expansion is based on |
3624 | // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that |
3625 | // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further |
3626 | // expanded if needed. Shift expansion has a special case for filling with |
3627 | // sign bits so that we will only end up with one SRA. |
3628 | bool HasSubCarry = TLI.isOperationLegalOrCustom( |
3629 | Op: ISD::USUBO_CARRY, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3630 | if (HasSubCarry) { |
3631 | SDValue Sign = DAG.getNode( |
3632 | Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Hi, |
3633 | N2: DAG.getShiftAmountConstant(Val: NVT.getSizeInBits() - 1, VT: NVT, DL: dl)); |
3634 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT)); |
3635 | Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Sign); |
3636 | Hi = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Hi, N2: Sign); |
3637 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: Lo, N2: Sign); |
3638 | Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: Hi, N2: Sign, N3: Lo.getValue(R: 1)); |
3639 | return; |
3640 | } |
3641 | |
3642 | // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) |
3643 | EVT VT = N->getValueType(ResNo: 0); |
3644 | SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, |
3645 | N1: DAG.getConstant(Val: 0, DL: dl, VT), N2: N0); |
3646 | SDValue NegLo, NegHi; |
3647 | SplitInteger(Op: Neg, Lo&: NegLo, Hi&: NegHi); |
3648 | |
3649 | SDValue HiIsNeg = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi, |
3650 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETLT); |
3651 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegLo, RHS: Lo); |
3652 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegHi, RHS: Hi); |
3653 | } |
3654 | |
3655 | void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
3656 | SDValue &Lo, SDValue &Hi) { |
3657 | SDLoc dl(N); |
3658 | // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) |
3659 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3660 | EVT NVT = Lo.getValueType(); |
3661 | |
3662 | SDValue HiNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi, |
3663 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3664 | |
3665 | SDValue LoLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Lo); |
3666 | SDValue HiLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Hi); |
3667 | |
3668 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiNotZero, LHS: HiLZ, |
3669 | RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: LoLZ, |
3670 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
3671 | VT: NVT))); |
3672 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3673 | } |
3674 | |
3675 | void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
3676 | SDValue &Lo, SDValue &Hi) { |
3677 | SDLoc dl(N); |
3678 | // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) |
3679 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3680 | EVT NVT = Lo.getValueType(); |
3681 | Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT: NVT, Operand: Lo), |
3682 | N2: DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT: NVT, Operand: Hi)); |
3683 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3684 | } |
3685 | |
3686 | void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
3687 | SDValue &Lo, SDValue &Hi) { |
3688 | SDLoc dl(N); |
3689 | // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) |
3690 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3691 | EVT NVT = Lo.getValueType(); |
3692 | |
3693 | SDValue LoNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, |
3694 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3695 | |
3696 | SDValue LoLZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Lo); |
3697 | SDValue HiLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Hi); |
3698 | |
3699 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: LoNotZero, LHS: LoLZ, |
3700 | RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: HiLZ, |
3701 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
3702 | VT: NVT))); |
3703 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3704 | } |
3705 | |
3706 | void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, |
3707 | SDValue &Hi) { |
3708 | SDLoc dl(N); |
3709 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3710 | unsigned NBitWidth = NVT.getSizeInBits(); |
3711 | |
3712 | Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); |
3713 | SDValue Chain = Lo.getValue(R: 1); |
3714 | // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING |
3715 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3716 | N2: DAG.getShiftAmountConstant(Val: NBitWidth - 1, VT: NVT, DL: dl)); |
3717 | |
3718 | // Legalize the chain result - switch anything that used the old chain to |
3719 | // use the new one. |
3720 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
3721 | } |
3722 | |
3723 | // Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op. |
3724 | static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, |
3725 | SDLoc DL, SelectionDAG &DAG) { |
3726 | if (IsStrict) { |
3727 | Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}); |
3728 | Chain = Op.getValue(R: 1); |
3729 | return Op; |
3730 | } |
3731 | return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op); |
3732 | } |
3733 | |
3734 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo, |
3735 | SDValue &Hi) { |
3736 | SDLoc dl(N); |
3737 | EVT VT = N->getValueType(ResNo: 0); |
3738 | |
3739 | bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || |
3740 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
3741 | bool IsStrict = N->isStrictFPOpcode(); |
3742 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
3743 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
3744 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) |
3745 | Op = GetPromotedFloat(Op); |
3746 | |
3747 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
3748 | EVT OFPVT = Op.getValueType(); |
3749 | EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OFPVT); |
3750 | Op = GetSoftPromotedHalf(Op); |
3751 | Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl, |
3752 | NFPVT, Op); |
3753 | Op = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT, Operand: Op); |
3754 | SplitInteger(Op, Lo, Hi); |
3755 | return; |
3756 | } |
3757 | |
3758 | if (Op.getValueType() == MVT::bf16) { |
3759 | // Extend to f32 as there is no bf16 libcall. |
3760 | Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG); |
3761 | } |
3762 | |
3763 | RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(OpVT: Op.getValueType(), RetVT: VT) |
3764 | : RTLIB::getFPTOUINT(OpVT: Op.getValueType(), RetVT: VT); |
3765 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!" ); |
3766 | TargetLowering::MakeLibCallOptions CallOptions; |
3767 | CallOptions.setSExt(true); |
3768 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, |
3769 | CallOptions, dl, Chain); |
3770 | SplitInteger(Op: Tmp.first, Lo, Hi); |
3771 | |
3772 | if (IsStrict) |
3773 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
3774 | } |
3775 | |
3776 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
3777 | SDValue &Hi) { |
3778 | SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
3779 | SplitInteger(Op: Res, Lo, Hi); |
3780 | } |
3781 | |
3782 | void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, |
3783 | SDValue &Hi) { |
3784 | SDLoc dl(N); |
3785 | bool IsStrict = N->isStrictFPOpcode(); |
3786 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
3787 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
3788 | |
3789 | assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && |
3790 | "Input type needs to be promoted!" ); |
3791 | |
3792 | EVT VT = Op.getValueType(); |
3793 | |
3794 | if (VT == MVT::f16) { |
3795 | // Extend to f32. |
3796 | VT = MVT::f32; |
3797 | Op = fpExtendHelper(Op, Chain, IsStrict, VT, DL: dl, DAG); |
3798 | } |
3799 | |
3800 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3801 | if (N->getOpcode() == ISD::LROUND || |
3802 | N->getOpcode() == ISD::STRICT_LROUND) { |
3803 | if (VT == MVT::f32) |
3804 | LC = RTLIB::LROUND_F32; |
3805 | else if (VT == MVT::f64) |
3806 | LC = RTLIB::LROUND_F64; |
3807 | else if (VT == MVT::f80) |
3808 | LC = RTLIB::LROUND_F80; |
3809 | else if (VT == MVT::f128) |
3810 | LC = RTLIB::LROUND_F128; |
3811 | else if (VT == MVT::ppcf128) |
3812 | LC = RTLIB::LROUND_PPCF128; |
3813 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!" ); |
3814 | } else if (N->getOpcode() == ISD::LRINT || |
3815 | N->getOpcode() == ISD::STRICT_LRINT) { |
3816 | if (VT == MVT::f32) |
3817 | LC = RTLIB::LRINT_F32; |
3818 | else if (VT == MVT::f64) |
3819 | LC = RTLIB::LRINT_F64; |
3820 | else if (VT == MVT::f80) |
3821 | LC = RTLIB::LRINT_F80; |
3822 | else if (VT == MVT::f128) |
3823 | LC = RTLIB::LRINT_F128; |
3824 | else if (VT == MVT::ppcf128) |
3825 | LC = RTLIB::LRINT_PPCF128; |
3826 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!" ); |
3827 | } else if (N->getOpcode() == ISD::LLROUND || |
3828 | N->getOpcode() == ISD::STRICT_LLROUND) { |
3829 | if (VT == MVT::f32) |
3830 | LC = RTLIB::LLROUND_F32; |
3831 | else if (VT == MVT::f64) |
3832 | LC = RTLIB::LLROUND_F64; |
3833 | else if (VT == MVT::f80) |
3834 | LC = RTLIB::LLROUND_F80; |
3835 | else if (VT == MVT::f128) |
3836 | LC = RTLIB::LLROUND_F128; |
3837 | else if (VT == MVT::ppcf128) |
3838 | LC = RTLIB::LLROUND_PPCF128; |
3839 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!" ); |
3840 | } else if (N->getOpcode() == ISD::LLRINT || |
3841 | N->getOpcode() == ISD::STRICT_LLRINT) { |
3842 | if (VT == MVT::f32) |
3843 | LC = RTLIB::LLRINT_F32; |
3844 | else if (VT == MVT::f64) |
3845 | LC = RTLIB::LLRINT_F64; |
3846 | else if (VT == MVT::f80) |
3847 | LC = RTLIB::LLRINT_F80; |
3848 | else if (VT == MVT::f128) |
3849 | LC = RTLIB::LLRINT_F128; |
3850 | else if (VT == MVT::ppcf128) |
3851 | LC = RTLIB::LLRINT_PPCF128; |
3852 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!" ); |
3853 | } else |
3854 | llvm_unreachable("Unexpected opcode!" ); |
3855 | |
3856 | EVT RetVT = N->getValueType(ResNo: 0); |
3857 | |
3858 | TargetLowering::MakeLibCallOptions CallOptions; |
3859 | CallOptions.setSExt(true); |
3860 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
3861 | Ops: Op, CallOptions, dl, |
3862 | Chain); |
3863 | SplitInteger(Op: Tmp.first, Lo, Hi); |
3864 | |
3865 | if (N->isStrictFPOpcode()) |
3866 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
3867 | } |
3868 | |
3869 | void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
3870 | SDValue &Lo, SDValue &Hi) { |
3871 | assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?" ); |
3872 | |
3873 | if (ISD::isNormalLoad(N)) { |
3874 | ExpandRes_NormalLoad(N, Lo, Hi); |
3875 | return; |
3876 | } |
3877 | |
3878 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!" ); |
3879 | |
3880 | EVT VT = N->getValueType(ResNo: 0); |
3881 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
3882 | SDValue Ch = N->getChain(); |
3883 | SDValue Ptr = N->getBasePtr(); |
3884 | ISD::LoadExtType ExtType = N->getExtensionType(); |
3885 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
3886 | AAMDNodes AAInfo = N->getAAInfo(); |
3887 | SDLoc dl(N); |
3888 | |
3889 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
3890 | |
3891 | if (N->getMemoryVT().bitsLE(VT: NVT)) { |
3892 | EVT MemVT = N->getMemoryVT(); |
3893 | |
3894 | Lo = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), MemVT, |
3895 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3896 | |
3897 | // Remember the chain. |
3898 | Ch = Lo.getValue(R: 1); |
3899 | |
3900 | if (ExtType == ISD::SEXTLOAD) { |
3901 | // The high part is obtained by SRA'ing all but one of the bits of the |
3902 | // lo part. |
3903 | unsigned LoSize = Lo.getValueSizeInBits(); |
3904 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3905 | N2: DAG.getConstant(Val: LoSize - 1, DL: dl, |
3906 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3907 | } else if (ExtType == ISD::ZEXTLOAD) { |
3908 | // The high part is just a zero. |
3909 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3910 | } else { |
3911 | assert(ExtType == ISD::EXTLOAD && "Unknown extload!" ); |
3912 | // The high part is undefined. |
3913 | Hi = DAG.getUNDEF(VT: NVT); |
3914 | } |
3915 | } else if (DAG.getDataLayout().isLittleEndian()) { |
3916 | // Little-endian - low bits are at low addresses. |
3917 | Lo = DAG.getLoad(VT: NVT, dl, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), |
3918 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3919 | |
3920 | unsigned ExcessBits = |
3921 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
3922 | EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits); |
3923 | |
3924 | // Increment the pointer to the other half. |
3925 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3926 | Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
3927 | Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, |
3928 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), MemVT: NEVT, |
3929 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3930 | |
3931 | // Build a factor node to remember that this load is independent of the |
3932 | // other one. |
3933 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
3934 | Hi.getValue(R: 1)); |
3935 | } else { |
3936 | // Big-endian - high bits are at low addresses. Favor aligned loads at |
3937 | // the cost of some bit-fiddling. |
3938 | EVT MemVT = N->getMemoryVT(); |
3939 | unsigned EBytes = MemVT.getStoreSize(); |
3940 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3941 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
3942 | |
3943 | // Load both the high bits and maybe some of the low bits. |
3944 | Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), |
3945 | MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
3946 | BitWidth: MemVT.getSizeInBits() - ExcessBits), |
3947 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3948 | |
3949 | // Increment the pointer to the other half. |
3950 | Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
3951 | // Load the rest of the low bits. |
3952 | Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: NVT, Chain: Ch, Ptr, |
3953 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
3954 | MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits), |
3955 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3956 | |
3957 | // Build a factor node to remember that this load is independent of the |
3958 | // other one. |
3959 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
3960 | Hi.getValue(R: 1)); |
3961 | |
3962 | if (ExcessBits < NVT.getSizeInBits()) { |
3963 | // Transfer low bits from the bottom of Hi to the top of Lo. |
3964 | Lo = DAG.getNode( |
3965 | Opcode: ISD::OR, DL: dl, VT: NVT, N1: Lo, |
3966 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi, |
3967 | N2: DAG.getConstant(Val: ExcessBits, DL: dl, |
3968 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())))); |
3969 | // Move high bits to the right position in Hi. |
3970 | Hi = DAG.getNode(Opcode: ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, DL: dl, VT: NVT, |
3971 | N1: Hi, |
3972 | N2: DAG.getConstant(Val: NVT.getSizeInBits() - ExcessBits, DL: dl, |
3973 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3974 | } |
3975 | } |
3976 | |
3977 | // Legalize the chain result - switch anything that used the old chain to |
3978 | // use the new one. |
3979 | ReplaceValueWith(From: SDValue(N, 1), To: Ch); |
3980 | } |
3981 | |
3982 | void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
3983 | SDValue &Lo, SDValue &Hi) { |
3984 | SDLoc dl(N); |
3985 | SDValue LL, LH, RL, RH; |
3986 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH); |
3987 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH); |
3988 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LL, N2: RL); |
3989 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LH, N2: RH); |
3990 | } |
3991 | |
3992 | void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
3993 | SDValue &Lo, SDValue &Hi) { |
3994 | EVT VT = N->getValueType(ResNo: 0); |
3995 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
3996 | SDLoc dl(N); |
3997 | |
3998 | SDValue LL, LH, RL, RH; |
3999 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH); |
4000 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH); |
4001 | |
4002 | if (TLI.expandMUL(N, Lo, Hi, HiLoVT: NVT, DAG, |
4003 | Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
4004 | LL, LH, RL, RH)) |
4005 | return; |
4006 | |
4007 | // If nothing else, we can make a libcall. |
4008 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4009 | if (VT == MVT::i16) |
4010 | LC = RTLIB::MUL_I16; |
4011 | else if (VT == MVT::i32) |
4012 | LC = RTLIB::MUL_I32; |
4013 | else if (VT == MVT::i64) |
4014 | LC = RTLIB::MUL_I64; |
4015 | else if (VT == MVT::i128) |
4016 | LC = RTLIB::MUL_I128; |
4017 | |
4018 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC)) { |
4019 | // Perform a wide multiplication where the wide type is the original VT and |
4020 | // the 4 parts are the split arguments. |
4021 | TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, WideVT: VT, LL, LH, RL, RH, Lo, |
4022 | Hi); |
4023 | return; |
4024 | } |
4025 | |
4026 | // Note that we don't need to do a wide MUL here since we don't care about the |
4027 | // upper half of the result if it exceeds VT. |
4028 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4029 | TargetLowering::MakeLibCallOptions CallOptions; |
4030 | CallOptions.setSExt(true); |
4031 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, |
4032 | Lo, Hi); |
4033 | } |
4034 | |
4035 | void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo, |
4036 | SDValue &Hi) { |
4037 | SDLoc DL(N); |
4038 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4039 | SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
4040 | SDValue R = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N: N->getOperand(Num: 0)); |
4041 | Lo = R.getValue(R: 0); |
4042 | Hi = R.getValue(R: 1); |
4043 | ReplaceValueWith(From: SDValue(N, 1), To: R.getValue(R: 2)); |
4044 | } |
4045 | |
4046 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, |
4047 | SDValue &Hi) { |
4048 | SDValue Result = TLI.expandAddSubSat(Node: N, DAG); |
4049 | SplitInteger(Op: Result, Lo, Hi); |
4050 | } |
4051 | |
4052 | void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, |
4053 | SDValue &Hi) { |
4054 | SDValue Result = TLI.expandShlSat(Node: N, DAG); |
4055 | SplitInteger(Op: Result, Lo, Hi); |
4056 | } |
4057 | |
4058 | /// This performs an expansion of the integer result for a fixed point |
4059 | /// multiplication. The default expansion performs rounding down towards |
4060 | /// negative infinity, though targets that do care about rounding should specify |
4061 | /// a target hook for rounding and provide their own expansion or lowering of |
4062 | /// fixed point multiplication to be consistent with rounding. |
4063 | void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, |
4064 | SDValue &Hi) { |
4065 | SDLoc dl(N); |
4066 | EVT VT = N->getValueType(ResNo: 0); |
4067 | unsigned VTSize = VT.getScalarSizeInBits(); |
4068 | SDValue LHS = N->getOperand(Num: 0); |
4069 | SDValue RHS = N->getOperand(Num: 1); |
4070 | uint64_t Scale = N->getConstantOperandVal(Num: 2); |
4071 | bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || |
4072 | N->getOpcode() == ISD::UMULFIXSAT); |
4073 | bool Signed = (N->getOpcode() == ISD::SMULFIX || |
4074 | N->getOpcode() == ISD::SMULFIXSAT); |
4075 | |
4076 | // Handle special case when scale is equal to zero. |
4077 | if (!Scale) { |
4078 | SDValue Result; |
4079 | if (!Saturating) { |
4080 | Result = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS); |
4081 | } else { |
4082 | EVT BoolVT = getSetCCResultType(VT); |
4083 | unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; |
4084 | Result = DAG.getNode(Opcode: MulOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS); |
4085 | SDValue Product = Result.getValue(R: 0); |
4086 | SDValue Overflow = Result.getValue(R: 1); |
4087 | if (Signed) { |
4088 | APInt MinVal = APInt::getSignedMinValue(numBits: VTSize); |
4089 | APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize); |
4090 | SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT); |
4091 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT); |
4092 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
4093 | // Xor the inputs, if resulting sign bit is 0 the product will be |
4094 | // positive, else negative. |
4095 | SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS); |
4096 | SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT); |
4097 | Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax); |
4098 | Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product); |
4099 | } else { |
4100 | // For unsigned multiplication, we only need to check the max since we |
4101 | // can't really overflow towards zero. |
4102 | APInt MaxVal = APInt::getMaxValue(numBits: VTSize); |
4103 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT); |
4104 | Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product); |
4105 | } |
4106 | } |
4107 | SplitInteger(Op: Result, Lo, Hi); |
4108 | return; |
4109 | } |
4110 | |
4111 | // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will |
4112 | // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. |
4113 | assert(Scale <= VTSize && "Scale can't be larger than the value type size." ); |
4114 | |
4115 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4116 | SDValue LL, LH, RL, RH; |
4117 | GetExpandedInteger(Op: LHS, Lo&: LL, Hi&: LH); |
4118 | GetExpandedInteger(Op: RHS, Lo&: RL, Hi&: RH); |
4119 | SmallVector<SDValue, 4> Result; |
4120 | |
4121 | unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; |
4122 | if (!TLI.expandMUL_LOHI(Opcode: LoHiOp, VT, dl, LHS, RHS, Result, HiLoVT: NVT, DAG, |
4123 | Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
4124 | LL, LH, RL, RH)) { |
4125 | Result.clear(); |
4126 | Result.resize(N: 4); |
4127 | |
4128 | SDValue LoTmp, HiTmp; |
4129 | TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo&: LoTmp, Hi&: HiTmp); |
4130 | SplitInteger(Op: LoTmp, Lo&: Result[0], Hi&: Result[1]); |
4131 | SplitInteger(Op: HiTmp, Lo&: Result[2], Hi&: Result[3]); |
4132 | } |
4133 | assert(Result.size() == 4 && "Unexpected number of partlets in the result" ); |
4134 | |
4135 | unsigned NVTSize = NVT.getScalarSizeInBits(); |
4136 | assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " |
4137 | "the size of the current value type" ); |
4138 | |
4139 | // After getting the multiplication result in 4 parts, we need to perform a |
4140 | // shift right by the amount of the scale to get the result in that scale. |
4141 | // |
4142 | // Let's say we multiply 2 64 bit numbers. The resulting value can be held in |
4143 | // 128 bits that are cut into 4 32-bit parts: |
4144 | // |
4145 | // HH HL LH LL |
4146 | // |---32---|---32---|---32---|---32---| |
4147 | // 128 96 64 32 0 |
4148 | // |
4149 | // |------VTSize-----| |
4150 | // |
4151 | // |NVTSize-| |
4152 | // |
4153 | // The resulting Lo and Hi would normally be in LL and LH after the shift. But |
4154 | // to avoid unneccessary shifting of all 4 parts, we can adjust the shift |
4155 | // amount and get Lo and Hi using two funnel shifts. Or for the special case |
4156 | // when Scale is a multiple of NVTSize we can just pick the result without |
4157 | // shifting. |
4158 | uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. |
4159 | if (Scale % NVTSize) { |
4160 | SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: Scale % NVTSize, VT: NVT, DL: dl); |
4161 | Lo = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 1], N2: Result[Part0], |
4162 | N3: ShiftAmount); |
4163 | Hi = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 2], N2: Result[Part0 + 1], |
4164 | N3: ShiftAmount); |
4165 | } else { |
4166 | Lo = Result[Part0]; |
4167 | Hi = Result[Part0 + 1]; |
4168 | } |
4169 | |
4170 | // Unless saturation is requested we are done. The result is in <Hi,Lo>. |
4171 | if (!Saturating) |
4172 | return; |
4173 | |
4174 | // Can not overflow when there is no integer part. |
4175 | if (Scale == VTSize) |
4176 | return; |
4177 | |
4178 | // To handle saturation we must check for overflow in the multiplication. |
4179 | // |
4180 | // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) |
4181 | // aren't all zeroes. |
4182 | // |
4183 | // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) |
4184 | // aren't all ones or all zeroes. |
4185 | // |
4186 | // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the |
4187 | // highest bit of HH determines saturation direction in the event of signed |
4188 | // saturation. |
4189 | |
4190 | SDValue ResultHL = Result[2]; |
4191 | SDValue ResultHH = Result[3]; |
4192 | |
4193 | SDValue SatMax, SatMin; |
4194 | SDValue NVTZero = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
4195 | SDValue NVTNeg1 = DAG.getConstant(Val: -1, DL: dl, VT: NVT); |
4196 | EVT BoolNVT = getSetCCResultType(VT: NVT); |
4197 | |
4198 | if (!Signed) { |
4199 | if (Scale < NVTSize) { |
4200 | // Overflow happened if ((HH | (HL >> Scale)) != 0). |
4201 | SDValue HLAdjusted = |
4202 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL, |
4203 | N2: DAG.getShiftAmountConstant(Val: Scale, VT: NVT, DL: dl)); |
4204 | SDValue Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: HLAdjusted, N2: ResultHH); |
4205 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: Tmp, RHS: NVTZero, Cond: ISD::SETNE); |
4206 | } else if (Scale == NVTSize) { |
4207 | // Overflow happened if (HH != 0). |
4208 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETNE); |
4209 | } else if (Scale < VTSize) { |
4210 | // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). |
4211 | SDValue HLAdjusted = |
4212 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL, |
4213 | N2: DAG.getShiftAmountConstant(Val: Scale - NVTSize, VT: NVT, DL: dl)); |
4214 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: HLAdjusted, RHS: NVTZero, Cond: ISD::SETNE); |
4215 | } else |
4216 | llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" |
4217 | "(and saturation can't happen with Scale==VTSize)." ); |
4218 | |
4219 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Hi); |
4220 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Lo); |
4221 | return; |
4222 | } |
4223 | |
4224 | if (Scale < NVTSize) { |
4225 | // The number of overflow bits we can check are VTSize - Scale + 1 (we |
4226 | // include the sign bit). If these top bits are > 0, then we overflowed past |
4227 | // the max value. If these top bits are < -1, then we overflowed past the |
4228 | // min value. Otherwise, we did not overflow. |
4229 | unsigned OverflowBits = VTSize - Scale + 1; |
4230 | assert(OverflowBits <= VTSize && OverflowBits > NVTSize && |
4231 | "Extent of overflow bits must start within HL" ); |
4232 | SDValue HLHiMask = DAG.getConstant( |
4233 | Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits - NVTSize), DL: dl, VT: NVT); |
4234 | SDValue HLLoMask = DAG.getConstant( |
4235 | Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: VTSize - OverflowBits), DL: dl, VT: NVT); |
4236 | // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). |
4237 | SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT); |
4238 | SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ); |
4239 | SDValue HLUGT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLLoMask, Cond: ISD::SETUGT); |
4240 | SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0, |
4241 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLUGT)); |
4242 | // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). |
4243 | SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT); |
4244 | SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ); |
4245 | SDValue HLULT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLHiMask, Cond: ISD::SETULT); |
4246 | SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT, |
4247 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLULT)); |
4248 | } else if (Scale == NVTSize) { |
4249 | // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). |
4250 | SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT); |
4251 | SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ); |
4252 | SDValue HLNeg = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETLT); |
4253 | SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0, |
4254 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLNeg)); |
4255 | // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). |
4256 | SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT); |
4257 | SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ); |
4258 | SDValue HLPos = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETGE); |
4259 | SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT, |
4260 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLPos)); |
4261 | } else if (Scale < VTSize) { |
4262 | // This is similar to the case when we saturate if Scale < NVTSize, but we |
4263 | // only need to check HH. |
4264 | unsigned OverflowBits = VTSize - Scale + 1; |
4265 | SDValue HHHiMask = DAG.getConstant( |
4266 | Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits), DL: dl, VT: NVT); |
4267 | SDValue HHLoMask = DAG.getConstant( |
4268 | Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: NVTSize - OverflowBits), DL: dl, VT: NVT); |
4269 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHLoMask, Cond: ISD::SETGT); |
4270 | SatMin = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHHiMask, Cond: ISD::SETLT); |
4271 | } else |
4272 | llvm_unreachable("Illegal scale for signed fixed point mul." ); |
4273 | |
4274 | // Saturate to signed maximum. |
4275 | APInt MaxHi = APInt::getSignedMaxValue(numBits: NVTSize); |
4276 | APInt MaxLo = APInt::getAllOnes(numBits: NVTSize); |
4277 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxHi, DL: dl, VT: NVT), RHS: Hi); |
4278 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxLo, DL: dl, VT: NVT), RHS: Lo); |
4279 | // Saturate to signed minimum. |
4280 | APInt MinHi = APInt::getSignedMinValue(numBits: NVTSize); |
4281 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: DAG.getConstant(Val: MinHi, DL: dl, VT: NVT), RHS: Hi); |
4282 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: NVTZero, RHS: Lo); |
4283 | } |
4284 | |
4285 | void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, |
4286 | SDValue &Hi) { |
4287 | SDLoc dl(N); |
4288 | // Try expanding in the existing type first. |
4289 | SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: N->getOperand(Num: 0), |
4290 | RHS: N->getOperand(Num: 1), |
4291 | Scale: N->getConstantOperandVal(Num: 2), DAG); |
4292 | |
4293 | if (!Res) |
4294 | Res = earlyExpandDIVFIX(N, LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), |
4295 | Scale: N->getConstantOperandVal(Num: 2), TLI, DAG); |
4296 | SplitInteger(Op: Res, Lo, Hi); |
4297 | } |
4298 | |
4299 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
4300 | SDValue &Lo, SDValue &Hi) { |
4301 | assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && |
4302 | "Node has unexpected Opcode" ); |
4303 | SDValue LHS = Node->getOperand(Num: 0); |
4304 | SDValue RHS = Node->getOperand(Num: 1); |
4305 | SDLoc dl(Node); |
4306 | |
4307 | SDValue Ovf; |
4308 | |
4309 | bool IsAdd = Node->getOpcode() == ISD::SADDO; |
4310 | unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; |
4311 | |
4312 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
4313 | Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType())); |
4314 | |
4315 | if (HasCarryOp) { |
4316 | // Expand the subcomponents. |
4317 | SDValue LHSL, LHSH, RHSL, RHSH; |
4318 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
4319 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
4320 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: Node->getValueType(ResNo: 1)); |
4321 | |
4322 | Lo = DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: dl, VTList, Ops: {LHSL, RHSL}); |
4323 | Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) }); |
4324 | |
4325 | Ovf = Hi.getValue(R: 1); |
4326 | } else { |
4327 | // Expand the result by simply replacing it with the equivalent |
4328 | // non-overflow-checking operation. |
4329 | SDValue Sum = DAG.getNode(Opcode: Node->getOpcode() == ISD::SADDO ? |
4330 | ISD::ADD : ISD::SUB, DL: dl, VT: LHS.getValueType(), |
4331 | N1: LHS, N2: RHS); |
4332 | SplitInteger(Op: Sum, Lo, Hi); |
4333 | |
4334 | // Compute the overflow. |
4335 | // |
4336 | // LHSSign -> LHS < 0 |
4337 | // RHSSign -> RHS < 0 |
4338 | // SumSign -> Sum < 0 |
4339 | // |
4340 | // Add: |
4341 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
4342 | // Sub: |
4343 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
4344 | // |
4345 | // To get better codegen we can rewrite this by doing bitwise math on |
4346 | // the integers and extract the final sign bit at the end. So the |
4347 | // above becomes: |
4348 | // |
4349 | // Add: |
4350 | // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 |
4351 | // Sub: |
4352 | // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 |
4353 | // |
4354 | // NOTE: This is different than the expansion we do in expandSADDSUBO |
4355 | // because it is more costly to determine the RHS is > 0 for SSUBO with the |
4356 | // integers split. |
4357 | EVT VT = LHS.getValueType(); |
4358 | SDValue SignsMatch = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS); |
4359 | if (IsAdd) |
4360 | SignsMatch = DAG.getNOT(DL: dl, Val: SignsMatch, VT); |
4361 | |
4362 | SDValue SumSignNE = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: Sum); |
4363 | Ovf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SignsMatch, N2: SumSignNE); |
4364 | EVT OType = Node->getValueType(ResNo: 1); |
4365 | Ovf = DAG.getSetCC(DL: dl, VT: OType, LHS: Ovf, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT); |
4366 | } |
4367 | |
4368 | // Use the calculated overflow everywhere. |
4369 | ReplaceValueWith(From: SDValue(Node, 1), To: Ovf); |
4370 | } |
4371 | |
4372 | void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
4373 | SDValue &Lo, SDValue &Hi) { |
4374 | EVT VT = N->getValueType(ResNo: 0); |
4375 | SDLoc dl(N); |
4376 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4377 | |
4378 | if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) { |
4379 | SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4380 | SplitInteger(Op: Res.getValue(R: 0), Lo, Hi); |
4381 | return; |
4382 | } |
4383 | |
4384 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4385 | if (VT == MVT::i16) |
4386 | LC = RTLIB::SDIV_I16; |
4387 | else if (VT == MVT::i32) |
4388 | LC = RTLIB::SDIV_I32; |
4389 | else if (VT == MVT::i64) |
4390 | LC = RTLIB::SDIV_I64; |
4391 | else if (VT == MVT::i128) |
4392 | LC = RTLIB::SDIV_I128; |
4393 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!" ); |
4394 | |
4395 | TargetLowering::MakeLibCallOptions CallOptions; |
4396 | CallOptions.setSExt(true); |
4397 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4398 | } |
4399 | |
4400 | void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, |
4401 | SDValue &Hi) { |
4402 | SDLoc dl(N); |
4403 | SDValue Shiftee = N->getOperand(Num: 0); |
4404 | EVT VT = Shiftee.getValueType(); |
4405 | SDValue ShAmt = N->getOperand(Num: 1); |
4406 | EVT ShAmtVT = ShAmt.getValueType(); |
4407 | |
4408 | // This legalization is optimal when the shift is by a multiple of byte width, |
4409 | // %x * 8 <-> %x << 3 so 3 low bits should be be known zero. |
4410 | bool ShiftByByteMultiple = |
4411 | DAG.computeKnownBits(Op: ShAmt).countMinTrailingZeros() >= 3; |
4412 | |
4413 | // If we can't do it as one step, we'll have two uses of shift amount, |
4414 | // and thus must freeze it. |
4415 | if (!ShiftByByteMultiple) |
4416 | ShAmt = DAG.getFreeze(V: ShAmt); |
4417 | |
4418 | unsigned VTBitWidth = VT.getScalarSizeInBits(); |
4419 | assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?" ); |
4420 | unsigned VTByteWidth = VTBitWidth / 8; |
4421 | assert(isPowerOf2_32(VTByteWidth) && |
4422 | "Shiftee type size is not a power of two!" ); |
4423 | unsigned StackSlotByteWidth = 2 * VTByteWidth; |
4424 | unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; |
4425 | EVT StackSlotVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StackSlotBitWidth); |
4426 | |
4427 | // Get a temporary stack slot 2x the width of our VT. |
4428 | // FIXME: reuse stack slots? |
4429 | // FIXME: should we be more picky about alignment? |
4430 | Align StackSlotAlignment(1); |
4431 | SDValue StackPtr = DAG.CreateStackTemporary( |
4432 | Bytes: TypeSize::getFixed(ExactSize: StackSlotByteWidth), Alignment: StackSlotAlignment); |
4433 | EVT PtrTy = StackPtr.getValueType(); |
4434 | SDValue Ch = DAG.getEntryNode(); |
4435 | |
4436 | MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( |
4437 | MF&: DAG.getMachineFunction(), |
4438 | FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex()); |
4439 | |
4440 | // Extend the value, that is being shifted, to the entire stack slot's width. |
4441 | SDValue Init; |
4442 | if (N->getOpcode() != ISD::SHL) { |
4443 | unsigned WideningOpc = |
4444 | N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4445 | Init = DAG.getNode(Opcode: WideningOpc, DL: dl, VT: StackSlotVT, Operand: Shiftee); |
4446 | } else { |
4447 | // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. |
4448 | SDValue AllZeros = DAG.getConstant(Val: 0, DL: dl, VT); |
4449 | Init = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: StackSlotVT, N1: AllZeros, N2: Shiftee); |
4450 | } |
4451 | // And spill it into the stack slot. |
4452 | Ch = DAG.getStore(Chain: Ch, dl, Val: Init, Ptr: StackPtr, PtrInfo: StackPtrInfo, Alignment: StackSlotAlignment); |
4453 | |
4454 | // Now, compute the full-byte offset into stack slot from where we can load. |
4455 | // We have shift amount, which is in bits, but in multiples of byte. |
4456 | // So just divide by CHAR_BIT. |
4457 | SDNodeFlags Flags; |
4458 | if (ShiftByByteMultiple) |
4459 | Flags.setExact(true); |
4460 | SDValue ByteOffset = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShAmtVT, N1: ShAmt, |
4461 | N2: DAG.getConstant(Val: 3, DL: dl, VT: ShAmtVT), Flags); |
4462 | // And clamp it, because OOB load is an immediate UB, |
4463 | // while shift overflow would have *just* been poison. |
4464 | ByteOffset = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ByteOffset, |
4465 | N2: DAG.getConstant(Val: VTByteWidth - 1, DL: dl, VT: ShAmtVT)); |
4466 | // We have exactly two strategies on indexing into stack slot here: |
4467 | // 1. upwards starting from the beginning of the slot |
4468 | // 2. downwards starting from the middle of the slot |
4469 | // On little-endian machine, we pick 1. for right shifts and 2. for left-shift |
4470 | // and vice versa on big-endian machine. |
4471 | bool WillIndexUpwards = N->getOpcode() != ISD::SHL; |
4472 | if (DAG.getDataLayout().isBigEndian()) |
4473 | WillIndexUpwards = !WillIndexUpwards; |
4474 | |
4475 | SDValue AdjStackPtr; |
4476 | if (WillIndexUpwards) { |
4477 | AdjStackPtr = StackPtr; |
4478 | } else { |
4479 | AdjStackPtr = DAG.getMemBasePlusOffset( |
4480 | Base: StackPtr, Offset: DAG.getConstant(Val: VTByteWidth, DL: dl, VT: PtrTy), DL: dl); |
4481 | ByteOffset = DAG.getNegative(Val: ByteOffset, DL: dl, VT: ShAmtVT); |
4482 | } |
4483 | |
4484 | // Get the pointer somewhere into the stack slot from which we need to load. |
4485 | ByteOffset = DAG.getSExtOrTrunc(Op: ByteOffset, DL: dl, VT: PtrTy); |
4486 | AdjStackPtr = DAG.getMemBasePlusOffset(Base: AdjStackPtr, Offset: ByteOffset, DL: dl); |
4487 | |
4488 | // And load it! While the load is not legal, legalizing it is obvious. |
4489 | SDValue Res = DAG.getLoad( |
4490 | VT, dl, Chain: Ch, Ptr: AdjStackPtr, |
4491 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), Alignment: Align(1)); |
4492 | // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] |
4493 | |
4494 | // If we may still have a less-than-CHAR_BIT to shift by, do so now. |
4495 | if (!ShiftByByteMultiple) { |
4496 | SDValue ShAmtRem = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt, |
4497 | N2: DAG.getConstant(Val: 7, DL: dl, VT: ShAmtVT)); |
4498 | Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: Res, N2: ShAmtRem); |
4499 | } |
4500 | |
4501 | // Finally, split the computed value. |
4502 | SplitInteger(Op: Res, Lo, Hi); |
4503 | } |
4504 | |
4505 | void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
4506 | SDValue &Lo, SDValue &Hi) { |
4507 | EVT VT = N->getValueType(ResNo: 0); |
4508 | SDLoc dl(N); |
4509 | |
4510 | // If we can emit an efficient shift operation, do so now. Check to see if |
4511 | // the RHS is a constant. |
4512 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) |
4513 | return ExpandShiftByConstant(N, Amt: CN->getAPIntValue(), Lo, Hi); |
4514 | |
4515 | // If we can determine that the high bit of the shift is zero or one, even if |
4516 | // the low bits are variable, emit this shift in an optimized form. |
4517 | if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) |
4518 | return; |
4519 | |
4520 | // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. |
4521 | unsigned PartsOpc; |
4522 | if (N->getOpcode() == ISD::SHL) { |
4523 | PartsOpc = ISD::SHL_PARTS; |
4524 | } else if (N->getOpcode() == ISD::SRL) { |
4525 | PartsOpc = ISD::SRL_PARTS; |
4526 | } else { |
4527 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!" ); |
4528 | PartsOpc = ISD::SRA_PARTS; |
4529 | } |
4530 | |
4531 | // Next check to see if the target supports this SHL_PARTS operation or if it |
4532 | // will custom expand it. Don't lower this to SHL_PARTS when we optimise for |
4533 | // size, but create a libcall instead. |
4534 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4535 | TargetLowering::LegalizeAction Action = TLI.getOperationAction(Op: PartsOpc, VT: NVT); |
4536 | const bool LegalOrCustom = |
4537 | (Action == TargetLowering::Legal && TLI.isTypeLegal(VT: NVT)) || |
4538 | Action == TargetLowering::Custom; |
4539 | |
4540 | unsigned ExpansionFactor = 1; |
4541 | // That VT->NVT expansion is one step. But will we re-expand NVT? |
4542 | for (EVT TmpVT = NVT;;) { |
4543 | EVT NewTMPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: TmpVT); |
4544 | if (NewTMPVT == TmpVT) |
4545 | break; |
4546 | TmpVT = NewTMPVT; |
4547 | ++ExpansionFactor; |
4548 | } |
4549 | |
4550 | TargetLowering::ShiftLegalizationStrategy S = |
4551 | TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); |
4552 | |
4553 | if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) |
4554 | return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); |
4555 | |
4556 | if (LegalOrCustom && |
4557 | S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { |
4558 | // Expand the subcomponents. |
4559 | SDValue LHSL, LHSH; |
4560 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
4561 | EVT VT = LHSL.getValueType(); |
4562 | |
4563 | // If the shift amount operand is coming from a vector legalization it may |
4564 | // have an illegal type. Fix that first by casting the operand, otherwise |
4565 | // the new SHL_PARTS operation would need further legalization. |
4566 | SDValue ShiftOp = N->getOperand(Num: 1); |
4567 | EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout()); |
4568 | if (ShiftOp.getValueType() != ShiftTy) |
4569 | ShiftOp = DAG.getZExtOrTrunc(Op: ShiftOp, DL: dl, VT: ShiftTy); |
4570 | |
4571 | SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
4572 | Lo = DAG.getNode(Opcode: PartsOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4573 | Hi = Lo.getValue(R: 1); |
4574 | return; |
4575 | } |
4576 | |
4577 | // Otherwise, emit a libcall. |
4578 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4579 | bool isSigned; |
4580 | if (N->getOpcode() == ISD::SHL) { |
4581 | isSigned = false; /*sign irrelevant*/ |
4582 | if (VT == MVT::i16) |
4583 | LC = RTLIB::SHL_I16; |
4584 | else if (VT == MVT::i32) |
4585 | LC = RTLIB::SHL_I32; |
4586 | else if (VT == MVT::i64) |
4587 | LC = RTLIB::SHL_I64; |
4588 | else if (VT == MVT::i128) |
4589 | LC = RTLIB::SHL_I128; |
4590 | } else if (N->getOpcode() == ISD::SRL) { |
4591 | isSigned = false; |
4592 | if (VT == MVT::i16) |
4593 | LC = RTLIB::SRL_I16; |
4594 | else if (VT == MVT::i32) |
4595 | LC = RTLIB::SRL_I32; |
4596 | else if (VT == MVT::i64) |
4597 | LC = RTLIB::SRL_I64; |
4598 | else if (VT == MVT::i128) |
4599 | LC = RTLIB::SRL_I128; |
4600 | } else { |
4601 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!" ); |
4602 | isSigned = true; |
4603 | if (VT == MVT::i16) |
4604 | LC = RTLIB::SRA_I16; |
4605 | else if (VT == MVT::i32) |
4606 | LC = RTLIB::SRA_I32; |
4607 | else if (VT == MVT::i64) |
4608 | LC = RTLIB::SRA_I64; |
4609 | else if (VT == MVT::i128) |
4610 | LC = RTLIB::SRA_I128; |
4611 | } |
4612 | |
4613 | if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(Call: LC)) { |
4614 | EVT ShAmtTy = |
4615 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize()); |
4616 | SDValue ShAmt = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl, VT: ShAmtTy); |
4617 | SDValue Ops[2] = {N->getOperand(Num: 0), ShAmt}; |
4618 | TargetLowering::MakeLibCallOptions CallOptions; |
4619 | CallOptions.setSExt(isSigned); |
4620 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4621 | return; |
4622 | } |
4623 | |
4624 | if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) |
4625 | llvm_unreachable("Unsupported shift!" ); |
4626 | } |
4627 | |
4628 | void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
4629 | SDValue &Lo, SDValue &Hi) { |
4630 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4631 | SDLoc dl(N); |
4632 | SDValue Op = N->getOperand(Num: 0); |
4633 | if (Op.getValueType().bitsLE(VT: NVT)) { |
4634 | // The low part is sign extension of the input (degenerates to a copy). |
4635 | Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4636 | // The high part is obtained by SRA'ing all but one of the bits of low part. |
4637 | unsigned LoSize = NVT.getSizeInBits(); |
4638 | Hi = DAG.getNode( |
4639 | Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
4640 | N2: DAG.getConstant(Val: LoSize - 1, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4641 | } else { |
4642 | // For example, extension of an i48 to an i64. The operand type necessarily |
4643 | // promotes to the result type, so will end up being expanded too. |
4644 | assert(getTypeAction(Op.getValueType()) == |
4645 | TargetLowering::TypePromoteInteger && |
4646 | "Only know how to promote this result!" ); |
4647 | SDValue Res = GetPromotedInteger(Op); |
4648 | assert(Res.getValueType() == N->getValueType(0) && |
4649 | "Operand over promoted?" ); |
4650 | // Split the promoted operand. This will simplify when it is expanded. |
4651 | SplitInteger(Op: Res, Lo, Hi); |
4652 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
4653 | Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi, |
4654 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
4655 | BitWidth: ExcessBits))); |
4656 | } |
4657 | } |
4658 | |
4659 | void DAGTypeLegalizer:: |
4660 | ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
4661 | SDLoc dl(N); |
4662 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
4663 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
4664 | |
4665 | if (EVT.bitsLE(VT: Lo.getValueType())) { |
4666 | // sext_inreg the low part if needed. |
4667 | Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Lo.getValueType(), N1: Lo, |
4668 | N2: N->getOperand(Num: 1)); |
4669 | |
4670 | // The high part gets the sign extension from the lo-part. This handles |
4671 | // things like sextinreg V:i64 from i8. |
4672 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Hi.getValueType(), N1: Lo, |
4673 | N2: DAG.getConstant(Val: Hi.getValueSizeInBits() - 1, DL: dl, |
4674 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4675 | } else { |
4676 | // For example, extension of an i48 to an i64. Leave the low part alone, |
4677 | // sext_inreg the high part. |
4678 | unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
4679 | Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi, |
4680 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
4681 | BitWidth: ExcessBits))); |
4682 | } |
4683 | } |
4684 | |
4685 | void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
4686 | SDValue &Lo, SDValue &Hi) { |
4687 | EVT VT = N->getValueType(ResNo: 0); |
4688 | SDLoc dl(N); |
4689 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4690 | |
4691 | if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) { |
4692 | SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4693 | SplitInteger(Op: Res.getValue(R: 1), Lo, Hi); |
4694 | return; |
4695 | } |
4696 | |
4697 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4698 | if (VT == MVT::i16) |
4699 | LC = RTLIB::SREM_I16; |
4700 | else if (VT == MVT::i32) |
4701 | LC = RTLIB::SREM_I32; |
4702 | else if (VT == MVT::i64) |
4703 | LC = RTLIB::SREM_I64; |
4704 | else if (VT == MVT::i128) |
4705 | LC = RTLIB::SREM_I128; |
4706 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!" ); |
4707 | |
4708 | TargetLowering::MakeLibCallOptions CallOptions; |
4709 | CallOptions.setSExt(true); |
4710 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4711 | } |
4712 | |
4713 | void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
4714 | SDValue &Lo, SDValue &Hi) { |
4715 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4716 | SDLoc dl(N); |
4717 | Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4718 | Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: N->getOperand(Num: 0).getValueType(), |
4719 | N1: N->getOperand(Num: 0), |
4720 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
4721 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4722 | Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Hi); |
4723 | } |
4724 | |
4725 | void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
4726 | SDValue &Lo, SDValue &Hi) { |
4727 | EVT VT = N->getValueType(ResNo: 0); |
4728 | SDLoc dl(N); |
4729 | |
4730 | if (N->getOpcode() == ISD::UMULO) { |
4731 | // This section expands the operation into the following sequence of |
4732 | // instructions. `iNh` here refers to a type which has half the bit width of |
4733 | // the type the original operation operated on. |
4734 | // |
4735 | // %0 = %LHS.HI != 0 && %RHS.HI != 0 |
4736 | // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) |
4737 | // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) |
4738 | // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) |
4739 | // %4 = add iNh %1.0, %2.0 as iN |
4740 | // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) |
4741 | // |
4742 | // %lo = %3.LO |
4743 | // %hi = %5.0 |
4744 | // %ovf = %0 || %1.1 || %2.1 || %5.1 |
4745 | SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1); |
4746 | SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; |
4747 | GetExpandedInteger(Op: LHS, Lo&: LHSLow, Hi&: LHSHigh); |
4748 | GetExpandedInteger(Op: RHS, Lo&: RHSLow, Hi&: RHSHigh); |
4749 | EVT HalfVT = LHSLow.getValueType(); |
4750 | EVT BitVT = N->getValueType(ResNo: 1); |
4751 | SDVTList VTHalfWithO = DAG.getVTList(VT1: HalfVT, VT2: BitVT); |
4752 | |
4753 | SDValue HalfZero = DAG.getConstant(Val: 0, DL: dl, VT: HalfVT); |
4754 | SDValue Overflow = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BitVT, |
4755 | N1: DAG.getSetCC(DL: dl, VT: BitVT, LHS: LHSHigh, RHS: HalfZero, Cond: ISD::SETNE), |
4756 | N2: DAG.getSetCC(DL: dl, VT: BitVT, LHS: RHSHigh, RHS: HalfZero, Cond: ISD::SETNE)); |
4757 | |
4758 | SDValue One = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: LHSHigh, N2: RHSLow); |
4759 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: One.getValue(R: 1)); |
4760 | |
4761 | SDValue Two = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: RHSHigh, N2: LHSLow); |
4762 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Two.getValue(R: 1)); |
4763 | |
4764 | SDValue HighSum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HalfVT, N1: One, N2: Two); |
4765 | |
4766 | // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not |
4767 | // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this |
4768 | // operation recursively legalized?). |
4769 | // |
4770 | // Many backends understand this pattern and will convert into LOHI |
4771 | // themselves, if applicable. |
4772 | SDValue Three = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, |
4773 | N1: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LHSLow), |
4774 | N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RHSLow)); |
4775 | SplitInteger(Op: Three, Lo, Hi); |
4776 | |
4777 | Hi = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: VTHalfWithO, N1: Hi, N2: HighSum); |
4778 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Hi.getValue(R: 1)); |
4779 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
4780 | return; |
4781 | } |
4782 | |
4783 | Type *RetTy = VT.getTypeForEVT(Context&: *DAG.getContext()); |
4784 | EVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout()); |
4785 | Type *PtrTy = PtrVT.getTypeForEVT(Context&: *DAG.getContext()); |
4786 | |
4787 | // Replace this with a libcall that will check overflow. |
4788 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4789 | if (VT == MVT::i32) |
4790 | LC = RTLIB::MULO_I32; |
4791 | else if (VT == MVT::i64) |
4792 | LC = RTLIB::MULO_I64; |
4793 | else if (VT == MVT::i128) |
4794 | LC = RTLIB::MULO_I128; |
4795 | |
4796 | // If we don't have the libcall or if the function we are compiling is the |
4797 | // implementation of the expected libcall (avoid inf-loop), expand inline. |
4798 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC) || |
4799 | TLI.getLibcallName(Call: LC) == DAG.getMachineFunction().getName()) { |
4800 | // FIXME: This is not an optimal expansion, but better than crashing. |
4801 | EVT WideVT = |
4802 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getScalarSizeInBits() * 2); |
4803 | SDValue LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: WideVT, Operand: N->getOperand(Num: 0)); |
4804 | SDValue RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: WideVT, Operand: N->getOperand(Num: 1)); |
4805 | SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS); |
4806 | SDValue MulLo, MulHi; |
4807 | SplitInteger(Op: Mul, Lo&: MulLo, Hi&: MulHi); |
4808 | SDValue SRA = |
4809 | DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: MulLo, |
4810 | N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1, DL: dl, VT)); |
4811 | SDValue Overflow = |
4812 | DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: MulHi, RHS: SRA, Cond: ISD::SETNE); |
4813 | SplitInteger(Op: MulLo, Lo, Hi); |
4814 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
4815 | return; |
4816 | } |
4817 | |
4818 | SDValue Temp = DAG.CreateStackTemporary(VT: PtrVT); |
4819 | // Temporary for the overflow value, default it to zero. |
4820 | SDValue Chain = |
4821 | DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), Ptr: Temp, |
4822 | PtrInfo: MachinePointerInfo()); |
4823 | |
4824 | TargetLowering::ArgListTy Args; |
4825 | TargetLowering::ArgListEntry Entry; |
4826 | for (const SDValue &Op : N->op_values()) { |
4827 | EVT ArgVT = Op.getValueType(); |
4828 | Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext()); |
4829 | Entry.Node = Op; |
4830 | Entry.Ty = ArgTy; |
4831 | Entry.IsSExt = true; |
4832 | Entry.IsZExt = false; |
4833 | Args.push_back(x: Entry); |
4834 | } |
4835 | |
4836 | // Also pass the address of the overflow check. |
4837 | Entry.Node = Temp; |
4838 | Entry.Ty = PointerType::getUnqual(C&: PtrTy->getContext()); |
4839 | Entry.IsSExt = true; |
4840 | Entry.IsZExt = false; |
4841 | Args.push_back(x: Entry); |
4842 | |
4843 | SDValue Func = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC), VT: PtrVT); |
4844 | |
4845 | TargetLowering::CallLoweringInfo CLI(DAG); |
4846 | CLI.setDebugLoc(dl) |
4847 | .setChain(Chain) |
4848 | .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Func, ArgsList: std::move(Args)) |
4849 | .setSExtResult(); |
4850 | |
4851 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
4852 | |
4853 | SplitInteger(Op: CallInfo.first, Lo, Hi); |
4854 | SDValue Temp2 = |
4855 | DAG.getLoad(VT: PtrVT, dl, Chain: CallInfo.second, Ptr: Temp, PtrInfo: MachinePointerInfo()); |
4856 | SDValue Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Temp2, |
4857 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), |
4858 | Cond: ISD::SETNE); |
4859 | // Use the overflow from the libcall everywhere. |
4860 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
4861 | } |
4862 | |
4863 | void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
4864 | SDValue &Lo, SDValue &Hi) { |
4865 | EVT VT = N->getValueType(ResNo: 0); |
4866 | SDLoc dl(N); |
4867 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4868 | |
4869 | if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4870 | SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4871 | SplitInteger(Op: Res.getValue(R: 0), Lo, Hi); |
4872 | return; |
4873 | } |
4874 | |
4875 | // Try to expand UDIV by constant. |
4876 | if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) { |
4877 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4878 | // Only if the new type is legal. |
4879 | if (isTypeLegal(VT: NVT)) { |
4880 | SDValue InL, InH; |
4881 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
4882 | SmallVector<SDValue> Result; |
4883 | if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) { |
4884 | Lo = Result[0]; |
4885 | Hi = Result[1]; |
4886 | return; |
4887 | } |
4888 | } |
4889 | } |
4890 | |
4891 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4892 | if (VT == MVT::i16) |
4893 | LC = RTLIB::UDIV_I16; |
4894 | else if (VT == MVT::i32) |
4895 | LC = RTLIB::UDIV_I32; |
4896 | else if (VT == MVT::i64) |
4897 | LC = RTLIB::UDIV_I64; |
4898 | else if (VT == MVT::i128) |
4899 | LC = RTLIB::UDIV_I128; |
4900 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!" ); |
4901 | |
4902 | TargetLowering::MakeLibCallOptions CallOptions; |
4903 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4904 | } |
4905 | |
4906 | void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
4907 | SDValue &Lo, SDValue &Hi) { |
4908 | EVT VT = N->getValueType(ResNo: 0); |
4909 | SDLoc dl(N); |
4910 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4911 | |
4912 | if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4913 | SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4914 | SplitInteger(Op: Res.getValue(R: 1), Lo, Hi); |
4915 | return; |
4916 | } |
4917 | |
4918 | // Try to expand UREM by constant. |
4919 | if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) { |
4920 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4921 | // Only if the new type is legal. |
4922 | if (isTypeLegal(VT: NVT)) { |
4923 | SDValue InL, InH; |
4924 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
4925 | SmallVector<SDValue> Result; |
4926 | if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) { |
4927 | Lo = Result[0]; |
4928 | Hi = Result[1]; |
4929 | return; |
4930 | } |
4931 | } |
4932 | } |
4933 | |
4934 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4935 | if (VT == MVT::i16) |
4936 | LC = RTLIB::UREM_I16; |
4937 | else if (VT == MVT::i32) |
4938 | LC = RTLIB::UREM_I32; |
4939 | else if (VT == MVT::i64) |
4940 | LC = RTLIB::UREM_I64; |
4941 | else if (VT == MVT::i128) |
4942 | LC = RTLIB::UREM_I128; |
4943 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!" ); |
4944 | |
4945 | TargetLowering::MakeLibCallOptions CallOptions; |
4946 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4947 | } |
4948 | |
4949 | void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
4950 | SDValue &Lo, SDValue &Hi) { |
4951 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4952 | SDLoc dl(N); |
4953 | SDValue Op = N->getOperand(Num: 0); |
4954 | if (Op.getValueType().bitsLE(VT: NVT)) { |
4955 | // The low part is zero extension of the input (degenerates to a copy). |
4956 | Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4957 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // The high part is just a zero. |
4958 | } else { |
4959 | // For example, extension of an i48 to an i64. The operand type necessarily |
4960 | // promotes to the result type, so will end up being expanded too. |
4961 | assert(getTypeAction(Op.getValueType()) == |
4962 | TargetLowering::TypePromoteInteger && |
4963 | "Only know how to promote this result!" ); |
4964 | SDValue Res = GetPromotedInteger(Op); |
4965 | assert(Res.getValueType() == N->getValueType(0) && |
4966 | "Operand over promoted?" ); |
4967 | // Split the promoted operand. This will simplify when it is expanded. |
4968 | SplitInteger(Op: Res, Lo, Hi); |
4969 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
4970 | Hi = DAG.getZeroExtendInReg(Op: Hi, DL: dl, |
4971 | VT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
4972 | BitWidth: ExcessBits)); |
4973 | } |
4974 | } |
4975 | |
4976 | void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
4977 | SDValue &Lo, SDValue &Hi) { |
4978 | SDLoc dl(N); |
4979 | EVT VT = cast<AtomicSDNode>(Val: N)->getMemoryVT(); |
4980 | SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
4981 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
4982 | SDValue Swap = DAG.getAtomicCmpSwap( |
4983 | Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
4984 | MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(), VTs, Chain: N->getOperand(Num: 0), |
4985 | Ptr: N->getOperand(Num: 1), Cmp: Zero, Swp: Zero, MMO: cast<AtomicSDNode>(Val: N)->getMemOperand()); |
4986 | |
4987 | ReplaceValueWith(From: SDValue(N, 0), To: Swap.getValue(R: 0)); |
4988 | ReplaceValueWith(From: SDValue(N, 1), To: Swap.getValue(R: 2)); |
4989 | } |
4990 | |
4991 | void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, |
4992 | SDValue &Lo, SDValue &Hi) { |
4993 | // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate |
4994 | // both halves independently. |
4995 | SDValue Res = TLI.expandVecReduce(Node: N, DAG); |
4996 | SplitInteger(Op: Res, Lo, Hi); |
4997 | } |
4998 | |
4999 | void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, |
5000 | SDValue &Lo, SDValue &Hi) { |
5001 | // Delegate to funnel-shift expansion. |
5002 | SDLoc DL(N); |
5003 | unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; |
5004 | SDValue Res = DAG.getNode(Opcode, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), |
5005 | N2: N->getOperand(Num: 0), N3: N->getOperand(Num: 1)); |
5006 | SplitInteger(Op: Res, Lo, Hi); |
5007 | } |
5008 | |
5009 | void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, |
5010 | SDValue &Hi) { |
5011 | // Values numbered from least significant to most significant. |
5012 | SDValue In1, In2, In3, In4; |
5013 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: In3, Hi&: In4); |
5014 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: In1, Hi&: In2); |
5015 | EVT HalfVT = In1.getValueType(); |
5016 | |
5017 | SDLoc DL(N); |
5018 | unsigned Opc = N->getOpcode(); |
5019 | SDValue ShAmt = N->getOperand(Num: 2); |
5020 | EVT ShAmtVT = ShAmt.getValueType(); |
5021 | EVT ShAmtCCVT = getSetCCResultType(VT: ShAmtVT); |
5022 | |
5023 | // If the shift amount is at least half the bitwidth, swap the inputs. |
5024 | unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); |
5025 | SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL, VT: ShAmtVT, N1: ShAmt, |
5026 | N2: DAG.getConstant(Val: HalfVTBits, DL, VT: ShAmtVT)); |
5027 | SDValue Cond = |
5028 | DAG.getSetCC(DL, VT: ShAmtCCVT, LHS: AndNode, RHS: DAG.getConstant(Val: 0, DL, VT: ShAmtVT), |
5029 | Cond: Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); |
5030 | |
5031 | // Expand to a pair of funnel shifts. |
5032 | EVT NewShAmtVT = TLI.getShiftAmountTy(LHSTy: HalfVT, DL: DAG.getDataLayout()); |
5033 | SDValue NewShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: NewShAmtVT); |
5034 | |
5035 | SDValue Select1 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In1, N3: In2); |
5036 | SDValue Select2 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In2, N3: In3); |
5037 | SDValue Select3 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In3, N3: In4); |
5038 | Lo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select2, N2: Select1, N3: NewShAmt); |
5039 | Hi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select3, N2: Select2, N3: NewShAmt); |
5040 | } |
5041 | |
5042 | void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, |
5043 | SDValue &Hi) { |
5044 | EVT VT = N->getValueType(ResNo: 0); |
5045 | EVT HalfVT = |
5046 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: N->getValueSizeInBits(ResNo: 0) / 2); |
5047 | SDLoc dl(N); |
5048 | |
5049 | // We assume VSCALE(1) fits into a legal integer. |
5050 | APInt One(HalfVT.getSizeInBits(), 1); |
5051 | SDValue VScaleBase = DAG.getVScale(DL: dl, VT: HalfVT, MulImm: One); |
5052 | VScaleBase = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: VScaleBase); |
5053 | SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: VScaleBase, N2: N->getOperand(Num: 0)); |
5054 | SplitInteger(Op: Res, Lo, Hi); |
5055 | } |
5056 | |
5057 | //===----------------------------------------------------------------------===// |
5058 | // Integer Operand Expansion |
5059 | //===----------------------------------------------------------------------===// |
5060 | |
5061 | /// ExpandIntegerOperand - This method is called when the specified operand of |
5062 | /// the specified node is found to need expansion. At this point, all of the |
5063 | /// result types of the node are known to be legal, but other operands of the |
5064 | /// node may need promotion or expansion as well as the specified one. |
5065 | bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
5066 | LLVM_DEBUG(dbgs() << "Expand integer operand: " ; N->dump(&DAG)); |
5067 | SDValue Res = SDValue(); |
5068 | |
5069 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) |
5070 | return false; |
5071 | |
5072 | switch (N->getOpcode()) { |
5073 | default: |
5074 | #ifndef NDEBUG |
5075 | dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": " ; |
5076 | N->dump(G: &DAG); dbgs() << "\n" ; |
5077 | #endif |
5078 | report_fatal_error(reason: "Do not know how to expand this operator's operand!" ); |
5079 | |
5080 | case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
5081 | case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
5082 | case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
5083 | case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
5084 | case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
5085 | case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
5086 | case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; |
5087 | case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
5088 | case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
5089 | case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
5090 | case ISD::STRICT_SINT_TO_FP: |
5091 | case ISD::SINT_TO_FP: |
5092 | case ISD::STRICT_UINT_TO_FP: |
5093 | case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break; |
5094 | case ISD::STORE: Res = ExpandIntOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo); break; |
5095 | case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
5096 | |
5097 | case ISD::SHL: |
5098 | case ISD::SRA: |
5099 | case ISD::SRL: |
5100 | case ISD::ROTL: |
5101 | case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
5102 | case ISD::RETURNADDR: |
5103 | case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
5104 | |
5105 | case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
5106 | case ISD::STACKMAP: |
5107 | Res = ExpandIntOp_STACKMAP(N, OpNo); |
5108 | break; |
5109 | case ISD::PATCHPOINT: |
5110 | Res = ExpandIntOp_PATCHPOINT(N, OpNo); |
5111 | break; |
5112 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
5113 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
5114 | Res = ExpandIntOp_VP_STRIDED(N, OpNo); |
5115 | break; |
5116 | } |
5117 | |
5118 | // If the result is null, the sub-method took care of registering results etc. |
5119 | if (!Res.getNode()) return false; |
5120 | |
5121 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
5122 | // core about this. |
5123 | if (Res.getNode() == N) |
5124 | return true; |
5125 | |
5126 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
5127 | "Invalid operand expansion" ); |
5128 | |
5129 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
5130 | return false; |
5131 | } |
5132 | |
5133 | /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code |
5134 | /// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
5135 | void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
5136 | SDValue &NewRHS, |
5137 | ISD::CondCode &CCCode, |
5138 | const SDLoc &dl) { |
5139 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
5140 | GetExpandedInteger(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi); |
5141 | GetExpandedInteger(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi); |
5142 | |
5143 | if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { |
5144 | if (RHSLo == RHSHi && isAllOnesConstant(V: RHSLo)) { |
5145 | // Equality comparison to -1. |
5146 | NewLHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: LHSHi); |
5147 | NewRHS = RHSLo; |
5148 | return; |
5149 | } |
5150 | |
5151 | NewLHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo); |
5152 | NewRHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSHi, N2: RHSHi); |
5153 | NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NewLHS.getValueType(), N1: NewLHS, N2: NewRHS); |
5154 | NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: NewLHS.getValueType()); |
5155 | return; |
5156 | } |
5157 | |
5158 | // If this is a comparison of the sign bit, just look at the top part. |
5159 | // X > -1, x < 0 |
5160 | if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: NewRHS)) |
5161 | if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 |
5162 | (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 |
5163 | NewLHS = LHSHi; |
5164 | NewRHS = RHSHi; |
5165 | return; |
5166 | } |
5167 | |
5168 | // FIXME: This generated code sucks. |
5169 | ISD::CondCode LowCC; |
5170 | switch (CCCode) { |
5171 | default: llvm_unreachable("Unknown integer setcc!" ); |
5172 | case ISD::SETLT: |
5173 | case ISD::SETULT: LowCC = ISD::SETULT; break; |
5174 | case ISD::SETGT: |
5175 | case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
5176 | case ISD::SETLE: |
5177 | case ISD::SETULE: LowCC = ISD::SETULE; break; |
5178 | case ISD::SETGE: |
5179 | case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
5180 | } |
5181 | |
5182 | // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison |
5183 | // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands |
5184 | // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; |
5185 | |
5186 | // NOTE: on targets without efficient SELECT of bools, we can always use |
5187 | // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) |
5188 | TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
5189 | nullptr); |
5190 | SDValue LoCmp, HiCmp; |
5191 | if (TLI.isTypeLegal(VT: LHSLo.getValueType()) && |
5192 | TLI.isTypeLegal(VT: RHSLo.getValueType())) |
5193 | LoCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSLo.getValueType()), N0: LHSLo, |
5194 | N1: RHSLo, Cond: LowCC, foldBooleans: false, DCI&: DagCombineInfo, dl); |
5195 | if (!LoCmp.getNode()) |
5196 | LoCmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo, |
5197 | RHS: RHSLo, Cond: LowCC); |
5198 | if (TLI.isTypeLegal(VT: LHSHi.getValueType()) && |
5199 | TLI.isTypeLegal(VT: RHSHi.getValueType())) |
5200 | HiCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSHi.getValueType()), N0: LHSHi, |
5201 | N1: RHSHi, Cond: CCCode, foldBooleans: false, DCI&: DagCombineInfo, dl); |
5202 | if (!HiCmp.getNode()) |
5203 | HiCmp = |
5204 | DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), |
5205 | N1: LHSHi, N2: RHSHi, N3: DAG.getCondCode(Cond: CCCode)); |
5206 | |
5207 | ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(Val: LoCmp.getNode()); |
5208 | ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(Val: HiCmp.getNode()); |
5209 | |
5210 | bool EqAllowed = ISD::isTrueWhenEqual(Cond: CCCode); |
5211 | |
5212 | // FIXME: Is the HiCmpC->isOne() here correct for |
5213 | // ZeroOrNegativeOneBooleanContent. |
5214 | if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || |
5215 | (!EqAllowed && |
5216 | ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { |
5217 | // For LE / GE, if high part is known false, ignore the low part. |
5218 | // For LT / GT: if low part is known false, return the high part. |
5219 | // if high part is known true, ignore the low part. |
5220 | NewLHS = HiCmp; |
5221 | NewRHS = SDValue(); |
5222 | return; |
5223 | } |
5224 | |
5225 | if (LHSHi == RHSHi) { |
5226 | // Comparing the low bits is enough. |
5227 | NewLHS = LoCmp; |
5228 | NewRHS = SDValue(); |
5229 | return; |
5230 | } |
5231 | |
5232 | // Lower with SETCCCARRY if the target supports it. |
5233 | EVT HiVT = LHSHi.getValueType(); |
5234 | EVT ExpandVT = TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: HiVT); |
5235 | bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(Op: ISD::SETCCCARRY, VT: ExpandVT); |
5236 | |
5237 | // FIXME: Make all targets support this, then remove the other lowering. |
5238 | if (HasSETCCCARRY) { |
5239 | // SETCCCARRY can detect < and >= directly. For > and <=, flip |
5240 | // operands and condition code. |
5241 | bool FlipOperands = false; |
5242 | switch (CCCode) { |
5243 | case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
5244 | case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
5245 | case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
5246 | case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
5247 | default: break; |
5248 | } |
5249 | if (FlipOperands) { |
5250 | std::swap(a&: LHSLo, b&: RHSLo); |
5251 | std::swap(a&: LHSHi, b&: RHSHi); |
5252 | } |
5253 | // Perform a wide subtraction, feeding the carry from the low part into |
5254 | // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high |
5255 | // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is |
5256 | // zero or positive iff LHS >= RHS. |
5257 | EVT LoVT = LHSLo.getValueType(); |
5258 | SDVTList VTList = DAG.getVTList(VT1: LoVT, VT2: getSetCCResultType(VT: LoVT)); |
5259 | SDValue LowCmp = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: LHSLo, N2: RHSLo); |
5260 | SDValue Res = DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: getSetCCResultType(VT: HiVT), |
5261 | N1: LHSHi, N2: RHSHi, N3: LowCmp.getValue(R: 1), |
5262 | N4: DAG.getCondCode(Cond: CCCode)); |
5263 | NewLHS = Res; |
5264 | NewRHS = SDValue(); |
5265 | return; |
5266 | } |
5267 | |
5268 | NewLHS = TLI.SimplifySetCC(VT: getSetCCResultType(VT: HiVT), N0: LHSHi, N1: RHSHi, Cond: ISD::SETEQ, |
5269 | foldBooleans: false, DCI&: DagCombineInfo, dl); |
5270 | if (!NewLHS.getNode()) |
5271 | NewLHS = |
5272 | DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: HiVT), LHS: LHSHi, RHS: RHSHi, Cond: ISD::SETEQ); |
5273 | NewLHS = DAG.getSelect(DL: dl, VT: LoCmp.getValueType(), Cond: NewLHS, LHS: LoCmp, RHS: HiCmp); |
5274 | NewRHS = SDValue(); |
5275 | } |
5276 | |
5277 | SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
5278 | SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3); |
5279 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
5280 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5281 | |
5282 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
5283 | // against zero to select between true and false values. |
5284 | if (!NewRHS.getNode()) { |
5285 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
5286 | CCCode = ISD::SETNE; |
5287 | } |
5288 | |
5289 | // Update N to have the operands specified. |
5290 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
5291 | Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS, |
5292 | Op5: N->getOperand(Num: 4)), 0); |
5293 | } |
5294 | |
5295 | SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
5296 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
5297 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get(); |
5298 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5299 | |
5300 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
5301 | // against zero to select between true and false values. |
5302 | if (!NewRHS.getNode()) { |
5303 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
5304 | CCCode = ISD::SETNE; |
5305 | } |
5306 | |
5307 | // Update N to have the operands specified. |
5308 | return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, |
5309 | Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3), |
5310 | Op5: DAG.getCondCode(Cond: CCCode)), 0); |
5311 | } |
5312 | |
5313 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
5314 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
5315 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get(); |
5316 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5317 | |
5318 | // If ExpandSetCCOperands returned a scalar, use it. |
5319 | if (!NewRHS.getNode()) { |
5320 | assert(NewLHS.getValueType() == N->getValueType(0) && |
5321 | "Unexpected setcc expansion!" ); |
5322 | return NewLHS; |
5323 | } |
5324 | |
5325 | // Otherwise, update N to have the operands specified. |
5326 | return SDValue( |
5327 | DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, Op3: DAG.getCondCode(Cond: CCCode)), 0); |
5328 | } |
5329 | |
5330 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
5331 | SDValue LHS = N->getOperand(Num: 0); |
5332 | SDValue RHS = N->getOperand(Num: 1); |
5333 | SDValue Carry = N->getOperand(Num: 2); |
5334 | SDValue Cond = N->getOperand(Num: 3); |
5335 | SDLoc dl = SDLoc(N); |
5336 | |
5337 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
5338 | GetExpandedInteger(Op: LHS, Lo&: LHSLo, Hi&: LHSHi); |
5339 | GetExpandedInteger(Op: RHS, Lo&: RHSLo, Hi&: RHSHi); |
5340 | |
5341 | // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. |
5342 | SDVTList VTList = DAG.getVTList(VT1: LHSLo.getValueType(), VT2: Carry.getValueType()); |
5343 | SDValue LowCmp = |
5344 | DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: LHSLo, N2: RHSLo, N3: Carry); |
5345 | return DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: N->getValueType(ResNo: 0), N1: LHSHi, N2: RHSHi, |
5346 | N3: LowCmp.getValue(R: 1), N4: Cond); |
5347 | } |
5348 | |
5349 | SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { |
5350 | // Split the operand and replace with SPLAT_VECTOR_PARTS. |
5351 | SDValue Lo, Hi; |
5352 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
5353 | return DAG.getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, |
5354 | N2: Hi); |
5355 | } |
5356 | |
5357 | SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
5358 | // The value being shifted is legal, but the shift amount is too big. |
5359 | // It follows that either the result of the shift is undefined, or the |
5360 | // upper half of the shift amount is zero. Just use the lower half. |
5361 | SDValue Lo, Hi; |
5362 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo, Hi); |
5363 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Lo), 0); |
5364 | } |
5365 | |
5366 | SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
5367 | // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This |
5368 | // surely makes pretty nice problems on 8/16 bit targets. Just truncate this |
5369 | // constant to valid type. |
5370 | SDValue Lo, Hi; |
5371 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
5372 | return SDValue(DAG.UpdateNodeOperands(N, Op: Lo), 0); |
5373 | } |
5374 | |
5375 | SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) { |
5376 | bool IsStrict = N->isStrictFPOpcode(); |
5377 | bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || |
5378 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
5379 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
5380 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
5381 | EVT DstVT = N->getValueType(ResNo: 0); |
5382 | RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT) |
5383 | : RTLIB::getUINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT); |
5384 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
5385 | "Don't know how to expand this XINT_TO_FP!" ); |
5386 | TargetLowering::MakeLibCallOptions CallOptions; |
5387 | CallOptions.setSExt(true); |
5388 | std::pair<SDValue, SDValue> Tmp = |
5389 | TLI.makeLibCall(DAG, LC, RetVT: DstVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain); |
5390 | |
5391 | if (!IsStrict) |
5392 | return Tmp.first; |
5393 | |
5394 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
5395 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
5396 | return SDValue(); |
5397 | } |
5398 | |
5399 | SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
5400 | assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?" ); |
5401 | |
5402 | if (ISD::isNormalStore(N)) |
5403 | return ExpandOp_NormalStore(N, OpNo); |
5404 | |
5405 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
5406 | assert(OpNo == 1 && "Can only expand the stored value so far" ); |
5407 | |
5408 | EVT VT = N->getOperand(Num: 1).getValueType(); |
5409 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5410 | SDValue Ch = N->getChain(); |
5411 | SDValue Ptr = N->getBasePtr(); |
5412 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
5413 | AAMDNodes AAInfo = N->getAAInfo(); |
5414 | SDLoc dl(N); |
5415 | SDValue Lo, Hi; |
5416 | |
5417 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
5418 | |
5419 | if (N->getMemoryVT().bitsLE(VT: NVT)) { |
5420 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5421 | return DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), |
5422 | SVT: N->getMemoryVT(), Alignment: N->getOriginalAlign(), MMOFlags, |
5423 | AAInfo); |
5424 | } |
5425 | |
5426 | if (DAG.getDataLayout().isLittleEndian()) { |
5427 | // Little-endian - low bits are at low addresses. |
5428 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5429 | |
5430 | Lo = DAG.getStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), |
5431 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5432 | |
5433 | unsigned ExcessBits = |
5434 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
5435 | EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits); |
5436 | |
5437 | // Increment the pointer to the other half. |
5438 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
5439 | Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize)); |
5440 | Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr, |
5441 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
5442 | SVT: NEVT, Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5443 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
5444 | } |
5445 | |
5446 | // Big-endian - high bits are at low addresses. Favor aligned stores at |
5447 | // the cost of some bit-fiddling. |
5448 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5449 | |
5450 | EVT ExtVT = N->getMemoryVT(); |
5451 | unsigned EBytes = ExtVT.getStoreSize(); |
5452 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
5453 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
5454 | EVT HiVT = EVT::getIntegerVT(Context&: *DAG.getContext(), |
5455 | BitWidth: ExtVT.getSizeInBits() - ExcessBits); |
5456 | |
5457 | if (ExcessBits < NVT.getSizeInBits()) { |
5458 | // Transfer high bits from the top of Lo to the bottom of Hi. |
5459 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi, |
5460 | N2: DAG.getConstant(Val: NVT.getSizeInBits() - ExcessBits, DL: dl, |
5461 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
5462 | Hi = DAG.getNode( |
5463 | Opcode: ISD::OR, DL: dl, VT: NVT, N1: Hi, |
5464 | N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Lo, |
5465 | N2: DAG.getConstant(Val: ExcessBits, DL: dl, |
5466 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())))); |
5467 | } |
5468 | |
5469 | // Store both the high bits and maybe some of the low bits. |
5470 | Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr, PtrInfo: N->getPointerInfo(), SVT: HiVT, |
5471 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5472 | |
5473 | // Increment the pointer to the other half. |
5474 | Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize)); |
5475 | // Store the lowest ExcessBits bits in the second half. |
5476 | Lo = DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr, |
5477 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
5478 | SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits), |
5479 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5480 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
5481 | } |
5482 | |
5483 | SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
5484 | SDValue InL, InH; |
5485 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
5486 | // Just truncate the low part of the source. |
5487 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InL); |
5488 | } |
5489 | |
5490 | SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
5491 | SDLoc dl(N); |
5492 | SDValue Swap = |
5493 | DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(), |
5494 | Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 2), Val: N->getOperand(Num: 1), |
5495 | MMO: cast<AtomicSDNode>(Val: N)->getMemOperand()); |
5496 | return Swap.getValue(R: 1); |
5497 | } |
5498 | |
5499 | SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
5500 | assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
5501 | (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
5502 | |
5503 | SDValue Hi; // The upper half is dropped out. |
5504 | SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); |
5505 | GetExpandedInteger(Op: NewOps[OpNo], Lo&: NewOps[OpNo], Hi); |
5506 | |
5507 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
5508 | } |
5509 | |
5510 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { |
5511 | SDLoc dl(N); |
5512 | |
5513 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5514 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5515 | EVT OutVT = V0.getValueType(); |
5516 | |
5517 | return DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: OutVT, N1: V0, N2: V1, N3: N->getOperand(Num: 2)); |
5518 | } |
5519 | |
5520 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { |
5521 | SDLoc dl(N); |
5522 | |
5523 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5524 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5525 | EVT ResVT = V0.getValueType(); |
5526 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, |
5527 | VTList: DAG.getVTList(VT1: ResVT, VT2: ResVT), N1: V0, N2: V1); |
5528 | SetPromotedInteger(Op: SDValue(N, 0), Result: Res.getValue(R: 0)); |
5529 | SetPromotedInteger(Op: SDValue(N, 1), Result: Res.getValue(R: 1)); |
5530 | return SDValue(); |
5531 | } |
5532 | |
5533 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5534 | |
5535 | EVT OutVT = N->getValueType(ResNo: 0); |
5536 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5537 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5538 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5539 | |
5540 | SDLoc dl(N); |
5541 | SDValue BaseIdx = N->getOperand(Num: 1); |
5542 | |
5543 | // TODO: We may be able to use this for types other than scalable |
5544 | // vectors and fix those tests that expect BUILD_VECTOR to be used |
5545 | if (OutVT.isScalableVector()) { |
5546 | SDValue InOp0 = N->getOperand(Num: 0); |
5547 | EVT InVT = InOp0.getValueType(); |
5548 | |
5549 | // Try and extract from a smaller type so that it eventually falls |
5550 | // into the promotion code below. |
5551 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector || |
5552 | getTypeAction(VT: InVT) == TargetLowering::TypeLegal) { |
5553 | EVT NInVT = InVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext()); |
5554 | unsigned NElts = NInVT.getVectorMinNumElements(); |
5555 | uint64_t IdxVal = BaseIdx->getAsZExtVal(); |
5556 | |
5557 | SDValue Step1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NInVT, N1: InOp0, |
5558 | N2: DAG.getConstant(Val: alignDown(Value: IdxVal, Align: NElts), DL: dl, |
5559 | VT: BaseIdx.getValueType())); |
5560 | SDValue Step2 = DAG.getNode( |
5561 | Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: Step1, |
5562 | N2: DAG.getConstant(Val: IdxVal % NElts, DL: dl, VT: BaseIdx.getValueType())); |
5563 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Step2); |
5564 | } |
5565 | |
5566 | // Try and extract from a widened type. |
5567 | if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) { |
5568 | SDValue Ops[] = {GetWidenedVector(Op: InOp0), BaseIdx}; |
5569 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: OutVT, Ops); |
5570 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext); |
5571 | } |
5572 | |
5573 | // Promote operands and see if this is handled by target lowering, |
5574 | // Otherwise, use the BUILD_VECTOR approach below |
5575 | if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) { |
5576 | // Collect the (promoted) operands |
5577 | SDValue Ops[] = { GetPromotedInteger(Op: InOp0), BaseIdx }; |
5578 | |
5579 | EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); |
5580 | assert(PromEltVT.bitsLE(NOutVTElem) && |
5581 | "Promoted operand has an element type greater than result" ); |
5582 | |
5583 | EVT ExtVT = NOutVT.changeVectorElementType(EltVT: PromEltVT); |
5584 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: ExtVT, Ops); |
5585 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext); |
5586 | } |
5587 | } |
5588 | |
5589 | if (OutVT.isScalableVector()) |
5590 | report_fatal_error(reason: "Unable to promote scalable types using BUILD_VECTOR" ); |
5591 | |
5592 | SDValue InOp0 = N->getOperand(Num: 0); |
5593 | if (getTypeAction(VT: InOp0.getValueType()) == TargetLowering::TypePromoteInteger) |
5594 | InOp0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5595 | |
5596 | EVT InVT = InOp0.getValueType(); |
5597 | |
5598 | unsigned OutNumElems = OutVT.getVectorNumElements(); |
5599 | SmallVector<SDValue, 8> Ops; |
5600 | Ops.reserve(N: OutNumElems); |
5601 | for (unsigned i = 0; i != OutNumElems; ++i) { |
5602 | |
5603 | // Extract the element from the original vector. |
5604 | SDValue Index = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BaseIdx.getValueType(), |
5605 | N1: BaseIdx, N2: DAG.getConstant(Val: i, DL: dl, VT: BaseIdx.getValueType())); |
5606 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
5607 | VT: InVT.getVectorElementType(), N1: N->getOperand(Num: 0), N2: Index); |
5608 | |
5609 | SDValue Op = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NOutVTElem); |
5610 | // Insert the converted element to the new vector. |
5611 | Ops.push_back(Elt: Op); |
5612 | } |
5613 | |
5614 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5615 | } |
5616 | |
5617 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { |
5618 | EVT OutVT = N->getValueType(ResNo: 0); |
5619 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5620 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5621 | |
5622 | SDLoc dl(N); |
5623 | SDValue Vec = N->getOperand(Num: 0); |
5624 | SDValue SubVec = N->getOperand(Num: 1); |
5625 | SDValue Idx = N->getOperand(Num: 2); |
5626 | |
5627 | EVT SubVecVT = SubVec.getValueType(); |
5628 | EVT NSubVT = |
5629 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: NOutVT.getVectorElementType(), |
5630 | EC: SubVecVT.getVectorElementCount()); |
5631 | |
5632 | Vec = GetPromotedInteger(Op: Vec); |
5633 | SubVec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NSubVT, Operand: SubVec); |
5634 | |
5635 | return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: NOutVT, N1: Vec, N2: SubVec, N3: Idx); |
5636 | } |
5637 | |
5638 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { |
5639 | SDLoc dl(N); |
5640 | |
5641 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5642 | EVT OutVT = V0.getValueType(); |
5643 | |
5644 | return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: OutVT, Operand: V0); |
5645 | } |
5646 | |
5647 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
5648 | ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(Val: N); |
5649 | EVT VT = N->getValueType(ResNo: 0); |
5650 | SDLoc dl(N); |
5651 | |
5652 | ArrayRef<int> NewMask = SV->getMask().slice(N: 0, M: VT.getVectorNumElements()); |
5653 | |
5654 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5655 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5656 | EVT OutVT = V0.getValueType(); |
5657 | |
5658 | return DAG.getVectorShuffle(VT: OutVT, dl, N1: V0, N2: V1, Mask: NewMask); |
5659 | } |
5660 | |
5661 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
5662 | EVT OutVT = N->getValueType(ResNo: 0); |
5663 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5664 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5665 | unsigned NumElems = N->getNumOperands(); |
5666 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5667 | TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(Type: NOutVT); |
5668 | unsigned NOutExtOpc = TargetLowering::getExtendForContent(Content: NOutBoolType); |
5669 | SDLoc dl(N); |
5670 | |
5671 | SmallVector<SDValue, 8> Ops; |
5672 | Ops.reserve(N: NumElems); |
5673 | for (unsigned i = 0; i != NumElems; ++i) { |
5674 | SDValue Op = N->getOperand(Num: i); |
5675 | EVT OpVT = Op.getValueType(); |
5676 | // BUILD_VECTOR integer operand types are allowed to be larger than the |
5677 | // result's element type. This may still be true after the promotion. For |
5678 | // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to |
5679 | // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. |
5680 | if (OpVT.bitsLT(VT: NOutVTElem)) { |
5681 | unsigned ExtOpc = ISD::ANY_EXTEND; |
5682 | // Attempt to extend constant bool vectors to match target's BooleanContent. |
5683 | // While not necessary, this improves chances of the constant correctly |
5684 | // folding with compare results (e.g. for NOT patterns). |
5685 | if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) |
5686 | ExtOpc = NOutExtOpc; |
5687 | Op = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: NOutVTElem, Operand: Op); |
5688 | } |
5689 | Ops.push_back(Elt: Op); |
5690 | } |
5691 | |
5692 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5693 | } |
5694 | |
5695 | SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { |
5696 | |
5697 | SDLoc dl(N); |
5698 | |
5699 | assert(!N->getOperand(0).getValueType().isVector() && |
5700 | "Input must be a scalar" ); |
5701 | |
5702 | EVT OutVT = N->getValueType(ResNo: 0); |
5703 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5704 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5705 | EVT NOutElemVT = NOutVT.getVectorElementType(); |
5706 | |
5707 | SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutElemVT, Operand: N->getOperand(Num: 0)); |
5708 | |
5709 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NOutVT, Operand: Op); |
5710 | } |
5711 | |
5712 | SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { |
5713 | SDLoc dl(N); |
5714 | EVT OutVT = N->getValueType(ResNo: 0); |
5715 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5716 | assert(NOutVT.isScalableVector() && |
5717 | "Type must be promoted to a scalable vector type" ); |
5718 | const APInt &StepVal = N->getConstantOperandAPInt(Num: 0); |
5719 | return DAG.getStepVector(DL: dl, ResVT: NOutVT, |
5720 | StepVal: StepVal.sext(width: NOutVT.getScalarSizeInBits())); |
5721 | } |
5722 | |
5723 | SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
5724 | SDLoc dl(N); |
5725 | |
5726 | EVT OutVT = N->getValueType(ResNo: 0); |
5727 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5728 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5729 | |
5730 | unsigned NumOperands = N->getNumOperands(); |
5731 | unsigned NumOutElem = NOutVT.getVectorMinNumElements(); |
5732 | EVT OutElemTy = NOutVT.getVectorElementType(); |
5733 | if (OutVT.isScalableVector()) { |
5734 | // Find the largest promoted element type for each of the operands. |
5735 | SDUse *MaxSizedValue = std::max_element( |
5736 | first: N->op_begin(), last: N->op_end(), comp: [](const SDValue &A, const SDValue &B) { |
5737 | EVT AVT = A.getValueType().getVectorElementType(); |
5738 | EVT BVT = B.getValueType().getVectorElementType(); |
5739 | return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); |
5740 | }); |
5741 | EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); |
5742 | |
5743 | // Then promote all vectors to the largest element type. |
5744 | SmallVector<SDValue, 8> Ops; |
5745 | for (unsigned I = 0; I < NumOperands; ++I) { |
5746 | SDValue Op = N->getOperand(Num: I); |
5747 | EVT OpVT = Op.getValueType(); |
5748 | if (getTypeAction(VT: OpVT) == TargetLowering::TypePromoteInteger) |
5749 | Op = GetPromotedInteger(Op); |
5750 | else |
5751 | assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && |
5752 | "Unhandled legalization type" ); |
5753 | |
5754 | if (OpVT.getVectorElementType().getScalarSizeInBits() < |
5755 | MaxElementVT.getScalarSizeInBits()) |
5756 | Op = DAG.getAnyExtOrTrunc(Op, DL: dl, |
5757 | VT: OpVT.changeVectorElementType(EltVT: MaxElementVT)); |
5758 | Ops.push_back(Elt: Op); |
5759 | } |
5760 | |
5761 | // Do the CONCAT on the promoted type and finally truncate to (the promoted) |
5762 | // NOutVT. |
5763 | return DAG.getAnyExtOrTrunc( |
5764 | Op: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, |
5765 | VT: OutVT.changeVectorElementType(EltVT: MaxElementVT), Ops), |
5766 | DL: dl, VT: NOutVT); |
5767 | } |
5768 | |
5769 | unsigned NumElem = N->getOperand(Num: 0).getValueType().getVectorNumElements(); |
5770 | assert(NumElem * NumOperands == NumOutElem && |
5771 | "Unexpected number of elements" ); |
5772 | |
5773 | // Take the elements from the first vector. |
5774 | SmallVector<SDValue, 8> Ops(NumOutElem); |
5775 | for (unsigned i = 0; i < NumOperands; ++i) { |
5776 | SDValue Op = N->getOperand(Num: i); |
5777 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteInteger) |
5778 | Op = GetPromotedInteger(Op); |
5779 | EVT SclrTy = Op.getValueType().getVectorElementType(); |
5780 | assert(NumElem == Op.getValueType().getVectorNumElements() && |
5781 | "Unexpected number of elements" ); |
5782 | |
5783 | for (unsigned j = 0; j < NumElem; ++j) { |
5784 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Op, |
5785 | N2: DAG.getVectorIdxConstant(Val: j, DL: dl)); |
5786 | Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: OutElemTy); |
5787 | } |
5788 | } |
5789 | |
5790 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5791 | } |
5792 | |
5793 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
5794 | EVT VT = N->getValueType(ResNo: 0); |
5795 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5796 | assert(NVT.isVector() && "This type must be promoted to a vector type" ); |
5797 | |
5798 | SDLoc dl(N); |
5799 | |
5800 | // For operands whose TypeAction is to promote, extend the promoted node |
5801 | // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion |
5802 | // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to |
5803 | // type.. |
5804 | if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) |
5805 | == TargetLowering::TypePromoteInteger) { |
5806 | SDValue Promoted; |
5807 | |
5808 | switch(N->getOpcode()) { |
5809 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
5810 | Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
5811 | break; |
5812 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
5813 | Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
5814 | break; |
5815 | case ISD::ANY_EXTEND_VECTOR_INREG: |
5816 | Promoted = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5817 | break; |
5818 | default: |
5819 | llvm_unreachable("Node has unexpected Opcode" ); |
5820 | } |
5821 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Promoted); |
5822 | } |
5823 | |
5824 | // Directly extend to the appropriate transform-to type. |
5825 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
5826 | } |
5827 | |
5828 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
5829 | EVT OutVT = N->getValueType(ResNo: 0); |
5830 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5831 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5832 | |
5833 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5834 | |
5835 | SDLoc dl(N); |
5836 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5837 | |
5838 | SDValue ConvElem = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, |
5839 | VT: NOutVTElem, Operand: N->getOperand(Num: 1)); |
5840 | return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NOutVT, |
5841 | N1: V0, N2: ConvElem, N3: N->getOperand(Num: 2)); |
5842 | } |
5843 | |
5844 | SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { |
5845 | // The VECREDUCE result size may be larger than the element size, so |
5846 | // we can simply change the result type. |
5847 | SDLoc dl(N); |
5848 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5849 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Ops: N->ops()); |
5850 | } |
5851 | |
5852 | SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { |
5853 | // The VP_REDUCE result size may be larger than the element size, so we can |
5854 | // simply change the result type. However the start value and result must be |
5855 | // the same. |
5856 | SDLoc DL(N); |
5857 | SDValue Start = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0)); |
5858 | return DAG.getNode(Opcode: N->getOpcode(), DL, VT: Start.getValueType(), N1: Start, |
5859 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
5860 | } |
5861 | |
5862 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5863 | SDLoc dl(N); |
5864 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5865 | SDValue V1 = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl, |
5866 | VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())); |
5867 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
5868 | VT: V0->getValueType(ResNo: 0).getScalarType(), N1: V0, N2: V1); |
5869 | |
5870 | // EXTRACT_VECTOR_ELT can return types which are wider than the incoming |
5871 | // element types. If this is the case then we need to expand the outgoing |
5872 | // value and not truncate it. |
5873 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0)); |
5874 | } |
5875 | |
5876 | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { |
5877 | SDLoc dl(N); |
5878 | // The result type is equal to the first input operand's type, so the |
5879 | // type that needs promoting must be the second source vector. |
5880 | SDValue V0 = N->getOperand(Num: 0); |
5881 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5882 | SDValue Idx = N->getOperand(Num: 2); |
5883 | EVT PromVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5884 | VT: V1.getValueType().getVectorElementType(), |
5885 | EC: V0.getValueType().getVectorElementCount()); |
5886 | V0 = DAG.getAnyExtOrTrunc(Op: V0, DL: dl, VT: PromVT); |
5887 | SDValue Ext = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: PromVT, N1: V0, N2: V1, N3: Idx); |
5888 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0)); |
5889 | } |
5890 | |
5891 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5892 | SDLoc dl(N); |
5893 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5894 | MVT InVT = V0.getValueType().getSimpleVT(); |
5895 | MVT OutVT = MVT::getVectorVT(VT: InVT.getVectorElementType(), |
5896 | NumElements: N->getValueType(ResNo: 0).getVectorNumElements()); |
5897 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: V0, N2: N->getOperand(Num: 1)); |
5898 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Ext); |
5899 | } |
5900 | |
5901 | SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
5902 | SDLoc dl(N); |
5903 | |
5904 | EVT ResVT = N->getValueType(ResNo: 0); |
5905 | unsigned NumElems = N->getNumOperands(); |
5906 | |
5907 | if (ResVT.isScalableVector()) { |
5908 | SDValue ResVec = DAG.getUNDEF(VT: ResVT); |
5909 | |
5910 | for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { |
5911 | SDValue Op = N->getOperand(Num: OpIdx); |
5912 | unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); |
5913 | ResVec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: ResVec, N2: Op, |
5914 | N3: DAG.getIntPtrConstant(Val: OpIdx * OpNumElts, DL: dl)); |
5915 | } |
5916 | |
5917 | return ResVec; |
5918 | } |
5919 | |
5920 | EVT RetSclrTy = N->getValueType(ResNo: 0).getVectorElementType(); |
5921 | |
5922 | SmallVector<SDValue, 8> NewOps; |
5923 | NewOps.reserve(N: NumElems); |
5924 | |
5925 | // For each incoming vector |
5926 | for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { |
5927 | SDValue Incoming = GetPromotedInteger(Op: N->getOperand(Num: VecIdx)); |
5928 | EVT SclrTy = Incoming->getValueType(ResNo: 0).getVectorElementType(); |
5929 | unsigned NumElem = Incoming->getValueType(ResNo: 0).getVectorNumElements(); |
5930 | |
5931 | for (unsigned i=0; i<NumElem; ++i) { |
5932 | // Extract element from incoming vector |
5933 | SDValue Ex = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Incoming, |
5934 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
5935 | SDValue Tr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RetSclrTy, Operand: Ex); |
5936 | NewOps.push_back(Elt: Tr); |
5937 | } |
5938 | } |
5939 | |
5940 | return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: dl, Ops: NewOps); |
5941 | } |
5942 | |
5943 | SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
5944 | assert(OpNo > 1); |
5945 | SDValue Op = N->getOperand(Num: OpNo); |
5946 | |
5947 | // FIXME: Non-constant operands are not yet handled: |
5948 | // - https://github.com/llvm/llvm-project/issues/26431 |
5949 | // - https://github.com/llvm/llvm-project/issues/55957 |
5950 | ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op); |
5951 | if (!CN) |
5952 | return SDValue(); |
5953 | |
5954 | // Copy operands before the one being expanded. |
5955 | SmallVector<SDValue> NewOps; |
5956 | for (unsigned I = 0; I < OpNo; I++) |
5957 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
5958 | |
5959 | EVT Ty = Op.getValueType(); |
5960 | SDLoc DL = SDLoc(N); |
5961 | if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
5962 | NewOps.push_back( |
5963 | DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
5964 | NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty)); |
5965 | } else { |
5966 | // FIXME: https://github.com/llvm/llvm-project/issues/55609 |
5967 | return SDValue(); |
5968 | } |
5969 | |
5970 | // Copy remaining operands. |
5971 | for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
5972 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
5973 | |
5974 | SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps); |
5975 | |
5976 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
5977 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
5978 | |
5979 | return SDValue(); // Signal that we have replaced the node already. |
5980 | } |
5981 | |
5982 | SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
5983 | assert(OpNo >= 7); |
5984 | SDValue Op = N->getOperand(Num: OpNo); |
5985 | |
5986 | // FIXME: Non-constant operands are not yet handled: |
5987 | // - https://github.com/llvm/llvm-project/issues/26431 |
5988 | // - https://github.com/llvm/llvm-project/issues/55957 |
5989 | ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op); |
5990 | if (!CN) |
5991 | return SDValue(); |
5992 | |
5993 | // Copy operands before the one being expanded. |
5994 | SmallVector<SDValue> NewOps; |
5995 | for (unsigned I = 0; I < OpNo; I++) |
5996 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
5997 | |
5998 | EVT Ty = Op.getValueType(); |
5999 | SDLoc DL = SDLoc(N); |
6000 | if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
6001 | NewOps.push_back( |
6002 | DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
6003 | NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty)); |
6004 | } else { |
6005 | // FIXME: https://github.com/llvm/llvm-project/issues/55609 |
6006 | return SDValue(); |
6007 | } |
6008 | |
6009 | // Copy remaining operands. |
6010 | for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
6011 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
6012 | |
6013 | SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps); |
6014 | |
6015 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
6016 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
6017 | |
6018 | return SDValue(); // Signal that we have replaced the node already. |
6019 | } |
6020 | |