1 | //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements integer type expansion and promotion for LegalizeTypes. |
10 | // Promotion is the act of changing a computation in an illegal type into a |
11 | // computation in a larger type. For example, implementing i8 arithmetic in an |
12 | // i32 register (often needed on powerpc). |
13 | // Expansion is the act of changing a computation in an illegal type into a |
14 | // computation in two identical registers of a smaller type. For example, |
15 | // implementing i64 arithmetic in two i32 registers (often needed on 32-bit |
16 | // targets). |
17 | // |
18 | //===----------------------------------------------------------------------===// |
19 | |
20 | #include "LegalizeTypes.h" |
21 | #include "llvm/Analysis/TargetLibraryInfo.h" |
22 | #include "llvm/CodeGen/StackMaps.h" |
23 | #include "llvm/CodeGen/TargetLowering.h" |
24 | #include "llvm/IR/DerivedTypes.h" |
25 | #include "llvm/Support/ErrorHandling.h" |
26 | #include "llvm/Support/KnownBits.h" |
27 | #include "llvm/Support/raw_ostream.h" |
28 | #include <algorithm> |
29 | using namespace llvm; |
30 | |
31 | #define DEBUG_TYPE "legalize-types" |
32 | |
33 | //===----------------------------------------------------------------------===// |
34 | // Integer Result Promotion |
35 | //===----------------------------------------------------------------------===// |
36 | |
37 | /// PromoteIntegerResult - This method is called when a result of a node is |
38 | /// found to be in need of promotion to a larger type. At this point, the node |
39 | /// may also have invalid operands or may have other results that need |
40 | /// expansion, we just know that (at least) one result needs promotion. |
41 | void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
42 | LLVM_DEBUG(dbgs() << "Promote integer result: " ; N->dump(&DAG)); |
43 | SDValue Res = SDValue(); |
44 | |
45 | // See if the target wants to custom expand this node. |
46 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) { |
47 | LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n" ); |
48 | return; |
49 | } |
50 | |
51 | switch (N->getOpcode()) { |
52 | default: |
53 | #ifndef NDEBUG |
54 | dbgs() << "PromoteIntegerResult #" << ResNo << ": " ; |
55 | N->dump(G: &DAG); dbgs() << "\n" ; |
56 | #endif |
57 | report_fatal_error(reason: "Do not know how to promote this operator!" ); |
58 | case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
59 | case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
60 | case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
61 | case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
62 | case ISD::VP_BITREVERSE: |
63 | case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
64 | case ISD::VP_BSWAP: |
65 | case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
66 | case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
67 | case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
68 | case ISD::VP_CTLZ_ZERO_UNDEF: |
69 | case ISD::VP_CTLZ: |
70 | case ISD::CTLZ_ZERO_UNDEF: |
71 | case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
72 | case ISD::PARITY: |
73 | case ISD::VP_CTPOP: |
74 | case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; |
75 | case ISD::VP_CTTZ_ZERO_UNDEF: |
76 | case ISD::VP_CTTZ: |
77 | case ISD::CTTZ_ZERO_UNDEF: |
78 | case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
79 | case ISD::EXTRACT_VECTOR_ELT: |
80 | Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
81 | case ISD::LOAD: Res = PromoteIntRes_LOAD(N: cast<LoadSDNode>(Val: N)); break; |
82 | case ISD::MLOAD: Res = PromoteIntRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N)); |
83 | break; |
84 | case ISD::MGATHER: Res = PromoteIntRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N)); |
85 | break; |
86 | case ISD::SELECT: |
87 | case ISD::VSELECT: |
88 | case ISD::VP_SELECT: |
89 | case ISD::VP_MERGE: |
90 | Res = PromoteIntRes_Select(N); |
91 | break; |
92 | case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
93 | case ISD::STRICT_FSETCC: |
94 | case ISD::STRICT_FSETCCS: |
95 | case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
96 | case ISD::SMIN: |
97 | case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
98 | case ISD::UMIN: |
99 | case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; |
100 | |
101 | case ISD::SHL: |
102 | case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break; |
103 | case ISD::SIGN_EXTEND_INREG: |
104 | Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
105 | case ISD::SRA: |
106 | case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break; |
107 | case ISD::SRL: |
108 | case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break; |
109 | case ISD::VP_TRUNCATE: |
110 | case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
111 | case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
112 | case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
113 | case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; |
114 | |
115 | case ISD::EXTRACT_SUBVECTOR: |
116 | Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
117 | case ISD::INSERT_SUBVECTOR: |
118 | Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; |
119 | case ISD::VECTOR_REVERSE: |
120 | Res = PromoteIntRes_VECTOR_REVERSE(N); break; |
121 | case ISD::VECTOR_SHUFFLE: |
122 | Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
123 | case ISD::VECTOR_SPLICE: |
124 | Res = PromoteIntRes_VECTOR_SPLICE(N); break; |
125 | case ISD::VECTOR_INTERLEAVE: |
126 | case ISD::VECTOR_DEINTERLEAVE: |
127 | Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N); |
128 | return; |
129 | case ISD::INSERT_VECTOR_ELT: |
130 | Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
131 | case ISD::BUILD_VECTOR: |
132 | Res = PromoteIntRes_BUILD_VECTOR(N); |
133 | break; |
134 | case ISD::SPLAT_VECTOR: |
135 | case ISD::SCALAR_TO_VECTOR: |
136 | Res = PromoteIntRes_ScalarOp(N); |
137 | break; |
138 | case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; |
139 | case ISD::CONCAT_VECTORS: |
140 | Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
141 | |
142 | case ISD::ANY_EXTEND_VECTOR_INREG: |
143 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
144 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
145 | Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
146 | |
147 | case ISD::SIGN_EXTEND: |
148 | case ISD::VP_SIGN_EXTEND: |
149 | case ISD::ZERO_EXTEND: |
150 | case ISD::VP_ZERO_EXTEND: |
151 | case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
152 | |
153 | case ISD::VP_FP_TO_SINT: |
154 | case ISD::VP_FP_TO_UINT: |
155 | case ISD::STRICT_FP_TO_SINT: |
156 | case ISD::STRICT_FP_TO_UINT: |
157 | case ISD::FP_TO_SINT: |
158 | case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
159 | |
160 | case ISD::FP_TO_SINT_SAT: |
161 | case ISD::FP_TO_UINT_SAT: |
162 | Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; |
163 | |
164 | case ISD::FP_TO_BF16: |
165 | case ISD::FP_TO_FP16: |
166 | Res = PromoteIntRes_FP_TO_FP16_BF16(N); |
167 | break; |
168 | case ISD::STRICT_FP_TO_BF16: |
169 | case ISD::STRICT_FP_TO_FP16: |
170 | Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N); |
171 | break; |
172 | case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; |
173 | |
174 | case ISD::AND: |
175 | case ISD::OR: |
176 | case ISD::XOR: |
177 | case ISD::ADD: |
178 | case ISD::SUB: |
179 | case ISD::MUL: |
180 | case ISD::VP_AND: |
181 | case ISD::VP_OR: |
182 | case ISD::VP_XOR: |
183 | case ISD::VP_ADD: |
184 | case ISD::VP_SUB: |
185 | case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
186 | |
187 | case ISD::VP_SMIN: |
188 | case ISD::VP_SMAX: |
189 | case ISD::SDIV: |
190 | case ISD::SREM: |
191 | case ISD::VP_SDIV: |
192 | case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
193 | |
194 | case ISD::VP_UMIN: |
195 | case ISD::VP_UMAX: |
196 | case ISD::UDIV: |
197 | case ISD::UREM: |
198 | case ISD::VP_UDIV: |
199 | case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
200 | |
201 | case ISD::SADDO: |
202 | case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
203 | case ISD::UADDO: |
204 | case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
205 | case ISD::SMULO: |
206 | case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
207 | |
208 | case ISD::ADDE: |
209 | case ISD::SUBE: |
210 | case ISD::UADDO_CARRY: |
211 | case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break; |
212 | |
213 | case ISD::SADDO_CARRY: |
214 | case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; |
215 | |
216 | case ISD::SADDSAT: |
217 | case ISD::UADDSAT: |
218 | case ISD::SSUBSAT: |
219 | case ISD::USUBSAT: |
220 | case ISD::SSHLSAT: |
221 | case ISD::USHLSAT: |
222 | Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N); |
223 | break; |
224 | case ISD::VP_SADDSAT: |
225 | case ISD::VP_UADDSAT: |
226 | case ISD::VP_SSUBSAT: |
227 | case ISD::VP_USUBSAT: |
228 | Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N); |
229 | break; |
230 | |
231 | case ISD::SMULFIX: |
232 | case ISD::SMULFIXSAT: |
233 | case ISD::UMULFIX: |
234 | case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; |
235 | |
236 | case ISD::SDIVFIX: |
237 | case ISD::SDIVFIXSAT: |
238 | case ISD::UDIVFIX: |
239 | case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; |
240 | |
241 | case ISD::ABS: Res = PromoteIntRes_ABS(N); break; |
242 | |
243 | case ISD::ATOMIC_LOAD: |
244 | Res = PromoteIntRes_Atomic0(N: cast<AtomicSDNode>(Val: N)); break; |
245 | |
246 | case ISD::ATOMIC_LOAD_ADD: |
247 | case ISD::ATOMIC_LOAD_SUB: |
248 | case ISD::ATOMIC_LOAD_AND: |
249 | case ISD::ATOMIC_LOAD_CLR: |
250 | case ISD::ATOMIC_LOAD_OR: |
251 | case ISD::ATOMIC_LOAD_XOR: |
252 | case ISD::ATOMIC_LOAD_NAND: |
253 | case ISD::ATOMIC_LOAD_MIN: |
254 | case ISD::ATOMIC_LOAD_MAX: |
255 | case ISD::ATOMIC_LOAD_UMIN: |
256 | case ISD::ATOMIC_LOAD_UMAX: |
257 | case ISD::ATOMIC_SWAP: |
258 | Res = PromoteIntRes_Atomic1(N: cast<AtomicSDNode>(Val: N)); break; |
259 | |
260 | case ISD::ATOMIC_CMP_SWAP: |
261 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
262 | Res = PromoteIntRes_AtomicCmpSwap(N: cast<AtomicSDNode>(Val: N), ResNo); |
263 | break; |
264 | |
265 | case ISD::VECREDUCE_ADD: |
266 | case ISD::VECREDUCE_MUL: |
267 | case ISD::VECREDUCE_AND: |
268 | case ISD::VECREDUCE_OR: |
269 | case ISD::VECREDUCE_XOR: |
270 | case ISD::VECREDUCE_SMAX: |
271 | case ISD::VECREDUCE_SMIN: |
272 | case ISD::VECREDUCE_UMAX: |
273 | case ISD::VECREDUCE_UMIN: |
274 | Res = PromoteIntRes_VECREDUCE(N); |
275 | break; |
276 | |
277 | case ISD::VP_REDUCE_ADD: |
278 | case ISD::VP_REDUCE_MUL: |
279 | case ISD::VP_REDUCE_AND: |
280 | case ISD::VP_REDUCE_OR: |
281 | case ISD::VP_REDUCE_XOR: |
282 | case ISD::VP_REDUCE_SMAX: |
283 | case ISD::VP_REDUCE_SMIN: |
284 | case ISD::VP_REDUCE_UMAX: |
285 | case ISD::VP_REDUCE_UMIN: |
286 | Res = PromoteIntRes_VP_REDUCE(N); |
287 | break; |
288 | |
289 | case ISD::FREEZE: |
290 | Res = PromoteIntRes_FREEZE(N); |
291 | break; |
292 | |
293 | case ISD::ROTL: |
294 | case ISD::ROTR: |
295 | Res = PromoteIntRes_Rotate(N); |
296 | break; |
297 | |
298 | case ISD::FSHL: |
299 | case ISD::FSHR: |
300 | Res = PromoteIntRes_FunnelShift(N); |
301 | break; |
302 | |
303 | case ISD::VP_FSHL: |
304 | case ISD::VP_FSHR: |
305 | Res = PromoteIntRes_VPFunnelShift(N); |
306 | break; |
307 | |
308 | case ISD::IS_FPCLASS: |
309 | Res = PromoteIntRes_IS_FPCLASS(N); |
310 | break; |
311 | case ISD::FFREXP: |
312 | Res = PromoteIntRes_FFREXP(N); |
313 | break; |
314 | |
315 | case ISD::LRINT: |
316 | case ISD::LLRINT: |
317 | Res = PromoteIntRes_XRINT(N); |
318 | break; |
319 | } |
320 | |
321 | // If the result is null then the sub-method took care of registering it. |
322 | if (Res.getNode()) |
323 | SetPromotedInteger(Op: SDValue(N, ResNo), Result: Res); |
324 | } |
325 | |
326 | SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
327 | unsigned ResNo) { |
328 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
329 | return GetPromotedInteger(Op); |
330 | } |
331 | |
332 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
333 | // Sign-extend the new bits, and continue the assertion. |
334 | SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
335 | return DAG.getNode(Opcode: ISD::AssertSext, DL: SDLoc(N), |
336 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
337 | } |
338 | |
339 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
340 | // Zero the new bits, and continue the assertion. |
341 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
342 | return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), |
343 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
344 | } |
345 | |
346 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
347 | EVT ResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
348 | SDValue Res = DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), |
349 | MemVT: N->getMemoryVT(), VT: ResVT, |
350 | Chain: N->getChain(), Ptr: N->getBasePtr(), |
351 | MMO: N->getMemOperand()); |
352 | if (N->getOpcode() == ISD::ATOMIC_LOAD) { |
353 | ISD::LoadExtType ETy = cast<AtomicSDNode>(Val: N)->getExtensionType(); |
354 | if (ETy == ISD::NON_EXTLOAD) { |
355 | switch (TLI.getExtendForAtomicOps()) { |
356 | case ISD::SIGN_EXTEND: |
357 | ETy = ISD::SEXTLOAD; |
358 | break; |
359 | case ISD::ZERO_EXTEND: |
360 | ETy = ISD::ZEXTLOAD; |
361 | break; |
362 | case ISD::ANY_EXTEND: |
363 | ETy = ISD::EXTLOAD; |
364 | break; |
365 | default: |
366 | llvm_unreachable("Invalid atomic op extension" ); |
367 | } |
368 | } |
369 | cast<AtomicSDNode>(Val&: Res)->setExtensionType(ETy); |
370 | } |
371 | |
372 | // Legalize the chain result - switch anything that used the old chain to |
373 | // use the new one. |
374 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
375 | return Res; |
376 | } |
377 | |
378 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
379 | SDValue Op2 = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
380 | SDValue Res = DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), |
381 | MemVT: N->getMemoryVT(), |
382 | Chain: N->getChain(), Ptr: N->getBasePtr(), |
383 | Val: Op2, MMO: N->getMemOperand()); |
384 | // Legalize the chain result - switch anything that used the old chain to |
385 | // use the new one. |
386 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
387 | return Res; |
388 | } |
389 | |
390 | SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
391 | unsigned ResNo) { |
392 | if (ResNo == 1) { |
393 | assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
394 | EVT SVT = getSetCCResultType(VT: N->getOperand(Num: 2).getValueType()); |
395 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
396 | |
397 | // Only use the result of getSetCCResultType if it is legal, |
398 | // otherwise just use the promoted result type (NVT). |
399 | if (!TLI.isTypeLegal(VT: SVT)) |
400 | SVT = NVT; |
401 | |
402 | SDVTList VTs = DAG.getVTList(N->getValueType(ResNo: 0), SVT, MVT::Other); |
403 | SDValue Res = DAG.getAtomicCmpSwap( |
404 | Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs, |
405 | Chain: N->getChain(), Ptr: N->getBasePtr(), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3), |
406 | MMO: N->getMemOperand()); |
407 | ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0)); |
408 | ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2)); |
409 | return DAG.getSExtOrTrunc(Op: Res.getValue(R: 1), DL: SDLoc(N), VT: NVT); |
410 | } |
411 | |
412 | // Op2 is used for the comparison and thus must be extended according to the |
413 | // target's atomic operations. Op3 is merely stored and so can be left alone. |
414 | SDValue Op2 = N->getOperand(Num: 2); |
415 | SDValue Op3 = GetPromotedInteger(Op: N->getOperand(Num: 3)); |
416 | switch (TLI.getExtendForAtomicCmpSwapArg()) { |
417 | case ISD::SIGN_EXTEND: |
418 | Op2 = SExtPromotedInteger(Op: Op2); |
419 | break; |
420 | case ISD::ZERO_EXTEND: |
421 | Op2 = ZExtPromotedInteger(Op: Op2); |
422 | break; |
423 | case ISD::ANY_EXTEND: |
424 | Op2 = GetPromotedInteger(Op: Op2); |
425 | break; |
426 | default: |
427 | llvm_unreachable("Invalid atomic op extension" ); |
428 | } |
429 | |
430 | SDVTList VTs = |
431 | DAG.getVTList(Op2.getValueType(), N->getValueType(ResNo: 1), MVT::Other); |
432 | SDValue Res = DAG.getAtomicCmpSwap( |
433 | Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(), VTs, Chain: N->getChain(), |
434 | Ptr: N->getBasePtr(), Cmp: Op2, Swp: Op3, MMO: N->getMemOperand()); |
435 | // Update the use to N with the newly created Res. |
436 | for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) |
437 | ReplaceValueWith(From: SDValue(N, i), To: Res.getValue(R: i)); |
438 | return Res; |
439 | } |
440 | |
441 | SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
442 | SDValue InOp = N->getOperand(Num: 0); |
443 | EVT InVT = InOp.getValueType(); |
444 | EVT NInVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT); |
445 | EVT OutVT = N->getValueType(ResNo: 0); |
446 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
447 | SDLoc dl(N); |
448 | |
449 | switch (getTypeAction(VT: InVT)) { |
450 | case TargetLowering::TypeLegal: |
451 | break; |
452 | case TargetLowering::TypePromoteInteger: |
453 | if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector() && !NInVT.isVector()) |
454 | // The input promotes to the same size. Convert the promoted value. |
455 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetPromotedInteger(Op: InOp)); |
456 | break; |
457 | case TargetLowering::TypeSoftenFloat: |
458 | // Promote the integer operand by hand. |
459 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftenedFloat(Op: InOp)); |
460 | case TargetLowering::TypeSoftPromoteHalf: |
461 | // Promote the integer operand by hand. |
462 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: GetSoftPromotedHalf(Op: InOp)); |
463 | case TargetLowering::TypePromoteFloat: { |
464 | // Convert the promoted float by hand. |
465 | if (!NOutVT.isVector()) |
466 | return DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: NOutVT, Operand: GetPromotedFloat(Op: InOp)); |
467 | break; |
468 | } |
469 | case TargetLowering::TypeExpandInteger: |
470 | case TargetLowering::TypeExpandFloat: |
471 | break; |
472 | case TargetLowering::TypeScalarizeVector: |
473 | // Convert the element to an integer and promote it by hand. |
474 | if (!NOutVT.isVector()) |
475 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, |
476 | Operand: BitConvertToInteger(Op: GetScalarizedVector(Op: InOp))); |
477 | break; |
478 | case TargetLowering::TypeScalarizeScalableVector: |
479 | report_fatal_error(reason: "Scalarization of scalable vectors is not supported." ); |
480 | case TargetLowering::TypeSplitVector: { |
481 | if (!NOutVT.isVector()) { |
482 | // For example, i32 = BITCAST v2i16 on alpha. Convert the split |
483 | // pieces of the input into integers and reassemble in the final type. |
484 | SDValue Lo, Hi; |
485 | GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi); |
486 | Lo = BitConvertToInteger(Op: Lo); |
487 | Hi = BitConvertToInteger(Op: Hi); |
488 | |
489 | if (DAG.getDataLayout().isBigEndian()) |
490 | std::swap(a&: Lo, b&: Hi); |
491 | |
492 | InOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, |
493 | VT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
494 | BitWidth: NOutVT.getSizeInBits()), |
495 | Operand: JoinIntegers(Lo, Hi)); |
496 | return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: InOp); |
497 | } |
498 | break; |
499 | } |
500 | case TargetLowering::TypeWidenVector: |
501 | // The input is widened to the same size. Convert to the widened value. |
502 | // Make sure that the outgoing value is not a vector, because this would |
503 | // make us bitcast between two vectors which are legalized in different ways. |
504 | if (NOutVT.bitsEq(VT: NInVT) && !NOutVT.isVector()) { |
505 | SDValue Res = |
506 | DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NOutVT, Operand: GetWidenedVector(Op: InOp)); |
507 | |
508 | // For big endian targets we need to shift the casted value or the |
509 | // interesting bits will end up at the wrong place. |
510 | if (DAG.getDataLayout().isBigEndian()) { |
511 | unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
512 | assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!" ); |
513 | Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NOutVT, N1: Res, |
514 | N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NOutVT, DL: dl)); |
515 | } |
516 | return Res; |
517 | } |
518 | // If the output type is also a vector and widening it to the same size |
519 | // as the widened input type would be a legal type, we can widen the bitcast |
520 | // and handle the promotion after. |
521 | if (NOutVT.isVector()) { |
522 | TypeSize WidenInSize = NInVT.getSizeInBits(); |
523 | TypeSize OutSize = OutVT.getSizeInBits(); |
524 | if (WidenInSize.hasKnownScalarFactor(RHS: OutSize)) { |
525 | unsigned Scale = WidenInSize.getKnownScalarFactor(RHS: OutSize); |
526 | EVT WideOutVT = |
527 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: OutVT.getVectorElementType(), |
528 | EC: OutVT.getVectorElementCount() * Scale); |
529 | if (isTypeLegal(VT: WideOutVT)) { |
530 | InOp = DAG.getBitcast(VT: WideOutVT, V: GetWidenedVector(Op: InOp)); |
531 | InOp = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: InOp, |
532 | N2: DAG.getVectorIdxConstant(Val: 0, DL: dl)); |
533 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: InOp); |
534 | } |
535 | } |
536 | } |
537 | } |
538 | |
539 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, |
540 | Operand: CreateStackStoreLoad(Op: InOp, DestVT: OutVT)); |
541 | } |
542 | |
543 | SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { |
544 | SDValue V = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
545 | return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), |
546 | VT: V.getValueType(), Operand: V); |
547 | } |
548 | |
549 | SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
550 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
551 | EVT OVT = N->getValueType(ResNo: 0); |
552 | EVT NVT = Op.getValueType(); |
553 | SDLoc dl(N); |
554 | |
555 | // If the larger BSWAP isn't supported by the target, try to expand now. |
556 | // If we expand later we'll end up with more operations since we lost the |
557 | // original type. We only do this for scalars since we have a shuffle |
558 | // based lowering for vectors in LegalizeVectorOps. |
559 | if (!OVT.isVector() && |
560 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BSWAP, VT: NVT)) { |
561 | if (SDValue Res = TLI.expandBSWAP(N, DAG)) |
562 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res); |
563 | } |
564 | |
565 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
566 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl); |
567 | if (N->getOpcode() == ISD::BSWAP) |
568 | return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: NVT, Operand: Op), |
569 | N2: ShAmt); |
570 | SDValue Mask = N->getOperand(Num: 1); |
571 | SDValue EVL = N->getOperand(Num: 2); |
572 | return DAG.getNode(Opcode: ISD::VP_LSHR, DL: dl, VT: NVT, |
573 | N1: DAG.getNode(Opcode: ISD::VP_BSWAP, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), N2: ShAmt, |
574 | N3: Mask, N4: EVL); |
575 | } |
576 | |
577 | SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
578 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
579 | EVT OVT = N->getValueType(ResNo: 0); |
580 | EVT NVT = Op.getValueType(); |
581 | SDLoc dl(N); |
582 | |
583 | // If the larger BITREVERSE isn't supported by the target, try to expand now. |
584 | // If we expand later we'll end up with more operations since we lost the |
585 | // original type. We only do this for scalars since we have a shuffle |
586 | // based lowering for vectors in LegalizeVectorOps. |
587 | if (!OVT.isVector() && OVT.isSimple() && |
588 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::BITREVERSE, VT: NVT)) { |
589 | if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) |
590 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Res); |
591 | } |
592 | |
593 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
594 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl); |
595 | if (N->getOpcode() == ISD::BITREVERSE) |
596 | return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, |
597 | N1: DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: NVT, Operand: Op), N2: ShAmt); |
598 | SDValue Mask = N->getOperand(Num: 1); |
599 | SDValue EVL = N->getOperand(Num: 2); |
600 | return DAG.getNode(Opcode: ISD::VP_LSHR, DL: dl, VT: NVT, |
601 | N1: DAG.getNode(Opcode: ISD::VP_BITREVERSE, DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), |
602 | N2: ShAmt, N3: Mask, N4: EVL); |
603 | } |
604 | |
605 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
606 | // The pair element type may be legal, or may not promote to the same type as |
607 | // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. |
608 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), |
609 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
610 | VT: N->getValueType(ResNo: 0)), Operand: JoinIntegers(Lo: N->getOperand(Num: 0), |
611 | Hi: N->getOperand(Num: 1))); |
612 | } |
613 | |
614 | SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
615 | EVT VT = N->getValueType(ResNo: 0); |
616 | // FIXME there is no actual debug info here |
617 | SDLoc dl(N); |
618 | // Zero extend things like i1, sign extend everything else. It shouldn't |
619 | // matter in theory which one we pick, but this tends to give better code? |
620 | unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
621 | SDValue Result = DAG.getNode(Opcode: Opc, DL: dl, |
622 | VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT), |
623 | Operand: SDValue(N, 0)); |
624 | assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?" ); |
625 | return Result; |
626 | } |
627 | |
628 | SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
629 | EVT OVT = N->getValueType(ResNo: 0); |
630 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
631 | SDLoc dl(N); |
632 | |
633 | // If the larger CTLZ isn't supported by the target, try to expand now. |
634 | // If we expand later we'll end up with more operations since we lost the |
635 | // original type. |
636 | if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
637 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ, VT: NVT) && |
638 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTLZ_ZERO_UNDEF, VT: NVT)) { |
639 | if (SDValue Result = TLI.expandCTLZ(N, DAG)) { |
640 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result); |
641 | return Result; |
642 | } |
643 | } |
644 | |
645 | // Zero extend to the promoted type and do the count there. |
646 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
647 | |
648 | // Subtract off the extra leading bits in the bigger type. |
649 | SDValue = DAG.getConstant( |
650 | Val: NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), DL: dl, VT: NVT); |
651 | if (!N->isVPOpcode()) |
652 | return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, |
653 | N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op), |
654 | N2: ExtractLeadingBits); |
655 | SDValue Mask = N->getOperand(Num: 1); |
656 | SDValue EVL = N->getOperand(Num: 2); |
657 | return DAG.getNode(Opcode: ISD::VP_SUB, DL: dl, VT: NVT, |
658 | N1: DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op, N2: Mask, N3: EVL), |
659 | N2: ExtractLeadingBits, N3: Mask, N4: EVL); |
660 | } |
661 | |
662 | SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { |
663 | EVT OVT = N->getValueType(ResNo: 0); |
664 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
665 | |
666 | // If the larger CTPOP isn't supported by the target, try to expand now. |
667 | // If we expand later we'll end up with more operations since we lost the |
668 | // original type. |
669 | // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to |
670 | // TargetLowering. |
671 | if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
672 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT: NVT)) { |
673 | if (SDValue Result = TLI.expandCTPOP(N, DAG)) { |
674 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Result); |
675 | return Result; |
676 | } |
677 | } |
678 | |
679 | // Zero extend to the promoted type and do the count or parity there. |
680 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
681 | if (!N->isVPOpcode()) |
682 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), Operand: Op); |
683 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op, |
684 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
685 | } |
686 | |
687 | SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
688 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
689 | EVT OVT = N->getValueType(ResNo: 0); |
690 | EVT NVT = Op.getValueType(); |
691 | SDLoc dl(N); |
692 | |
693 | // If the larger CTTZ isn't supported by the target, try to expand now. |
694 | // If we expand later we'll end up with more operations since we lost the |
695 | // original type. Don't expand if we can use CTPOP or CTLZ expansion on the |
696 | // larger type. |
697 | if (!OVT.isVector() && TLI.isTypeLegal(VT: NVT) && |
698 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ, VT: NVT) && |
699 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::CTTZ_ZERO_UNDEF, VT: NVT) && |
700 | !TLI.isOperationLegal(Op: ISD::CTPOP, VT: NVT) && |
701 | !TLI.isOperationLegal(Op: ISD::CTLZ, VT: NVT)) { |
702 | if (SDValue Result = TLI.expandCTTZ(N, DAG)) { |
703 | Result = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Result); |
704 | return Result; |
705 | } |
706 | } |
707 | |
708 | if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) { |
709 | // The count is the same in the promoted type except if the original |
710 | // value was zero. This can be handled by setting the bit just off |
711 | // the top of the original type. |
712 | auto TopBit = APInt::getOneBitSet(numBits: NVT.getScalarSizeInBits(), |
713 | BitNo: OVT.getScalarSizeInBits()); |
714 | if (N->getOpcode() == ISD::CTTZ) |
715 | Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT)); |
716 | else |
717 | Op = |
718 | DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT: NVT, N1: Op, N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT), |
719 | N3: N->getOperand(Num: 1), N4: N->getOperand(Num: 2)); |
720 | } |
721 | if (!N->isVPOpcode()) |
722 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op); |
723 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op, N2: N->getOperand(Num: 1), |
724 | N3: N->getOperand(Num: 2)); |
725 | } |
726 | |
727 | SDValue DAGTypeLegalizer::(SDNode *N) { |
728 | SDLoc dl(N); |
729 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
730 | |
731 | SDValue Op0 = N->getOperand(Num: 0); |
732 | SDValue Op1 = N->getOperand(Num: 1); |
733 | |
734 | // If the input also needs to be promoted, do that first so we can get a |
735 | // get a good idea for the output type. |
736 | if (TLI.getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) |
737 | == TargetLowering::TypePromoteInteger) { |
738 | SDValue In = GetPromotedInteger(Op: Op0); |
739 | |
740 | // If the new type is larger than NVT, use it. We probably won't need to |
741 | // promote it again. |
742 | EVT SVT = In.getValueType().getScalarType(); |
743 | if (SVT.bitsGE(VT: NVT)) { |
744 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SVT, N1: In, N2: Op1); |
745 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NVT); |
746 | } |
747 | } |
748 | |
749 | return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NVT, N1: Op0, N2: Op1); |
750 | } |
751 | |
752 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
753 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
754 | unsigned NewOpc = N->getOpcode(); |
755 | SDLoc dl(N); |
756 | |
757 | // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is |
758 | // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT |
759 | // and SINT conversions are Custom, there is no way to tell which is |
760 | // preferable. We choose SINT because that's the right thing on PPC.) |
761 | if (N->getOpcode() == ISD::FP_TO_UINT && |
762 | !TLI.isOperationLegal(Op: ISD::FP_TO_UINT, VT: NVT) && |
763 | TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_SINT, VT: NVT)) |
764 | NewOpc = ISD::FP_TO_SINT; |
765 | |
766 | if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && |
767 | !TLI.isOperationLegal(Op: ISD::STRICT_FP_TO_UINT, VT: NVT) && |
768 | TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FP_TO_SINT, VT: NVT)) |
769 | NewOpc = ISD::STRICT_FP_TO_SINT; |
770 | |
771 | if (N->getOpcode() == ISD::VP_FP_TO_UINT && |
772 | !TLI.isOperationLegal(Op: ISD::VP_FP_TO_UINT, VT: NVT) && |
773 | TLI.isOperationLegalOrCustom(Op: ISD::VP_FP_TO_SINT, VT: NVT)) |
774 | NewOpc = ISD::VP_FP_TO_SINT; |
775 | |
776 | SDValue Res; |
777 | if (N->isStrictFPOpcode()) { |
778 | Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, |
779 | {N->getOperand(0), N->getOperand(1)}); |
780 | // Legalize the chain result - switch anything that used the old chain to |
781 | // use the new one. |
782 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
783 | } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) { |
784 | Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1), |
785 | N->getOperand(Num: 2)}); |
786 | } else { |
787 | Res = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
788 | } |
789 | |
790 | // Assert that the converted value fits in the original type. If it doesn't |
791 | // (eg: because the value being converted is too big), then the result of the |
792 | // original operation was undefined anyway, so the assert is still correct. |
793 | // |
794 | // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: |
795 | // before legalization: fp-to-uint16, 65534. -> 0xfffe |
796 | // after legalization: fp-to-sint32, 65534. -> 0x0000fffe |
797 | return DAG.getNode(Opcode: (N->getOpcode() == ISD::FP_TO_UINT || |
798 | N->getOpcode() == ISD::STRICT_FP_TO_UINT || |
799 | N->getOpcode() == ISD::VP_FP_TO_UINT) |
800 | ? ISD::AssertZext |
801 | : ISD::AssertSext, |
802 | DL: dl, VT: NVT, N1: Res, |
803 | N2: DAG.getValueType(N->getValueType(ResNo: 0).getScalarType())); |
804 | } |
805 | |
806 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { |
807 | // Promote the result type, while keeping the original width in Op1. |
808 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
809 | SDLoc dl(N); |
810 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0), |
811 | N2: N->getOperand(Num: 1)); |
812 | } |
813 | |
814 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { |
815 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
816 | SDLoc dl(N); |
817 | |
818 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
819 | } |
820 | |
821 | SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) { |
822 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
823 | SDLoc dl(N); |
824 | |
825 | SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), |
826 | N->getOperand(Num: 0), N->getOperand(Num: 1)); |
827 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
828 | return Res; |
829 | } |
830 | |
831 | SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) { |
832 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
833 | SDLoc dl(N); |
834 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
835 | } |
836 | |
837 | SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) { |
838 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
839 | SDLoc dl(N); |
840 | |
841 | SDValue Res = |
842 | DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); |
843 | |
844 | // Legalize the chain result - switch anything that used the old chain to |
845 | // use the new one. |
846 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
847 | return Res; |
848 | } |
849 | |
850 | SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
851 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
852 | SDLoc dl(N); |
853 | |
854 | if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) |
855 | == TargetLowering::TypePromoteInteger) { |
856 | SDValue Res = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
857 | assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!" ); |
858 | |
859 | // If the result and operand types are the same after promotion, simplify |
860 | // to an in-register extension. Unless this is a VP_*_EXTEND. |
861 | if (NVT == Res.getValueType() && N->getNumOperands() == 1) { |
862 | // The high bits are not guaranteed to be anything. Insert an extend. |
863 | if (N->getOpcode() == ISD::SIGN_EXTEND) |
864 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res, |
865 | N2: DAG.getValueType(N->getOperand(Num: 0).getValueType())); |
866 | if (N->getOpcode() == ISD::ZERO_EXTEND) |
867 | return DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: N->getOperand(Num: 0).getValueType()); |
868 | assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!" ); |
869 | return Res; |
870 | } |
871 | } |
872 | |
873 | // Otherwise, just extend the original operand all the way to the larger type. |
874 | if (N->getNumOperands() != 1) { |
875 | assert(N->getNumOperands() == 3 && "Unexpected number of operands!" ); |
876 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
877 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: N->getOperand(Num: 0), |
878 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
879 | } |
880 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
881 | } |
882 | |
883 | SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
884 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!" ); |
885 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
886 | ISD::LoadExtType ExtType = |
887 | ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); |
888 | SDLoc dl(N); |
889 | SDValue Res = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: N->getChain(), Ptr: N->getBasePtr(), |
890 | MemVT: N->getMemoryVT(), MMO: N->getMemOperand()); |
891 | |
892 | // Legalize the chain result - switch anything that used the old chain to |
893 | // use the new one. |
894 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
895 | return Res; |
896 | } |
897 | |
898 | SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
899 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
900 | SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru()); |
901 | |
902 | ISD::LoadExtType ExtType = N->getExtensionType(); |
903 | if (ExtType == ISD::NON_EXTLOAD) |
904 | ExtType = ISD::EXTLOAD; |
905 | |
906 | SDLoc dl(N); |
907 | SDValue Res = DAG.getMaskedLoad(VT: NVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), |
908 | Offset: N->getOffset(), Mask: N->getMask(), Src0: ExtPassThru, |
909 | MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), |
910 | AM: N->getAddressingMode(), ExtType, |
911 | IsExpanding: N->isExpandingLoad()); |
912 | // Legalize the chain result - switch anything that used the old chain to |
913 | // use the new one. |
914 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
915 | return Res; |
916 | } |
917 | |
918 | SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
919 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
920 | SDValue ExtPassThru = GetPromotedInteger(Op: N->getPassThru()); |
921 | assert(NVT == ExtPassThru.getValueType() && |
922 | "Gather result type and the passThru argument type should be the same" ); |
923 | |
924 | ISD::LoadExtType ExtType = N->getExtensionType(); |
925 | if (ExtType == ISD::NON_EXTLOAD) |
926 | ExtType = ISD::EXTLOAD; |
927 | |
928 | SDLoc dl(N); |
929 | SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), |
930 | N->getIndex(), N->getScale() }; |
931 | SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(NVT, MVT::Other), |
932 | MemVT: N->getMemoryVT(), dl, Ops, |
933 | MMO: N->getMemOperand(), IndexType: N->getIndexType(), |
934 | ExtTy: ExtType); |
935 | // Legalize the chain result - switch anything that used the old chain to |
936 | // use the new one. |
937 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
938 | return Res; |
939 | } |
940 | |
941 | /// Promote the overflow flag of an overflowing arithmetic node. |
942 | SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
943 | // Change the return type of the boolean result while obeying |
944 | // getSetCCResultType. |
945 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
946 | EVT VT = N->getValueType(ResNo: 0); |
947 | EVT SVT = getSetCCResultType(VT); |
948 | SDValue Ops[3] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
949 | unsigned NumOps = N->getNumOperands(); |
950 | assert(NumOps <= 3 && "Too many operands" ); |
951 | if (NumOps == 3) |
952 | Ops[2] = PromoteTargetBoolean(Bool: N->getOperand(Num: 2), ValVT: VT); |
953 | |
954 | SDLoc dl(N); |
955 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: SVT), |
956 | Ops: ArrayRef(Ops, NumOps)); |
957 | |
958 | // Modified the sum result - switch anything that used the old sum to use |
959 | // the new one. |
960 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
961 | |
962 | // Convert to the expected type. |
963 | return DAG.getBoolExtOrTrunc(Op: Res.getValue(R: 1), SL: dl, VT: NVT, OpVT: VT); |
964 | } |
965 | |
966 | template <class MatchContextClass> |
967 | SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { |
968 | // If the promoted type is legal, we can convert this to: |
969 | // 1. ANY_EXTEND iN to iM |
970 | // 2. SHL by M-N |
971 | // 3. [US][ADD|SUB|SHL]SAT |
972 | // 4. L/ASHR by M-N |
973 | // Else it is more efficient to convert this to a min and a max |
974 | // operation in the higher precision arithmetic. |
975 | SDLoc dl(N); |
976 | SDValue Op1 = N->getOperand(Num: 0); |
977 | SDValue Op2 = N->getOperand(Num: 1); |
978 | MatchContextClass matcher(DAG, TLI, N); |
979 | unsigned OldBits = Op1.getScalarValueSizeInBits(); |
980 | |
981 | unsigned Opcode = matcher.getRootBaseOpcode(); |
982 | bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; |
983 | |
984 | // FIXME: We need vp-aware PromotedInteger functions. |
985 | SDValue Op1Promoted, Op2Promoted; |
986 | if (IsShift) { |
987 | Op1Promoted = GetPromotedInteger(Op: Op1); |
988 | Op2Promoted = ZExtPromotedInteger(Op: Op2); |
989 | } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { |
990 | Op1Promoted = ZExtPromotedInteger(Op: Op1); |
991 | Op2Promoted = ZExtPromotedInteger(Op: Op2); |
992 | } else { |
993 | Op1Promoted = SExtPromotedInteger(Op: Op1); |
994 | Op2Promoted = SExtPromotedInteger(Op: Op2); |
995 | } |
996 | EVT PromotedType = Op1Promoted.getValueType(); |
997 | unsigned NewBits = PromotedType.getScalarSizeInBits(); |
998 | |
999 | if (Opcode == ISD::UADDSAT) { |
1000 | APInt MaxVal = APInt::getAllOnes(numBits: OldBits).zext(width: NewBits); |
1001 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: PromotedType); |
1002 | SDValue Add = |
1003 | matcher.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); |
1004 | return matcher.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); |
1005 | } |
1006 | |
1007 | // USUBSAT can always be promoted as long as we have zero-extended the args. |
1008 | if (Opcode == ISD::USUBSAT) |
1009 | return matcher.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, |
1010 | Op2Promoted); |
1011 | |
1012 | // Shift cannot use a min/max expansion, we can't detect overflow if all of |
1013 | // the bits have been shifted out. |
1014 | if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) { |
1015 | unsigned ShiftOp; |
1016 | switch (Opcode) { |
1017 | case ISD::SADDSAT: |
1018 | case ISD::SSUBSAT: |
1019 | case ISD::SSHLSAT: |
1020 | ShiftOp = ISD::SRA; |
1021 | break; |
1022 | case ISD::USHLSAT: |
1023 | ShiftOp = ISD::SRL; |
1024 | break; |
1025 | default: |
1026 | llvm_unreachable("Expected opcode to be signed or unsigned saturation " |
1027 | "addition, subtraction or left shift" ); |
1028 | } |
1029 | |
1030 | unsigned SHLAmount = NewBits - OldBits; |
1031 | SDValue ShiftAmount = |
1032 | DAG.getShiftAmountConstant(Val: SHLAmount, VT: PromotedType, DL: dl); |
1033 | Op1Promoted = |
1034 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, N2: ShiftAmount); |
1035 | if (!IsShift) |
1036 | Op2Promoted = |
1037 | matcher.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); |
1038 | |
1039 | SDValue Result = |
1040 | matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); |
1041 | return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); |
1042 | } |
1043 | |
1044 | unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; |
1045 | APInt MinVal = APInt::getSignedMinValue(numBits: OldBits).sext(width: NewBits); |
1046 | APInt MaxVal = APInt::getSignedMaxValue(numBits: OldBits).sext(width: NewBits); |
1047 | SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT: PromotedType); |
1048 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT: PromotedType); |
1049 | SDValue Result = |
1050 | matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); |
1051 | Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); |
1052 | Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); |
1053 | return Result; |
1054 | } |
1055 | |
1056 | SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { |
1057 | // Can just promote the operands then continue with operation. |
1058 | SDLoc dl(N); |
1059 | SDValue Op1Promoted, Op2Promoted; |
1060 | bool Signed = |
1061 | N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; |
1062 | bool Saturating = |
1063 | N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; |
1064 | if (Signed) { |
1065 | Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1066 | Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1067 | } else { |
1068 | Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1069 | Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1070 | } |
1071 | EVT OldType = N->getOperand(Num: 0).getValueType(); |
1072 | EVT PromotedType = Op1Promoted.getValueType(); |
1073 | unsigned DiffSize = |
1074 | PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); |
1075 | |
1076 | if (Saturating) { |
1077 | // Promoting the operand and result values changes the saturation width, |
1078 | // which is extends the values that we clamp to on saturation. This could be |
1079 | // resolved by shifting one of the operands the same amount, which would |
1080 | // also shift the result we compare against, then shifting back. |
1081 | Op1Promoted = |
1082 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, |
1083 | N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl)); |
1084 | SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, |
1085 | N2: Op2Promoted, N3: N->getOperand(Num: 2)); |
1086 | unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; |
1087 | return DAG.getNode(Opcode: ShiftOp, DL: dl, VT: PromotedType, N1: Result, |
1088 | N2: DAG.getShiftAmountConstant(Val: DiffSize, VT: PromotedType, DL: dl)); |
1089 | } |
1090 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, N2: Op2Promoted, |
1091 | N3: N->getOperand(Num: 2)); |
1092 | } |
1093 | |
1094 | static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, |
1095 | unsigned SatW, bool Signed, |
1096 | const TargetLowering &TLI, |
1097 | SelectionDAG &DAG) { |
1098 | EVT VT = V.getValueType(); |
1099 | unsigned VTW = VT.getScalarSizeInBits(); |
1100 | |
1101 | if (!Signed) { |
1102 | // Saturate to the unsigned maximum by getting the minimum of V and the |
1103 | // maximum. |
1104 | return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: V, |
1105 | N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW), |
1106 | DL: dl, VT)); |
1107 | } |
1108 | |
1109 | // Saturate to the signed maximum (the low SatW - 1 bits) by taking the |
1110 | // signed minimum of it and V. |
1111 | V = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: V, |
1112 | N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTW, loBitsSet: SatW - 1), |
1113 | DL: dl, VT)); |
1114 | // Saturate to the signed minimum (the high SatW + 1 bits) by taking the |
1115 | // signed maximum of it and V. |
1116 | V = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: V, |
1117 | N2: DAG.getConstant(Val: APInt::getHighBitsSet(numBits: VTW, hiBitsSet: VTW - SatW + 1), |
1118 | DL: dl, VT)); |
1119 | return V; |
1120 | } |
1121 | |
1122 | static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, |
1123 | unsigned Scale, const TargetLowering &TLI, |
1124 | SelectionDAG &DAG, unsigned SatW = 0) { |
1125 | EVT VT = LHS.getValueType(); |
1126 | unsigned VTSize = VT.getScalarSizeInBits(); |
1127 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
1128 | N->getOpcode() == ISD::SDIVFIXSAT; |
1129 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
1130 | N->getOpcode() == ISD::UDIVFIXSAT; |
1131 | |
1132 | SDLoc dl(N); |
1133 | // Widen the types by a factor of two. This is guaranteed to expand, since it |
1134 | // will always have enough high bits in the LHS to shift into. |
1135 | EVT WideVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VTSize * 2); |
1136 | if (VT.isVector()) |
1137 | WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT, |
1138 | EC: VT.getVectorElementCount()); |
1139 | LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL: dl, VT: WideVT); |
1140 | RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL: dl, VT: WideVT); |
1141 | SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS, RHS, Scale, |
1142 | DAG); |
1143 | assert(Res && "Expanding DIVFIX with wide type failed?" ); |
1144 | if (Saturating) { |
1145 | // If the caller has told us to saturate at something less, use that width |
1146 | // instead of the type before doubling. However, it cannot be more than |
1147 | // what we just widened! |
1148 | assert(SatW <= VTSize && |
1149 | "Tried to saturate to more than the original type?" ); |
1150 | Res = SaturateWidenedDIVFIX(V: Res, dl, SatW: SatW == 0 ? VTSize : SatW, Signed, |
1151 | TLI, DAG); |
1152 | } |
1153 | return DAG.getZExtOrTrunc(Op: Res, DL: dl, VT); |
1154 | } |
1155 | |
1156 | SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { |
1157 | SDLoc dl(N); |
1158 | SDValue Op1Promoted, Op2Promoted; |
1159 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
1160 | N->getOpcode() == ISD::SDIVFIXSAT; |
1161 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
1162 | N->getOpcode() == ISD::UDIVFIXSAT; |
1163 | if (Signed) { |
1164 | Op1Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1165 | Op2Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1166 | } else { |
1167 | Op1Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1168 | Op2Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1169 | } |
1170 | EVT PromotedType = Op1Promoted.getValueType(); |
1171 | unsigned Scale = N->getConstantOperandVal(Num: 2); |
1172 | |
1173 | // If the type is already legal and the operation is legal in that type, we |
1174 | // should not early expand. |
1175 | if (TLI.isTypeLegal(VT: PromotedType)) { |
1176 | TargetLowering::LegalizeAction Action = |
1177 | TLI.getFixedPointOperationAction(Op: N->getOpcode(), VT: PromotedType, Scale); |
1178 | if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { |
1179 | unsigned Diff = PromotedType.getScalarSizeInBits() - |
1180 | N->getValueType(ResNo: 0).getScalarSizeInBits(); |
1181 | if (Saturating) |
1182 | Op1Promoted = |
1183 | DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: PromotedType, N1: Op1Promoted, |
1184 | N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl)); |
1185 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: PromotedType, N1: Op1Promoted, |
1186 | N2: Op2Promoted, N3: N->getOperand(Num: 2)); |
1187 | if (Saturating) |
1188 | Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT: PromotedType, N1: Res, |
1189 | N2: DAG.getShiftAmountConstant(Val: Diff, VT: PromotedType, DL: dl)); |
1190 | return Res; |
1191 | } |
1192 | } |
1193 | |
1194 | // See if we can perform the division in this type without expanding. |
1195 | if (SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: Op1Promoted, |
1196 | RHS: Op2Promoted, Scale, DAG)) { |
1197 | if (Saturating) |
1198 | Res = SaturateWidenedDIVFIX(V: Res, dl, |
1199 | SatW: N->getValueType(ResNo: 0).getScalarSizeInBits(), |
1200 | Signed, TLI, DAG); |
1201 | return Res; |
1202 | } |
1203 | // If we cannot, expand it to twice the type width. If we are saturating, give |
1204 | // it the original width as a saturating width so we don't need to emit |
1205 | // two saturations. |
1206 | return earlyExpandDIVFIX(N, LHS: Op1Promoted, RHS: Op2Promoted, Scale, TLI, DAG, |
1207 | SatW: N->getValueType(ResNo: 0).getScalarSizeInBits()); |
1208 | } |
1209 | |
1210 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
1211 | if (ResNo == 1) |
1212 | return PromoteIntRes_Overflow(N); |
1213 | |
1214 | // The operation overflowed iff the result in the larger type is not the |
1215 | // sign extension of its truncation to the original type. |
1216 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1217 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1218 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
1219 | EVT NVT = LHS.getValueType(); |
1220 | SDLoc dl(N); |
1221 | |
1222 | // Do the arithmetic in the larger type. |
1223 | unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; |
1224 | SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS); |
1225 | |
1226 | // Calculate the overflow flag: sign extend the arithmetic result from |
1227 | // the original type. |
1228 | SDValue Ofl = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NVT, N1: Res, |
1229 | N2: DAG.getValueType(OVT)); |
1230 | // Overflowed if and only if this is not equal to Res. |
1231 | Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE); |
1232 | |
1233 | // Use the calculated overflow everywhere. |
1234 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
1235 | |
1236 | return Res; |
1237 | } |
1238 | |
1239 | SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) { |
1240 | SDValue Mask = N->getOperand(Num: 0); |
1241 | |
1242 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1243 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
1244 | |
1245 | unsigned Opcode = N->getOpcode(); |
1246 | if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE) |
1247 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS, |
1248 | N4: N->getOperand(Num: 3)); |
1249 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: LHS.getValueType(), N1: Mask, N2: LHS, N3: RHS); |
1250 | } |
1251 | |
1252 | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
1253 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 2)); |
1254 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 3)); |
1255 | return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), |
1256 | VT: LHS.getValueType(), N1: N->getOperand(Num: 0), |
1257 | N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4)); |
1258 | } |
1259 | |
1260 | SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
1261 | unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
1262 | EVT InVT = N->getOperand(Num: OpNo).getValueType(); |
1263 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1264 | |
1265 | EVT SVT = getSetCCResultType(VT: InVT); |
1266 | |
1267 | // If we got back a type that needs to be promoted, this likely means the |
1268 | // the input type also needs to be promoted. So get the promoted type for |
1269 | // the input and try the query again. |
1270 | if (getTypeAction(VT: SVT) == TargetLowering::TypePromoteInteger) { |
1271 | if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) { |
1272 | InVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT); |
1273 | SVT = getSetCCResultType(VT: InVT); |
1274 | } else { |
1275 | // Input type isn't promoted, just use the default promoted type. |
1276 | SVT = NVT; |
1277 | } |
1278 | } |
1279 | |
1280 | SDLoc dl(N); |
1281 | assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && |
1282 | "Vector compare must return a vector result!" ); |
1283 | |
1284 | // Get the SETCC result using the canonical SETCC type. |
1285 | SDValue SetCC; |
1286 | if (N->isStrictFPOpcode()) { |
1287 | SDVTList VTs = DAG.getVTList({SVT, MVT::Other}); |
1288 | SDValue Opers[] = {N->getOperand(Num: 0), N->getOperand(Num: 1), |
1289 | N->getOperand(Num: 2), N->getOperand(Num: 3)}; |
1290 | SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, Ops: Opers, Flags: N->getFlags()); |
1291 | // Legalize the chain result - switch anything that used the old chain to |
1292 | // use the new one. |
1293 | ReplaceValueWith(From: SDValue(N, 1), To: SetCC.getValue(R: 1)); |
1294 | } else |
1295 | SetCC = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SVT, N1: N->getOperand(Num: 0), |
1296 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), Flags: N->getFlags()); |
1297 | |
1298 | // Convert to the expected type. |
1299 | return DAG.getSExtOrTrunc(Op: SetCC, DL: dl, VT: NVT); |
1300 | } |
1301 | |
1302 | SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) { |
1303 | SDLoc DL(N); |
1304 | SDValue Arg = N->getOperand(Num: 0); |
1305 | SDValue Test = N->getOperand(Num: 1); |
1306 | EVT NResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1307 | return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: NResVT, N1: Arg, N2: Test); |
1308 | } |
1309 | |
1310 | SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) { |
1311 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 1)); |
1312 | EVT VT = N->getValueType(ResNo: 0); |
1313 | |
1314 | SDLoc dl(N); |
1315 | SDValue Res = |
1316 | DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: NVT), N: N->getOperand(Num: 0)); |
1317 | |
1318 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1319 | return Res.getValue(R: 1); |
1320 | } |
1321 | |
1322 | SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
1323 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1324 | SDValue RHS = N->getOperand(Num: 1); |
1325 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1326 | RHS = ZExtPromotedInteger(Op: RHS); |
1327 | if (N->getOpcode() != ISD::VP_SHL) |
1328 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1329 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1330 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1331 | } |
1332 | |
1333 | SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
1334 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1335 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), |
1336 | VT: Op.getValueType(), N1: Op, N2: N->getOperand(Num: 1)); |
1337 | } |
1338 | |
1339 | SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
1340 | // The input may have strange things in the top bits of the registers, but |
1341 | // these operations don't care. They may have weird bits going out, but |
1342 | // that too is okay if they are integer operations. |
1343 | SDValue LHS = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1344 | SDValue RHS = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1345 | if (N->getNumOperands() == 2) |
1346 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1347 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1348 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1349 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1350 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1351 | } |
1352 | |
1353 | SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
1354 | // Sign extend the input. |
1355 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1356 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1357 | if (N->getNumOperands() == 2) |
1358 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1359 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1360 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1361 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1362 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1363 | } |
1364 | |
1365 | SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
1366 | // Zero extend the input. |
1367 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1368 | SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1369 | if (N->getNumOperands() == 2) |
1370 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1371 | assert(N->getNumOperands() == 4 && "Unexpected number of operands!" ); |
1372 | assert(N->isVPOpcode() && "Expected VP opcode" ); |
1373 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1374 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1375 | } |
1376 | |
1377 | SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { |
1378 | SDValue LHS = N->getOperand(Num: 0); |
1379 | SDValue RHS = N->getOperand(Num: 1); |
1380 | |
1381 | // It doesn't matter if we sign extend or zero extend in the inputs. So do |
1382 | // whatever is best for the target and the promoted operands. |
1383 | SExtOrZExtPromotedOperands(LHS, RHS); |
1384 | |
1385 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), |
1386 | VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1387 | } |
1388 | |
1389 | SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
1390 | // The input value must be properly sign extended. |
1391 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1392 | SDValue RHS = N->getOperand(Num: 1); |
1393 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1394 | RHS = ZExtPromotedInteger(Op: RHS); |
1395 | if (N->getOpcode() != ISD::VP_ASHR) |
1396 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1397 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1398 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1399 | } |
1400 | |
1401 | SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
1402 | // The input value must be properly zero extended. |
1403 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1404 | SDValue RHS = N->getOperand(Num: 1); |
1405 | if (getTypeAction(VT: RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1406 | RHS = ZExtPromotedInteger(Op: RHS); |
1407 | if (N->getOpcode() != ISD::VP_LSHR) |
1408 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS); |
1409 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: LHS.getValueType(), N1: LHS, N2: RHS, |
1410 | N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
1411 | } |
1412 | |
1413 | SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { |
1414 | // Lower the rotate to shifts and ORs which can be promoted. |
1415 | SDValue Res = TLI.expandROT(N, AllowVectorOps: true /*AllowVectorOps*/, DAG); |
1416 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1417 | return SDValue(); |
1418 | } |
1419 | |
1420 | SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { |
1421 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1422 | SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1423 | SDValue Amt = N->getOperand(Num: 2); |
1424 | if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
1425 | Amt = ZExtPromotedInteger(Op: Amt); |
1426 | EVT AmtVT = Amt.getValueType(); |
1427 | |
1428 | SDLoc DL(N); |
1429 | EVT OldVT = N->getOperand(Num: 0).getValueType(); |
1430 | EVT VT = Lo.getValueType(); |
1431 | unsigned Opcode = N->getOpcode(); |
1432 | bool IsFSHR = Opcode == ISD::FSHR; |
1433 | unsigned OldBits = OldVT.getScalarSizeInBits(); |
1434 | unsigned NewBits = VT.getScalarSizeInBits(); |
1435 | |
1436 | // Amount has to be interpreted modulo the old bit width. |
1437 | Amt = DAG.getNode(Opcode: ISD::UREM, DL, VT: AmtVT, N1: Amt, |
1438 | N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT)); |
1439 | |
1440 | // If the promoted type is twice the size (or more), then we use the |
1441 | // traditional funnel 'double' shift codegen. This isn't necessary if the |
1442 | // shift amount is constant. |
1443 | // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
1444 | // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
1445 | if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) && |
1446 | !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) { |
1447 | SDValue HiShift = DAG.getConstant(Val: OldBits, DL, VT); |
1448 | Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: HiShift); |
1449 | Lo = DAG.getZeroExtendInReg(Op: Lo, DL, VT: OldVT); |
1450 | SDValue Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Hi, N2: Lo); |
1451 | Res = DAG.getNode(Opcode: IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, N1: Res, N2: Amt); |
1452 | if (!IsFSHR) |
1453 | Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res, N2: HiShift); |
1454 | return Res; |
1455 | } |
1456 | |
1457 | // Shift Lo up to occupy the upper bits of the promoted type. |
1458 | SDValue ShiftOffset = DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT); |
1459 | Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShiftOffset); |
1460 | |
1461 | // Increase Amount to shift the result into the lower bits of the promoted |
1462 | // type. |
1463 | if (IsFSHR) |
1464 | Amt = DAG.getNode(Opcode: ISD::ADD, DL, VT: AmtVT, N1: Amt, N2: ShiftOffset); |
1465 | |
1466 | return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt); |
1467 | } |
1468 | |
1469 | // A vp version of PromoteIntRes_FunnelShift. |
1470 | SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) { |
1471 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
1472 | SDValue Lo = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
1473 | SDValue Amt = N->getOperand(Num: 2); |
1474 | SDValue Mask = N->getOperand(Num: 3); |
1475 | SDValue EVL = N->getOperand(Num: 4); |
1476 | if (getTypeAction(VT: Amt.getValueType()) == TargetLowering::TypePromoteInteger) |
1477 | Amt = ZExtPromotedInteger(Op: Amt); |
1478 | EVT AmtVT = Amt.getValueType(); |
1479 | |
1480 | SDLoc DL(N); |
1481 | EVT OldVT = N->getOperand(Num: 0).getValueType(); |
1482 | EVT VT = Lo.getValueType(); |
1483 | unsigned Opcode = N->getOpcode(); |
1484 | bool IsFSHR = Opcode == ISD::VP_FSHR; |
1485 | unsigned OldBits = OldVT.getScalarSizeInBits(); |
1486 | unsigned NewBits = VT.getScalarSizeInBits(); |
1487 | |
1488 | // Amount has to be interpreted modulo the old bit width. |
1489 | Amt = DAG.getNode(Opcode: ISD::VP_UREM, DL, VT: AmtVT, N1: Amt, |
1490 | N2: DAG.getConstant(Val: OldBits, DL, VT: AmtVT), N3: Mask, N4: EVL); |
1491 | |
1492 | // If the promoted type is twice the size (or more), then we use the |
1493 | // traditional funnel 'double' shift codegen. This isn't necessary if the |
1494 | // shift amount is constant. |
1495 | // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw. |
1496 | // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)). |
1497 | if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Val: Amt) && |
1498 | !TLI.isOperationLegalOrCustom(Op: Opcode, VT)) { |
1499 | SDValue HiShift = DAG.getConstant(Val: OldBits, DL, VT); |
1500 | Hi = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Hi, N2: HiShift, N3: Mask, N4: EVL); |
1501 | // FIXME: Replace it by vp operations. |
1502 | Lo = DAG.getZeroExtendInReg(Op: Lo, DL, VT: OldVT); |
1503 | SDValue Res = DAG.getNode(Opcode: ISD::VP_OR, DL, VT, N1: Hi, N2: Lo, N3: Mask, N4: EVL); |
1504 | Res = DAG.getNode(Opcode: IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, N1: Res, N2: Amt, |
1505 | N3: Mask, N4: EVL); |
1506 | if (!IsFSHR) |
1507 | Res = DAG.getNode(Opcode: ISD::VP_LSHR, DL, VT, N1: Res, N2: HiShift, N3: Mask, N4: EVL); |
1508 | return Res; |
1509 | } |
1510 | |
1511 | // Shift Lo up to occupy the upper bits of the promoted type. |
1512 | SDValue ShiftOffset = DAG.getConstant(Val: NewBits - OldBits, DL, VT: AmtVT); |
1513 | Lo = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: Lo, N2: ShiftOffset, N3: Mask, N4: EVL); |
1514 | |
1515 | // Increase Amount to shift the result into the lower bits of the promoted |
1516 | // type. |
1517 | if (IsFSHR) |
1518 | Amt = DAG.getNode(Opcode: ISD::VP_ADD, DL, VT: AmtVT, N1: Amt, N2: ShiftOffset, N3: Mask, N4: EVL); |
1519 | |
1520 | return DAG.getNode(Opcode, DL, VT, N1: Hi, N2: Lo, N3: Amt, N4: Mask, N5: EVL); |
1521 | } |
1522 | |
1523 | SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
1524 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1525 | SDValue Res; |
1526 | SDValue InOp = N->getOperand(Num: 0); |
1527 | SDLoc dl(N); |
1528 | |
1529 | switch (getTypeAction(VT: InOp.getValueType())) { |
1530 | default: llvm_unreachable("Unknown type action!" ); |
1531 | case TargetLowering::TypeLegal: |
1532 | case TargetLowering::TypeExpandInteger: |
1533 | Res = InOp; |
1534 | break; |
1535 | case TargetLowering::TypePromoteInteger: |
1536 | Res = GetPromotedInteger(Op: InOp); |
1537 | break; |
1538 | case TargetLowering::TypeSplitVector: { |
1539 | EVT InVT = InOp.getValueType(); |
1540 | assert(InVT.isVector() && "Cannot split scalar types" ); |
1541 | ElementCount NumElts = InVT.getVectorElementCount(); |
1542 | assert(NumElts == NVT.getVectorElementCount() && |
1543 | "Dst and Src must have the same number of elements" ); |
1544 | assert(isPowerOf2_32(NumElts.getKnownMinValue()) && |
1545 | "Promoted vector type must be a power of two" ); |
1546 | |
1547 | SDValue EOp1, EOp2; |
1548 | GetSplitVector(Op: InOp, Lo&: EOp1, Hi&: EOp2); |
1549 | |
1550 | EVT HalfNVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getScalarType(), |
1551 | EC: NumElts.divideCoefficientBy(RHS: 2)); |
1552 | if (N->getOpcode() == ISD::TRUNCATE) { |
1553 | EOp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp1); |
1554 | EOp2 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfNVT, Operand: EOp2); |
1555 | } else { |
1556 | assert(N->getOpcode() == ISD::VP_TRUNCATE && |
1557 | "Expected VP_TRUNCATE opcode" ); |
1558 | SDValue MaskLo, MaskHi, EVLLo, EVLHi; |
1559 | std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 1)); |
1560 | std::tie(args&: EVLLo, args&: EVLHi) = |
1561 | DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: N->getValueType(ResNo: 0), DL: dl); |
1562 | EOp1 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp1, N2: MaskLo, N3: EVLLo); |
1563 | EOp2 = DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: HalfNVT, N1: EOp2, N2: MaskHi, N3: EVLHi); |
1564 | } |
1565 | return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, N1: EOp1, N2: EOp2); |
1566 | } |
1567 | // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some |
1568 | // targets. |
1569 | case TargetLowering::TypeWidenVector: { |
1570 | SDValue WideInOp = GetWidenedVector(Op: InOp); |
1571 | |
1572 | // Truncate widened InOp. |
1573 | unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
1574 | EVT TruncVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
1575 | VT: N->getValueType(ResNo: 0).getScalarType(), NumElements: NumElem); |
1576 | SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncVT, Operand: WideInOp); |
1577 | |
1578 | // Zero extend so that the elements are of same type as those of NVT |
1579 | EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(), |
1580 | NumElements: NumElem); |
1581 | SDValue WideExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: ExtVT, Operand: WideTrunc); |
1582 | |
1583 | // Extract the low NVT subvector. |
1584 | SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: dl); |
1585 | return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NVT, N1: WideExt, N2: ZeroIdx); |
1586 | } |
1587 | } |
1588 | |
1589 | // Truncate to NVT instead of VT |
1590 | if (N->getOpcode() == ISD::VP_TRUNCATE) |
1591 | return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: dl, VT: NVT, N1: Res, N2: N->getOperand(Num: 1), |
1592 | N3: N->getOperand(Num: 2)); |
1593 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Res); |
1594 | } |
1595 | |
1596 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
1597 | if (ResNo == 1) |
1598 | return PromoteIntRes_Overflow(N); |
1599 | |
1600 | // The operation overflowed iff the result in the larger type is not the |
1601 | // zero extension of its truncation to the original type. |
1602 | SDValue LHS = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1603 | SDValue RHS = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1604 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
1605 | EVT NVT = LHS.getValueType(); |
1606 | SDLoc dl(N); |
1607 | |
1608 | // Do the arithmetic in the larger type. |
1609 | unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; |
1610 | SDValue Res = DAG.getNode(Opcode, DL: dl, VT: NVT, N1: LHS, N2: RHS); |
1611 | |
1612 | // Calculate the overflow flag: zero extend the arithmetic result from |
1613 | // the original type. |
1614 | SDValue Ofl = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: OVT); |
1615 | // Overflowed if and only if this is not equal to Res. |
1616 | Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Ofl, RHS: Res, Cond: ISD::SETNE); |
1617 | |
1618 | // Use the calculated overflow everywhere. |
1619 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
1620 | |
1621 | return Res; |
1622 | } |
1623 | |
1624 | // Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that |
1625 | // the third operand of ADDE/SUBE nodes is carry flag, which differs from |
1626 | // the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean. |
1627 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N, |
1628 | unsigned ResNo) { |
1629 | if (ResNo == 1) |
1630 | return PromoteIntRes_Overflow(N); |
1631 | |
1632 | // We need to sign-extend the operands so the carry value computed by the |
1633 | // wide operation will be equivalent to the carry value computed by the |
1634 | // narrow operation. |
1635 | // An UADDO_CARRY can generate carry only if any of the operands has its |
1636 | // most significant bit set. Sign extension propagates the most significant |
1637 | // bit into the higher bits which means the extra bit that the narrow |
1638 | // addition would need (i.e. the carry) will be propagated through the higher |
1639 | // bits of the wide addition. |
1640 | // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will |
1641 | // be preserved by sign extension. |
1642 | SDValue LHS = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1643 | SDValue RHS = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
1644 | |
1645 | EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(ResNo: 1)}; |
1646 | |
1647 | // Do the arithmetic in the wide type. |
1648 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: DAG.getVTList(VTs: ValueVTs), |
1649 | N1: LHS, N2: RHS, N3: N->getOperand(Num: 2)); |
1650 | |
1651 | // Update the users of the original carry/borrow value. |
1652 | ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1)); |
1653 | |
1654 | return SDValue(Res.getNode(), 0); |
1655 | } |
1656 | |
1657 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, |
1658 | unsigned ResNo) { |
1659 | assert(ResNo == 1 && "Don't know how to promote other results yet." ); |
1660 | return PromoteIntRes_Overflow(N); |
1661 | } |
1662 | |
1663 | SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { |
1664 | EVT OVT = N->getValueType(ResNo: 0); |
1665 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT); |
1666 | |
1667 | // If a larger ABS or SMAX isn't supported by the target, try to expand now. |
1668 | // If we expand later we'll end up sign extending more than just the sra input |
1669 | // in sra+xor+sub expansion. |
1670 | if (!OVT.isVector() && |
1671 | !TLI.isOperationLegalOrCustomOrPromote(Op: ISD::ABS, VT: NVT) && |
1672 | !TLI.isOperationLegal(Op: ISD::SMAX, VT: NVT)) { |
1673 | if (SDValue Res = TLI.expandABS(N, DAG)) |
1674 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand: Res); |
1675 | } |
1676 | |
1677 | SDValue Op0 = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
1678 | return DAG.getNode(Opcode: ISD::ABS, DL: SDLoc(N), VT: Op0.getValueType(), Operand: Op0); |
1679 | } |
1680 | |
1681 | SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
1682 | // Promote the overflow bit trivially. |
1683 | if (ResNo == 1) |
1684 | return PromoteIntRes_Overflow(N); |
1685 | |
1686 | SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1); |
1687 | SDLoc DL(N); |
1688 | EVT SmallVT = LHS.getValueType(); |
1689 | |
1690 | // To determine if the result overflowed in a larger type, we extend the |
1691 | // input to the larger type, do the multiply (checking if it overflows), |
1692 | // then also check the high bits of the result to see if overflow happened |
1693 | // there. |
1694 | if (N->getOpcode() == ISD::SMULO) { |
1695 | LHS = SExtPromotedInteger(Op: LHS); |
1696 | RHS = SExtPromotedInteger(Op: RHS); |
1697 | } else { |
1698 | LHS = ZExtPromotedInteger(Op: LHS); |
1699 | RHS = ZExtPromotedInteger(Op: RHS); |
1700 | } |
1701 | SDVTList VTs = DAG.getVTList(VT1: LHS.getValueType(), VT2: N->getValueType(ResNo: 1)); |
1702 | SDValue Mul = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N1: LHS, N2: RHS); |
1703 | |
1704 | // Overflow occurred if it occurred in the larger type, or if the high part |
1705 | // of the result does not zero/sign-extend the low part. Check this second |
1706 | // possibility first. |
1707 | SDValue Overflow; |
1708 | if (N->getOpcode() == ISD::UMULO) { |
1709 | // Unsigned overflow occurred if the high part is non-zero. |
1710 | unsigned Shift = SmallVT.getScalarSizeInBits(); |
1711 | SDValue Hi = |
1712 | DAG.getNode(Opcode: ISD::SRL, DL, VT: Mul.getValueType(), N1: Mul, |
1713 | N2: DAG.getShiftAmountConstant(Val: Shift, VT: Mul.getValueType(), DL)); |
1714 | Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Hi, |
1715 | RHS: DAG.getConstant(Val: 0, DL, VT: Hi.getValueType()), |
1716 | Cond: ISD::SETNE); |
1717 | } else { |
1718 | // Signed overflow occurred if the high part does not sign extend the low. |
1719 | SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: Mul.getValueType(), |
1720 | N1: Mul, N2: DAG.getValueType(SmallVT)); |
1721 | Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: SExt, RHS: Mul, Cond: ISD::SETNE); |
1722 | } |
1723 | |
1724 | // The only other way for overflow to occur is if the multiplication in the |
1725 | // larger type itself overflowed. |
1726 | Overflow = DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 1), N1: Overflow, |
1727 | N2: SDValue(Mul.getNode(), 1)); |
1728 | |
1729 | // Use the calculated overflow everywhere. |
1730 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
1731 | return Mul; |
1732 | } |
1733 | |
1734 | SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
1735 | return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), |
1736 | VT: N->getValueType(ResNo: 0))); |
1737 | } |
1738 | |
1739 | SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { |
1740 | EVT VT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1741 | |
1742 | const APInt &MulImm = N->getConstantOperandAPInt(Num: 0); |
1743 | return DAG.getVScale(DL: SDLoc(N), VT, MulImm: MulImm.sext(width: VT.getSizeInBits())); |
1744 | } |
1745 | |
1746 | SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
1747 | SDValue Chain = N->getOperand(Num: 0); // Get the chain. |
1748 | SDValue Ptr = N->getOperand(Num: 1); // Get the pointer. |
1749 | EVT VT = N->getValueType(ResNo: 0); |
1750 | SDLoc dl(N); |
1751 | |
1752 | MVT RegVT = TLI.getRegisterType(Context&: *DAG.getContext(), VT); |
1753 | unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT); |
1754 | // The argument is passed as NumRegs registers of type RegVT. |
1755 | |
1756 | SmallVector<SDValue, 8> Parts(NumRegs); |
1757 | for (unsigned i = 0; i < NumRegs; ++i) { |
1758 | Parts[i] = DAG.getVAArg(VT: RegVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2), |
1759 | Align: N->getConstantOperandVal(Num: 3)); |
1760 | Chain = Parts[i].getValue(R: 1); |
1761 | } |
1762 | |
1763 | // Handle endianness of the load. |
1764 | if (DAG.getDataLayout().isBigEndian()) |
1765 | std::reverse(first: Parts.begin(), last: Parts.end()); |
1766 | |
1767 | // Assemble the parts in the promoted type. |
1768 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
1769 | SDValue Res = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[0]); |
1770 | for (unsigned i = 1; i < NumRegs; ++i) { |
1771 | SDValue Part = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Parts[i]); |
1772 | // Shift it to the right position and "or" it in. |
1773 | Part = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Part, |
1774 | N2: DAG.getConstant(Val: i * RegVT.getSizeInBits(), DL: dl, |
1775 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
1776 | Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Res, N2: Part); |
1777 | } |
1778 | |
1779 | // Modified the chain result - switch anything that used the old chain to |
1780 | // use the new one. |
1781 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
1782 | |
1783 | return Res; |
1784 | } |
1785 | |
1786 | //===----------------------------------------------------------------------===// |
1787 | // Integer Operand Promotion |
1788 | //===----------------------------------------------------------------------===// |
1789 | |
1790 | /// PromoteIntegerOperand - This method is called when the specified operand of |
1791 | /// the specified node is found to need promotion. At this point, all of the |
1792 | /// result types of the node are known to be legal, but other operands of the |
1793 | /// node may need promotion or expansion as well as the specified one. |
1794 | bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
1795 | LLVM_DEBUG(dbgs() << "Promote integer operand: " ; N->dump(&DAG)); |
1796 | SDValue Res = SDValue(); |
1797 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) { |
1798 | LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n" ); |
1799 | return false; |
1800 | } |
1801 | |
1802 | switch (N->getOpcode()) { |
1803 | default: |
1804 | #ifndef NDEBUG |
1805 | dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": " ; |
1806 | N->dump(G: &DAG); dbgs() << "\n" ; |
1807 | #endif |
1808 | report_fatal_error(reason: "Do not know how to promote this operator's operand!" ); |
1809 | |
1810 | case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
1811 | case ISD::ATOMIC_STORE: |
1812 | Res = PromoteIntOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N)); |
1813 | break; |
1814 | case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
1815 | case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
1816 | case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
1817 | case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
1818 | case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
1819 | case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
1820 | case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
1821 | case ISD::INSERT_VECTOR_ELT: |
1822 | Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo); |
1823 | break; |
1824 | case ISD::SPLAT_VECTOR: |
1825 | case ISD::SCALAR_TO_VECTOR: |
1826 | Res = PromoteIntOp_ScalarOp(N); |
1827 | break; |
1828 | case ISD::VSELECT: |
1829 | case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
1830 | case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
1831 | case ISD::VP_SETCC: |
1832 | case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
1833 | case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
1834 | case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break; |
1835 | case ISD::VP_SINT_TO_FP: |
1836 | case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
1837 | case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; |
1838 | case ISD::STORE: Res = PromoteIntOp_STORE(N: cast<StoreSDNode>(Val: N), |
1839 | OpNo); break; |
1840 | case ISD::MSTORE: Res = PromoteIntOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), |
1841 | OpNo); break; |
1842 | case ISD::MLOAD: Res = PromoteIntOp_MLOAD(N: cast<MaskedLoadSDNode>(Val: N), |
1843 | OpNo); break; |
1844 | case ISD::MGATHER: Res = PromoteIntOp_MGATHER(N: cast<MaskedGatherSDNode>(Val: N), |
1845 | OpNo); break; |
1846 | case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(N: cast<MaskedScatterSDNode>(Val: N), |
1847 | OpNo); break; |
1848 | case ISD::VP_TRUNCATE: |
1849 | case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
1850 | case ISD::BF16_TO_FP: |
1851 | case ISD::FP16_TO_FP: |
1852 | case ISD::VP_UINT_TO_FP: |
1853 | case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
1854 | case ISD::STRICT_FP16_TO_FP: |
1855 | case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; |
1856 | case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
1857 | case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; |
1858 | case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
1859 | case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; |
1860 | |
1861 | case ISD::SHL: |
1862 | case ISD::SRA: |
1863 | case ISD::SRL: |
1864 | case ISD::ROTL: |
1865 | case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
1866 | |
1867 | case ISD::FSHL: |
1868 | case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break; |
1869 | |
1870 | case ISD::FRAMEADDR: |
1871 | case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; |
1872 | |
1873 | case ISD::SMULFIX: |
1874 | case ISD::SMULFIXSAT: |
1875 | case ISD::UMULFIX: |
1876 | case ISD::UMULFIXSAT: |
1877 | case ISD::SDIVFIX: |
1878 | case ISD::SDIVFIXSAT: |
1879 | case ISD::UDIVFIX: |
1880 | case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; |
1881 | case ISD::FPOWI: |
1882 | case ISD::STRICT_FPOWI: |
1883 | case ISD::FLDEXP: |
1884 | case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break; |
1885 | case ISD::VECREDUCE_ADD: |
1886 | case ISD::VECREDUCE_MUL: |
1887 | case ISD::VECREDUCE_AND: |
1888 | case ISD::VECREDUCE_OR: |
1889 | case ISD::VECREDUCE_XOR: |
1890 | case ISD::VECREDUCE_SMAX: |
1891 | case ISD::VECREDUCE_SMIN: |
1892 | case ISD::VECREDUCE_UMAX: |
1893 | case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; |
1894 | case ISD::VP_REDUCE_ADD: |
1895 | case ISD::VP_REDUCE_MUL: |
1896 | case ISD::VP_REDUCE_AND: |
1897 | case ISD::VP_REDUCE_OR: |
1898 | case ISD::VP_REDUCE_XOR: |
1899 | case ISD::VP_REDUCE_SMAX: |
1900 | case ISD::VP_REDUCE_SMIN: |
1901 | case ISD::VP_REDUCE_UMAX: |
1902 | case ISD::VP_REDUCE_UMIN: |
1903 | Res = PromoteIntOp_VP_REDUCE(N, OpNo); |
1904 | break; |
1905 | |
1906 | case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; |
1907 | case ISD::STACKMAP: |
1908 | Res = PromoteIntOp_STACKMAP(N, OpNo); |
1909 | break; |
1910 | case ISD::PATCHPOINT: |
1911 | Res = PromoteIntOp_PATCHPOINT(N, OpNo); |
1912 | break; |
1913 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
1914 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
1915 | Res = PromoteIntOp_VP_STRIDED(N, OpNo); |
1916 | break; |
1917 | case ISD::EXPERIMENTAL_VP_SPLICE: |
1918 | Res = PromoteIntOp_VP_SPLICE(N, OpNo); |
1919 | break; |
1920 | } |
1921 | |
1922 | // If the result is null, the sub-method took care of registering results etc. |
1923 | if (!Res.getNode()) return false; |
1924 | |
1925 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
1926 | // core about this. |
1927 | if (Res.getNode() == N) |
1928 | return true; |
1929 | |
1930 | const bool IsStrictFp = N->isStrictFPOpcode(); |
1931 | assert(Res.getValueType() == N->getValueType(0) && |
1932 | N->getNumValues() == (IsStrictFp ? 2 : 1) && |
1933 | "Invalid operand expansion" ); |
1934 | LLVM_DEBUG(dbgs() << "Replacing: " ; N->dump(&DAG); dbgs() << " with: " ; |
1935 | Res.dump()); |
1936 | |
1937 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
1938 | if (IsStrictFp) |
1939 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1)); |
1940 | |
1941 | return false; |
1942 | } |
1943 | |
1944 | // These operands can be either sign extended or zero extended as long as we |
1945 | // treat them the same. If an extension is free, choose that. Otherwise, follow |
1946 | // target preference. |
1947 | void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) { |
1948 | SDValue OpL = GetPromotedInteger(Op: LHS); |
1949 | SDValue OpR = GetPromotedInteger(Op: RHS); |
1950 | |
1951 | if (TLI.isSExtCheaperThanZExt(FromTy: LHS.getValueType(), ToTy: OpL.getValueType())) { |
1952 | // The target would prefer to promote the comparison operand with sign |
1953 | // extension. Honor that unless the promoted values are already zero |
1954 | // extended. |
1955 | unsigned OpLEffectiveBits = |
1956 | DAG.computeKnownBits(Op: OpL).countMaxActiveBits(); |
1957 | unsigned OpREffectiveBits = |
1958 | DAG.computeKnownBits(Op: OpR).countMaxActiveBits(); |
1959 | if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
1960 | OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
1961 | LHS = OpL; |
1962 | RHS = OpR; |
1963 | return; |
1964 | } |
1965 | |
1966 | // The promoted values aren't zero extended, use a sext_inreg. |
1967 | LHS = SExtPromotedInteger(Op: LHS); |
1968 | RHS = SExtPromotedInteger(Op: RHS); |
1969 | return; |
1970 | } |
1971 | |
1972 | // Prefer to promote the comparison operand with zero extension. |
1973 | |
1974 | // If the width of OpL/OpR excluding the duplicated sign bits is no greater |
1975 | // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation |
1976 | // that we might not be able to remove. |
1977 | unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpL); |
1978 | unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(Op: OpR); |
1979 | if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() && |
1980 | OpREffectiveBits <= RHS.getScalarValueSizeInBits()) { |
1981 | LHS = OpL; |
1982 | RHS = OpR; |
1983 | return; |
1984 | } |
1985 | |
1986 | // Otherwise, use zext_inreg. |
1987 | LHS = ZExtPromotedInteger(Op: LHS); |
1988 | RHS = ZExtPromotedInteger(Op: RHS); |
1989 | } |
1990 | |
1991 | /// PromoteSetCCOperands - Promote the operands of a comparison. This code is |
1992 | /// shared among BR_CC, SELECT_CC, and SETCC handlers. |
1993 | void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS, |
1994 | ISD::CondCode CCCode) { |
1995 | // We have to insert explicit sign or zero extends. Note that we could |
1996 | // insert sign extends for ALL conditions. For those operations where either |
1997 | // zero or sign extension would be valid, we ask the target which extension |
1998 | // it would prefer. |
1999 | |
2000 | // Signed comparisons always require sign extension. |
2001 | if (ISD::isSignedIntSetCC(Code: CCCode)) { |
2002 | LHS = SExtPromotedInteger(Op: LHS); |
2003 | RHS = SExtPromotedInteger(Op: RHS); |
2004 | return; |
2005 | } |
2006 | |
2007 | assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && |
2008 | "Unknown integer comparison!" ); |
2009 | |
2010 | SExtOrZExtPromotedOperands(LHS, RHS); |
2011 | } |
2012 | |
2013 | SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
2014 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2015 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
2016 | } |
2017 | |
2018 | SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
2019 | SDValue Op1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
2020 | return DAG.getAtomic(Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT(), |
2021 | Chain: N->getChain(), Ptr: Op1, Val: N->getBasePtr(), MMO: N->getMemOperand()); |
2022 | } |
2023 | |
2024 | SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
2025 | // This should only occur in unusual situations like bitcasting to an |
2026 | // x86_fp80, so just turn it into a store+load |
2027 | return CreateStackStoreLoad(Op: N->getOperand(Num: 0), DestVT: N->getValueType(ResNo: 0)); |
2028 | } |
2029 | |
2030 | SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
2031 | assert(OpNo == 2 && "Don't know how to promote this operand!" ); |
2032 | |
2033 | SDValue LHS = N->getOperand(Num: 2); |
2034 | SDValue RHS = N->getOperand(Num: 3); |
2035 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get()); |
2036 | |
2037 | // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always |
2038 | // legal types. |
2039 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2040 | Op2: N->getOperand(Num: 1), Op3: LHS, Op4: RHS, Op5: N->getOperand(Num: 4)), |
2041 | 0); |
2042 | } |
2043 | |
2044 | SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
2045 | assert(OpNo == 1 && "only know how to promote condition" ); |
2046 | |
2047 | // Promote all the way up to the canonical SetCC type. |
2048 | SDValue Cond = PromoteTargetBoolean(Bool: N->getOperand(Num: 1), MVT::ValVT: Other); |
2049 | |
2050 | // The chain (Op#0) and basic block destination (Op#2) are always legal types. |
2051 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Cond, |
2052 | Op3: N->getOperand(Num: 2)), 0); |
2053 | } |
2054 | |
2055 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
2056 | // Since the result type is legal, the operands must promote to it. |
2057 | EVT OVT = N->getOperand(Num: 0).getValueType(); |
2058 | SDValue Lo = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2059 | SDValue Hi = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
2060 | assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?" ); |
2061 | SDLoc dl(N); |
2062 | |
2063 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: Hi, |
2064 | N2: DAG.getConstant(Val: OVT.getSizeInBits(), DL: dl, |
2065 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
2066 | return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi); |
2067 | } |
2068 | |
2069 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
2070 | // The vector type is legal but the element type is not. This implies |
2071 | // that the vector is a power-of-two in length and that the element |
2072 | // type does not have a strange size (eg: it is not i1). |
2073 | EVT VecVT = N->getValueType(ResNo: 0); |
2074 | unsigned NumElts = VecVT.getVectorNumElements(); |
2075 | assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
2076 | "Legal vector of one illegal element?" ); |
2077 | |
2078 | // Promote the inserted value. The type does not need to match the |
2079 | // vector element type. Check that any extra bits introduced will be |
2080 | // truncated away. |
2081 | assert(N->getOperand(0).getValueSizeInBits() >= |
2082 | N->getValueType(0).getScalarSizeInBits() && |
2083 | "Type of inserted value narrower than vector element type!" ); |
2084 | |
2085 | SmallVector<SDValue, 16> NewOps; |
2086 | for (unsigned i = 0; i < NumElts; ++i) |
2087 | NewOps.push_back(Elt: GetPromotedInteger(Op: N->getOperand(Num: i))); |
2088 | |
2089 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2090 | } |
2091 | |
2092 | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
2093 | unsigned OpNo) { |
2094 | if (OpNo == 1) { |
2095 | // Promote the inserted value. This is valid because the type does not |
2096 | // have to match the vector element type. |
2097 | |
2098 | // Check that any extra bits introduced will be truncated away. |
2099 | assert(N->getOperand(1).getValueSizeInBits() >= |
2100 | N->getValueType(0).getScalarSizeInBits() && |
2101 | "Type of inserted value narrower than vector element type!" ); |
2102 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2103 | Op2: GetPromotedInteger(Op: N->getOperand(Num: 1)), |
2104 | Op3: N->getOperand(Num: 2)), |
2105 | 0); |
2106 | } |
2107 | |
2108 | assert(OpNo == 2 && "Different operand and result vector types?" ); |
2109 | |
2110 | // Promote the index. |
2111 | SDValue Idx = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 2), DL: SDLoc(N), |
2112 | VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())); |
2113 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2114 | Op2: N->getOperand(Num: 1), Op3: Idx), 0); |
2115 | } |
2116 | |
2117 | SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) { |
2118 | // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated, |
2119 | // so just promote the operand in place. |
2120 | return SDValue(DAG.UpdateNodeOperands(N, |
2121 | Op: GetPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2122 | } |
2123 | |
2124 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
2125 | assert(OpNo == 0 && "Only know how to promote the condition!" ); |
2126 | SDValue Cond = N->getOperand(Num: 0); |
2127 | EVT OpTy = N->getOperand(Num: 1).getValueType(); |
2128 | |
2129 | if (N->getOpcode() == ISD::VSELECT) |
2130 | if (SDValue Res = WidenVSELECTMask(N)) |
2131 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), |
2132 | N1: Res, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
2133 | |
2134 | // Promote all the way up to the canonical SetCC type. |
2135 | EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; |
2136 | Cond = PromoteTargetBoolean(Bool: Cond, ValVT: OpVT); |
2137 | |
2138 | return SDValue(DAG.UpdateNodeOperands(N, Op1: Cond, Op2: N->getOperand(Num: 1), |
2139 | Op3: N->getOperand(Num: 2)), 0); |
2140 | } |
2141 | |
2142 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
2143 | assert(OpNo == 0 && "Don't know how to promote this operand!" ); |
2144 | |
2145 | SDValue LHS = N->getOperand(Num: 0); |
2146 | SDValue RHS = N->getOperand(Num: 1); |
2147 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get()); |
2148 | |
2149 | // The CC (#4) and the possible return values (#2 and #3) have legal types. |
2150 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2), |
2151 | Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)), 0); |
2152 | } |
2153 | |
2154 | SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
2155 | assert(OpNo == 0 && "Don't know how to promote this operand!" ); |
2156 | |
2157 | SDValue LHS = N->getOperand(Num: 0); |
2158 | SDValue RHS = N->getOperand(Num: 1); |
2159 | PromoteSetCCOperands(LHS, RHS, CCCode: cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get()); |
2160 | |
2161 | // The CC (#2) is always legal. |
2162 | if (N->getOpcode() == ISD::SETCC) |
2163 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2)), 0); |
2164 | |
2165 | assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode" ); |
2166 | |
2167 | return SDValue(DAG.UpdateNodeOperands(N, Op1: LHS, Op2: RHS, Op3: N->getOperand(Num: 2), |
2168 | Op4: N->getOperand(Num: 3), Op5: N->getOperand(Num: 4)), |
2169 | 0); |
2170 | } |
2171 | |
2172 | SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
2173 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2174 | Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2175 | } |
2176 | |
2177 | SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) { |
2178 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1), |
2179 | Op3: ZExtPromotedInteger(Op: N->getOperand(Num: 2))), 0); |
2180 | } |
2181 | |
2182 | SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
2183 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2184 | SDLoc dl(N); |
2185 | Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op); |
2186 | return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Op.getValueType(), |
2187 | N1: Op, N2: DAG.getValueType(N->getOperand(Num: 0).getValueType())); |
2188 | } |
2189 | |
2190 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) { |
2191 | SDLoc dl(N); |
2192 | EVT VT = N->getValueType(ResNo: 0); |
2193 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2194 | // FIXME: There is no VP_ANY_EXTEND yet. |
2195 | Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), |
2196 | N3: N->getOperand(Num: 2)); |
2197 | unsigned Diff = |
2198 | VT.getScalarSizeInBits() - N->getOperand(Num: 0).getScalarValueSizeInBits(); |
2199 | SDValue ShAmt = DAG.getShiftAmountConstant(Val: Diff, VT, DL: dl); |
2200 | // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts. |
2201 | SDValue Shl = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: ShAmt, N3: N->getOperand(Num: 1), |
2202 | N4: N->getOperand(Num: 2)); |
2203 | return DAG.getNode(Opcode: ISD::VP_ASHR, DL: dl, VT, N1: Shl, N2: ShAmt, N3: N->getOperand(Num: 1), |
2204 | N4: N->getOperand(Num: 2)); |
2205 | } |
2206 | |
2207 | SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
2208 | if (N->getOpcode() == ISD::VP_SINT_TO_FP) |
2209 | return SDValue(DAG.UpdateNodeOperands(N, |
2210 | Op1: SExtPromotedInteger(Op: N->getOperand(Num: 0)), |
2211 | Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)), |
2212 | 0); |
2213 | return SDValue(DAG.UpdateNodeOperands(N, |
2214 | Op: SExtPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2215 | } |
2216 | |
2217 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { |
2218 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2219 | Op2: SExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2220 | } |
2221 | |
2222 | SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
2223 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
2224 | SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
2225 | SDLoc dl(N); |
2226 | |
2227 | SDValue Val = GetPromotedInteger(Op: N->getValue()); // Get promoted value. |
2228 | |
2229 | // Truncate the value and store the result. |
2230 | return DAG.getTruncStore(Chain: Ch, dl, Val, Ptr, |
2231 | SVT: N->getMemoryVT(), MMO: N->getMemOperand()); |
2232 | } |
2233 | |
2234 | SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
2235 | unsigned OpNo) { |
2236 | SDValue DataOp = N->getValue(); |
2237 | SDValue Mask = N->getMask(); |
2238 | |
2239 | if (OpNo == 4) { |
2240 | // The Mask. Update in place. |
2241 | EVT DataVT = DataOp.getValueType(); |
2242 | Mask = PromoteTargetBoolean(Bool: Mask, ValVT: DataVT); |
2243 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2244 | NewOps[4] = Mask; |
2245 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2246 | } |
2247 | |
2248 | assert(OpNo == 1 && "Unexpected operand for promotion" ); |
2249 | DataOp = GetPromotedInteger(Op: DataOp); |
2250 | |
2251 | return DAG.getMaskedStore(Chain: N->getChain(), dl: SDLoc(N), Val: DataOp, Base: N->getBasePtr(), |
2252 | Offset: N->getOffset(), Mask, MemVT: N->getMemoryVT(), |
2253 | MMO: N->getMemOperand(), AM: N->getAddressingMode(), |
2254 | /*IsTruncating*/ true, IsCompressing: N->isCompressingStore()); |
2255 | } |
2256 | |
2257 | SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
2258 | unsigned OpNo) { |
2259 | assert(OpNo == 3 && "Only know how to promote the mask!" ); |
2260 | EVT DataVT = N->getValueType(ResNo: 0); |
2261 | SDValue Mask = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2262 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2263 | NewOps[OpNo] = Mask; |
2264 | SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps); |
2265 | if (Res == N) |
2266 | return SDValue(Res, 0); |
2267 | |
2268 | // Update triggered CSE, do our own replacement since caller can't. |
2269 | ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0)); |
2270 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1)); |
2271 | return SDValue(); |
2272 | } |
2273 | |
2274 | SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
2275 | unsigned OpNo) { |
2276 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
2277 | |
2278 | if (OpNo == 2) { |
2279 | // The Mask |
2280 | EVT DataVT = N->getValueType(ResNo: 0); |
2281 | NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2282 | } else if (OpNo == 4) { |
2283 | // The Index |
2284 | if (N->isIndexSigned()) |
2285 | // Need to sign extend the index since the bits will likely be used. |
2286 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2287 | else |
2288 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2289 | } else |
2290 | NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2291 | |
2292 | SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps); |
2293 | if (Res == N) |
2294 | return SDValue(Res, 0); |
2295 | |
2296 | // Update triggered CSE, do our own replacement since caller can't. |
2297 | ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0)); |
2298 | ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1)); |
2299 | return SDValue(); |
2300 | } |
2301 | |
2302 | SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
2303 | unsigned OpNo) { |
2304 | bool TruncateStore = N->isTruncatingStore(); |
2305 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
2306 | |
2307 | if (OpNo == 2) { |
2308 | // The Mask |
2309 | EVT DataVT = N->getValue().getValueType(); |
2310 | NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT); |
2311 | } else if (OpNo == 4) { |
2312 | // The Index |
2313 | if (N->isIndexSigned()) |
2314 | // Need to sign extend the index since the bits will likely be used. |
2315 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2316 | else |
2317 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2318 | } else { |
2319 | NewOps[OpNo] = GetPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2320 | TruncateStore = true; |
2321 | } |
2322 | |
2323 | return DAG.getMaskedScatter(VTs: DAG.getVTList(MVT::Other), MemVT: N->getMemoryVT(), |
2324 | dl: SDLoc(N), Ops: NewOps, MMO: N->getMemOperand(), |
2325 | IndexType: N->getIndexType(), IsTruncating: TruncateStore); |
2326 | } |
2327 | |
2328 | SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
2329 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2330 | if (N->getOpcode() == ISD::VP_TRUNCATE) |
2331 | return DAG.getNode(Opcode: ISD::VP_TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op, |
2332 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2)); |
2333 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op); |
2334 | } |
2335 | |
2336 | SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
2337 | if (N->getOpcode() == ISD::VP_UINT_TO_FP) |
2338 | return SDValue(DAG.UpdateNodeOperands(N, |
2339 | Op1: ZExtPromotedInteger(Op: N->getOperand(Num: 0)), |
2340 | Op2: N->getOperand(Num: 1), Op3: N->getOperand(Num: 2)), |
2341 | 0); |
2342 | return SDValue(DAG.UpdateNodeOperands(N, |
2343 | Op: ZExtPromotedInteger(Op: N->getOperand(Num: 0))), 0); |
2344 | } |
2345 | |
2346 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { |
2347 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
2348 | Op2: ZExtPromotedInteger(Op: N->getOperand(Num: 1))), 0); |
2349 | } |
2350 | |
2351 | SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
2352 | SDLoc dl(N); |
2353 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2354 | Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Op); |
2355 | return DAG.getZeroExtendInReg(Op, DL: dl, VT: N->getOperand(Num: 0).getValueType()); |
2356 | } |
2357 | |
2358 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) { |
2359 | SDLoc dl(N); |
2360 | EVT VT = N->getValueType(ResNo: 0); |
2361 | SDValue Op = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
2362 | // FIXME: There is no VP_ANY_EXTEND yet. |
2363 | Op = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), |
2364 | N3: N->getOperand(Num: 2)); |
2365 | APInt Imm = APInt::getLowBitsSet(numBits: VT.getScalarSizeInBits(), |
2366 | loBitsSet: N->getOperand(Num: 0).getScalarValueSizeInBits()); |
2367 | return DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: Imm, DL: dl, VT), |
2368 | N3: N->getOperand(Num: 1), N4: N->getOperand(Num: 2)); |
2369 | } |
2370 | |
2371 | SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { |
2372 | SDValue Op2 = ZExtPromotedInteger(Op: N->getOperand(Num: 2)); |
2373 | return SDValue( |
2374 | DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: N->getOperand(Num: 1), Op3: Op2), 0); |
2375 | } |
2376 | |
2377 | SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { |
2378 | // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width. |
2379 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2380 | return SDValue(DAG.UpdateNodeOperands(N, Op), 0); |
2381 | } |
2382 | |
2383 | SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) { |
2384 | bool IsStrict = N->isStrictFPOpcode(); |
2385 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
2386 | |
2387 | bool IsPowI = |
2388 | N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; |
2389 | |
2390 | // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result |
2391 | // and floating point operand is already type legalized). |
2392 | RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)) |
2393 | : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)); |
2394 | |
2395 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC)) { |
2396 | SDValue Op = SExtPromotedInteger(Op: N->getOperand(Num: 1)); |
2397 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Op), 0); |
2398 | } |
2399 | |
2400 | // We can't just promote the exponent type in FPOWI, since we want to lower |
2401 | // the node to a libcall and we if we promote to a type larger than |
2402 | // sizeof(int) the libcall might not be according to the targets ABI. Instead |
2403 | // we rewrite to a libcall here directly, letting makeLibCall handle promotion |
2404 | // if the target accepts it according to shouldSignExtendTypeInLibCall. |
2405 | |
2406 | unsigned OpOffset = IsStrict ? 1 : 0; |
2407 | // The exponent should fit in a sizeof(int) type for the libcall to be valid. |
2408 | assert(DAG.getLibInfo().getIntSize() == |
2409 | N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && |
2410 | "POWI exponent should match with sizeof(int) when doing the libcall." ); |
2411 | TargetLowering::MakeLibCallOptions CallOptions; |
2412 | CallOptions.setSExt(true); |
2413 | SDValue Ops[2] = {N->getOperand(Num: 0 + OpOffset), N->getOperand(Num: 1 + OpOffset)}; |
2414 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall( |
2415 | DAG, LC, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain); |
2416 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
2417 | if (IsStrict) |
2418 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
2419 | return SDValue(); |
2420 | } |
2421 | |
2422 | static unsigned getExtendForIntVecReduction(SDNode *N) { |
2423 | switch (N->getOpcode()) { |
2424 | default: |
2425 | llvm_unreachable("Expected integer vector reduction" ); |
2426 | case ISD::VECREDUCE_ADD: |
2427 | case ISD::VECREDUCE_MUL: |
2428 | case ISD::VECREDUCE_AND: |
2429 | case ISD::VECREDUCE_OR: |
2430 | case ISD::VECREDUCE_XOR: |
2431 | case ISD::VP_REDUCE_ADD: |
2432 | case ISD::VP_REDUCE_MUL: |
2433 | case ISD::VP_REDUCE_AND: |
2434 | case ISD::VP_REDUCE_OR: |
2435 | case ISD::VP_REDUCE_XOR: |
2436 | return ISD::ANY_EXTEND; |
2437 | case ISD::VECREDUCE_SMAX: |
2438 | case ISD::VECREDUCE_SMIN: |
2439 | case ISD::VP_REDUCE_SMAX: |
2440 | case ISD::VP_REDUCE_SMIN: |
2441 | return ISD::SIGN_EXTEND; |
2442 | case ISD::VECREDUCE_UMAX: |
2443 | case ISD::VECREDUCE_UMIN: |
2444 | case ISD::VP_REDUCE_UMAX: |
2445 | case ISD::VP_REDUCE_UMIN: |
2446 | return ISD::ZERO_EXTEND; |
2447 | } |
2448 | } |
2449 | |
2450 | SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) { |
2451 | switch (getExtendForIntVecReduction(N)) { |
2452 | default: |
2453 | llvm_unreachable("Impossible extension kind for integer reduction" ); |
2454 | case ISD::ANY_EXTEND: |
2455 | return GetPromotedInteger(Op: V); |
2456 | case ISD::SIGN_EXTEND: |
2457 | return SExtPromotedInteger(Op: V); |
2458 | case ISD::ZERO_EXTEND: |
2459 | return ZExtPromotedInteger(Op: V); |
2460 | } |
2461 | } |
2462 | |
2463 | SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { |
2464 | SDLoc dl(N); |
2465 | SDValue Op = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0)); |
2466 | |
2467 | EVT OrigEltVT = N->getOperand(Num: 0).getValueType().getVectorElementType(); |
2468 | EVT InVT = Op.getValueType(); |
2469 | EVT EltVT = InVT.getVectorElementType(); |
2470 | EVT ResVT = N->getValueType(ResNo: 0); |
2471 | unsigned Opcode = N->getOpcode(); |
2472 | |
2473 | // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if |
2474 | // vecreduce_xor is not legal |
2475 | if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 && |
2476 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_XOR, VT: InVT) && |
2477 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_ADD, VT: InVT)) |
2478 | Opcode = ISD::VECREDUCE_ADD; |
2479 | |
2480 | // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if |
2481 | // vecreduce_or is not legal |
2482 | else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 && |
2483 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_OR, VT: InVT) && |
2484 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMAX, VT: InVT)) { |
2485 | Opcode = ISD::VECREDUCE_UMAX; |
2486 | // Can't use promoteTargetBoolean here because we still need |
2487 | // to either sign_ext or zero_ext in the undefined case. |
2488 | switch (TLI.getBooleanContents(Type: InVT)) { |
2489 | case TargetLoweringBase::UndefinedBooleanContent: |
2490 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
2491 | Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2492 | break; |
2493 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
2494 | Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2495 | break; |
2496 | } |
2497 | } |
2498 | |
2499 | // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if |
2500 | // vecreduce_and is not legal |
2501 | else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 && |
2502 | !TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_AND, VT: InVT) && |
2503 | TLI.isOperationLegalOrCustom(Op: ISD::VECREDUCE_UMIN, VT: InVT)) { |
2504 | Opcode = ISD::VECREDUCE_UMIN; |
2505 | // Can't use promoteTargetBoolean here because we still need |
2506 | // to either sign_ext or zero_ext in the undefined case. |
2507 | switch (TLI.getBooleanContents(Type: InVT)) { |
2508 | case TargetLoweringBase::UndefinedBooleanContent: |
2509 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
2510 | Op = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2511 | break; |
2512 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
2513 | Op = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
2514 | break; |
2515 | } |
2516 | } |
2517 | |
2518 | if (ResVT.bitsGE(VT: EltVT)) |
2519 | return DAG.getNode(Opcode, DL: SDLoc(N), VT: ResVT, Operand: Op); |
2520 | |
2521 | // Result size must be >= element size. If this is not the case after |
2522 | // promotion, also promote the result type and then truncate. |
2523 | SDValue Reduce = DAG.getNode(Opcode, DL: dl, VT: EltVT, Operand: Op); |
2524 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Reduce); |
2525 | } |
2526 | |
2527 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) { |
2528 | SDLoc DL(N); |
2529 | SDValue Op = N->getOperand(Num: OpNo); |
2530 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
2531 | |
2532 | if (OpNo == 2) { // Mask |
2533 | // Update in place. |
2534 | NewOps[2] = PromoteTargetBoolean(Bool: Op, ValVT: N->getOperand(Num: 1).getValueType()); |
2535 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2536 | } |
2537 | |
2538 | assert(OpNo == 1 && "Unexpected operand for promotion" ); |
2539 | |
2540 | Op = PromoteIntOpVectorReduction(N, V: Op); |
2541 | |
2542 | NewOps[OpNo] = Op; |
2543 | |
2544 | EVT VT = N->getValueType(ResNo: 0); |
2545 | EVT EltVT = Op.getValueType().getScalarType(); |
2546 | |
2547 | if (VT.bitsGE(VT: EltVT)) |
2548 | return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, Ops: NewOps); |
2549 | |
2550 | // Result size must be >= element/start-value size. If this is not the case |
2551 | // after promotion, also promote both the start value and result type and |
2552 | // then truncate. |
2553 | NewOps[0] = |
2554 | DAG.getNode(Opcode: getExtendForIntVecReduction(N), DL, VT: EltVT, Operand: N->getOperand(Num: 0)); |
2555 | SDValue Reduce = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, Ops: NewOps); |
2556 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Reduce); |
2557 | } |
2558 | |
2559 | SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { |
2560 | SDValue Op = ZExtPromotedInteger(Op: N->getOperand(Num: 1)); |
2561 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Op), 0); |
2562 | } |
2563 | |
2564 | SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
2565 | assert(OpNo > 1); // Because the first two arguments are guaranteed legal. |
2566 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
2567 | SDValue Operand = N->getOperand(Num: OpNo); |
2568 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Operand.getValueType()); |
2569 | NewOps[OpNo] = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand); |
2570 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2571 | } |
2572 | |
2573 | SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
2574 | assert(OpNo >= 7); |
2575 | SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); |
2576 | SDValue Operand = N->getOperand(Num: OpNo); |
2577 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Operand.getValueType()); |
2578 | NewOps[OpNo] = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: NVT, Operand); |
2579 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2580 | } |
2581 | |
2582 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
2583 | assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
2584 | (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
2585 | |
2586 | SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); |
2587 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2588 | |
2589 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2590 | } |
2591 | |
2592 | SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) { |
2593 | SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end()); |
2594 | |
2595 | if (OpNo == 2) { // Offset operand |
2596 | NewOps[OpNo] = SExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2597 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2598 | } |
2599 | |
2600 | assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion" ); |
2601 | |
2602 | NewOps[OpNo] = ZExtPromotedInteger(Op: N->getOperand(Num: OpNo)); |
2603 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
2604 | } |
2605 | |
2606 | //===----------------------------------------------------------------------===// |
2607 | // Integer Result Expansion |
2608 | //===----------------------------------------------------------------------===// |
2609 | |
2610 | /// ExpandIntegerResult - This method is called when the specified result of the |
2611 | /// specified node is found to need expansion. At this point, the node may also |
2612 | /// have invalid operands or may have other results that need promotion, we just |
2613 | /// know that (at least) one result needs expansion. |
2614 | void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
2615 | LLVM_DEBUG(dbgs() << "Expand integer result: " ; N->dump(&DAG)); |
2616 | SDValue Lo, Hi; |
2617 | Lo = Hi = SDValue(); |
2618 | |
2619 | // See if the target wants to custom expand this node. |
2620 | if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) |
2621 | return; |
2622 | |
2623 | switch (N->getOpcode()) { |
2624 | default: |
2625 | #ifndef NDEBUG |
2626 | dbgs() << "ExpandIntegerResult #" << ResNo << ": " ; |
2627 | N->dump(G: &DAG); dbgs() << "\n" ; |
2628 | #endif |
2629 | report_fatal_error(reason: "Do not know how to expand the result of this " |
2630 | "operator!" ); |
2631 | |
2632 | case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break; |
2633 | case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
2634 | case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; |
2635 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
2636 | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
2637 | case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; |
2638 | |
2639 | case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
2640 | case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
2641 | case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
2642 | case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
2643 | case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
2644 | |
2645 | case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
2646 | case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
2647 | case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
2648 | case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
2649 | case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
2650 | case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; |
2651 | case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
2652 | case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; |
2653 | case ISD::CTLZ_ZERO_UNDEF: |
2654 | case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
2655 | case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
2656 | case ISD::CTTZ_ZERO_UNDEF: |
2657 | case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
2658 | case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break; |
2659 | case ISD::STRICT_FP_TO_SINT: |
2660 | case ISD::FP_TO_SINT: |
2661 | case ISD::STRICT_FP_TO_UINT: |
2662 | case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break; |
2663 | case ISD::FP_TO_SINT_SAT: |
2664 | case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; |
2665 | case ISD::STRICT_LROUND: |
2666 | case ISD::STRICT_LRINT: |
2667 | case ISD::LROUND: |
2668 | case ISD::LRINT: |
2669 | case ISD::STRICT_LLROUND: |
2670 | case ISD::STRICT_LLRINT: |
2671 | case ISD::LLROUND: |
2672 | case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break; |
2673 | case ISD::LOAD: ExpandIntRes_LOAD(N: cast<LoadSDNode>(Val: N), Lo, Hi); break; |
2674 | case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
2675 | case ISD::READCYCLECOUNTER: |
2676 | case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break; |
2677 | case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
2678 | case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
2679 | case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
2680 | case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
2681 | case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
2682 | case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
2683 | case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
2684 | case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
2685 | case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
2686 | |
2687 | case ISD::ATOMIC_LOAD_ADD: |
2688 | case ISD::ATOMIC_LOAD_SUB: |
2689 | case ISD::ATOMIC_LOAD_AND: |
2690 | case ISD::ATOMIC_LOAD_CLR: |
2691 | case ISD::ATOMIC_LOAD_OR: |
2692 | case ISD::ATOMIC_LOAD_XOR: |
2693 | case ISD::ATOMIC_LOAD_NAND: |
2694 | case ISD::ATOMIC_LOAD_MIN: |
2695 | case ISD::ATOMIC_LOAD_MAX: |
2696 | case ISD::ATOMIC_LOAD_UMIN: |
2697 | case ISD::ATOMIC_LOAD_UMAX: |
2698 | case ISD::ATOMIC_SWAP: |
2699 | case ISD::ATOMIC_CMP_SWAP: { |
2700 | std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node: N); |
2701 | SplitInteger(Op: Tmp.first, Lo, Hi); |
2702 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
2703 | break; |
2704 | } |
2705 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2706 | AtomicSDNode *AN = cast<AtomicSDNode>(Val: N); |
2707 | SDVTList VTs = DAG.getVTList(N->getValueType(ResNo: 0), MVT::Other); |
2708 | SDValue Tmp = DAG.getAtomicCmpSwap( |
2709 | Opcode: ISD::ATOMIC_CMP_SWAP, dl: SDLoc(N), MemVT: AN->getMemoryVT(), VTs, |
2710 | Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 1), Cmp: N->getOperand(Num: 2), Swp: N->getOperand(Num: 3), |
2711 | MMO: AN->getMemOperand()); |
2712 | |
2713 | // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine |
2714 | // success simply by comparing the loaded value against the ingoing |
2715 | // comparison. |
2716 | SDValue Success = DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 1), LHS: Tmp, |
2717 | RHS: N->getOperand(Num: 2), Cond: ISD::SETEQ); |
2718 | |
2719 | SplitInteger(Op: Tmp, Lo, Hi); |
2720 | ReplaceValueWith(From: SDValue(N, 1), To: Success); |
2721 | ReplaceValueWith(From: SDValue(N, 2), To: Tmp.getValue(R: 1)); |
2722 | break; |
2723 | } |
2724 | |
2725 | case ISD::AND: |
2726 | case ISD::OR: |
2727 | case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
2728 | |
2729 | case ISD::UMAX: |
2730 | case ISD::SMAX: |
2731 | case ISD::UMIN: |
2732 | case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
2733 | |
2734 | case ISD::ADD: |
2735 | case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
2736 | |
2737 | case ISD::ADDC: |
2738 | case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
2739 | |
2740 | case ISD::ADDE: |
2741 | case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
2742 | |
2743 | case ISD::UADDO_CARRY: |
2744 | case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break; |
2745 | |
2746 | case ISD::SADDO_CARRY: |
2747 | case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; |
2748 | |
2749 | case ISD::SHL: |
2750 | case ISD::SRA: |
2751 | case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
2752 | |
2753 | case ISD::SADDO: |
2754 | case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
2755 | case ISD::UADDO: |
2756 | case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
2757 | case ISD::UMULO: |
2758 | case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
2759 | |
2760 | case ISD::SADDSAT: |
2761 | case ISD::UADDSAT: |
2762 | case ISD::SSUBSAT: |
2763 | case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; |
2764 | |
2765 | case ISD::SSHLSAT: |
2766 | case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; |
2767 | |
2768 | case ISD::SMULFIX: |
2769 | case ISD::SMULFIXSAT: |
2770 | case ISD::UMULFIX: |
2771 | case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; |
2772 | |
2773 | case ISD::SDIVFIX: |
2774 | case ISD::SDIVFIXSAT: |
2775 | case ISD::UDIVFIX: |
2776 | case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; |
2777 | |
2778 | case ISD::VECREDUCE_ADD: |
2779 | case ISD::VECREDUCE_MUL: |
2780 | case ISD::VECREDUCE_AND: |
2781 | case ISD::VECREDUCE_OR: |
2782 | case ISD::VECREDUCE_XOR: |
2783 | case ISD::VECREDUCE_SMAX: |
2784 | case ISD::VECREDUCE_SMIN: |
2785 | case ISD::VECREDUCE_UMAX: |
2786 | case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; |
2787 | |
2788 | case ISD::ROTL: |
2789 | case ISD::ROTR: |
2790 | ExpandIntRes_Rotate(N, Lo, Hi); |
2791 | break; |
2792 | |
2793 | case ISD::FSHL: |
2794 | case ISD::FSHR: |
2795 | ExpandIntRes_FunnelShift(N, Lo, Hi); |
2796 | break; |
2797 | |
2798 | case ISD::VSCALE: |
2799 | ExpandIntRes_VSCALE(N, Lo, Hi); |
2800 | break; |
2801 | } |
2802 | |
2803 | // If Lo/Hi is null, the sub-method took care of registering results etc. |
2804 | if (Lo.getNode()) |
2805 | SetExpandedInteger(Op: SDValue(N, ResNo), Lo, Hi); |
2806 | } |
2807 | |
2808 | /// Lower an atomic node to the appropriate builtin call. |
2809 | std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
2810 | unsigned Opc = Node->getOpcode(); |
2811 | MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT(); |
2812 | AtomicOrdering order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering(); |
2813 | // Lower to outline atomic libcall if outline atomics enabled, |
2814 | // or to sync libcall otherwise |
2815 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order: order, VT); |
2816 | EVT RetVT = Node->getValueType(ResNo: 0); |
2817 | TargetLowering::MakeLibCallOptions CallOptions; |
2818 | SmallVector<SDValue, 4> Ops; |
2819 | if (TLI.getLibcallName(Call: LC)) { |
2820 | Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end()); |
2821 | Ops.push_back(Elt: Node->getOperand(Num: 1)); |
2822 | } else { |
2823 | LC = RTLIB::getSYNC(Opc, VT); |
2824 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
2825 | "Unexpected atomic op or value type!" ); |
2826 | Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end()); |
2827 | } |
2828 | return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, dl: SDLoc(Node), |
2829 | Chain: Node->getOperand(Num: 0)); |
2830 | } |
2831 | |
2832 | /// N is a shift by a value that needs to be expanded, |
2833 | /// and the shift amount is a constant 'Amt'. Expand the operation. |
2834 | void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
2835 | SDValue &Lo, SDValue &Hi) { |
2836 | SDLoc DL(N); |
2837 | // Expand the incoming operand to be shifted, so that we have its parts |
2838 | SDValue InL, InH; |
2839 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
2840 | |
2841 | // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization |
2842 | // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. |
2843 | if (!Amt) { |
2844 | Lo = InL; |
2845 | Hi = InH; |
2846 | return; |
2847 | } |
2848 | |
2849 | EVT NVT = InL.getValueType(); |
2850 | unsigned VTBits = N->getValueType(ResNo: 0).getSizeInBits(); |
2851 | unsigned NVTBits = NVT.getSizeInBits(); |
2852 | |
2853 | if (N->getOpcode() == ISD::SHL) { |
2854 | if (Amt.uge(RHS: VTBits)) { |
2855 | Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2856 | } else if (Amt.ugt(RHS: NVTBits)) { |
2857 | Lo = DAG.getConstant(Val: 0, DL, VT: NVT); |
2858 | Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL, |
2859 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2860 | } else if (Amt == NVTBits) { |
2861 | Lo = DAG.getConstant(Val: 0, DL, VT: NVT); |
2862 | Hi = InL; |
2863 | } else { |
2864 | Lo = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InL, |
2865 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2866 | Hi = DAG.getNode( |
2867 | Opcode: ISD::OR, DL, VT: NVT, |
2868 | N1: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2869 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2870 | N2: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2871 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2872 | } |
2873 | return; |
2874 | } |
2875 | |
2876 | if (N->getOpcode() == ISD::SRL) { |
2877 | if (Amt.uge(RHS: VTBits)) { |
2878 | Lo = Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2879 | } else if (Amt.ugt(RHS: NVTBits)) { |
2880 | Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH, |
2881 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2882 | Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2883 | } else if (Amt == NVTBits) { |
2884 | Lo = InH; |
2885 | Hi = DAG.getConstant(Val: 0, DL, VT: NVT); |
2886 | } else { |
2887 | Lo = DAG.getNode( |
2888 | Opcode: ISD::OR, DL, VT: NVT, |
2889 | N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2890 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2891 | N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2892 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2893 | Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InH, |
2894 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2895 | } |
2896 | return; |
2897 | } |
2898 | |
2899 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!" ); |
2900 | if (Amt.uge(RHS: VTBits)) { |
2901 | Hi = Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2902 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2903 | } else if (Amt.ugt(RHS: NVTBits)) { |
2904 | Lo = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2905 | N2: DAG.getShiftAmountConstant(Val: Amt - NVTBits, VT: NVT, DL)); |
2906 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2907 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2908 | } else if (Amt == NVTBits) { |
2909 | Lo = InH; |
2910 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2911 | N2: DAG.getShiftAmountConstant(Val: NVTBits - 1, VT: NVT, DL)); |
2912 | } else { |
2913 | Lo = DAG.getNode( |
2914 | Opcode: ISD::OR, DL, VT: NVT, |
2915 | N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: NVT, N1: InL, |
2916 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)), |
2917 | N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: InH, |
2918 | N2: DAG.getShiftAmountConstant(Val: -Amt + NVTBits, VT: NVT, DL))); |
2919 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: InH, |
2920 | N2: DAG.getShiftAmountConstant(Val: Amt, VT: NVT, DL)); |
2921 | } |
2922 | } |
2923 | |
2924 | /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify |
2925 | /// this shift based on knowledge of the high bit of the shift amount. If we |
2926 | /// can tell this, we know that it is >= 32 or < 32, without knowing the actual |
2927 | /// shift amount. |
2928 | bool DAGTypeLegalizer:: |
2929 | ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2930 | unsigned Opc = N->getOpcode(); |
2931 | SDValue In = N->getOperand(Num: 0); |
2932 | SDValue Amt = N->getOperand(Num: 1); |
2933 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
2934 | EVT ShTy = Amt.getValueType(); |
2935 | unsigned ShBits = ShTy.getScalarSizeInBits(); |
2936 | unsigned NVTBits = NVT.getScalarSizeInBits(); |
2937 | assert(isPowerOf2_32(NVTBits) && |
2938 | "Expanded integer type size not a power of two!" ); |
2939 | SDLoc dl(N); |
2940 | |
2941 | APInt HighBitMask = APInt::getHighBitsSet(numBits: ShBits, hiBitsSet: ShBits - Log2_32(Value: NVTBits)); |
2942 | KnownBits Known = DAG.computeKnownBits(Op: Amt); |
2943 | |
2944 | // If we don't know anything about the high bits, exit. |
2945 | if (((Known.Zero | Known.One) & HighBitMask) == 0) |
2946 | return false; |
2947 | |
2948 | // Get the incoming operand to be shifted. |
2949 | SDValue InL, InH; |
2950 | GetExpandedInteger(Op: In, Lo&: InL, Hi&: InH); |
2951 | |
2952 | // If we know that any of the high bits of the shift amount are one, then we |
2953 | // can do this as a couple of simple shifts. |
2954 | if (Known.One.intersects(RHS: HighBitMask)) { |
2955 | // Mask out the high bit, which we know is set. |
2956 | Amt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShTy, N1: Amt, |
2957 | N2: DAG.getConstant(Val: ~HighBitMask, DL: dl, VT: ShTy)); |
2958 | |
2959 | switch (Opc) { |
2960 | default: llvm_unreachable("Unknown shift" ); |
2961 | case ISD::SHL: |
2962 | Lo = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Low part is zero. |
2963 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt); // High part from Lo part. |
2964 | return true; |
2965 | case ISD::SRL: |
2966 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero. |
2967 | Lo = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part. |
2968 | return true; |
2969 | case ISD::SRA: |
2970 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign extend high part. |
2971 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
2972 | Lo = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt); // Lo part from Hi part. |
2973 | return true; |
2974 | } |
2975 | } |
2976 | |
2977 | // If we know that all of the high bits of the shift amount are zero, then we |
2978 | // can do this as a couple of simple shifts. |
2979 | if (HighBitMask.isSubsetOf(RHS: Known.Zero)) { |
2980 | // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined |
2981 | // shift if x is zero. We can use XOR here because x is known to be smaller |
2982 | // than 32. |
2983 | SDValue Amt2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ShTy, N1: Amt, |
2984 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
2985 | |
2986 | unsigned Op1, Op2; |
2987 | switch (Opc) { |
2988 | default: llvm_unreachable("Unknown shift" ); |
2989 | case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
2990 | case ISD::SRL: |
2991 | case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
2992 | } |
2993 | |
2994 | // When shifting right the arithmetic for Lo and Hi is swapped. |
2995 | if (Opc != ISD::SHL) |
2996 | std::swap(a&: InL, b&: InH); |
2997 | |
2998 | // Use a little trick to get the bits that move from Lo to Hi. First |
2999 | // shift by one bit. |
3000 | SDValue Sh1 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: InL, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShTy)); |
3001 | // Then compute the remaining shift with amount-1. |
3002 | SDValue Sh2 = DAG.getNode(Opcode: Op2, DL: dl, VT: NVT, N1: Sh1, N2: Amt2); |
3003 | |
3004 | Lo = DAG.getNode(Opcode: Opc, DL: dl, VT: NVT, N1: InL, N2: Amt); |
3005 | Hi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: Op1, DL: dl, VT: NVT, N1: InH, N2: Amt),N2: Sh2); |
3006 | |
3007 | if (Opc != ISD::SHL) |
3008 | std::swap(a&: Hi, b&: Lo); |
3009 | return true; |
3010 | } |
3011 | |
3012 | return false; |
3013 | } |
3014 | |
3015 | /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift |
3016 | /// of any size. |
3017 | bool DAGTypeLegalizer:: |
3018 | ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
3019 | SDValue Amt = N->getOperand(Num: 1); |
3020 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3021 | EVT ShTy = Amt.getValueType(); |
3022 | unsigned NVTBits = NVT.getSizeInBits(); |
3023 | assert(isPowerOf2_32(NVTBits) && |
3024 | "Expanded integer type size not a power of two!" ); |
3025 | SDLoc dl(N); |
3026 | |
3027 | // Get the incoming operand to be shifted. |
3028 | SDValue InL, InH; |
3029 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
3030 | |
3031 | SDValue NVBitsNode = DAG.getConstant(Val: NVTBits, DL: dl, VT: ShTy); |
3032 | SDValue AmtExcess = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: Amt, N2: NVBitsNode); |
3033 | SDValue AmtLack = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ShTy, N1: NVBitsNode, N2: Amt); |
3034 | SDValue isShort = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy), |
3035 | LHS: Amt, RHS: NVBitsNode, Cond: ISD::SETULT); |
3036 | SDValue isZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: ShTy), |
3037 | LHS: Amt, RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShTy), |
3038 | Cond: ISD::SETEQ); |
3039 | |
3040 | SDValue LoS, HiS, LoL, HiL; |
3041 | switch (N->getOpcode()) { |
3042 | default: llvm_unreachable("Unknown shift" ); |
3043 | case ISD::SHL: |
3044 | // Short: ShAmt < NVTBits |
3045 | LoS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: Amt); |
3046 | HiS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3047 | N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: Amt), |
3048 | N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: AmtLack)); |
3049 | |
3050 | // Long: ShAmt >= NVTBits |
3051 | LoL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Lo part is zero. |
3052 | HiL = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InL, N2: AmtExcess); // Hi from Lo part. |
3053 | |
3054 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL); |
3055 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InH, |
3056 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL)); |
3057 | return true; |
3058 | case ISD::SRL: |
3059 | // Short: ShAmt < NVTBits |
3060 | HiS = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: Amt); |
3061 | LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3062 | N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt), |
3063 | // FIXME: If Amt is zero, the following shift generates an undefined result |
3064 | // on some architectures. |
3065 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack)); |
3066 | |
3067 | // Long: ShAmt >= NVTBits |
3068 | HiL = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // Hi part is zero. |
3069 | LoL = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part. |
3070 | |
3071 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL, |
3072 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL)); |
3073 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL); |
3074 | return true; |
3075 | case ISD::SRA: |
3076 | // Short: ShAmt < NVTBits |
3077 | HiS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: Amt); |
3078 | LoS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, |
3079 | N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: InL, N2: Amt), |
3080 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: InH, N2: AmtLack)); |
3081 | |
3082 | // Long: ShAmt >= NVTBits |
3083 | HiL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, // Sign of Hi part. |
3084 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, VT: ShTy)); |
3085 | LoL = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: InH, N2: AmtExcess); // Lo from Hi part. |
3086 | |
3087 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: isZero, LHS: InL, |
3088 | RHS: DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: LoS, RHS: LoL)); |
3089 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: isShort, LHS: HiS, RHS: HiL); |
3090 | return true; |
3091 | } |
3092 | } |
3093 | |
3094 | static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { |
3095 | |
3096 | switch (Op) { |
3097 | default: llvm_unreachable("invalid min/max opcode" ); |
3098 | case ISD::SMAX: |
3099 | return std::make_pair(x: ISD::SETGT, y: ISD::UMAX); |
3100 | case ISD::UMAX: |
3101 | return std::make_pair(x: ISD::SETUGT, y: ISD::UMAX); |
3102 | case ISD::SMIN: |
3103 | return std::make_pair(x: ISD::SETLT, y: ISD::UMIN); |
3104 | case ISD::UMIN: |
3105 | return std::make_pair(x: ISD::SETULT, y: ISD::UMIN); |
3106 | } |
3107 | } |
3108 | |
3109 | void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
3110 | SDValue &Lo, SDValue &Hi) { |
3111 | SDLoc DL(N); |
3112 | |
3113 | SDValue LHS = N->getOperand(Num: 0); |
3114 | SDValue RHS = N->getOperand(Num: 1); |
3115 | |
3116 | // If the upper halves are all sign bits, then we can perform the MINMAX on |
3117 | // the lower half and sign-extend the result to the upper half. |
3118 | unsigned NumBits = N->getValueType(ResNo: 0).getScalarSizeInBits(); |
3119 | unsigned NumHalfBits = NumBits / 2; |
3120 | if (DAG.ComputeNumSignBits(Op: LHS) > NumHalfBits && |
3121 | DAG.ComputeNumSignBits(Op: RHS) > NumHalfBits) { |
3122 | SDValue LHSL, LHSH, RHSL, RHSH; |
3123 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3124 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3125 | EVT NVT = LHSL.getValueType(); |
3126 | |
3127 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, N1: LHSL, N2: RHSL); |
3128 | Hi = DAG.getNode(Opcode: ISD::SRA, DL, VT: NVT, N1: Lo, |
3129 | N2: DAG.getShiftAmountConstant(Val: NumHalfBits - 1, VT: NVT, DL)); |
3130 | return; |
3131 | } |
3132 | |
3133 | // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1. |
3134 | // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL. |
3135 | if ((N->getOpcode() == ISD::SMAX && isNullConstant(V: RHS)) || |
3136 | (N->getOpcode() == ISD::SMIN && isAllOnesConstant(V: RHS))) { |
3137 | SDValue LHSL, LHSH, RHSL, RHSH; |
3138 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3139 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3140 | EVT NVT = LHSL.getValueType(); |
3141 | EVT CCT = getSetCCResultType(VT: NVT); |
3142 | |
3143 | SDValue HiNeg = |
3144 | DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: DAG.getConstant(Val: 0, DL, VT: NVT), Cond: ISD::SETLT); |
3145 | if (N->getOpcode() == ISD::SMIN) { |
3146 | Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: LHSL, RHS: DAG.getConstant(Val: -1, DL, VT: NVT)); |
3147 | } else { |
3148 | Lo = DAG.getSelect(DL, VT: NVT, Cond: HiNeg, LHS: DAG.getConstant(Val: 0, DL, VT: NVT), RHS: LHSL); |
3149 | } |
3150 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH}); |
3151 | return; |
3152 | } |
3153 | |
3154 | const APInt *RHSVal = nullptr; |
3155 | if (auto *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS)) |
3156 | RHSVal = &RHSConst->getAPIntValue(); |
3157 | |
3158 | // The high half of MIN/MAX is always just the the MIN/MAX of the |
3159 | // high halves of the operands. Expand this way if it appears profitable. |
3160 | if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) && |
3161 | (RHSVal->countLeadingOnes() >= NumHalfBits || |
3162 | RHSVal->countLeadingZeros() >= NumHalfBits)) { |
3163 | SDValue LHSL, LHSH, RHSL, RHSH; |
3164 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3165 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3166 | EVT NVT = LHSL.getValueType(); |
3167 | EVT CCT = getSetCCResultType(VT: NVT); |
3168 | |
3169 | ISD::NodeType LoOpc; |
3170 | ISD::CondCode CondC; |
3171 | std::tie(args&: CondC, args&: LoOpc) = getExpandedMinMaxOps(Op: N->getOpcode()); |
3172 | |
3173 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Ops: {LHSH, RHSH}); |
3174 | // We need to know whether to select Lo part that corresponds to 'winning' |
3175 | // Hi part or if Hi parts are equal. |
3176 | SDValue IsHiLeft = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: CondC); |
3177 | SDValue IsHiEq = DAG.getSetCC(DL, VT: CCT, LHS: LHSH, RHS: RHSH, Cond: ISD::SETEQ); |
3178 | |
3179 | // Lo part corresponding to the 'winning' Hi part |
3180 | SDValue LoCmp = DAG.getSelect(DL, VT: NVT, Cond: IsHiLeft, LHS: LHSL, RHS: RHSL); |
3181 | |
3182 | // Recursed Lo part if Hi parts are equal, this uses unsigned version |
3183 | SDValue LoMinMax = DAG.getNode(Opcode: LoOpc, DL, VT: NVT, Ops: {LHSL, RHSL}); |
3184 | |
3185 | Lo = DAG.getSelect(DL, VT: NVT, Cond: IsHiEq, LHS: LoMinMax, RHS: LoCmp); |
3186 | return; |
3187 | } |
3188 | |
3189 | // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies |
3190 | // the compare. |
3191 | ISD::CondCode Pred; |
3192 | switch (N->getOpcode()) { |
3193 | default: llvm_unreachable("How did we get here?" ); |
3194 | case ISD::SMAX: |
3195 | if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
3196 | Pred = ISD::SETGE; |
3197 | else |
3198 | Pred = ISD::SETGT; |
3199 | break; |
3200 | case ISD::SMIN: |
3201 | if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
3202 | Pred = ISD::SETLE; |
3203 | else |
3204 | Pred = ISD::SETLT; |
3205 | break; |
3206 | case ISD::UMAX: |
3207 | if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits) |
3208 | Pred = ISD::SETUGE; |
3209 | else |
3210 | Pred = ISD::SETUGT; |
3211 | break; |
3212 | case ISD::UMIN: |
3213 | if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits) |
3214 | Pred = ISD::SETULE; |
3215 | else |
3216 | Pred = ISD::SETULT; |
3217 | break; |
3218 | } |
3219 | EVT VT = N->getValueType(ResNo: 0); |
3220 | EVT CCT = getSetCCResultType(VT); |
3221 | SDValue Cond = DAG.getSetCC(DL, VT: CCT, LHS, RHS, Cond: Pred); |
3222 | SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS); |
3223 | SplitInteger(Op: Result, Lo, Hi); |
3224 | } |
3225 | |
3226 | void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
3227 | SDValue &Lo, SDValue &Hi) { |
3228 | SDLoc dl(N); |
3229 | // Expand the subcomponents. |
3230 | SDValue LHSL, LHSH, RHSL, RHSH; |
3231 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3232 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3233 | |
3234 | EVT NVT = LHSL.getValueType(); |
3235 | SDValue LoOps[2] = { LHSL, RHSL }; |
3236 | SDValue HiOps[3] = { LHSH, RHSH }; |
3237 | |
3238 | bool HasOpCarry = TLI.isOperationLegalOrCustom( |
3239 | Op: N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, |
3240 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3241 | if (HasOpCarry) { |
3242 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT)); |
3243 | if (N->getOpcode() == ISD::ADD) { |
3244 | Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps); |
3245 | HiOps[2] = Lo.getValue(R: 1); |
3246 | Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero() |
3247 | ? DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: ArrayRef(HiOps, 2)) |
3248 | : DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, Ops: HiOps); |
3249 | } else { |
3250 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps); |
3251 | HiOps[2] = Lo.getValue(R: 1); |
3252 | Hi = DAG.computeKnownBits(Op: HiOps[2]).isZero() |
3253 | ? DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: ArrayRef(HiOps, 2)) |
3254 | : DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, Ops: HiOps); |
3255 | } |
3256 | return; |
3257 | } |
3258 | |
3259 | // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support |
3260 | // them. TODO: Teach operation legalization how to expand unsupported |
3261 | // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate |
3262 | // a carry of type MVT::Glue, but there doesn't seem to be any way to |
3263 | // generate a value of this type in the expanded code sequence. |
3264 | bool hasCarry = |
3265 | TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ? |
3266 | ISD::ADDC : ISD::SUBC, |
3267 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3268 | |
3269 | if (hasCarry) { |
3270 | SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
3271 | if (N->getOpcode() == ISD::ADD) { |
3272 | Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps); |
3273 | HiOps[2] = Lo.getValue(R: 1); |
3274 | Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps); |
3275 | } else { |
3276 | Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps); |
3277 | HiOps[2] = Lo.getValue(R: 1); |
3278 | Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps); |
3279 | } |
3280 | return; |
3281 | } |
3282 | |
3283 | bool hasOVF = |
3284 | TLI.isOperationLegalOrCustom(Op: N->getOpcode() == ISD::ADD ? |
3285 | ISD::UADDO : ISD::USUBO, |
3286 | VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3287 | TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(Type: NVT); |
3288 | |
3289 | if (hasOVF) { |
3290 | EVT OvfVT = getSetCCResultType(VT: NVT); |
3291 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: OvfVT); |
3292 | int RevOpc; |
3293 | if (N->getOpcode() == ISD::ADD) { |
3294 | RevOpc = ISD::SUB; |
3295 | Lo = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, Ops: LoOps); |
3296 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3297 | } else { |
3298 | RevOpc = ISD::ADD; |
3299 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, Ops: LoOps); |
3300 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3301 | } |
3302 | SDValue OVF = Lo.getValue(R: 1); |
3303 | |
3304 | switch (BoolType) { |
3305 | case TargetLoweringBase::UndefinedBooleanContent: |
3306 | OVF = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OvfVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: OvfVT), N2: OVF); |
3307 | [[fallthrough]]; |
3308 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
3309 | OVF = DAG.getZExtOrTrunc(Op: OVF, DL: dl, VT: NVT); |
3310 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Hi, N2: OVF); |
3311 | break; |
3312 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
3313 | OVF = DAG.getSExtOrTrunc(Op: OVF, DL: dl, VT: NVT); |
3314 | Hi = DAG.getNode(Opcode: RevOpc, DL: dl, VT: NVT, N1: Hi, N2: OVF); |
3315 | } |
3316 | return; |
3317 | } |
3318 | |
3319 | if (N->getOpcode() == ISD::ADD) { |
3320 | Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: LoOps); |
3321 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3322 | SDValue Cmp; |
3323 | // Special case: X+1 has a carry out if X+1==0. This may reduce the live |
3324 | // range of X. We assume comparing with 0 is cheap. |
3325 | if (isOneConstant(V: LoOps[1])) |
3326 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, |
3327 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ); |
3328 | else if (isAllOnesConstant(V: LoOps[1])) { |
3329 | if (isAllOnesConstant(V: HiOps[1])) |
3330 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0], |
3331 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETEQ); |
3332 | else |
3333 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: LoOps[0], |
3334 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3335 | } else |
3336 | Cmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, RHS: LoOps[0], |
3337 | Cond: ISD::SETULT); |
3338 | |
3339 | SDValue Carry; |
3340 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
3341 | Carry = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT); |
3342 | else |
3343 | Carry = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT), |
3344 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT)); |
3345 | |
3346 | if (isAllOnesConstant(V: LoOps[1]) && isAllOnesConstant(V: HiOps[1])) |
3347 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: HiOps[0], N2: Carry); |
3348 | else |
3349 | Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: Hi, N2: Carry); |
3350 | } else { |
3351 | Lo = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: LoOps); |
3352 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, Ops: ArrayRef(HiOps, 2)); |
3353 | SDValue Cmp = |
3354 | DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LoOps[0].getValueType()), |
3355 | LHS: LoOps[0], RHS: LoOps[1], Cond: ISD::SETULT); |
3356 | |
3357 | SDValue Borrow; |
3358 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
3359 | Borrow = DAG.getZExtOrTrunc(Op: Cmp, DL: dl, VT: NVT); |
3360 | else |
3361 | Borrow = DAG.getSelect(DL: dl, VT: NVT, Cond: Cmp, LHS: DAG.getConstant(Val: 1, DL: dl, VT: NVT), |
3362 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT)); |
3363 | |
3364 | Hi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Hi, N2: Borrow); |
3365 | } |
3366 | } |
3367 | |
3368 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
3369 | SDValue &Lo, SDValue &Hi) { |
3370 | // Expand the subcomponents. |
3371 | SDValue LHSL, LHSH, RHSL, RHSH; |
3372 | SDLoc dl(N); |
3373 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3374 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3375 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
3376 | SDValue LoOps[2] = { LHSL, RHSL }; |
3377 | SDValue HiOps[3] = { LHSH, RHSH }; |
3378 | |
3379 | if (N->getOpcode() == ISD::ADDC) { |
3380 | Lo = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList, Ops: LoOps); |
3381 | HiOps[2] = Lo.getValue(R: 1); |
3382 | Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList, Ops: HiOps); |
3383 | } else { |
3384 | Lo = DAG.getNode(Opcode: ISD::SUBC, DL: dl, VTList, Ops: LoOps); |
3385 | HiOps[2] = Lo.getValue(R: 1); |
3386 | Hi = DAG.getNode(Opcode: ISD::SUBE, DL: dl, VTList, Ops: HiOps); |
3387 | } |
3388 | |
3389 | // Legalized the flag result - switch anything that used the old flag to |
3390 | // use the new one. |
3391 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3392 | } |
3393 | |
3394 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
3395 | SDValue &Lo, SDValue &Hi) { |
3396 | // Expand the subcomponents. |
3397 | SDValue LHSL, LHSH, RHSL, RHSH; |
3398 | SDLoc dl(N); |
3399 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3400 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3401 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
3402 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) }; |
3403 | SDValue HiOps[3] = { LHSH, RHSH }; |
3404 | |
3405 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3406 | HiOps[2] = Lo.getValue(R: 1); |
3407 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps); |
3408 | |
3409 | // Legalized the flag result - switch anything that used the old flag to |
3410 | // use the new one. |
3411 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3412 | } |
3413 | |
3414 | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
3415 | SDValue &Lo, SDValue &Hi) { |
3416 | SDValue LHS = N->getOperand(Num: 0); |
3417 | SDValue RHS = N->getOperand(Num: 1); |
3418 | SDLoc dl(N); |
3419 | |
3420 | SDValue Ovf; |
3421 | |
3422 | unsigned CarryOp, NoCarryOp; |
3423 | ISD::CondCode Cond; |
3424 | switch(N->getOpcode()) { |
3425 | case ISD::UADDO: |
3426 | CarryOp = ISD::UADDO_CARRY; |
3427 | NoCarryOp = ISD::ADD; |
3428 | Cond = ISD::SETULT; |
3429 | break; |
3430 | case ISD::USUBO: |
3431 | CarryOp = ISD::USUBO_CARRY; |
3432 | NoCarryOp = ISD::SUB; |
3433 | Cond = ISD::SETUGT; |
3434 | break; |
3435 | default: |
3436 | llvm_unreachable("Node has unexpected Opcode" ); |
3437 | } |
3438 | |
3439 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
3440 | Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType())); |
3441 | |
3442 | if (HasCarryOp) { |
3443 | // Expand the subcomponents. |
3444 | SDValue LHSL, LHSH, RHSL, RHSH; |
3445 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
3446 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
3447 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3448 | SDValue LoOps[2] = { LHSL, RHSL }; |
3449 | SDValue HiOps[3] = { LHSH, RHSH }; |
3450 | |
3451 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3452 | HiOps[2] = Lo.getValue(R: 1); |
3453 | Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: HiOps); |
3454 | |
3455 | Ovf = Hi.getValue(R: 1); |
3456 | } else { |
3457 | // Expand the result by simply replacing it with the equivalent |
3458 | // non-overflow-checking operation. |
3459 | SDValue Sum = DAG.getNode(Opcode: NoCarryOp, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS); |
3460 | SplitInteger(Op: Sum, Lo, Hi); |
3461 | |
3462 | if (N->getOpcode() == ISD::UADDO && isOneConstant(V: RHS)) { |
3463 | // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this |
3464 | // with (Lo | Hi) == 0. |
3465 | SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Lo.getValueType(), N1: Lo, N2: Hi); |
3466 | Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Or, |
3467 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: Lo.getValueType()), Cond: ISD::SETEQ); |
3468 | } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(V: RHS)) { |
3469 | // Special case: uaddo X, -1 overflows if X == 0. |
3470 | Ovf = |
3471 | DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS, |
3472 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType()), Cond: ISD::SETNE); |
3473 | } else { |
3474 | // Calculate the overflow: addition overflows iff a + b < a, and |
3475 | // subtraction overflows iff a - b > a. |
3476 | Ovf = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Sum, RHS: LHS, Cond); |
3477 | } |
3478 | } |
3479 | |
3480 | // Legalized the flag result - switch anything that used the old flag to |
3481 | // use the new one. |
3482 | ReplaceValueWith(From: SDValue(N, 1), To: Ovf); |
3483 | } |
3484 | |
3485 | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo, |
3486 | SDValue &Hi) { |
3487 | // Expand the subcomponents. |
3488 | SDValue LHSL, LHSH, RHSL, RHSH; |
3489 | SDLoc dl(N); |
3490 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3491 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3492 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3493 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(Num: 2) }; |
3494 | SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
3495 | |
3496 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: LoOps); |
3497 | HiOps[2] = Lo.getValue(R: 1); |
3498 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: HiOps); |
3499 | |
3500 | // Legalized the flag result - switch anything that used the old flag to |
3501 | // use the new one. |
3502 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3503 | } |
3504 | |
3505 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, |
3506 | SDValue &Lo, SDValue &Hi) { |
3507 | // Expand the subcomponents. |
3508 | SDValue LHSL, LHSH, RHSL, RHSH; |
3509 | SDLoc dl(N); |
3510 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
3511 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RHSL, Hi&: RHSH); |
3512 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: N->getValueType(ResNo: 1)); |
3513 | |
3514 | // We need to use an unsigned carry op for the lo part. |
3515 | unsigned CarryOp = |
3516 | N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; |
3517 | Lo = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSL, RHSL, N->getOperand(Num: 2) }); |
3518 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) }); |
3519 | |
3520 | // Legalized the flag result - switch anything that used the old flag to |
3521 | // use the new one. |
3522 | ReplaceValueWith(From: SDValue(N, 1), To: Hi.getValue(R: 1)); |
3523 | } |
3524 | |
3525 | void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
3526 | SDValue &Lo, SDValue &Hi) { |
3527 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3528 | SDLoc dl(N); |
3529 | SDValue Op = N->getOperand(Num: 0); |
3530 | if (Op.getValueType().bitsLE(VT: NVT)) { |
3531 | // The low part is any extension of the input (which degenerates to a copy). |
3532 | Lo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Op); |
3533 | Hi = DAG.getUNDEF(VT: NVT); // The high part is undefined. |
3534 | } else { |
3535 | // For example, extension of an i48 to an i64. The operand type necessarily |
3536 | // promotes to the result type, so will end up being expanded too. |
3537 | assert(getTypeAction(Op.getValueType()) == |
3538 | TargetLowering::TypePromoteInteger && |
3539 | "Only know how to promote this result!" ); |
3540 | SDValue Res = GetPromotedInteger(Op); |
3541 | assert(Res.getValueType() == N->getValueType(0) && |
3542 | "Operand over promoted?" ); |
3543 | // Split the promoted operand. This will simplify when it is expanded. |
3544 | SplitInteger(Op: Res, Lo, Hi); |
3545 | } |
3546 | } |
3547 | |
3548 | void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
3549 | SDValue &Lo, SDValue &Hi) { |
3550 | SDLoc dl(N); |
3551 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3552 | EVT NVT = Lo.getValueType(); |
3553 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
3554 | unsigned NVTBits = NVT.getSizeInBits(); |
3555 | unsigned EVTBits = EVT.getSizeInBits(); |
3556 | |
3557 | if (NVTBits < EVTBits) { |
3558 | Hi = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Hi, |
3559 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
3560 | BitWidth: EVTBits - NVTBits))); |
3561 | } else { |
3562 | Lo = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT)); |
3563 | // The high part replicates the sign bit of Lo, make it explicit. |
3564 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3565 | N2: DAG.getConstant(Val: NVTBits - 1, DL: dl, |
3566 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3567 | } |
3568 | } |
3569 | |
3570 | void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
3571 | SDValue &Lo, SDValue &Hi) { |
3572 | SDLoc dl(N); |
3573 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3574 | EVT NVT = Lo.getValueType(); |
3575 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
3576 | unsigned NVTBits = NVT.getSizeInBits(); |
3577 | unsigned EVTBits = EVT.getSizeInBits(); |
3578 | |
3579 | if (NVTBits < EVTBits) { |
3580 | Hi = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Hi, |
3581 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
3582 | BitWidth: EVTBits - NVTBits))); |
3583 | } else { |
3584 | Lo = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: NVT, N1: Lo, N2: DAG.getValueType(EVT)); |
3585 | // The high part must be zero, make it explicit. |
3586 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3587 | } |
3588 | } |
3589 | |
3590 | void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
3591 | SDValue &Lo, SDValue &Hi) { |
3592 | SDLoc dl(N); |
3593 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands. |
3594 | Lo = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
3595 | Hi = DAG.getNode(Opcode: ISD::BITREVERSE, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
3596 | } |
3597 | |
3598 | void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
3599 | SDValue &Lo, SDValue &Hi) { |
3600 | SDLoc dl(N); |
3601 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: Hi, Hi&: Lo); // Note swapped operands. |
3602 | Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Lo.getValueType(), Operand: Lo); |
3603 | Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: Hi.getValueType(), Operand: Hi); |
3604 | } |
3605 | |
3606 | void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, |
3607 | SDValue &Hi) { |
3608 | SDLoc dl(N); |
3609 | // parity(HiLo) -> parity(Lo^Hi) |
3610 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3611 | EVT NVT = Lo.getValueType(); |
3612 | Lo = |
3613 | DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT: NVT, Operand: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Hi)); |
3614 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3615 | } |
3616 | |
3617 | void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
3618 | SDValue &Lo, SDValue &Hi) { |
3619 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3620 | unsigned NBitWidth = NVT.getSizeInBits(); |
3621 | auto Constant = cast<ConstantSDNode>(Val: N); |
3622 | const APInt &Cst = Constant->getAPIntValue(); |
3623 | bool IsTarget = Constant->isTargetOpcode(); |
3624 | bool IsOpaque = Constant->isOpaque(); |
3625 | SDLoc dl(N); |
3626 | Lo = DAG.getConstant(Val: Cst.trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget, isOpaque: IsOpaque); |
3627 | Hi = DAG.getConstant(Val: Cst.lshr(shiftAmt: NBitWidth).trunc(width: NBitWidth), DL: dl, VT: NVT, isTarget: IsTarget, |
3628 | isOpaque: IsOpaque); |
3629 | } |
3630 | |
3631 | void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { |
3632 | SDLoc dl(N); |
3633 | |
3634 | SDValue N0 = N->getOperand(Num: 0); |
3635 | GetExpandedInteger(Op: N0, Lo, Hi); |
3636 | EVT NVT = Lo.getValueType(); |
3637 | |
3638 | // If the upper half is all sign bits, then we can perform the ABS on the |
3639 | // lower half and zero-extend. |
3640 | if (DAG.ComputeNumSignBits(Op: N0) > NVT.getScalarSizeInBits()) { |
3641 | Lo = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: NVT, Operand: Lo); |
3642 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3643 | return; |
3644 | } |
3645 | |
3646 | // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence |
3647 | // we use in LegalizeDAG. The SUB part of the expansion is based on |
3648 | // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that |
3649 | // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further |
3650 | // expanded if needed. Shift expansion has a special case for filling with |
3651 | // sign bits so that we will only end up with one SRA. |
3652 | bool HasSubCarry = TLI.isOperationLegalOrCustom( |
3653 | Op: ISD::USUBO_CARRY, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: NVT)); |
3654 | if (HasSubCarry) { |
3655 | SDValue Sign = DAG.getNode( |
3656 | Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Hi, |
3657 | N2: DAG.getShiftAmountConstant(Val: NVT.getSizeInBits() - 1, VT: NVT, DL: dl)); |
3658 | SDVTList VTList = DAG.getVTList(VT1: NVT, VT2: getSetCCResultType(VT: NVT)); |
3659 | Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Lo, N2: Sign); |
3660 | Hi = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: Hi, N2: Sign); |
3661 | Lo = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: Lo, N2: Sign); |
3662 | Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: Hi, N2: Sign, N3: Lo.getValue(R: 1)); |
3663 | return; |
3664 | } |
3665 | |
3666 | // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) |
3667 | EVT VT = N->getValueType(ResNo: 0); |
3668 | SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, |
3669 | N1: DAG.getConstant(Val: 0, DL: dl, VT), N2: N0); |
3670 | SDValue NegLo, NegHi; |
3671 | SplitInteger(Op: Neg, Lo&: NegLo, Hi&: NegHi); |
3672 | |
3673 | SDValue HiIsNeg = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi, |
3674 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETLT); |
3675 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegLo, RHS: Lo); |
3676 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: HiIsNeg, LHS: NegHi, RHS: Hi); |
3677 | } |
3678 | |
3679 | void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
3680 | SDValue &Lo, SDValue &Hi) { |
3681 | SDLoc dl(N); |
3682 | // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) |
3683 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3684 | EVT NVT = Lo.getValueType(); |
3685 | |
3686 | SDValue HiNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Hi, |
3687 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3688 | |
3689 | SDValue LoLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Lo); |
3690 | SDValue HiLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Hi); |
3691 | |
3692 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: HiNotZero, LHS: HiLZ, |
3693 | RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: LoLZ, |
3694 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
3695 | VT: NVT))); |
3696 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3697 | } |
3698 | |
3699 | void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
3700 | SDValue &Lo, SDValue &Hi) { |
3701 | SDLoc dl(N); |
3702 | // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) |
3703 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3704 | EVT NVT = Lo.getValueType(); |
3705 | Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT: NVT, Operand: Lo), |
3706 | N2: DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT: NVT, Operand: Hi)); |
3707 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3708 | } |
3709 | |
3710 | void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
3711 | SDValue &Lo, SDValue &Hi) { |
3712 | SDLoc dl(N); |
3713 | // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) |
3714 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
3715 | EVT NVT = Lo.getValueType(); |
3716 | |
3717 | SDValue LoNotZero = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: NVT), LHS: Lo, |
3718 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: NVT), Cond: ISD::SETNE); |
3719 | |
3720 | SDValue LoLZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_UNDEF, DL: dl, VT: NVT, Operand: Lo); |
3721 | SDValue HiLZ = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Hi); |
3722 | |
3723 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: LoNotZero, LHS: LoLZ, |
3724 | RHS: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: NVT, N1: HiLZ, |
3725 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
3726 | VT: NVT))); |
3727 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3728 | } |
3729 | |
3730 | void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo, |
3731 | SDValue &Hi) { |
3732 | SDLoc dl(N); |
3733 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
3734 | unsigned NBitWidth = NVT.getSizeInBits(); |
3735 | |
3736 | Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0)); |
3737 | SDValue Chain = Lo.getValue(R: 1); |
3738 | // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING |
3739 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3740 | N2: DAG.getShiftAmountConstant(Val: NBitWidth - 1, VT: NVT, DL: dl)); |
3741 | |
3742 | // Legalize the chain result - switch anything that used the old chain to |
3743 | // use the new one. |
3744 | ReplaceValueWith(From: SDValue(N, 1), To: Chain); |
3745 | } |
3746 | |
3747 | // Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op. |
3748 | static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, |
3749 | SDLoc DL, SelectionDAG &DAG) { |
3750 | if (IsStrict) { |
3751 | Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}); |
3752 | Chain = Op.getValue(R: 1); |
3753 | return Op; |
3754 | } |
3755 | return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op); |
3756 | } |
3757 | |
3758 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo, |
3759 | SDValue &Hi) { |
3760 | SDLoc dl(N); |
3761 | EVT VT = N->getValueType(ResNo: 0); |
3762 | |
3763 | bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || |
3764 | N->getOpcode() == ISD::STRICT_FP_TO_SINT; |
3765 | bool IsStrict = N->isStrictFPOpcode(); |
3766 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
3767 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
3768 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) |
3769 | Op = GetPromotedFloat(Op); |
3770 | |
3771 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
3772 | EVT OFPVT = Op.getValueType(); |
3773 | EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OFPVT); |
3774 | Op = GetSoftPromotedHalf(Op); |
3775 | Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl, |
3776 | NFPVT, Op); |
3777 | Op = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT, Operand: Op); |
3778 | SplitInteger(Op, Lo, Hi); |
3779 | return; |
3780 | } |
3781 | |
3782 | if (Op.getValueType() == MVT::bf16) { |
3783 | // Extend to f32 as there is no bf16 libcall. |
3784 | Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG); |
3785 | } |
3786 | |
3787 | RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(OpVT: Op.getValueType(), RetVT: VT) |
3788 | : RTLIB::getFPTOUINT(OpVT: Op.getValueType(), RetVT: VT); |
3789 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!" ); |
3790 | TargetLowering::MakeLibCallOptions CallOptions; |
3791 | CallOptions.setSExt(true); |
3792 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, |
3793 | CallOptions, dl, Chain); |
3794 | SplitInteger(Op: Tmp.first, Lo, Hi); |
3795 | |
3796 | if (IsStrict) |
3797 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
3798 | } |
3799 | |
3800 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
3801 | SDValue &Hi) { |
3802 | SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
3803 | SplitInteger(Op: Res, Lo, Hi); |
3804 | } |
3805 | |
3806 | void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo, |
3807 | SDValue &Hi) { |
3808 | SDLoc dl(N); |
3809 | bool IsStrict = N->isStrictFPOpcode(); |
3810 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
3811 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
3812 | |
3813 | assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && |
3814 | "Input type needs to be promoted!" ); |
3815 | |
3816 | EVT VT = Op.getValueType(); |
3817 | |
3818 | if (VT == MVT::f16) { |
3819 | // Extend to f32. |
3820 | VT = MVT::f32; |
3821 | Op = fpExtendHelper(Op, Chain, IsStrict, VT, DL: dl, DAG); |
3822 | } |
3823 | |
3824 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3825 | if (N->getOpcode() == ISD::LROUND || |
3826 | N->getOpcode() == ISD::STRICT_LROUND) { |
3827 | if (VT == MVT::f32) |
3828 | LC = RTLIB::LROUND_F32; |
3829 | else if (VT == MVT::f64) |
3830 | LC = RTLIB::LROUND_F64; |
3831 | else if (VT == MVT::f80) |
3832 | LC = RTLIB::LROUND_F80; |
3833 | else if (VT == MVT::f128) |
3834 | LC = RTLIB::LROUND_F128; |
3835 | else if (VT == MVT::ppcf128) |
3836 | LC = RTLIB::LROUND_PPCF128; |
3837 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!" ); |
3838 | } else if (N->getOpcode() == ISD::LRINT || |
3839 | N->getOpcode() == ISD::STRICT_LRINT) { |
3840 | if (VT == MVT::f32) |
3841 | LC = RTLIB::LRINT_F32; |
3842 | else if (VT == MVT::f64) |
3843 | LC = RTLIB::LRINT_F64; |
3844 | else if (VT == MVT::f80) |
3845 | LC = RTLIB::LRINT_F80; |
3846 | else if (VT == MVT::f128) |
3847 | LC = RTLIB::LRINT_F128; |
3848 | else if (VT == MVT::ppcf128) |
3849 | LC = RTLIB::LRINT_PPCF128; |
3850 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!" ); |
3851 | } else if (N->getOpcode() == ISD::LLROUND || |
3852 | N->getOpcode() == ISD::STRICT_LLROUND) { |
3853 | if (VT == MVT::f32) |
3854 | LC = RTLIB::LLROUND_F32; |
3855 | else if (VT == MVT::f64) |
3856 | LC = RTLIB::LLROUND_F64; |
3857 | else if (VT == MVT::f80) |
3858 | LC = RTLIB::LLROUND_F80; |
3859 | else if (VT == MVT::f128) |
3860 | LC = RTLIB::LLROUND_F128; |
3861 | else if (VT == MVT::ppcf128) |
3862 | LC = RTLIB::LLROUND_PPCF128; |
3863 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!" ); |
3864 | } else if (N->getOpcode() == ISD::LLRINT || |
3865 | N->getOpcode() == ISD::STRICT_LLRINT) { |
3866 | if (VT == MVT::f32) |
3867 | LC = RTLIB::LLRINT_F32; |
3868 | else if (VT == MVT::f64) |
3869 | LC = RTLIB::LLRINT_F64; |
3870 | else if (VT == MVT::f80) |
3871 | LC = RTLIB::LLRINT_F80; |
3872 | else if (VT == MVT::f128) |
3873 | LC = RTLIB::LLRINT_F128; |
3874 | else if (VT == MVT::ppcf128) |
3875 | LC = RTLIB::LLRINT_PPCF128; |
3876 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!" ); |
3877 | } else |
3878 | llvm_unreachable("Unexpected opcode!" ); |
3879 | |
3880 | EVT RetVT = N->getValueType(ResNo: 0); |
3881 | |
3882 | TargetLowering::MakeLibCallOptions CallOptions; |
3883 | CallOptions.setSExt(true); |
3884 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
3885 | Ops: Op, CallOptions, dl, |
3886 | Chain); |
3887 | SplitInteger(Op: Tmp.first, Lo, Hi); |
3888 | |
3889 | if (N->isStrictFPOpcode()) |
3890 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
3891 | } |
3892 | |
3893 | void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
3894 | SDValue &Lo, SDValue &Hi) { |
3895 | assert(!N->isAtomic() && "Should have been a ATOMIC_LOAD?" ); |
3896 | |
3897 | if (ISD::isNormalLoad(N)) { |
3898 | ExpandRes_NormalLoad(N, Lo, Hi); |
3899 | return; |
3900 | } |
3901 | |
3902 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!" ); |
3903 | |
3904 | EVT VT = N->getValueType(ResNo: 0); |
3905 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
3906 | SDValue Ch = N->getChain(); |
3907 | SDValue Ptr = N->getBasePtr(); |
3908 | ISD::LoadExtType ExtType = N->getExtensionType(); |
3909 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
3910 | AAMDNodes AAInfo = N->getAAInfo(); |
3911 | SDLoc dl(N); |
3912 | |
3913 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
3914 | |
3915 | if (N->getMemoryVT().bitsLE(VT: NVT)) { |
3916 | EVT MemVT = N->getMemoryVT(); |
3917 | |
3918 | Lo = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), MemVT, |
3919 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3920 | |
3921 | // Remember the chain. |
3922 | Ch = Lo.getValue(R: 1); |
3923 | |
3924 | if (ExtType == ISD::SEXTLOAD) { |
3925 | // The high part is obtained by SRA'ing all but one of the bits of the |
3926 | // lo part. |
3927 | unsigned LoSize = Lo.getValueSizeInBits(); |
3928 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
3929 | N2: DAG.getConstant(Val: LoSize - 1, DL: dl, |
3930 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3931 | } else if (ExtType == ISD::ZEXTLOAD) { |
3932 | // The high part is just a zero. |
3933 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
3934 | } else { |
3935 | assert(ExtType == ISD::EXTLOAD && "Unknown extload!" ); |
3936 | // The high part is undefined. |
3937 | Hi = DAG.getUNDEF(VT: NVT); |
3938 | } |
3939 | } else if (DAG.getDataLayout().isLittleEndian()) { |
3940 | // Little-endian - low bits are at low addresses. |
3941 | Lo = DAG.getLoad(VT: NVT, dl, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), |
3942 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3943 | |
3944 | unsigned ExcessBits = |
3945 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
3946 | EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits); |
3947 | |
3948 | // Increment the pointer to the other half. |
3949 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3950 | Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
3951 | Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, |
3952 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), MemVT: NEVT, |
3953 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3954 | |
3955 | // Build a factor node to remember that this load is independent of the |
3956 | // other one. |
3957 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
3958 | Hi.getValue(R: 1)); |
3959 | } else { |
3960 | // Big-endian - high bits are at low addresses. Favor aligned loads at |
3961 | // the cost of some bit-fiddling. |
3962 | EVT MemVT = N->getMemoryVT(); |
3963 | unsigned EBytes = MemVT.getStoreSize(); |
3964 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3965 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
3966 | |
3967 | // Load both the high bits and maybe some of the low bits. |
3968 | Hi = DAG.getExtLoad(ExtType, dl, VT: NVT, Chain: Ch, Ptr, PtrInfo: N->getPointerInfo(), |
3969 | MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
3970 | BitWidth: MemVT.getSizeInBits() - ExcessBits), |
3971 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3972 | |
3973 | // Increment the pointer to the other half. |
3974 | Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl); |
3975 | // Load the rest of the low bits. |
3976 | Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: NVT, Chain: Ch, Ptr, |
3977 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
3978 | MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits), |
3979 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
3980 | |
3981 | // Build a factor node to remember that this load is independent of the |
3982 | // other one. |
3983 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(R: 1), |
3984 | Hi.getValue(R: 1)); |
3985 | |
3986 | if (ExcessBits < NVT.getSizeInBits()) { |
3987 | // Transfer low bits from the bottom of Hi to the top of Lo. |
3988 | Lo = DAG.getNode( |
3989 | Opcode: ISD::OR, DL: dl, VT: NVT, N1: Lo, |
3990 | N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi, |
3991 | N2: DAG.getConstant(Val: ExcessBits, DL: dl, |
3992 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())))); |
3993 | // Move high bits to the right position in Hi. |
3994 | Hi = DAG.getNode(Opcode: ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, DL: dl, VT: NVT, |
3995 | N1: Hi, |
3996 | N2: DAG.getConstant(Val: NVT.getSizeInBits() - ExcessBits, DL: dl, |
3997 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
3998 | } |
3999 | } |
4000 | |
4001 | // Legalize the chain result - switch anything that used the old chain to |
4002 | // use the new one. |
4003 | ReplaceValueWith(From: SDValue(N, 1), To: Ch); |
4004 | } |
4005 | |
4006 | void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
4007 | SDValue &Lo, SDValue &Hi) { |
4008 | SDLoc dl(N); |
4009 | SDValue LL, LH, RL, RH; |
4010 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH); |
4011 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH); |
4012 | Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LL, N2: RL); |
4013 | Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LL.getValueType(), N1: LH, N2: RH); |
4014 | } |
4015 | |
4016 | void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
4017 | SDValue &Lo, SDValue &Hi) { |
4018 | EVT VT = N->getValueType(ResNo: 0); |
4019 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4020 | SDLoc dl(N); |
4021 | |
4022 | SDValue LL, LH, RL, RH; |
4023 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH); |
4024 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH); |
4025 | |
4026 | if (TLI.expandMUL(N, Lo, Hi, HiLoVT: NVT, DAG, |
4027 | Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
4028 | LL, LH, RL, RH)) |
4029 | return; |
4030 | |
4031 | // If nothing else, we can make a libcall. |
4032 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4033 | if (VT == MVT::i16) |
4034 | LC = RTLIB::MUL_I16; |
4035 | else if (VT == MVT::i32) |
4036 | LC = RTLIB::MUL_I32; |
4037 | else if (VT == MVT::i64) |
4038 | LC = RTLIB::MUL_I64; |
4039 | else if (VT == MVT::i128) |
4040 | LC = RTLIB::MUL_I128; |
4041 | |
4042 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC)) { |
4043 | // Perform a wide multiplication where the wide type is the original VT and |
4044 | // the 4 parts are the split arguments. |
4045 | TLI.forceExpandWideMUL(DAG, dl, /*Signed=*/true, WideVT: VT, LL, LH, RL, RH, Lo, |
4046 | Hi); |
4047 | return; |
4048 | } |
4049 | |
4050 | // Note that we don't need to do a wide MUL here since we don't care about the |
4051 | // upper half of the result if it exceeds VT. |
4052 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4053 | TargetLowering::MakeLibCallOptions CallOptions; |
4054 | CallOptions.setSExt(true); |
4055 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, |
4056 | Lo, Hi); |
4057 | } |
4058 | |
4059 | void DAGTypeLegalizer::ExpandIntRes_READCOUNTER(SDNode *N, SDValue &Lo, |
4060 | SDValue &Hi) { |
4061 | SDLoc DL(N); |
4062 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4063 | SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
4064 | SDValue R = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: VTs, N: N->getOperand(Num: 0)); |
4065 | Lo = R.getValue(R: 0); |
4066 | Hi = R.getValue(R: 1); |
4067 | ReplaceValueWith(From: SDValue(N, 1), To: R.getValue(R: 2)); |
4068 | } |
4069 | |
4070 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, |
4071 | SDValue &Hi) { |
4072 | SDValue Result = TLI.expandAddSubSat(Node: N, DAG); |
4073 | SplitInteger(Op: Result, Lo, Hi); |
4074 | } |
4075 | |
4076 | void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, |
4077 | SDValue &Hi) { |
4078 | SDValue Result = TLI.expandShlSat(Node: N, DAG); |
4079 | SplitInteger(Op: Result, Lo, Hi); |
4080 | } |
4081 | |
4082 | /// This performs an expansion of the integer result for a fixed point |
4083 | /// multiplication. The default expansion performs rounding down towards |
4084 | /// negative infinity, though targets that do care about rounding should specify |
4085 | /// a target hook for rounding and provide their own expansion or lowering of |
4086 | /// fixed point multiplication to be consistent with rounding. |
4087 | void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, |
4088 | SDValue &Hi) { |
4089 | SDLoc dl(N); |
4090 | EVT VT = N->getValueType(ResNo: 0); |
4091 | unsigned VTSize = VT.getScalarSizeInBits(); |
4092 | SDValue LHS = N->getOperand(Num: 0); |
4093 | SDValue RHS = N->getOperand(Num: 1); |
4094 | uint64_t Scale = N->getConstantOperandVal(Num: 2); |
4095 | bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || |
4096 | N->getOpcode() == ISD::UMULFIXSAT); |
4097 | bool Signed = (N->getOpcode() == ISD::SMULFIX || |
4098 | N->getOpcode() == ISD::SMULFIXSAT); |
4099 | |
4100 | // Handle special case when scale is equal to zero. |
4101 | if (!Scale) { |
4102 | SDValue Result; |
4103 | if (!Saturating) { |
4104 | Result = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS); |
4105 | } else { |
4106 | EVT BoolVT = getSetCCResultType(VT); |
4107 | unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; |
4108 | Result = DAG.getNode(Opcode: MulOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS); |
4109 | SDValue Product = Result.getValue(R: 0); |
4110 | SDValue Overflow = Result.getValue(R: 1); |
4111 | if (Signed) { |
4112 | APInt MinVal = APInt::getSignedMinValue(numBits: VTSize); |
4113 | APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize); |
4114 | SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT); |
4115 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT); |
4116 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
4117 | // Xor the inputs, if resulting sign bit is 0 the product will be |
4118 | // positive, else negative. |
4119 | SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS); |
4120 | SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT); |
4121 | Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax); |
4122 | Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product); |
4123 | } else { |
4124 | // For unsigned multiplication, we only need to check the max since we |
4125 | // can't really overflow towards zero. |
4126 | APInt MaxVal = APInt::getMaxValue(numBits: VTSize); |
4127 | SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT); |
4128 | Result = DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product); |
4129 | } |
4130 | } |
4131 | SplitInteger(Op: Result, Lo, Hi); |
4132 | return; |
4133 | } |
4134 | |
4135 | // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will |
4136 | // cover for unhandled cases below, while still being valid for UMULFIX[SAT]. |
4137 | assert(Scale <= VTSize && "Scale can't be larger than the value type size." ); |
4138 | |
4139 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4140 | SDValue LL, LH, RL, RH; |
4141 | GetExpandedInteger(Op: LHS, Lo&: LL, Hi&: LH); |
4142 | GetExpandedInteger(Op: RHS, Lo&: RL, Hi&: RH); |
4143 | SmallVector<SDValue, 4> Result; |
4144 | |
4145 | unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; |
4146 | if (!TLI.expandMUL_LOHI(Opcode: LoHiOp, VT, dl, LHS, RHS, Result, HiLoVT: NVT, DAG, |
4147 | Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
4148 | LL, LH, RL, RH)) { |
4149 | Result.clear(); |
4150 | Result.resize(N: 4); |
4151 | |
4152 | SDValue LoTmp, HiTmp; |
4153 | TLI.forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo&: LoTmp, Hi&: HiTmp); |
4154 | SplitInteger(Op: LoTmp, Lo&: Result[0], Hi&: Result[1]); |
4155 | SplitInteger(Op: HiTmp, Lo&: Result[2], Hi&: Result[3]); |
4156 | } |
4157 | assert(Result.size() == 4 && "Unexpected number of partlets in the result" ); |
4158 | |
4159 | unsigned NVTSize = NVT.getScalarSizeInBits(); |
4160 | assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " |
4161 | "the size of the current value type" ); |
4162 | |
4163 | // After getting the multiplication result in 4 parts, we need to perform a |
4164 | // shift right by the amount of the scale to get the result in that scale. |
4165 | // |
4166 | // Let's say we multiply 2 64 bit numbers. The resulting value can be held in |
4167 | // 128 bits that are cut into 4 32-bit parts: |
4168 | // |
4169 | // HH HL LH LL |
4170 | // |---32---|---32---|---32---|---32---| |
4171 | // 128 96 64 32 0 |
4172 | // |
4173 | // |------VTSize-----| |
4174 | // |
4175 | // |NVTSize-| |
4176 | // |
4177 | // The resulting Lo and Hi would normally be in LL and LH after the shift. But |
4178 | // to avoid unneccessary shifting of all 4 parts, we can adjust the shift |
4179 | // amount and get Lo and Hi using two funnel shifts. Or for the special case |
4180 | // when Scale is a multiple of NVTSize we can just pick the result without |
4181 | // shifting. |
4182 | uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed. |
4183 | if (Scale % NVTSize) { |
4184 | SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: Scale % NVTSize, VT: NVT, DL: dl); |
4185 | Lo = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 1], N2: Result[Part0], |
4186 | N3: ShiftAmount); |
4187 | Hi = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: NVT, N1: Result[Part0 + 2], N2: Result[Part0 + 1], |
4188 | N3: ShiftAmount); |
4189 | } else { |
4190 | Lo = Result[Part0]; |
4191 | Hi = Result[Part0 + 1]; |
4192 | } |
4193 | |
4194 | // Unless saturation is requested we are done. The result is in <Hi,Lo>. |
4195 | if (!Saturating) |
4196 | return; |
4197 | |
4198 | // Can not overflow when there is no integer part. |
4199 | if (Scale == VTSize) |
4200 | return; |
4201 | |
4202 | // To handle saturation we must check for overflow in the multiplication. |
4203 | // |
4204 | // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result) |
4205 | // aren't all zeroes. |
4206 | // |
4207 | // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result) |
4208 | // aren't all ones or all zeroes. |
4209 | // |
4210 | // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the |
4211 | // highest bit of HH determines saturation direction in the event of signed |
4212 | // saturation. |
4213 | |
4214 | SDValue ResultHL = Result[2]; |
4215 | SDValue ResultHH = Result[3]; |
4216 | |
4217 | SDValue SatMax, SatMin; |
4218 | SDValue NVTZero = DAG.getConstant(Val: 0, DL: dl, VT: NVT); |
4219 | SDValue NVTNeg1 = DAG.getConstant(Val: -1, DL: dl, VT: NVT); |
4220 | EVT BoolNVT = getSetCCResultType(VT: NVT); |
4221 | |
4222 | if (!Signed) { |
4223 | if (Scale < NVTSize) { |
4224 | // Overflow happened if ((HH | (HL >> Scale)) != 0). |
4225 | SDValue HLAdjusted = |
4226 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL, |
4227 | N2: DAG.getShiftAmountConstant(Val: Scale, VT: NVT, DL: dl)); |
4228 | SDValue Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: HLAdjusted, N2: ResultHH); |
4229 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: Tmp, RHS: NVTZero, Cond: ISD::SETNE); |
4230 | } else if (Scale == NVTSize) { |
4231 | // Overflow happened if (HH != 0). |
4232 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETNE); |
4233 | } else if (Scale < VTSize) { |
4234 | // Overflow happened if ((HH >> (Scale - NVTSize)) != 0). |
4235 | SDValue HLAdjusted = |
4236 | DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: ResultHL, |
4237 | N2: DAG.getShiftAmountConstant(Val: Scale - NVTSize, VT: NVT, DL: dl)); |
4238 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: HLAdjusted, RHS: NVTZero, Cond: ISD::SETNE); |
4239 | } else |
4240 | llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" |
4241 | "(and saturation can't happen with Scale==VTSize)." ); |
4242 | |
4243 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Hi); |
4244 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: NVTNeg1, RHS: Lo); |
4245 | return; |
4246 | } |
4247 | |
4248 | if (Scale < NVTSize) { |
4249 | // The number of overflow bits we can check are VTSize - Scale + 1 (we |
4250 | // include the sign bit). If these top bits are > 0, then we overflowed past |
4251 | // the max value. If these top bits are < -1, then we overflowed past the |
4252 | // min value. Otherwise, we did not overflow. |
4253 | unsigned OverflowBits = VTSize - Scale + 1; |
4254 | assert(OverflowBits <= VTSize && OverflowBits > NVTSize && |
4255 | "Extent of overflow bits must start within HL" ); |
4256 | SDValue HLHiMask = DAG.getConstant( |
4257 | Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits - NVTSize), DL: dl, VT: NVT); |
4258 | SDValue HLLoMask = DAG.getConstant( |
4259 | Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: VTSize - OverflowBits), DL: dl, VT: NVT); |
4260 | // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask). |
4261 | SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT); |
4262 | SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ); |
4263 | SDValue HLUGT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLLoMask, Cond: ISD::SETUGT); |
4264 | SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0, |
4265 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLUGT)); |
4266 | // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask). |
4267 | SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT); |
4268 | SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ); |
4269 | SDValue HLULT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: HLHiMask, Cond: ISD::SETULT); |
4270 | SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT, |
4271 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLULT)); |
4272 | } else if (Scale == NVTSize) { |
4273 | // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1). |
4274 | SDValue HHGT0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETGT); |
4275 | SDValue HHEQ0 = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTZero, Cond: ISD::SETEQ); |
4276 | SDValue HLNeg = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETLT); |
4277 | SatMax = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHGT0, |
4278 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ0, N2: HLNeg)); |
4279 | // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0). |
4280 | SDValue HHLT = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETLT); |
4281 | SDValue HHEQ = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: NVTNeg1, Cond: ISD::SETEQ); |
4282 | SDValue HLPos = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHL, RHS: NVTZero, Cond: ISD::SETGE); |
4283 | SatMin = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BoolNVT, N1: HHLT, |
4284 | N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolNVT, N1: HHEQ, N2: HLPos)); |
4285 | } else if (Scale < VTSize) { |
4286 | // This is similar to the case when we saturate if Scale < NVTSize, but we |
4287 | // only need to check HH. |
4288 | unsigned OverflowBits = VTSize - Scale + 1; |
4289 | SDValue HHHiMask = DAG.getConstant( |
4290 | Val: APInt::getHighBitsSet(numBits: NVTSize, hiBitsSet: OverflowBits), DL: dl, VT: NVT); |
4291 | SDValue HHLoMask = DAG.getConstant( |
4292 | Val: APInt::getLowBitsSet(numBits: NVTSize, loBitsSet: NVTSize - OverflowBits), DL: dl, VT: NVT); |
4293 | SatMax = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHLoMask, Cond: ISD::SETGT); |
4294 | SatMin = DAG.getSetCC(DL: dl, VT: BoolNVT, LHS: ResultHH, RHS: HHHiMask, Cond: ISD::SETLT); |
4295 | } else |
4296 | llvm_unreachable("Illegal scale for signed fixed point mul." ); |
4297 | |
4298 | // Saturate to signed maximum. |
4299 | APInt MaxHi = APInt::getSignedMaxValue(numBits: NVTSize); |
4300 | APInt MaxLo = APInt::getAllOnes(numBits: NVTSize); |
4301 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxHi, DL: dl, VT: NVT), RHS: Hi); |
4302 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMax, LHS: DAG.getConstant(Val: MaxLo, DL: dl, VT: NVT), RHS: Lo); |
4303 | // Saturate to signed minimum. |
4304 | APInt MinHi = APInt::getSignedMinValue(numBits: NVTSize); |
4305 | Hi = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: DAG.getConstant(Val: MinHi, DL: dl, VT: NVT), RHS: Hi); |
4306 | Lo = DAG.getSelect(DL: dl, VT: NVT, Cond: SatMin, LHS: NVTZero, RHS: Lo); |
4307 | } |
4308 | |
4309 | void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, |
4310 | SDValue &Hi) { |
4311 | SDLoc dl(N); |
4312 | // Try expanding in the existing type first. |
4313 | SDValue Res = TLI.expandFixedPointDiv(Opcode: N->getOpcode(), dl, LHS: N->getOperand(Num: 0), |
4314 | RHS: N->getOperand(Num: 1), |
4315 | Scale: N->getConstantOperandVal(Num: 2), DAG); |
4316 | |
4317 | if (!Res) |
4318 | Res = earlyExpandDIVFIX(N, LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), |
4319 | Scale: N->getConstantOperandVal(Num: 2), TLI, DAG); |
4320 | SplitInteger(Op: Res, Lo, Hi); |
4321 | } |
4322 | |
4323 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
4324 | SDValue &Lo, SDValue &Hi) { |
4325 | assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && |
4326 | "Node has unexpected Opcode" ); |
4327 | SDValue LHS = Node->getOperand(Num: 0); |
4328 | SDValue RHS = Node->getOperand(Num: 1); |
4329 | SDLoc dl(Node); |
4330 | |
4331 | SDValue Ovf; |
4332 | |
4333 | bool IsAdd = Node->getOpcode() == ISD::SADDO; |
4334 | unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; |
4335 | |
4336 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
4337 | Op: CarryOp, VT: TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: LHS.getValueType())); |
4338 | |
4339 | if (HasCarryOp) { |
4340 | // Expand the subcomponents. |
4341 | SDValue LHSL, LHSH, RHSL, RHSH; |
4342 | GetExpandedInteger(Op: LHS, Lo&: LHSL, Hi&: LHSH); |
4343 | GetExpandedInteger(Op: RHS, Lo&: RHSL, Hi&: RHSH); |
4344 | SDVTList VTList = DAG.getVTList(VT1: LHSL.getValueType(), VT2: Node->getValueType(ResNo: 1)); |
4345 | |
4346 | Lo = DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: dl, VTList, Ops: {LHSL, RHSL}); |
4347 | Hi = DAG.getNode(Opcode: CarryOp, DL: dl, VTList, Ops: { LHSH, RHSH, Lo.getValue(R: 1) }); |
4348 | |
4349 | Ovf = Hi.getValue(R: 1); |
4350 | } else { |
4351 | // Expand the result by simply replacing it with the equivalent |
4352 | // non-overflow-checking operation. |
4353 | SDValue Sum = DAG.getNode(Opcode: Node->getOpcode() == ISD::SADDO ? |
4354 | ISD::ADD : ISD::SUB, DL: dl, VT: LHS.getValueType(), |
4355 | N1: LHS, N2: RHS); |
4356 | SplitInteger(Op: Sum, Lo, Hi); |
4357 | |
4358 | // Compute the overflow. |
4359 | // |
4360 | // LHSSign -> LHS < 0 |
4361 | // RHSSign -> RHS < 0 |
4362 | // SumSign -> Sum < 0 |
4363 | // |
4364 | // Add: |
4365 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
4366 | // Sub: |
4367 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
4368 | // |
4369 | // To get better codegen we can rewrite this by doing bitwise math on |
4370 | // the integers and extract the final sign bit at the end. So the |
4371 | // above becomes: |
4372 | // |
4373 | // Add: |
4374 | // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0 |
4375 | // Sub: |
4376 | // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0 |
4377 | // |
4378 | // NOTE: This is different than the expansion we do in expandSADDSUBO |
4379 | // because it is more costly to determine the RHS is > 0 for SSUBO with the |
4380 | // integers split. |
4381 | EVT VT = LHS.getValueType(); |
4382 | SDValue SignsMatch = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS); |
4383 | if (IsAdd) |
4384 | SignsMatch = DAG.getNOT(DL: dl, Val: SignsMatch, VT); |
4385 | |
4386 | SDValue SumSignNE = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: Sum); |
4387 | Ovf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SignsMatch, N2: SumSignNE); |
4388 | EVT OType = Node->getValueType(ResNo: 1); |
4389 | Ovf = DAG.getSetCC(DL: dl, VT: OType, LHS: Ovf, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT); |
4390 | } |
4391 | |
4392 | // Use the calculated overflow everywhere. |
4393 | ReplaceValueWith(From: SDValue(Node, 1), To: Ovf); |
4394 | } |
4395 | |
4396 | void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
4397 | SDValue &Lo, SDValue &Hi) { |
4398 | EVT VT = N->getValueType(ResNo: 0); |
4399 | SDLoc dl(N); |
4400 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4401 | |
4402 | if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) { |
4403 | SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4404 | SplitInteger(Op: Res.getValue(R: 0), Lo, Hi); |
4405 | return; |
4406 | } |
4407 | |
4408 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4409 | if (VT == MVT::i16) |
4410 | LC = RTLIB::SDIV_I16; |
4411 | else if (VT == MVT::i32) |
4412 | LC = RTLIB::SDIV_I32; |
4413 | else if (VT == MVT::i64) |
4414 | LC = RTLIB::SDIV_I64; |
4415 | else if (VT == MVT::i128) |
4416 | LC = RTLIB::SDIV_I128; |
4417 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!" ); |
4418 | |
4419 | TargetLowering::MakeLibCallOptions CallOptions; |
4420 | CallOptions.setSExt(true); |
4421 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4422 | } |
4423 | |
4424 | void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(SDNode *N, SDValue &Lo, |
4425 | SDValue &Hi) { |
4426 | SDLoc dl(N); |
4427 | SDValue Shiftee = N->getOperand(Num: 0); |
4428 | EVT VT = Shiftee.getValueType(); |
4429 | SDValue ShAmt = N->getOperand(Num: 1); |
4430 | EVT ShAmtVT = ShAmt.getValueType(); |
4431 | |
4432 | // This legalization is optimal when the shift is by a multiple of byte width, |
4433 | // %x * 8 <-> %x << 3 so 3 low bits should be be known zero. |
4434 | bool ShiftByByteMultiple = |
4435 | DAG.computeKnownBits(Op: ShAmt).countMinTrailingZeros() >= 3; |
4436 | |
4437 | // If we can't do it as one step, we'll have two uses of shift amount, |
4438 | // and thus must freeze it. |
4439 | if (!ShiftByByteMultiple) |
4440 | ShAmt = DAG.getFreeze(V: ShAmt); |
4441 | |
4442 | unsigned VTBitWidth = VT.getScalarSizeInBits(); |
4443 | assert(VTBitWidth % 8 == 0 && "Shifting a not byte multiple value?" ); |
4444 | unsigned VTByteWidth = VTBitWidth / 8; |
4445 | assert(isPowerOf2_32(VTByteWidth) && |
4446 | "Shiftee type size is not a power of two!" ); |
4447 | unsigned StackSlotByteWidth = 2 * VTByteWidth; |
4448 | unsigned StackSlotBitWidth = 8 * StackSlotByteWidth; |
4449 | EVT StackSlotVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StackSlotBitWidth); |
4450 | |
4451 | // Get a temporary stack slot 2x the width of our VT. |
4452 | // FIXME: reuse stack slots? |
4453 | // FIXME: should we be more picky about alignment? |
4454 | Align StackSlotAlignment(1); |
4455 | SDValue StackPtr = DAG.CreateStackTemporary( |
4456 | Bytes: TypeSize::getFixed(ExactSize: StackSlotByteWidth), Alignment: StackSlotAlignment); |
4457 | EVT PtrTy = StackPtr.getValueType(); |
4458 | SDValue Ch = DAG.getEntryNode(); |
4459 | |
4460 | MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack( |
4461 | MF&: DAG.getMachineFunction(), |
4462 | FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex()); |
4463 | |
4464 | // Extend the value, that is being shifted, to the entire stack slot's width. |
4465 | SDValue Init; |
4466 | if (N->getOpcode() != ISD::SHL) { |
4467 | unsigned WideningOpc = |
4468 | N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4469 | Init = DAG.getNode(Opcode: WideningOpc, DL: dl, VT: StackSlotVT, Operand: Shiftee); |
4470 | } else { |
4471 | // For left-shifts, pad the Shiftee's LSB with zeros to twice it's width. |
4472 | SDValue AllZeros = DAG.getConstant(Val: 0, DL: dl, VT); |
4473 | Init = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: StackSlotVT, N1: AllZeros, N2: Shiftee); |
4474 | } |
4475 | // And spill it into the stack slot. |
4476 | Ch = DAG.getStore(Chain: Ch, dl, Val: Init, Ptr: StackPtr, PtrInfo: StackPtrInfo, Alignment: StackSlotAlignment); |
4477 | |
4478 | // Now, compute the full-byte offset into stack slot from where we can load. |
4479 | // We have shift amount, which is in bits, but in multiples of byte. |
4480 | // So just divide by CHAR_BIT. |
4481 | SDNodeFlags Flags; |
4482 | if (ShiftByByteMultiple) |
4483 | Flags.setExact(true); |
4484 | SDValue ByteOffset = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShAmtVT, N1: ShAmt, |
4485 | N2: DAG.getConstant(Val: 3, DL: dl, VT: ShAmtVT), Flags); |
4486 | // And clamp it, because OOB load is an immediate UB, |
4487 | // while shift overflow would have *just* been poison. |
4488 | ByteOffset = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ByteOffset, |
4489 | N2: DAG.getConstant(Val: VTByteWidth - 1, DL: dl, VT: ShAmtVT)); |
4490 | // We have exactly two strategies on indexing into stack slot here: |
4491 | // 1. upwards starting from the beginning of the slot |
4492 | // 2. downwards starting from the middle of the slot |
4493 | // On little-endian machine, we pick 1. for right shifts and 2. for left-shift |
4494 | // and vice versa on big-endian machine. |
4495 | bool WillIndexUpwards = N->getOpcode() != ISD::SHL; |
4496 | if (DAG.getDataLayout().isBigEndian()) |
4497 | WillIndexUpwards = !WillIndexUpwards; |
4498 | |
4499 | SDValue AdjStackPtr; |
4500 | if (WillIndexUpwards) { |
4501 | AdjStackPtr = StackPtr; |
4502 | } else { |
4503 | AdjStackPtr = DAG.getMemBasePlusOffset( |
4504 | Base: StackPtr, Offset: DAG.getConstant(Val: VTByteWidth, DL: dl, VT: PtrTy), DL: dl); |
4505 | ByteOffset = DAG.getNegative(Val: ByteOffset, DL: dl, VT: ShAmtVT); |
4506 | } |
4507 | |
4508 | // Get the pointer somewhere into the stack slot from which we need to load. |
4509 | ByteOffset = DAG.getSExtOrTrunc(Op: ByteOffset, DL: dl, VT: PtrTy); |
4510 | AdjStackPtr = DAG.getMemBasePlusOffset(Base: AdjStackPtr, Offset: ByteOffset, DL: dl); |
4511 | |
4512 | // And load it! While the load is not legal, legalizing it is obvious. |
4513 | SDValue Res = DAG.getLoad( |
4514 | VT, dl, Chain: Ch, Ptr: AdjStackPtr, |
4515 | PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), Alignment: Align(1)); |
4516 | // We've performed the shift by a CHAR_BIT * [_ShAmt / CHAR_BIT_] |
4517 | |
4518 | // If we may still have a less-than-CHAR_BIT to shift by, do so now. |
4519 | if (!ShiftByByteMultiple) { |
4520 | SDValue ShAmtRem = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt, |
4521 | N2: DAG.getConstant(Val: 7, DL: dl, VT: ShAmtVT)); |
4522 | Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: Res, N2: ShAmtRem); |
4523 | } |
4524 | |
4525 | // Finally, split the computed value. |
4526 | SplitInteger(Op: Res, Lo, Hi); |
4527 | } |
4528 | |
4529 | void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
4530 | SDValue &Lo, SDValue &Hi) { |
4531 | EVT VT = N->getValueType(ResNo: 0); |
4532 | unsigned Opc = N->getOpcode(); |
4533 | SDLoc dl(N); |
4534 | |
4535 | // If we can emit an efficient shift operation, do so now. Check to see if |
4536 | // the RHS is a constant. |
4537 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) |
4538 | return ExpandShiftByConstant(N, Amt: CN->getAPIntValue(), Lo, Hi); |
4539 | |
4540 | // If we can determine that the high bit of the shift is zero or one, even if |
4541 | // the low bits are variable, emit this shift in an optimized form. |
4542 | if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) |
4543 | return; |
4544 | |
4545 | // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. |
4546 | unsigned PartsOpc; |
4547 | if (Opc == ISD::SHL) { |
4548 | PartsOpc = ISD::SHL_PARTS; |
4549 | } else if (Opc == ISD::SRL) { |
4550 | PartsOpc = ISD::SRL_PARTS; |
4551 | } else { |
4552 | assert(Opc == ISD::SRA && "Unknown shift!" ); |
4553 | PartsOpc = ISD::SRA_PARTS; |
4554 | } |
4555 | |
4556 | // Next check to see if the target supports this SHL_PARTS operation or if it |
4557 | // will custom expand it. Don't lower this to SHL_PARTS when we optimise for |
4558 | // size, but create a libcall instead. |
4559 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
4560 | TargetLowering::LegalizeAction Action = TLI.getOperationAction(Op: PartsOpc, VT: NVT); |
4561 | const bool LegalOrCustom = |
4562 | (Action == TargetLowering::Legal && TLI.isTypeLegal(VT: NVT)) || |
4563 | Action == TargetLowering::Custom; |
4564 | |
4565 | unsigned ExpansionFactor = 1; |
4566 | // That VT->NVT expansion is one step. But will we re-expand NVT? |
4567 | for (EVT TmpVT = NVT;;) { |
4568 | EVT NewTMPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: TmpVT); |
4569 | if (NewTMPVT == TmpVT) |
4570 | break; |
4571 | TmpVT = NewTMPVT; |
4572 | ++ExpansionFactor; |
4573 | } |
4574 | |
4575 | TargetLowering::ShiftLegalizationStrategy S = |
4576 | TLI.preferredShiftLegalizationStrategy(DAG, N, ExpansionFactor); |
4577 | |
4578 | if (S == TargetLowering::ShiftLegalizationStrategy::ExpandThroughStack) |
4579 | return ExpandIntRes_ShiftThroughStack(N, Lo, Hi); |
4580 | |
4581 | if (LegalOrCustom && |
4582 | S != TargetLowering::ShiftLegalizationStrategy::LowerToLibcall) { |
4583 | // Expand the subcomponents. |
4584 | SDValue LHSL, LHSH; |
4585 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: LHSL, Hi&: LHSH); |
4586 | EVT VT = LHSL.getValueType(); |
4587 | |
4588 | // If the shift amount operand is coming from a vector legalization it may |
4589 | // have an illegal type. Fix that first by casting the operand, otherwise |
4590 | // the new SHL_PARTS operation would need further legalization. |
4591 | SDValue ShiftOp = N->getOperand(Num: 1); |
4592 | EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout()); |
4593 | if (ShiftOp.getValueType() != ShiftTy) |
4594 | ShiftOp = DAG.getZExtOrTrunc(Op: ShiftOp, DL: dl, VT: ShiftTy); |
4595 | |
4596 | SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
4597 | Lo = DAG.getNode(Opcode: PartsOpc, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4598 | Hi = Lo.getValue(R: 1); |
4599 | return; |
4600 | } |
4601 | |
4602 | // Otherwise, emit a libcall. |
4603 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4604 | bool isSigned; |
4605 | if (Opc == ISD::SHL) { |
4606 | isSigned = false; /*sign irrelevant*/ |
4607 | if (VT == MVT::i16) |
4608 | LC = RTLIB::SHL_I16; |
4609 | else if (VT == MVT::i32) |
4610 | LC = RTLIB::SHL_I32; |
4611 | else if (VT == MVT::i64) |
4612 | LC = RTLIB::SHL_I64; |
4613 | else if (VT == MVT::i128) |
4614 | LC = RTLIB::SHL_I128; |
4615 | } else if (Opc == ISD::SRL) { |
4616 | isSigned = false; |
4617 | if (VT == MVT::i16) |
4618 | LC = RTLIB::SRL_I16; |
4619 | else if (VT == MVT::i32) |
4620 | LC = RTLIB::SRL_I32; |
4621 | else if (VT == MVT::i64) |
4622 | LC = RTLIB::SRL_I64; |
4623 | else if (VT == MVT::i128) |
4624 | LC = RTLIB::SRL_I128; |
4625 | } else { |
4626 | assert(Opc == ISD::SRA && "Unknown shift!" ); |
4627 | isSigned = true; |
4628 | if (VT == MVT::i16) |
4629 | LC = RTLIB::SRA_I16; |
4630 | else if (VT == MVT::i32) |
4631 | LC = RTLIB::SRA_I32; |
4632 | else if (VT == MVT::i64) |
4633 | LC = RTLIB::SRA_I64; |
4634 | else if (VT == MVT::i128) |
4635 | LC = RTLIB::SRA_I128; |
4636 | } |
4637 | |
4638 | if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(Call: LC)) { |
4639 | EVT ShAmtTy = |
4640 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize()); |
4641 | SDValue ShAmt = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl, VT: ShAmtTy); |
4642 | SDValue Ops[2] = {N->getOperand(Num: 0), ShAmt}; |
4643 | TargetLowering::MakeLibCallOptions CallOptions; |
4644 | CallOptions.setSExt(isSigned); |
4645 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4646 | return; |
4647 | } |
4648 | |
4649 | if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) |
4650 | llvm_unreachable("Unsupported shift!" ); |
4651 | } |
4652 | |
4653 | void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
4654 | SDValue &Lo, SDValue &Hi) { |
4655 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4656 | SDLoc dl(N); |
4657 | SDValue Op = N->getOperand(Num: 0); |
4658 | if (Op.getValueType().bitsLE(VT: NVT)) { |
4659 | // The low part is sign extension of the input (degenerates to a copy). |
4660 | Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4661 | // The high part is obtained by SRA'ing all but one of the bits of low part. |
4662 | unsigned LoSize = NVT.getSizeInBits(); |
4663 | Hi = DAG.getNode( |
4664 | Opcode: ISD::SRA, DL: dl, VT: NVT, N1: Lo, |
4665 | N2: DAG.getConstant(Val: LoSize - 1, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4666 | } else { |
4667 | // For example, extension of an i48 to an i64. The operand type necessarily |
4668 | // promotes to the result type, so will end up being expanded too. |
4669 | assert(getTypeAction(Op.getValueType()) == |
4670 | TargetLowering::TypePromoteInteger && |
4671 | "Only know how to promote this result!" ); |
4672 | SDValue Res = GetPromotedInteger(Op); |
4673 | assert(Res.getValueType() == N->getValueType(0) && |
4674 | "Operand over promoted?" ); |
4675 | // Split the promoted operand. This will simplify when it is expanded. |
4676 | SplitInteger(Op: Res, Lo, Hi); |
4677 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
4678 | Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi, |
4679 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
4680 | BitWidth: ExcessBits))); |
4681 | } |
4682 | } |
4683 | |
4684 | void DAGTypeLegalizer:: |
4685 | ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
4686 | SDLoc dl(N); |
4687 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
4688 | EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT(); |
4689 | |
4690 | if (EVT.bitsLE(VT: Lo.getValueType())) { |
4691 | // sext_inreg the low part if needed. |
4692 | Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Lo.getValueType(), N1: Lo, |
4693 | N2: N->getOperand(Num: 1)); |
4694 | |
4695 | // The high part gets the sign extension from the lo-part. This handles |
4696 | // things like sextinreg V:i64 from i8. |
4697 | Hi = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: Hi.getValueType(), N1: Lo, |
4698 | N2: DAG.getConstant(Val: Hi.getValueSizeInBits() - 1, DL: dl, |
4699 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4700 | } else { |
4701 | // For example, extension of an i48 to an i64. Leave the low part alone, |
4702 | // sext_inreg the high part. |
4703 | unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
4704 | Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: Hi.getValueType(), N1: Hi, |
4705 | N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), |
4706 | BitWidth: ExcessBits))); |
4707 | } |
4708 | } |
4709 | |
4710 | void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
4711 | SDValue &Lo, SDValue &Hi) { |
4712 | EVT VT = N->getValueType(ResNo: 0); |
4713 | SDLoc dl(N); |
4714 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4715 | |
4716 | if (TLI.getOperationAction(Op: ISD::SDIVREM, VT) == TargetLowering::Custom) { |
4717 | SDValue Res = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4718 | SplitInteger(Op: Res.getValue(R: 1), Lo, Hi); |
4719 | return; |
4720 | } |
4721 | |
4722 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4723 | if (VT == MVT::i16) |
4724 | LC = RTLIB::SREM_I16; |
4725 | else if (VT == MVT::i32) |
4726 | LC = RTLIB::SREM_I32; |
4727 | else if (VT == MVT::i64) |
4728 | LC = RTLIB::SREM_I64; |
4729 | else if (VT == MVT::i128) |
4730 | LC = RTLIB::SREM_I128; |
4731 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!" ); |
4732 | |
4733 | TargetLowering::MakeLibCallOptions CallOptions; |
4734 | CallOptions.setSExt(true); |
4735 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4736 | } |
4737 | |
4738 | void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
4739 | SDValue &Lo, SDValue &Hi) { |
4740 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4741 | SDLoc dl(N); |
4742 | Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4743 | Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: N->getOperand(Num: 0).getValueType(), |
4744 | N1: N->getOperand(Num: 0), |
4745 | N2: DAG.getConstant(Val: NVT.getSizeInBits(), DL: dl, |
4746 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
4747 | Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: NVT, Operand: Hi); |
4748 | } |
4749 | |
4750 | void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
4751 | SDValue &Lo, SDValue &Hi) { |
4752 | EVT VT = N->getValueType(ResNo: 0); |
4753 | SDLoc dl(N); |
4754 | |
4755 | if (N->getOpcode() == ISD::UMULO) { |
4756 | // This section expands the operation into the following sequence of |
4757 | // instructions. `iNh` here refers to a type which has half the bit width of |
4758 | // the type the original operation operated on. |
4759 | // |
4760 | // %0 = %LHS.HI != 0 && %RHS.HI != 0 |
4761 | // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO) |
4762 | // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO) |
4763 | // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN) |
4764 | // %4 = add iNh %1.0, %2.0 as iN |
4765 | // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH) |
4766 | // |
4767 | // %lo = %3.LO |
4768 | // %hi = %5.0 |
4769 | // %ovf = %0 || %1.1 || %2.1 || %5.1 |
4770 | SDValue LHS = N->getOperand(Num: 0), RHS = N->getOperand(Num: 1); |
4771 | SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; |
4772 | GetExpandedInteger(Op: LHS, Lo&: LHSLow, Hi&: LHSHigh); |
4773 | GetExpandedInteger(Op: RHS, Lo&: RHSLow, Hi&: RHSHigh); |
4774 | EVT HalfVT = LHSLow.getValueType(); |
4775 | EVT BitVT = N->getValueType(ResNo: 1); |
4776 | SDVTList VTHalfWithO = DAG.getVTList(VT1: HalfVT, VT2: BitVT); |
4777 | |
4778 | SDValue HalfZero = DAG.getConstant(Val: 0, DL: dl, VT: HalfVT); |
4779 | SDValue Overflow = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BitVT, |
4780 | N1: DAG.getSetCC(DL: dl, VT: BitVT, LHS: LHSHigh, RHS: HalfZero, Cond: ISD::SETNE), |
4781 | N2: DAG.getSetCC(DL: dl, VT: BitVT, LHS: RHSHigh, RHS: HalfZero, Cond: ISD::SETNE)); |
4782 | |
4783 | SDValue One = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: LHSHigh, N2: RHSLow); |
4784 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: One.getValue(R: 1)); |
4785 | |
4786 | SDValue Two = DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: VTHalfWithO, N1: RHSHigh, N2: LHSLow); |
4787 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Two.getValue(R: 1)); |
4788 | |
4789 | SDValue HighSum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HalfVT, N1: One, N2: Two); |
4790 | |
4791 | // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not |
4792 | // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this |
4793 | // operation recursively legalized?). |
4794 | // |
4795 | // Many backends understand this pattern and will convert into LOHI |
4796 | // themselves, if applicable. |
4797 | SDValue Three = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, |
4798 | N1: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LHSLow), |
4799 | N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RHSLow)); |
4800 | SplitInteger(Op: Three, Lo, Hi); |
4801 | |
4802 | Hi = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: VTHalfWithO, N1: Hi, N2: HighSum); |
4803 | Overflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: BitVT, N1: Overflow, N2: Hi.getValue(R: 1)); |
4804 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
4805 | return; |
4806 | } |
4807 | |
4808 | Type *RetTy = VT.getTypeForEVT(Context&: *DAG.getContext()); |
4809 | EVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout()); |
4810 | Type *PtrTy = PtrVT.getTypeForEVT(Context&: *DAG.getContext()); |
4811 | |
4812 | // Replace this with a libcall that will check overflow. |
4813 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4814 | if (VT == MVT::i32) |
4815 | LC = RTLIB::MULO_I32; |
4816 | else if (VT == MVT::i64) |
4817 | LC = RTLIB::MULO_I64; |
4818 | else if (VT == MVT::i128) |
4819 | LC = RTLIB::MULO_I128; |
4820 | |
4821 | // If we don't have the libcall or if the function we are compiling is the |
4822 | // implementation of the expected libcall (avoid inf-loop), expand inline. |
4823 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(Call: LC) || |
4824 | TLI.getLibcallName(Call: LC) == DAG.getMachineFunction().getName()) { |
4825 | // FIXME: This is not an optimal expansion, but better than crashing. |
4826 | EVT WideVT = |
4827 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getScalarSizeInBits() * 2); |
4828 | SDValue LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: WideVT, Operand: N->getOperand(Num: 0)); |
4829 | SDValue RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: WideVT, Operand: N->getOperand(Num: 1)); |
4830 | SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS); |
4831 | SDValue MulLo, MulHi; |
4832 | SplitInteger(Op: Mul, Lo&: MulLo, Hi&: MulHi); |
4833 | SDValue SRA = |
4834 | DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: MulLo, |
4835 | N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1, DL: dl, VT)); |
4836 | SDValue Overflow = |
4837 | DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: MulHi, RHS: SRA, Cond: ISD::SETNE); |
4838 | SplitInteger(Op: MulLo, Lo, Hi); |
4839 | ReplaceValueWith(From: SDValue(N, 1), To: Overflow); |
4840 | return; |
4841 | } |
4842 | |
4843 | SDValue Temp = DAG.CreateStackTemporary(VT: PtrVT); |
4844 | // Temporary for the overflow value, default it to zero. |
4845 | SDValue Chain = |
4846 | DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), Ptr: Temp, |
4847 | PtrInfo: MachinePointerInfo()); |
4848 | |
4849 | TargetLowering::ArgListTy Args; |
4850 | TargetLowering::ArgListEntry Entry; |
4851 | for (const SDValue &Op : N->op_values()) { |
4852 | EVT ArgVT = Op.getValueType(); |
4853 | Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext()); |
4854 | Entry.Node = Op; |
4855 | Entry.Ty = ArgTy; |
4856 | Entry.IsSExt = true; |
4857 | Entry.IsZExt = false; |
4858 | Args.push_back(x: Entry); |
4859 | } |
4860 | |
4861 | // Also pass the address of the overflow check. |
4862 | Entry.Node = Temp; |
4863 | Entry.Ty = PointerType::getUnqual(C&: PtrTy->getContext()); |
4864 | Entry.IsSExt = true; |
4865 | Entry.IsZExt = false; |
4866 | Args.push_back(x: Entry); |
4867 | |
4868 | SDValue Func = DAG.getExternalSymbol(Sym: TLI.getLibcallName(Call: LC), VT: PtrVT); |
4869 | |
4870 | TargetLowering::CallLoweringInfo CLI(DAG); |
4871 | CLI.setDebugLoc(dl) |
4872 | .setChain(Chain) |
4873 | .setLibCallee(CC: TLI.getLibcallCallingConv(Call: LC), ResultType: RetTy, Target: Func, ArgsList: std::move(Args)) |
4874 | .setSExtResult(); |
4875 | |
4876 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
4877 | |
4878 | SplitInteger(Op: CallInfo.first, Lo, Hi); |
4879 | SDValue Temp2 = |
4880 | DAG.getLoad(VT: PtrVT, dl, Chain: CallInfo.second, Ptr: Temp, PtrInfo: MachinePointerInfo()); |
4881 | SDValue Ofl = DAG.getSetCC(DL: dl, VT: N->getValueType(ResNo: 1), LHS: Temp2, |
4882 | RHS: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), |
4883 | Cond: ISD::SETNE); |
4884 | // Use the overflow from the libcall everywhere. |
4885 | ReplaceValueWith(From: SDValue(N, 1), To: Ofl); |
4886 | } |
4887 | |
4888 | void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
4889 | SDValue &Lo, SDValue &Hi) { |
4890 | EVT VT = N->getValueType(ResNo: 0); |
4891 | SDLoc dl(N); |
4892 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4893 | |
4894 | if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4895 | SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4896 | SplitInteger(Op: Res.getValue(R: 0), Lo, Hi); |
4897 | return; |
4898 | } |
4899 | |
4900 | // Try to expand UDIV by constant. |
4901 | if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) { |
4902 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4903 | // Only if the new type is legal. |
4904 | if (isTypeLegal(VT: NVT)) { |
4905 | SDValue InL, InH; |
4906 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
4907 | SmallVector<SDValue> Result; |
4908 | if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) { |
4909 | Lo = Result[0]; |
4910 | Hi = Result[1]; |
4911 | return; |
4912 | } |
4913 | } |
4914 | } |
4915 | |
4916 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4917 | if (VT == MVT::i16) |
4918 | LC = RTLIB::UDIV_I16; |
4919 | else if (VT == MVT::i32) |
4920 | LC = RTLIB::UDIV_I32; |
4921 | else if (VT == MVT::i64) |
4922 | LC = RTLIB::UDIV_I64; |
4923 | else if (VT == MVT::i128) |
4924 | LC = RTLIB::UDIV_I128; |
4925 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!" ); |
4926 | |
4927 | TargetLowering::MakeLibCallOptions CallOptions; |
4928 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4929 | } |
4930 | |
4931 | void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
4932 | SDValue &Lo, SDValue &Hi) { |
4933 | EVT VT = N->getValueType(ResNo: 0); |
4934 | SDLoc dl(N); |
4935 | SDValue Ops[2] = { N->getOperand(Num: 0), N->getOperand(Num: 1) }; |
4936 | |
4937 | if (TLI.getOperationAction(Op: ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4938 | SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), Ops); |
4939 | SplitInteger(Op: Res.getValue(R: 1), Lo, Hi); |
4940 | return; |
4941 | } |
4942 | |
4943 | // Try to expand UREM by constant. |
4944 | if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) { |
4945 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4946 | // Only if the new type is legal. |
4947 | if (isTypeLegal(VT: NVT)) { |
4948 | SDValue InL, InH; |
4949 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
4950 | SmallVector<SDValue> Result; |
4951 | if (TLI.expandDIVREMByConstant(N, Result, HiLoVT: NVT, DAG, LL: InL, LH: InH)) { |
4952 | Lo = Result[0]; |
4953 | Hi = Result[1]; |
4954 | return; |
4955 | } |
4956 | } |
4957 | } |
4958 | |
4959 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4960 | if (VT == MVT::i16) |
4961 | LC = RTLIB::UREM_I16; |
4962 | else if (VT == MVT::i32) |
4963 | LC = RTLIB::UREM_I32; |
4964 | else if (VT == MVT::i64) |
4965 | LC = RTLIB::UREM_I64; |
4966 | else if (VT == MVT::i128) |
4967 | LC = RTLIB::UREM_I128; |
4968 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!" ); |
4969 | |
4970 | TargetLowering::MakeLibCallOptions CallOptions; |
4971 | SplitInteger(Op: TLI.makeLibCall(DAG, LC, RetVT: VT, Ops, CallOptions, dl).first, Lo, Hi); |
4972 | } |
4973 | |
4974 | void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
4975 | SDValue &Lo, SDValue &Hi) { |
4976 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
4977 | SDLoc dl(N); |
4978 | SDValue Op = N->getOperand(Num: 0); |
4979 | if (Op.getValueType().bitsLE(VT: NVT)) { |
4980 | // The low part is zero extension of the input (degenerates to a copy). |
4981 | Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
4982 | Hi = DAG.getConstant(Val: 0, DL: dl, VT: NVT); // The high part is just a zero. |
4983 | } else { |
4984 | // For example, extension of an i48 to an i64. The operand type necessarily |
4985 | // promotes to the result type, so will end up being expanded too. |
4986 | assert(getTypeAction(Op.getValueType()) == |
4987 | TargetLowering::TypePromoteInteger && |
4988 | "Only know how to promote this result!" ); |
4989 | SDValue Res = GetPromotedInteger(Op); |
4990 | assert(Res.getValueType() == N->getValueType(0) && |
4991 | "Operand over promoted?" ); |
4992 | // Split the promoted operand. This will simplify when it is expanded. |
4993 | SplitInteger(Op: Res, Lo, Hi); |
4994 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
4995 | Hi = DAG.getZeroExtendInReg(Op: Hi, DL: dl, |
4996 | VT: EVT::getIntegerVT(Context&: *DAG.getContext(), |
4997 | BitWidth: ExcessBits)); |
4998 | } |
4999 | } |
5000 | |
5001 | void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
5002 | SDValue &Lo, SDValue &Hi) { |
5003 | SDLoc dl(N); |
5004 | EVT VT = cast<AtomicSDNode>(Val: N)->getMemoryVT(); |
5005 | SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
5006 | SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT); |
5007 | SDValue Swap = DAG.getAtomicCmpSwap( |
5008 | Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
5009 | MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(), VTs, Chain: N->getOperand(Num: 0), |
5010 | Ptr: N->getOperand(Num: 1), Cmp: Zero, Swp: Zero, MMO: cast<AtomicSDNode>(Val: N)->getMemOperand()); |
5011 | |
5012 | ReplaceValueWith(From: SDValue(N, 0), To: Swap.getValue(R: 0)); |
5013 | ReplaceValueWith(From: SDValue(N, 1), To: Swap.getValue(R: 2)); |
5014 | } |
5015 | |
5016 | void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, |
5017 | SDValue &Lo, SDValue &Hi) { |
5018 | // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate |
5019 | // both halves independently. |
5020 | SDValue Res = TLI.expandVecReduce(Node: N, DAG); |
5021 | SplitInteger(Op: Res, Lo, Hi); |
5022 | } |
5023 | |
5024 | void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, |
5025 | SDValue &Lo, SDValue &Hi) { |
5026 | // Delegate to funnel-shift expansion. |
5027 | SDLoc DL(N); |
5028 | unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR; |
5029 | SDValue Res = DAG.getNode(Opcode, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), |
5030 | N2: N->getOperand(Num: 0), N3: N->getOperand(Num: 1)); |
5031 | SplitInteger(Op: Res, Lo, Hi); |
5032 | } |
5033 | |
5034 | void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo, |
5035 | SDValue &Hi) { |
5036 | // Values numbered from least significant to most significant. |
5037 | SDValue In1, In2, In3, In4; |
5038 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: In3, Hi&: In4); |
5039 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo&: In1, Hi&: In2); |
5040 | EVT HalfVT = In1.getValueType(); |
5041 | |
5042 | SDLoc DL(N); |
5043 | unsigned Opc = N->getOpcode(); |
5044 | SDValue ShAmt = N->getOperand(Num: 2); |
5045 | EVT ShAmtVT = ShAmt.getValueType(); |
5046 | EVT ShAmtCCVT = getSetCCResultType(VT: ShAmtVT); |
5047 | |
5048 | // If the shift amount is at least half the bitwidth, swap the inputs. |
5049 | unsigned HalfVTBits = HalfVT.getScalarSizeInBits(); |
5050 | SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL, VT: ShAmtVT, N1: ShAmt, |
5051 | N2: DAG.getConstant(Val: HalfVTBits, DL, VT: ShAmtVT)); |
5052 | SDValue Cond = |
5053 | DAG.getSetCC(DL, VT: ShAmtCCVT, LHS: AndNode, RHS: DAG.getConstant(Val: 0, DL, VT: ShAmtVT), |
5054 | Cond: Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ); |
5055 | |
5056 | // Expand to a pair of funnel shifts. |
5057 | EVT NewShAmtVT = TLI.getShiftAmountTy(LHSTy: HalfVT, DL: DAG.getDataLayout()); |
5058 | SDValue NewShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: NewShAmtVT); |
5059 | |
5060 | SDValue Select1 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In1, N3: In2); |
5061 | SDValue Select2 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In2, N3: In3); |
5062 | SDValue Select3 = DAG.getNode(Opcode: ISD::SELECT, DL, VT: HalfVT, N1: Cond, N2: In3, N3: In4); |
5063 | Lo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select2, N2: Select1, N3: NewShAmt); |
5064 | Hi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Select3, N2: Select2, N3: NewShAmt); |
5065 | } |
5066 | |
5067 | void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, |
5068 | SDValue &Hi) { |
5069 | EVT VT = N->getValueType(ResNo: 0); |
5070 | EVT HalfVT = |
5071 | EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: N->getValueSizeInBits(ResNo: 0) / 2); |
5072 | SDLoc dl(N); |
5073 | |
5074 | // We assume VSCALE(1) fits into a legal integer. |
5075 | APInt One(HalfVT.getSizeInBits(), 1); |
5076 | SDValue VScaleBase = DAG.getVScale(DL: dl, VT: HalfVT, MulImm: One); |
5077 | VScaleBase = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: VScaleBase); |
5078 | SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: VScaleBase, N2: N->getOperand(Num: 0)); |
5079 | SplitInteger(Op: Res, Lo, Hi); |
5080 | } |
5081 | |
5082 | //===----------------------------------------------------------------------===// |
5083 | // Integer Operand Expansion |
5084 | //===----------------------------------------------------------------------===// |
5085 | |
5086 | /// ExpandIntegerOperand - This method is called when the specified operand of |
5087 | /// the specified node is found to need expansion. At this point, all of the |
5088 | /// result types of the node are known to be legal, but other operands of the |
5089 | /// node may need promotion or expansion as well as the specified one. |
5090 | bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
5091 | LLVM_DEBUG(dbgs() << "Expand integer operand: " ; N->dump(&DAG)); |
5092 | SDValue Res = SDValue(); |
5093 | |
5094 | if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) |
5095 | return false; |
5096 | |
5097 | switch (N->getOpcode()) { |
5098 | default: |
5099 | #ifndef NDEBUG |
5100 | dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": " ; |
5101 | N->dump(G: &DAG); dbgs() << "\n" ; |
5102 | #endif |
5103 | report_fatal_error(reason: "Do not know how to expand this operator's operand!" ); |
5104 | |
5105 | case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
5106 | case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
5107 | case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
5108 | case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
5109 | case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
5110 | case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
5111 | case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; |
5112 | case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
5113 | case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
5114 | case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
5115 | case ISD::STRICT_SINT_TO_FP: |
5116 | case ISD::SINT_TO_FP: |
5117 | case ISD::STRICT_UINT_TO_FP: |
5118 | case ISD::UINT_TO_FP: Res = ExpandIntOp_XINT_TO_FP(N); break; |
5119 | case ISD::STORE: Res = ExpandIntOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo); break; |
5120 | case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
5121 | |
5122 | case ISD::SHL: |
5123 | case ISD::SRA: |
5124 | case ISD::SRL: |
5125 | case ISD::ROTL: |
5126 | case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
5127 | case ISD::RETURNADDR: |
5128 | case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
5129 | |
5130 | case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
5131 | case ISD::STACKMAP: |
5132 | Res = ExpandIntOp_STACKMAP(N, OpNo); |
5133 | break; |
5134 | case ISD::PATCHPOINT: |
5135 | Res = ExpandIntOp_PATCHPOINT(N, OpNo); |
5136 | break; |
5137 | case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: |
5138 | case ISD::EXPERIMENTAL_VP_STRIDED_STORE: |
5139 | Res = ExpandIntOp_VP_STRIDED(N, OpNo); |
5140 | break; |
5141 | } |
5142 | |
5143 | // If the result is null, the sub-method took care of registering results etc. |
5144 | if (!Res.getNode()) return false; |
5145 | |
5146 | // If the result is N, the sub-method updated N in place. Tell the legalizer |
5147 | // core about this. |
5148 | if (Res.getNode() == N) |
5149 | return true; |
5150 | |
5151 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
5152 | "Invalid operand expansion" ); |
5153 | |
5154 | ReplaceValueWith(From: SDValue(N, 0), To: Res); |
5155 | return false; |
5156 | } |
5157 | |
5158 | /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code |
5159 | /// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
5160 | void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
5161 | SDValue &NewRHS, |
5162 | ISD::CondCode &CCCode, |
5163 | const SDLoc &dl) { |
5164 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
5165 | GetExpandedInteger(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi); |
5166 | GetExpandedInteger(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi); |
5167 | |
5168 | if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { |
5169 | if (RHSLo == RHSHi && isAllOnesConstant(V: RHSLo)) { |
5170 | // Equality comparison to -1. |
5171 | NewLHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: LHSHi); |
5172 | NewRHS = RHSLo; |
5173 | return; |
5174 | } |
5175 | |
5176 | NewLHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo); |
5177 | NewRHS = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSLo.getValueType(), N1: LHSHi, N2: RHSHi); |
5178 | NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NewLHS.getValueType(), N1: NewLHS, N2: NewRHS); |
5179 | NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: NewLHS.getValueType()); |
5180 | return; |
5181 | } |
5182 | |
5183 | // If this is a comparison of the sign bit, just look at the top part. |
5184 | // X > -1, x < 0 |
5185 | if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: NewRHS)) |
5186 | if ((CCCode == ISD::SETLT && CST->isZero()) || // X < 0 |
5187 | (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1 |
5188 | NewLHS = LHSHi; |
5189 | NewRHS = RHSHi; |
5190 | return; |
5191 | } |
5192 | |
5193 | // FIXME: This generated code sucks. |
5194 | ISD::CondCode LowCC; |
5195 | switch (CCCode) { |
5196 | default: llvm_unreachable("Unknown integer setcc!" ); |
5197 | case ISD::SETLT: |
5198 | case ISD::SETULT: LowCC = ISD::SETULT; break; |
5199 | case ISD::SETGT: |
5200 | case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
5201 | case ISD::SETLE: |
5202 | case ISD::SETULE: LowCC = ISD::SETULE; break; |
5203 | case ISD::SETGE: |
5204 | case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
5205 | } |
5206 | |
5207 | // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison |
5208 | // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands |
5209 | // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; |
5210 | |
5211 | // NOTE: on targets without efficient SELECT of bools, we can always use |
5212 | // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) |
5213 | TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
5214 | nullptr); |
5215 | SDValue LoCmp, HiCmp; |
5216 | if (TLI.isTypeLegal(VT: LHSLo.getValueType()) && |
5217 | TLI.isTypeLegal(VT: RHSLo.getValueType())) |
5218 | LoCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSLo.getValueType()), N0: LHSLo, |
5219 | N1: RHSLo, Cond: LowCC, foldBooleans: false, DCI&: DagCombineInfo, dl); |
5220 | if (!LoCmp.getNode()) |
5221 | LoCmp = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo, |
5222 | RHS: RHSLo, Cond: LowCC); |
5223 | if (TLI.isTypeLegal(VT: LHSHi.getValueType()) && |
5224 | TLI.isTypeLegal(VT: RHSHi.getValueType())) |
5225 | HiCmp = TLI.SimplifySetCC(VT: getSetCCResultType(VT: LHSHi.getValueType()), N0: LHSHi, |
5226 | N1: RHSHi, Cond: CCCode, foldBooleans: false, DCI&: DagCombineInfo, dl); |
5227 | if (!HiCmp.getNode()) |
5228 | HiCmp = |
5229 | DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), |
5230 | N1: LHSHi, N2: RHSHi, N3: DAG.getCondCode(Cond: CCCode)); |
5231 | |
5232 | ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(Val: LoCmp.getNode()); |
5233 | ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(Val: HiCmp.getNode()); |
5234 | |
5235 | bool EqAllowed = ISD::isTrueWhenEqual(Cond: CCCode); |
5236 | |
5237 | // FIXME: Is the HiCmpC->isOne() here correct for |
5238 | // ZeroOrNegativeOneBooleanContent. |
5239 | if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) || |
5240 | (!EqAllowed && |
5241 | ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) { |
5242 | // For LE / GE, if high part is known false, ignore the low part. |
5243 | // For LT / GT: if low part is known false, return the high part. |
5244 | // if high part is known true, ignore the low part. |
5245 | NewLHS = HiCmp; |
5246 | NewRHS = SDValue(); |
5247 | return; |
5248 | } |
5249 | |
5250 | if (LHSHi == RHSHi) { |
5251 | // Comparing the low bits is enough. |
5252 | NewLHS = LoCmp; |
5253 | NewRHS = SDValue(); |
5254 | return; |
5255 | } |
5256 | |
5257 | // Lower with SETCCCARRY if the target supports it. |
5258 | EVT HiVT = LHSHi.getValueType(); |
5259 | EVT ExpandVT = TLI.getTypeToExpandTo(Context&: *DAG.getContext(), VT: HiVT); |
5260 | bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(Op: ISD::SETCCCARRY, VT: ExpandVT); |
5261 | |
5262 | // FIXME: Make all targets support this, then remove the other lowering. |
5263 | if (HasSETCCCARRY) { |
5264 | // SETCCCARRY can detect < and >= directly. For > and <=, flip |
5265 | // operands and condition code. |
5266 | bool FlipOperands = false; |
5267 | switch (CCCode) { |
5268 | case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
5269 | case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
5270 | case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
5271 | case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
5272 | default: break; |
5273 | } |
5274 | if (FlipOperands) { |
5275 | std::swap(a&: LHSLo, b&: RHSLo); |
5276 | std::swap(a&: LHSHi, b&: RHSHi); |
5277 | } |
5278 | // Perform a wide subtraction, feeding the carry from the low part into |
5279 | // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high |
5280 | // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is |
5281 | // zero or positive iff LHS >= RHS. |
5282 | EVT LoVT = LHSLo.getValueType(); |
5283 | SDVTList VTList = DAG.getVTList(VT1: LoVT, VT2: getSetCCResultType(VT: LoVT)); |
5284 | SDValue LowCmp = DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList, N1: LHSLo, N2: RHSLo); |
5285 | SDValue Res = DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: getSetCCResultType(VT: HiVT), |
5286 | N1: LHSHi, N2: RHSHi, N3: LowCmp.getValue(R: 1), |
5287 | N4: DAG.getCondCode(Cond: CCCode)); |
5288 | NewLHS = Res; |
5289 | NewRHS = SDValue(); |
5290 | return; |
5291 | } |
5292 | |
5293 | NewLHS = TLI.SimplifySetCC(VT: getSetCCResultType(VT: HiVT), N0: LHSHi, N1: RHSHi, Cond: ISD::SETEQ, |
5294 | foldBooleans: false, DCI&: DagCombineInfo, dl); |
5295 | if (!NewLHS.getNode()) |
5296 | NewLHS = |
5297 | DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: HiVT), LHS: LHSHi, RHS: RHSHi, Cond: ISD::SETEQ); |
5298 | NewLHS = DAG.getSelect(DL: dl, VT: LoCmp.getValueType(), Cond: NewLHS, LHS: LoCmp, RHS: HiCmp); |
5299 | NewRHS = SDValue(); |
5300 | } |
5301 | |
5302 | SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
5303 | SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3); |
5304 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
5305 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5306 | |
5307 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
5308 | // against zero to select between true and false values. |
5309 | if (!NewRHS.getNode()) { |
5310 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
5311 | CCCode = ISD::SETNE; |
5312 | } |
5313 | |
5314 | // Update N to have the operands specified. |
5315 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), |
5316 | Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS, |
5317 | Op5: N->getOperand(Num: 4)), 0); |
5318 | } |
5319 | |
5320 | SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
5321 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
5322 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get(); |
5323 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5324 | |
5325 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
5326 | // against zero to select between true and false values. |
5327 | if (!NewRHS.getNode()) { |
5328 | NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType()); |
5329 | CCCode = ISD::SETNE; |
5330 | } |
5331 | |
5332 | // Update N to have the operands specified. |
5333 | return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, |
5334 | Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3), |
5335 | Op5: DAG.getCondCode(Cond: CCCode)), 0); |
5336 | } |
5337 | |
5338 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
5339 | SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1); |
5340 | ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get(); |
5341 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N)); |
5342 | |
5343 | // If ExpandSetCCOperands returned a scalar, use it. |
5344 | if (!NewRHS.getNode()) { |
5345 | assert(NewLHS.getValueType() == N->getValueType(0) && |
5346 | "Unexpected setcc expansion!" ); |
5347 | return NewLHS; |
5348 | } |
5349 | |
5350 | // Otherwise, update N to have the operands specified. |
5351 | return SDValue( |
5352 | DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS, Op3: DAG.getCondCode(Cond: CCCode)), 0); |
5353 | } |
5354 | |
5355 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
5356 | SDValue LHS = N->getOperand(Num: 0); |
5357 | SDValue RHS = N->getOperand(Num: 1); |
5358 | SDValue Carry = N->getOperand(Num: 2); |
5359 | SDValue Cond = N->getOperand(Num: 3); |
5360 | SDLoc dl = SDLoc(N); |
5361 | |
5362 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
5363 | GetExpandedInteger(Op: LHS, Lo&: LHSLo, Hi&: LHSHi); |
5364 | GetExpandedInteger(Op: RHS, Lo&: RHSLo, Hi&: RHSHi); |
5365 | |
5366 | // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high. |
5367 | SDVTList VTList = DAG.getVTList(VT1: LHSLo.getValueType(), VT2: Carry.getValueType()); |
5368 | SDValue LowCmp = |
5369 | DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: dl, VTList, N1: LHSLo, N2: RHSLo, N3: Carry); |
5370 | return DAG.getNode(Opcode: ISD::SETCCCARRY, DL: dl, VT: N->getValueType(ResNo: 0), N1: LHSHi, N2: RHSHi, |
5371 | N3: LowCmp.getValue(R: 1), N4: Cond); |
5372 | } |
5373 | |
5374 | SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { |
5375 | // Split the operand and replace with SPLAT_VECTOR_PARTS. |
5376 | SDValue Lo, Hi; |
5377 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
5378 | return DAG.getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, |
5379 | N2: Hi); |
5380 | } |
5381 | |
5382 | SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
5383 | // The value being shifted is legal, but the shift amount is too big. |
5384 | // It follows that either the result of the shift is undefined, or the |
5385 | // upper half of the shift amount is zero. Just use the lower half. |
5386 | SDValue Lo, Hi; |
5387 | GetExpandedInteger(Op: N->getOperand(Num: 1), Lo, Hi); |
5388 | return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: Lo), 0); |
5389 | } |
5390 | |
5391 | SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
5392 | // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This |
5393 | // surely makes pretty nice problems on 8/16 bit targets. Just truncate this |
5394 | // constant to valid type. |
5395 | SDValue Lo, Hi; |
5396 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo, Hi); |
5397 | return SDValue(DAG.UpdateNodeOperands(N, Op: Lo), 0); |
5398 | } |
5399 | |
5400 | SDValue DAGTypeLegalizer::ExpandIntOp_XINT_TO_FP(SDNode *N) { |
5401 | bool IsStrict = N->isStrictFPOpcode(); |
5402 | bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || |
5403 | N->getOpcode() == ISD::STRICT_SINT_TO_FP; |
5404 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
5405 | SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0); |
5406 | EVT DstVT = N->getValueType(ResNo: 0); |
5407 | RTLIB::Libcall LC = IsSigned ? RTLIB::getSINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT) |
5408 | : RTLIB::getUINTTOFP(OpVT: Op.getValueType(), RetVT: DstVT); |
5409 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
5410 | "Don't know how to expand this XINT_TO_FP!" ); |
5411 | TargetLowering::MakeLibCallOptions CallOptions; |
5412 | CallOptions.setSExt(true); |
5413 | std::pair<SDValue, SDValue> Tmp = |
5414 | TLI.makeLibCall(DAG, LC, RetVT: DstVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain); |
5415 | |
5416 | if (!IsStrict) |
5417 | return Tmp.first; |
5418 | |
5419 | ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second); |
5420 | ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first); |
5421 | return SDValue(); |
5422 | } |
5423 | |
5424 | SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
5425 | assert(!N->isAtomic() && "Should have been a ATOMIC_STORE?" ); |
5426 | |
5427 | if (ISD::isNormalStore(N)) |
5428 | return ExpandOp_NormalStore(N, OpNo); |
5429 | |
5430 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!" ); |
5431 | assert(OpNo == 1 && "Can only expand the stored value so far" ); |
5432 | |
5433 | EVT VT = N->getOperand(Num: 1).getValueType(); |
5434 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5435 | SDValue Ch = N->getChain(); |
5436 | SDValue Ptr = N->getBasePtr(); |
5437 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
5438 | AAMDNodes AAInfo = N->getAAInfo(); |
5439 | SDLoc dl(N); |
5440 | SDValue Lo, Hi; |
5441 | |
5442 | assert(NVT.isByteSized() && "Expanded type not byte sized!" ); |
5443 | |
5444 | if (N->getMemoryVT().bitsLE(VT: NVT)) { |
5445 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5446 | return DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), |
5447 | SVT: N->getMemoryVT(), Alignment: N->getOriginalAlign(), MMOFlags, |
5448 | AAInfo); |
5449 | } |
5450 | |
5451 | if (DAG.getDataLayout().isLittleEndian()) { |
5452 | // Little-endian - low bits are at low addresses. |
5453 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5454 | |
5455 | Lo = DAG.getStore(Chain: Ch, dl, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), |
5456 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5457 | |
5458 | unsigned ExcessBits = |
5459 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
5460 | EVT NEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits); |
5461 | |
5462 | // Increment the pointer to the other half. |
5463 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
5464 | Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize)); |
5465 | Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr, |
5466 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
5467 | SVT: NEVT, Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5468 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
5469 | } |
5470 | |
5471 | // Big-endian - high bits are at low addresses. Favor aligned stores at |
5472 | // the cost of some bit-fiddling. |
5473 | GetExpandedInteger(Op: N->getValue(), Lo, Hi); |
5474 | |
5475 | EVT ExtVT = N->getMemoryVT(); |
5476 | unsigned EBytes = ExtVT.getStoreSize(); |
5477 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
5478 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
5479 | EVT HiVT = EVT::getIntegerVT(Context&: *DAG.getContext(), |
5480 | BitWidth: ExtVT.getSizeInBits() - ExcessBits); |
5481 | |
5482 | if (ExcessBits < NVT.getSizeInBits()) { |
5483 | // Transfer high bits from the top of Lo to the bottom of Hi. |
5484 | Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: Hi, |
5485 | N2: DAG.getConstant(Val: NVT.getSizeInBits() - ExcessBits, DL: dl, |
5486 | VT: TLI.getPointerTy(DL: DAG.getDataLayout()))); |
5487 | Hi = DAG.getNode( |
5488 | Opcode: ISD::OR, DL: dl, VT: NVT, N1: Hi, |
5489 | N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Lo, |
5490 | N2: DAG.getConstant(Val: ExcessBits, DL: dl, |
5491 | VT: TLI.getPointerTy(DL: DAG.getDataLayout())))); |
5492 | } |
5493 | |
5494 | // Store both the high bits and maybe some of the low bits. |
5495 | Hi = DAG.getTruncStore(Chain: Ch, dl, Val: Hi, Ptr, PtrInfo: N->getPointerInfo(), SVT: HiVT, |
5496 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5497 | |
5498 | // Increment the pointer to the other half. |
5499 | Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize)); |
5500 | // Store the lowest ExcessBits bits in the second half. |
5501 | Lo = DAG.getTruncStore(Chain: Ch, dl, Val: Lo, Ptr, |
5502 | PtrInfo: N->getPointerInfo().getWithOffset(O: IncrementSize), |
5503 | SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExcessBits), |
5504 | Alignment: N->getOriginalAlign(), MMOFlags, AAInfo); |
5505 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
5506 | } |
5507 | |
5508 | SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
5509 | SDValue InL, InH; |
5510 | GetExpandedInteger(Op: N->getOperand(Num: 0), Lo&: InL, Hi&: InH); |
5511 | // Just truncate the low part of the source. |
5512 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: InL); |
5513 | } |
5514 | |
5515 | SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
5516 | SDLoc dl(N); |
5517 | SDValue Swap = |
5518 | DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: N)->getMemoryVT(), |
5519 | Chain: N->getOperand(Num: 0), Ptr: N->getOperand(Num: 2), Val: N->getOperand(Num: 1), |
5520 | MMO: cast<AtomicSDNode>(Val: N)->getMemOperand()); |
5521 | return Swap.getValue(R: 1); |
5522 | } |
5523 | |
5524 | SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) { |
5525 | assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) || |
5526 | (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4)); |
5527 | |
5528 | SDValue Hi; // The upper half is dropped out. |
5529 | SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end()); |
5530 | GetExpandedInteger(Op: NewOps[OpNo], Lo&: NewOps[OpNo], Hi); |
5531 | |
5532 | return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0); |
5533 | } |
5534 | |
5535 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { |
5536 | SDLoc dl(N); |
5537 | |
5538 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5539 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5540 | EVT OutVT = V0.getValueType(); |
5541 | |
5542 | return DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL: dl, VT: OutVT, N1: V0, N2: V1, N3: N->getOperand(Num: 2)); |
5543 | } |
5544 | |
5545 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) { |
5546 | SDLoc dl(N); |
5547 | |
5548 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5549 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5550 | EVT ResVT = V0.getValueType(); |
5551 | SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, |
5552 | VTList: DAG.getVTList(VT1: ResVT, VT2: ResVT), N1: V0, N2: V1); |
5553 | SetPromotedInteger(Op: SDValue(N, 0), Result: Res.getValue(R: 0)); |
5554 | SetPromotedInteger(Op: SDValue(N, 1), Result: Res.getValue(R: 1)); |
5555 | return SDValue(); |
5556 | } |
5557 | |
5558 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5559 | |
5560 | EVT OutVT = N->getValueType(ResNo: 0); |
5561 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5562 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5563 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5564 | |
5565 | SDLoc dl(N); |
5566 | SDValue BaseIdx = N->getOperand(Num: 1); |
5567 | |
5568 | // TODO: We may be able to use this for types other than scalable |
5569 | // vectors and fix those tests that expect BUILD_VECTOR to be used |
5570 | if (OutVT.isScalableVector()) { |
5571 | SDValue InOp0 = N->getOperand(Num: 0); |
5572 | EVT InVT = InOp0.getValueType(); |
5573 | |
5574 | // Try and extract from a smaller type so that it eventually falls |
5575 | // into the promotion code below. |
5576 | if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector || |
5577 | getTypeAction(VT: InVT) == TargetLowering::TypeLegal) { |
5578 | EVT NInVT = InVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext()); |
5579 | unsigned NElts = NInVT.getVectorMinNumElements(); |
5580 | uint64_t IdxVal = BaseIdx->getAsZExtVal(); |
5581 | |
5582 | SDValue Step1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: NInVT, N1: InOp0, |
5583 | N2: DAG.getConstant(Val: alignDown(Value: IdxVal, Align: NElts), DL: dl, |
5584 | VT: BaseIdx.getValueType())); |
5585 | SDValue Step2 = DAG.getNode( |
5586 | Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: Step1, |
5587 | N2: DAG.getConstant(Val: IdxVal % NElts, DL: dl, VT: BaseIdx.getValueType())); |
5588 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Step2); |
5589 | } |
5590 | |
5591 | // Try and extract from a widened type. |
5592 | if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) { |
5593 | SDValue Ops[] = {GetWidenedVector(Op: InOp0), BaseIdx}; |
5594 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: OutVT, Ops); |
5595 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext); |
5596 | } |
5597 | |
5598 | // Promote operands and see if this is handled by target lowering, |
5599 | // Otherwise, use the BUILD_VECTOR approach below |
5600 | if (getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger) { |
5601 | // Collect the (promoted) operands |
5602 | SDValue Ops[] = { GetPromotedInteger(Op: InOp0), BaseIdx }; |
5603 | |
5604 | EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); |
5605 | assert(PromEltVT.bitsLE(NOutVTElem) && |
5606 | "Promoted operand has an element type greater than result" ); |
5607 | |
5608 | EVT ExtVT = NOutVT.changeVectorElementType(EltVT: PromEltVT); |
5609 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: ExtVT, Ops); |
5610 | return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutVT, Operand: Ext); |
5611 | } |
5612 | } |
5613 | |
5614 | if (OutVT.isScalableVector()) |
5615 | report_fatal_error(reason: "Unable to promote scalable types using BUILD_VECTOR" ); |
5616 | |
5617 | SDValue InOp0 = N->getOperand(Num: 0); |
5618 | if (getTypeAction(VT: InOp0.getValueType()) == TargetLowering::TypePromoteInteger) |
5619 | InOp0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5620 | |
5621 | EVT InVT = InOp0.getValueType(); |
5622 | |
5623 | unsigned OutNumElems = OutVT.getVectorNumElements(); |
5624 | SmallVector<SDValue, 8> Ops; |
5625 | Ops.reserve(N: OutNumElems); |
5626 | for (unsigned i = 0; i != OutNumElems; ++i) { |
5627 | |
5628 | // Extract the element from the original vector. |
5629 | SDValue Index = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BaseIdx.getValueType(), |
5630 | N1: BaseIdx, N2: DAG.getConstant(Val: i, DL: dl, VT: BaseIdx.getValueType())); |
5631 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
5632 | VT: InVT.getVectorElementType(), N1: N->getOperand(Num: 0), N2: Index); |
5633 | |
5634 | SDValue Op = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: NOutVTElem); |
5635 | // Insert the converted element to the new vector. |
5636 | Ops.push_back(Elt: Op); |
5637 | } |
5638 | |
5639 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5640 | } |
5641 | |
5642 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { |
5643 | EVT OutVT = N->getValueType(ResNo: 0); |
5644 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5645 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5646 | |
5647 | SDLoc dl(N); |
5648 | SDValue Vec = N->getOperand(Num: 0); |
5649 | SDValue SubVec = N->getOperand(Num: 1); |
5650 | SDValue Idx = N->getOperand(Num: 2); |
5651 | |
5652 | EVT SubVecVT = SubVec.getValueType(); |
5653 | EVT NSubVT = |
5654 | EVT::getVectorVT(Context&: *DAG.getContext(), VT: NOutVT.getVectorElementType(), |
5655 | EC: SubVecVT.getVectorElementCount()); |
5656 | |
5657 | Vec = GetPromotedInteger(Op: Vec); |
5658 | SubVec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NSubVT, Operand: SubVec); |
5659 | |
5660 | return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: NOutVT, N1: Vec, N2: SubVec, N3: Idx); |
5661 | } |
5662 | |
5663 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { |
5664 | SDLoc dl(N); |
5665 | |
5666 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5667 | EVT OutVT = V0.getValueType(); |
5668 | |
5669 | return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: OutVT, Operand: V0); |
5670 | } |
5671 | |
5672 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
5673 | ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(Val: N); |
5674 | EVT VT = N->getValueType(ResNo: 0); |
5675 | SDLoc dl(N); |
5676 | |
5677 | ArrayRef<int> NewMask = SV->getMask().slice(N: 0, M: VT.getVectorNumElements()); |
5678 | |
5679 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5680 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5681 | EVT OutVT = V0.getValueType(); |
5682 | |
5683 | return DAG.getVectorShuffle(VT: OutVT, dl, N1: V0, N2: V1, Mask: NewMask); |
5684 | } |
5685 | |
5686 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
5687 | EVT OutVT = N->getValueType(ResNo: 0); |
5688 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5689 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5690 | unsigned NumElems = N->getNumOperands(); |
5691 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5692 | TargetLoweringBase::BooleanContent NOutBoolType = TLI.getBooleanContents(Type: NOutVT); |
5693 | unsigned NOutExtOpc = TargetLowering::getExtendForContent(Content: NOutBoolType); |
5694 | SDLoc dl(N); |
5695 | |
5696 | SmallVector<SDValue, 8> Ops; |
5697 | Ops.reserve(N: NumElems); |
5698 | for (unsigned i = 0; i != NumElems; ++i) { |
5699 | SDValue Op = N->getOperand(Num: i); |
5700 | EVT OpVT = Op.getValueType(); |
5701 | // BUILD_VECTOR integer operand types are allowed to be larger than the |
5702 | // result's element type. This may still be true after the promotion. For |
5703 | // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to |
5704 | // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. |
5705 | if (OpVT.bitsLT(VT: NOutVTElem)) { |
5706 | unsigned ExtOpc = ISD::ANY_EXTEND; |
5707 | // Attempt to extend constant bool vectors to match target's BooleanContent. |
5708 | // While not necessary, this improves chances of the constant correctly |
5709 | // folding with compare results (e.g. for NOT patterns). |
5710 | if (OpVT == MVT::i1 && Op.getOpcode() == ISD::Constant) |
5711 | ExtOpc = NOutExtOpc; |
5712 | Op = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: NOutVTElem, Operand: Op); |
5713 | } |
5714 | Ops.push_back(Elt: Op); |
5715 | } |
5716 | |
5717 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5718 | } |
5719 | |
5720 | SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) { |
5721 | |
5722 | SDLoc dl(N); |
5723 | |
5724 | assert(!N->getOperand(0).getValueType().isVector() && |
5725 | "Input must be a scalar" ); |
5726 | |
5727 | EVT OutVT = N->getValueType(ResNo: 0); |
5728 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5729 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5730 | EVT NOutElemVT = NOutVT.getVectorElementType(); |
5731 | |
5732 | SDValue Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NOutElemVT, Operand: N->getOperand(Num: 0)); |
5733 | |
5734 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NOutVT, Operand: Op); |
5735 | } |
5736 | |
5737 | SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { |
5738 | SDLoc dl(N); |
5739 | EVT OutVT = N->getValueType(ResNo: 0); |
5740 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5741 | assert(NOutVT.isScalableVector() && |
5742 | "Type must be promoted to a scalable vector type" ); |
5743 | const APInt &StepVal = N->getConstantOperandAPInt(Num: 0); |
5744 | return DAG.getStepVector(DL: dl, ResVT: NOutVT, |
5745 | StepVal: StepVal.sext(width: NOutVT.getScalarSizeInBits())); |
5746 | } |
5747 | |
5748 | SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
5749 | SDLoc dl(N); |
5750 | |
5751 | EVT OutVT = N->getValueType(ResNo: 0); |
5752 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5753 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5754 | |
5755 | unsigned NumOperands = N->getNumOperands(); |
5756 | unsigned NumOutElem = NOutVT.getVectorMinNumElements(); |
5757 | EVT OutElemTy = NOutVT.getVectorElementType(); |
5758 | if (OutVT.isScalableVector()) { |
5759 | // Find the largest promoted element type for each of the operands. |
5760 | SDUse *MaxSizedValue = std::max_element( |
5761 | first: N->op_begin(), last: N->op_end(), comp: [](const SDValue &A, const SDValue &B) { |
5762 | EVT AVT = A.getValueType().getVectorElementType(); |
5763 | EVT BVT = B.getValueType().getVectorElementType(); |
5764 | return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits(); |
5765 | }); |
5766 | EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType(); |
5767 | |
5768 | // Then promote all vectors to the largest element type. |
5769 | SmallVector<SDValue, 8> Ops; |
5770 | for (unsigned I = 0; I < NumOperands; ++I) { |
5771 | SDValue Op = N->getOperand(Num: I); |
5772 | EVT OpVT = Op.getValueType(); |
5773 | if (getTypeAction(VT: OpVT) == TargetLowering::TypePromoteInteger) |
5774 | Op = GetPromotedInteger(Op); |
5775 | else |
5776 | assert(getTypeAction(OpVT) == TargetLowering::TypeLegal && |
5777 | "Unhandled legalization type" ); |
5778 | |
5779 | if (OpVT.getVectorElementType().getScalarSizeInBits() < |
5780 | MaxElementVT.getScalarSizeInBits()) |
5781 | Op = DAG.getAnyExtOrTrunc(Op, DL: dl, |
5782 | VT: OpVT.changeVectorElementType(EltVT: MaxElementVT)); |
5783 | Ops.push_back(Elt: Op); |
5784 | } |
5785 | |
5786 | // Do the CONCAT on the promoted type and finally truncate to (the promoted) |
5787 | // NOutVT. |
5788 | return DAG.getAnyExtOrTrunc( |
5789 | Op: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, |
5790 | VT: OutVT.changeVectorElementType(EltVT: MaxElementVT), Ops), |
5791 | DL: dl, VT: NOutVT); |
5792 | } |
5793 | |
5794 | unsigned NumElem = N->getOperand(Num: 0).getValueType().getVectorNumElements(); |
5795 | assert(NumElem * NumOperands == NumOutElem && |
5796 | "Unexpected number of elements" ); |
5797 | |
5798 | // Take the elements from the first vector. |
5799 | SmallVector<SDValue, 8> Ops(NumOutElem); |
5800 | for (unsigned i = 0; i < NumOperands; ++i) { |
5801 | SDValue Op = N->getOperand(Num: i); |
5802 | if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteInteger) |
5803 | Op = GetPromotedInteger(Op); |
5804 | EVT SclrTy = Op.getValueType().getVectorElementType(); |
5805 | assert(NumElem == Op.getValueType().getVectorNumElements() && |
5806 | "Unexpected number of elements" ); |
5807 | |
5808 | for (unsigned j = 0; j < NumElem; ++j) { |
5809 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Op, |
5810 | N2: DAG.getVectorIdxConstant(Val: j, DL: dl)); |
5811 | Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: OutElemTy); |
5812 | } |
5813 | } |
5814 | |
5815 | return DAG.getBuildVector(VT: NOutVT, DL: dl, Ops); |
5816 | } |
5817 | |
5818 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
5819 | EVT VT = N->getValueType(ResNo: 0); |
5820 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT); |
5821 | assert(NVT.isVector() && "This type must be promoted to a vector type" ); |
5822 | |
5823 | SDLoc dl(N); |
5824 | |
5825 | // For operands whose TypeAction is to promote, extend the promoted node |
5826 | // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion |
5827 | // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to |
5828 | // type.. |
5829 | if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) |
5830 | == TargetLowering::TypePromoteInteger) { |
5831 | SDValue Promoted; |
5832 | |
5833 | switch(N->getOpcode()) { |
5834 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
5835 | Promoted = SExtPromotedInteger(Op: N->getOperand(Num: 0)); |
5836 | break; |
5837 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
5838 | Promoted = ZExtPromotedInteger(Op: N->getOperand(Num: 0)); |
5839 | break; |
5840 | case ISD::ANY_EXTEND_VECTOR_INREG: |
5841 | Promoted = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5842 | break; |
5843 | default: |
5844 | llvm_unreachable("Node has unexpected Opcode" ); |
5845 | } |
5846 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Promoted); |
5847 | } |
5848 | |
5849 | // Directly extend to the appropriate transform-to type. |
5850 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0)); |
5851 | } |
5852 | |
5853 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
5854 | EVT OutVT = N->getValueType(ResNo: 0); |
5855 | EVT NOutVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OutVT); |
5856 | assert(NOutVT.isVector() && "This type must be promoted to a vector type" ); |
5857 | |
5858 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
5859 | |
5860 | SDLoc dl(N); |
5861 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5862 | |
5863 | SDValue ConvElem = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, |
5864 | VT: NOutVTElem, Operand: N->getOperand(Num: 1)); |
5865 | return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: NOutVT, |
5866 | N1: V0, N2: ConvElem, N3: N->getOperand(Num: 2)); |
5867 | } |
5868 | |
5869 | SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { |
5870 | // The VECREDUCE result size may be larger than the element size, so |
5871 | // we can simply change the result type. |
5872 | SDLoc dl(N); |
5873 | EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)); |
5874 | return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Ops: N->ops()); |
5875 | } |
5876 | |
5877 | SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) { |
5878 | // The VP_REDUCE result size may be larger than the element size, so we can |
5879 | // simply change the result type. However the start value and result must be |
5880 | // the same. |
5881 | SDLoc DL(N); |
5882 | SDValue Start = PromoteIntOpVectorReduction(N, V: N->getOperand(Num: 0)); |
5883 | return DAG.getNode(Opcode: N->getOpcode(), DL, VT: Start.getValueType(), N1: Start, |
5884 | N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3)); |
5885 | } |
5886 | |
5887 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5888 | SDLoc dl(N); |
5889 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5890 | SDValue V1 = DAG.getZExtOrTrunc(Op: N->getOperand(Num: 1), DL: dl, |
5891 | VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())); |
5892 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, |
5893 | VT: V0->getValueType(ResNo: 0).getScalarType(), N1: V0, N2: V1); |
5894 | |
5895 | // EXTRACT_VECTOR_ELT can return types which are wider than the incoming |
5896 | // element types. If this is the case then we need to expand the outgoing |
5897 | // value and not truncate it. |
5898 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0)); |
5899 | } |
5900 | |
5901 | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) { |
5902 | SDLoc dl(N); |
5903 | // The result type is equal to the first input operand's type, so the |
5904 | // type that needs promoting must be the second source vector. |
5905 | SDValue V0 = N->getOperand(Num: 0); |
5906 | SDValue V1 = GetPromotedInteger(Op: N->getOperand(Num: 1)); |
5907 | SDValue Idx = N->getOperand(Num: 2); |
5908 | EVT PromVT = EVT::getVectorVT(Context&: *DAG.getContext(), |
5909 | VT: V1.getValueType().getVectorElementType(), |
5910 | EC: V0.getValueType().getVectorElementCount()); |
5911 | V0 = DAG.getAnyExtOrTrunc(Op: V0, DL: dl, VT: PromVT); |
5912 | SDValue Ext = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: PromVT, N1: V0, N2: V1, N3: Idx); |
5913 | return DAG.getAnyExtOrTrunc(Op: Ext, DL: dl, VT: N->getValueType(ResNo: 0)); |
5914 | } |
5915 | |
5916 | SDValue DAGTypeLegalizer::(SDNode *N) { |
5917 | SDLoc dl(N); |
5918 | SDValue V0 = GetPromotedInteger(Op: N->getOperand(Num: 0)); |
5919 | MVT InVT = V0.getValueType().getSimpleVT(); |
5920 | MVT OutVT = MVT::getVectorVT(VT: InVT.getVectorElementType(), |
5921 | NumElements: N->getValueType(ResNo: 0).getVectorNumElements()); |
5922 | SDValue Ext = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: OutVT, N1: V0, N2: N->getOperand(Num: 1)); |
5923 | return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Ext); |
5924 | } |
5925 | |
5926 | SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
5927 | SDLoc dl(N); |
5928 | |
5929 | EVT ResVT = N->getValueType(ResNo: 0); |
5930 | unsigned NumElems = N->getNumOperands(); |
5931 | |
5932 | if (ResVT.isScalableVector()) { |
5933 | SDValue ResVec = DAG.getUNDEF(VT: ResVT); |
5934 | |
5935 | for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { |
5936 | SDValue Op = N->getOperand(Num: OpIdx); |
5937 | unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); |
5938 | ResVec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: ResVec, N2: Op, |
5939 | N3: DAG.getIntPtrConstant(Val: OpIdx * OpNumElts, DL: dl)); |
5940 | } |
5941 | |
5942 | return ResVec; |
5943 | } |
5944 | |
5945 | EVT RetSclrTy = N->getValueType(ResNo: 0).getVectorElementType(); |
5946 | |
5947 | SmallVector<SDValue, 8> NewOps; |
5948 | NewOps.reserve(N: NumElems); |
5949 | |
5950 | // For each incoming vector |
5951 | for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { |
5952 | SDValue Incoming = GetPromotedInteger(Op: N->getOperand(Num: VecIdx)); |
5953 | EVT SclrTy = Incoming->getValueType(ResNo: 0).getVectorElementType(); |
5954 | unsigned NumElem = Incoming->getValueType(ResNo: 0).getVectorNumElements(); |
5955 | |
5956 | for (unsigned i=0; i<NumElem; ++i) { |
5957 | // Extract element from incoming vector |
5958 | SDValue Ex = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SclrTy, N1: Incoming, |
5959 | N2: DAG.getVectorIdxConstant(Val: i, DL: dl)); |
5960 | SDValue Tr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RetSclrTy, Operand: Ex); |
5961 | NewOps.push_back(Elt: Tr); |
5962 | } |
5963 | } |
5964 | |
5965 | return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: dl, Ops: NewOps); |
5966 | } |
5967 | |
5968 | SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) { |
5969 | assert(OpNo > 1); |
5970 | SDValue Op = N->getOperand(Num: OpNo); |
5971 | |
5972 | // FIXME: Non-constant operands are not yet handled: |
5973 | // - https://github.com/llvm/llvm-project/issues/26431 |
5974 | // - https://github.com/llvm/llvm-project/issues/55957 |
5975 | ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op); |
5976 | if (!CN) |
5977 | return SDValue(); |
5978 | |
5979 | // Copy operands before the one being expanded. |
5980 | SmallVector<SDValue> NewOps; |
5981 | for (unsigned I = 0; I < OpNo; I++) |
5982 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
5983 | |
5984 | EVT Ty = Op.getValueType(); |
5985 | SDLoc DL = SDLoc(N); |
5986 | if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
5987 | NewOps.push_back( |
5988 | DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
5989 | NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty)); |
5990 | } else { |
5991 | // FIXME: https://github.com/llvm/llvm-project/issues/55609 |
5992 | return SDValue(); |
5993 | } |
5994 | |
5995 | // Copy remaining operands. |
5996 | for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
5997 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
5998 | |
5999 | SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps); |
6000 | |
6001 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
6002 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
6003 | |
6004 | return SDValue(); // Signal that we have replaced the node already. |
6005 | } |
6006 | |
6007 | SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) { |
6008 | assert(OpNo >= 7); |
6009 | SDValue Op = N->getOperand(Num: OpNo); |
6010 | |
6011 | // FIXME: Non-constant operands are not yet handled: |
6012 | // - https://github.com/llvm/llvm-project/issues/26431 |
6013 | // - https://github.com/llvm/llvm-project/issues/55957 |
6014 | ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op); |
6015 | if (!CN) |
6016 | return SDValue(); |
6017 | |
6018 | // Copy operands before the one being expanded. |
6019 | SmallVector<SDValue> NewOps; |
6020 | for (unsigned I = 0; I < OpNo; I++) |
6021 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
6022 | |
6023 | EVT Ty = Op.getValueType(); |
6024 | SDLoc DL = SDLoc(N); |
6025 | if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) { |
6026 | NewOps.push_back( |
6027 | DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); |
6028 | NewOps.push_back(Elt: DAG.getTargetConstant(Val: CN->getZExtValue(), DL, VT: Ty)); |
6029 | } else { |
6030 | // FIXME: https://github.com/llvm/llvm-project/issues/55609 |
6031 | return SDValue(); |
6032 | } |
6033 | |
6034 | // Copy remaining operands. |
6035 | for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++) |
6036 | NewOps.push_back(Elt: N->getOperand(Num: I)); |
6037 | |
6038 | SDValue NewNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), Ops: NewOps); |
6039 | |
6040 | for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) |
6041 | ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum)); |
6042 | |
6043 | return SDValue(); // Signal that we have replaced the node already. |
6044 | } |
6045 | |