1 | //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines a pattern matching instruction selector for PowerPC, |
10 | // converting from a legalized dag to a PPC dag. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "MCTargetDesc/PPCMCTargetDesc.h" |
15 | #include "MCTargetDesc/PPCPredicates.h" |
16 | #include "PPC.h" |
17 | #include "PPCISelLowering.h" |
18 | #include "PPCMachineFunctionInfo.h" |
19 | #include "PPCSubtarget.h" |
20 | #include "PPCTargetMachine.h" |
21 | #include "llvm/ADT/APInt.h" |
22 | #include "llvm/ADT/APSInt.h" |
23 | #include "llvm/ADT/DenseMap.h" |
24 | #include "llvm/ADT/STLExtras.h" |
25 | #include "llvm/ADT/SmallPtrSet.h" |
26 | #include "llvm/ADT/SmallVector.h" |
27 | #include "llvm/ADT/Statistic.h" |
28 | #include "llvm/Analysis/BranchProbabilityInfo.h" |
29 | #include "llvm/CodeGen/FunctionLoweringInfo.h" |
30 | #include "llvm/CodeGen/ISDOpcodes.h" |
31 | #include "llvm/CodeGen/MachineBasicBlock.h" |
32 | #include "llvm/CodeGen/MachineFrameInfo.h" |
33 | #include "llvm/CodeGen/MachineFunction.h" |
34 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
35 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
36 | #include "llvm/CodeGen/SelectionDAG.h" |
37 | #include "llvm/CodeGen/SelectionDAGISel.h" |
38 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
39 | #include "llvm/CodeGen/TargetInstrInfo.h" |
40 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
41 | #include "llvm/CodeGen/ValueTypes.h" |
42 | #include "llvm/CodeGenTypes/MachineValueType.h" |
43 | #include "llvm/IR/BasicBlock.h" |
44 | #include "llvm/IR/DebugLoc.h" |
45 | #include "llvm/IR/Function.h" |
46 | #include "llvm/IR/GlobalValue.h" |
47 | #include "llvm/IR/InlineAsm.h" |
48 | #include "llvm/IR/InstrTypes.h" |
49 | #include "llvm/IR/IntrinsicsPowerPC.h" |
50 | #include "llvm/IR/Module.h" |
51 | #include "llvm/Support/Casting.h" |
52 | #include "llvm/Support/CodeGen.h" |
53 | #include "llvm/Support/CommandLine.h" |
54 | #include "llvm/Support/Compiler.h" |
55 | #include "llvm/Support/Debug.h" |
56 | #include "llvm/Support/ErrorHandling.h" |
57 | #include "llvm/Support/KnownBits.h" |
58 | #include "llvm/Support/MathExtras.h" |
59 | #include "llvm/Support/raw_ostream.h" |
60 | #include <algorithm> |
61 | #include <cassert> |
62 | #include <cstdint> |
63 | #include <iterator> |
64 | #include <limits> |
65 | #include <memory> |
66 | #include <new> |
67 | #include <tuple> |
68 | #include <utility> |
69 | |
70 | using namespace llvm; |
71 | |
72 | #define DEBUG_TYPE "ppc-isel" |
73 | #define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection" |
74 | |
75 | STATISTIC(NumSextSetcc, |
76 | "Number of (sext(setcc)) nodes expanded into GPR sequence." ); |
77 | STATISTIC(NumZextSetcc, |
78 | "Number of (zext(setcc)) nodes expanded into GPR sequence." ); |
79 | STATISTIC(SignExtensionsAdded, |
80 | "Number of sign extensions for compare inputs added." ); |
81 | STATISTIC(ZeroExtensionsAdded, |
82 | "Number of zero extensions for compare inputs added." ); |
83 | STATISTIC(NumLogicOpsOnComparison, |
84 | "Number of logical ops on i1 values calculated in GPR." ); |
85 | STATISTIC(OmittedForNonExtendUses, |
86 | "Number of compares not eliminated as they have non-extending uses." ); |
87 | STATISTIC(NumP9Setb, |
88 | "Number of compares lowered to setb." ); |
89 | |
90 | // FIXME: Remove this once the bug has been fixed! |
91 | cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug" , |
92 | cl::desc("expose the ANDI glue bug on PPC" ), cl::Hidden); |
93 | |
94 | static cl::opt<bool> |
95 | UseBitPermRewriter("ppc-use-bit-perm-rewriter" , cl::init(Val: true), |
96 | cl::desc("use aggressive ppc isel for bit permutations" ), |
97 | cl::Hidden); |
98 | static cl::opt<bool> BPermRewriterNoMasking( |
99 | "ppc-bit-perm-rewriter-stress-rotates" , |
100 | cl::desc("stress rotate selection in aggressive ppc isel for " |
101 | "bit permutations" ), |
102 | cl::Hidden); |
103 | |
104 | static cl::opt<bool> EnableBranchHint( |
105 | "ppc-use-branch-hint" , cl::init(Val: true), |
106 | cl::desc("Enable static hinting of branches on ppc" ), |
107 | cl::Hidden); |
108 | |
109 | static cl::opt<bool> EnableTLSOpt( |
110 | "ppc-tls-opt" , cl::init(Val: true), |
111 | cl::desc("Enable tls optimization peephole" ), |
112 | cl::Hidden); |
113 | |
114 | enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64, |
115 | ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32, |
116 | ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 }; |
117 | |
118 | static cl::opt<ICmpInGPRType> CmpInGPR( |
119 | "ppc-gpr-icmps" , cl::Hidden, cl::init(Val: ICGPR_All), |
120 | cl::desc("Specify the types of comparisons to emit GPR-only code for." ), |
121 | cl::values(clEnumValN(ICGPR_None, "none" , "Do not modify integer comparisons." ), |
122 | clEnumValN(ICGPR_All, "all" , "All possible int comparisons in GPRs." ), |
123 | clEnumValN(ICGPR_I32, "i32" , "Only i32 comparisons in GPRs." ), |
124 | clEnumValN(ICGPR_I64, "i64" , "Only i64 comparisons in GPRs." ), |
125 | clEnumValN(ICGPR_NonExtIn, "nonextin" , |
126 | "Only comparisons where inputs don't need [sz]ext." ), |
127 | clEnumValN(ICGPR_Zext, "zext" , "Only comparisons with zext result." ), |
128 | clEnumValN(ICGPR_ZextI32, "zexti32" , |
129 | "Only i32 comparisons with zext result." ), |
130 | clEnumValN(ICGPR_ZextI64, "zexti64" , |
131 | "Only i64 comparisons with zext result." ), |
132 | clEnumValN(ICGPR_Sext, "sext" , "Only comparisons with sext result." ), |
133 | clEnumValN(ICGPR_SextI32, "sexti32" , |
134 | "Only i32 comparisons with sext result." ), |
135 | clEnumValN(ICGPR_SextI64, "sexti64" , |
136 | "Only i64 comparisons with sext result." ))); |
137 | namespace { |
138 | |
139 | //===--------------------------------------------------------------------===// |
140 | /// PPCDAGToDAGISel - PPC specific code to select PPC machine |
141 | /// instructions for SelectionDAG operations. |
142 | /// |
143 | class PPCDAGToDAGISel : public SelectionDAGISel { |
144 | const PPCTargetMachine &TM; |
145 | const PPCSubtarget *Subtarget = nullptr; |
146 | const PPCTargetLowering *PPCLowering = nullptr; |
147 | unsigned GlobalBaseReg = 0; |
148 | |
149 | public: |
150 | static char ID; |
151 | |
152 | PPCDAGToDAGISel() = delete; |
153 | |
154 | explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOptLevel OptLevel) |
155 | : SelectionDAGISel(ID, tm, OptLevel), TM(tm) {} |
156 | |
157 | bool runOnMachineFunction(MachineFunction &MF) override { |
158 | // Make sure we re-emit a set of the global base reg if necessary |
159 | GlobalBaseReg = 0; |
160 | Subtarget = &MF.getSubtarget<PPCSubtarget>(); |
161 | PPCLowering = Subtarget->getTargetLowering(); |
162 | if (Subtarget->hasROPProtect()) { |
163 | // Create a place on the stack for the ROP Protection Hash. |
164 | // The ROP Protection Hash will always be 8 bytes and aligned to 8 |
165 | // bytes. |
166 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
167 | PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); |
168 | const int Result = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false); |
169 | FI->setROPProtectionHashSaveIndex(Result); |
170 | } |
171 | SelectionDAGISel::runOnMachineFunction(MF); |
172 | |
173 | return true; |
174 | } |
175 | |
176 | void PreprocessISelDAG() override; |
177 | void PostprocessISelDAG() override; |
178 | |
179 | /// getI16Imm - Return a target constant with the specified value, of type |
180 | /// i16. |
181 | inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) { |
182 | return CurDAG->getTargetConstant(Imm, dl, MVT::i16); |
183 | } |
184 | |
185 | /// getI32Imm - Return a target constant with the specified value, of type |
186 | /// i32. |
187 | inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
188 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
189 | } |
190 | |
191 | /// getI64Imm - Return a target constant with the specified value, of type |
192 | /// i64. |
193 | inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) { |
194 | return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
195 | } |
196 | |
197 | /// getSmallIPtrImm - Return a target constant of pointer type. |
198 | inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) { |
199 | return CurDAG->getTargetConstant( |
200 | Val: Imm, DL: dl, VT: PPCLowering->getPointerTy(DL: CurDAG->getDataLayout())); |
201 | } |
202 | |
203 | /// isRotateAndMask - Returns true if Mask and Shift can be folded into a |
204 | /// rotate and mask opcode and mask operation. |
205 | static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, |
206 | unsigned &SH, unsigned &MB, unsigned &ME); |
207 | |
208 | /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC |
209 | /// base register. Return the virtual register that holds this value. |
210 | SDNode *getGlobalBaseReg(); |
211 | |
212 | void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0); |
213 | |
214 | // Select - Convert the specified operand from a target-independent to a |
215 | // target-specific node if it hasn't already been changed. |
216 | void Select(SDNode *N) override; |
217 | |
218 | bool tryBitfieldInsert(SDNode *N); |
219 | bool tryBitPermutation(SDNode *N); |
220 | bool tryIntCompareInGPR(SDNode *N); |
221 | |
222 | // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into |
223 | // an X-Form load instruction with the offset being a relocation coming from |
224 | // the PPCISD::ADD_TLS. |
225 | bool tryTLSXFormLoad(LoadSDNode *N); |
226 | // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into |
227 | // an X-Form store instruction with the offset being a relocation coming from |
228 | // the PPCISD::ADD_TLS. |
229 | bool tryTLSXFormStore(StoreSDNode *N); |
230 | /// SelectCC - Select a comparison of the specified values with the |
231 | /// specified condition code, returning the CR# of the expression. |
232 | SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
233 | const SDLoc &dl, SDValue Chain = SDValue()); |
234 | |
235 | /// SelectAddrImmOffs - Return true if the operand is valid for a preinc |
236 | /// immediate field. Note that the operand at this point is already the |
237 | /// result of a prior SelectAddressRegImm call. |
238 | bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { |
239 | if (N.getOpcode() == ISD::TargetConstant || |
240 | N.getOpcode() == ISD::TargetGlobalAddress) { |
241 | Out = N; |
242 | return true; |
243 | } |
244 | |
245 | return false; |
246 | } |
247 | |
248 | /// SelectDSForm - Returns true if address N can be represented by the |
249 | /// addressing mode of DSForm instructions (a base register, plus a signed |
250 | /// 16-bit displacement that is a multiple of 4. |
251 | bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
252 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
253 | Align: Align(4)) == PPC::AM_DSForm; |
254 | } |
255 | |
256 | /// SelectDQForm - Returns true if address N can be represented by the |
257 | /// addressing mode of DQForm instructions (a base register, plus a signed |
258 | /// 16-bit displacement that is a multiple of 16. |
259 | bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
260 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
261 | Align: Align(16)) == PPC::AM_DQForm; |
262 | } |
263 | |
264 | /// SelectDForm - Returns true if address N can be represented by |
265 | /// the addressing mode of DForm instructions (a base register, plus a |
266 | /// signed 16-bit immediate. |
267 | bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
268 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
269 | Align: std::nullopt) == PPC::AM_DForm; |
270 | } |
271 | |
272 | /// SelectPCRelForm - Returns true if address N can be represented by |
273 | /// PC-Relative addressing mode. |
274 | bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp, |
275 | SDValue &Base) { |
276 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
277 | Align: std::nullopt) == PPC::AM_PCRel; |
278 | } |
279 | |
280 | /// SelectPDForm - Returns true if address N can be represented by Prefixed |
281 | /// DForm addressing mode (a base register, plus a signed 34-bit immediate. |
282 | bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
283 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
284 | Align: std::nullopt) == |
285 | PPC::AM_PrefixDForm; |
286 | } |
287 | |
288 | /// SelectXForm - Returns true if address N can be represented by the |
289 | /// addressing mode of XForm instructions (an indexed [r+r] operation). |
290 | bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { |
291 | return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, DAG&: *CurDAG, |
292 | Align: std::nullopt) == PPC::AM_XForm; |
293 | } |
294 | |
295 | /// SelectForceXForm - Given the specified address, force it to be |
296 | /// represented as an indexed [r+r] operation (an XForm instruction). |
297 | bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp, |
298 | SDValue &Base) { |
299 | return PPCLowering->SelectForceXFormMode(N, Disp, Base, DAG&: *CurDAG) == |
300 | PPC::AM_XForm; |
301 | } |
302 | |
303 | /// SelectAddrIdx - Given the specified address, check to see if it can be |
304 | /// represented as an indexed [r+r] operation. |
305 | /// This is for xform instructions whose associated displacement form is D. |
306 | /// The last parameter \p 0 means associated D form has no requirment for 16 |
307 | /// bit signed displacement. |
308 | /// Returns false if it can be represented by [r+imm], which are preferred. |
309 | bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { |
310 | return PPCLowering->SelectAddressRegReg(N, Base, Index, DAG&: *CurDAG, |
311 | EncodingAlignment: std::nullopt); |
312 | } |
313 | |
314 | /// SelectAddrIdx4 - Given the specified address, check to see if it can be |
315 | /// represented as an indexed [r+r] operation. |
316 | /// This is for xform instructions whose associated displacement form is DS. |
317 | /// The last parameter \p 4 means associated DS form 16 bit signed |
318 | /// displacement must be a multiple of 4. |
319 | /// Returns false if it can be represented by [r+imm], which are preferred. |
320 | bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) { |
321 | return PPCLowering->SelectAddressRegReg(N, Base, Index, DAG&: *CurDAG, |
322 | EncodingAlignment: Align(4)); |
323 | } |
324 | |
325 | /// SelectAddrIdx16 - Given the specified address, check to see if it can be |
326 | /// represented as an indexed [r+r] operation. |
327 | /// This is for xform instructions whose associated displacement form is DQ. |
328 | /// The last parameter \p 16 means associated DQ form 16 bit signed |
329 | /// displacement must be a multiple of 16. |
330 | /// Returns false if it can be represented by [r+imm], which are preferred. |
331 | bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) { |
332 | return PPCLowering->SelectAddressRegReg(N, Base, Index, DAG&: *CurDAG, |
333 | EncodingAlignment: Align(16)); |
334 | } |
335 | |
336 | /// SelectAddrIdxOnly - Given the specified address, force it to be |
337 | /// represented as an indexed [r+r] operation. |
338 | bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { |
339 | return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, DAG&: *CurDAG); |
340 | } |
341 | |
342 | /// SelectAddrImm - Returns true if the address N can be represented by |
343 | /// a base register plus a signed 16-bit displacement [r+imm]. |
344 | /// The last parameter \p 0 means D form has no requirment for 16 bit signed |
345 | /// displacement. |
346 | bool SelectAddrImm(SDValue N, SDValue &Disp, |
347 | SDValue &Base) { |
348 | return PPCLowering->SelectAddressRegImm(N, Disp, Base, DAG&: *CurDAG, |
349 | EncodingAlignment: std::nullopt); |
350 | } |
351 | |
352 | /// SelectAddrImmX4 - Returns true if the address N can be represented by |
353 | /// a base register plus a signed 16-bit displacement that is a multiple of |
354 | /// 4 (last parameter). Suitable for use by STD and friends. |
355 | bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { |
356 | return PPCLowering->SelectAddressRegImm(N, Disp, Base, DAG&: *CurDAG, EncodingAlignment: Align(4)); |
357 | } |
358 | |
359 | /// SelectAddrImmX16 - Returns true if the address N can be represented by |
360 | /// a base register plus a signed 16-bit displacement that is a multiple of |
361 | /// 16(last parameter). Suitable for use by STXV and friends. |
362 | bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) { |
363 | return PPCLowering->SelectAddressRegImm(N, Disp, Base, DAG&: *CurDAG, |
364 | EncodingAlignment: Align(16)); |
365 | } |
366 | |
367 | /// SelectAddrImmX34 - Returns true if the address N can be represented by |
368 | /// a base register plus a signed 34-bit displacement. Suitable for use by |
369 | /// PSTXVP and friends. |
370 | bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) { |
371 | return PPCLowering->SelectAddressRegImm34(N, Disp, Base, DAG&: *CurDAG); |
372 | } |
373 | |
374 | // Select an address into a single register. |
375 | bool SelectAddr(SDValue N, SDValue &Base) { |
376 | Base = N; |
377 | return true; |
378 | } |
379 | |
380 | bool SelectAddrPCRel(SDValue N, SDValue &Base) { |
381 | return PPCLowering->SelectAddressPCRel(N, Base); |
382 | } |
383 | |
384 | /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for |
385 | /// inline asm expressions. It is always correct to compute the value into |
386 | /// a register. The case of adding a (possibly relocatable) constant to a |
387 | /// register can be improved, but it is wrong to substitute Reg+Reg for |
388 | /// Reg in an asm, because the load or store opcode would have to change. |
389 | bool SelectInlineAsmMemoryOperand(const SDValue &Op, |
390 | InlineAsm::ConstraintCode ConstraintID, |
391 | std::vector<SDValue> &OutOps) override { |
392 | switch(ConstraintID) { |
393 | default: |
394 | errs() << "ConstraintID: " |
395 | << InlineAsm::getMemConstraintName(C: ConstraintID) << "\n" ; |
396 | llvm_unreachable("Unexpected asm memory constraint" ); |
397 | case InlineAsm::ConstraintCode::es: |
398 | case InlineAsm::ConstraintCode::m: |
399 | case InlineAsm::ConstraintCode::o: |
400 | case InlineAsm::ConstraintCode::Q: |
401 | case InlineAsm::ConstraintCode::Z: |
402 | case InlineAsm::ConstraintCode::Zy: |
403 | // We need to make sure that this one operand does not end up in r0 |
404 | // (because we might end up lowering this as 0(%op)). |
405 | const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
406 | const TargetRegisterClass *TRC = TRI->getPointerRegClass(MF: *MF, /*Kind=*/1); |
407 | SDLoc dl(Op); |
408 | SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); |
409 | SDValue NewOp = |
410 | SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, |
411 | dl, VT: Op.getValueType(), |
412 | Op1: Op, Op2: RC), 0); |
413 | |
414 | OutOps.push_back(NewOp); |
415 | return false; |
416 | } |
417 | return true; |
418 | } |
419 | |
420 | // Include the pieces autogenerated from the target description. |
421 | #include "PPCGenDAGISel.inc" |
422 | |
423 | private: |
424 | bool trySETCC(SDNode *N); |
425 | bool tryFoldSWTestBRCC(SDNode *N); |
426 | bool trySelectLoopCountIntrinsic(SDNode *N); |
427 | bool tryAsSingleRLDICL(SDNode *N); |
428 | bool tryAsSingleRLDCL(SDNode *N); |
429 | bool tryAsSingleRLDICR(SDNode *N); |
430 | bool tryAsSingleRLWINM(SDNode *N); |
431 | bool tryAsSingleRLWINM8(SDNode *N); |
432 | bool tryAsSingleRLWIMI(SDNode *N); |
433 | bool tryAsPairOfRLDICL(SDNode *N); |
434 | bool tryAsSingleRLDIMI(SDNode *N); |
435 | |
436 | void PeepholePPC64(); |
437 | void PeepholePPC64ZExt(); |
438 | void PeepholeCROps(); |
439 | |
440 | SDValue combineToCMPB(SDNode *N); |
441 | void foldBoolExts(SDValue &Res, SDNode *&N); |
442 | |
443 | bool AllUsersSelectZero(SDNode *N); |
444 | void SwapAllSelectUsers(SDNode *N); |
445 | |
446 | bool isOffsetMultipleOf(SDNode *N, unsigned Val) const; |
447 | void transferMemOperands(SDNode *N, SDNode *Result); |
448 | }; |
449 | |
450 | } // end anonymous namespace |
451 | |
452 | char PPCDAGToDAGISel::ID = 0; |
453 | |
454 | INITIALIZE_PASS(PPCDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) |
455 | |
456 | /// getGlobalBaseReg - Output the instructions required to put the |
457 | /// base address to use for accessing globals into a register. |
458 | /// |
459 | SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { |
460 | if (!GlobalBaseReg) { |
461 | const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); |
462 | // Insert the set of GlobalBaseReg into the first MBB of the function |
463 | MachineBasicBlock &FirstMBB = MF->front(); |
464 | MachineBasicBlock::iterator MBBI = FirstMBB.begin(); |
465 | const Module *M = MF->getFunction().getParent(); |
466 | DebugLoc dl; |
467 | |
468 | if (PPCLowering->getPointerTy(DL: CurDAG->getDataLayout()) == MVT::i32) { |
469 | if (Subtarget->isTargetELF()) { |
470 | GlobalBaseReg = PPC::R30; |
471 | if (!Subtarget->isSecurePlt() && |
472 | M->getPICLevel() == PICLevel::SmallPIC) { |
473 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MoveGOTtoLR)); |
474 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MFLR), GlobalBaseReg); |
475 | MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); |
476 | } else { |
477 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MovePCtoLR)); |
478 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MFLR), GlobalBaseReg); |
479 | Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); |
480 | BuildMI(FirstMBB, MBBI, dl, |
481 | TII.get(PPC::Opcode: UpdateGBR), GlobalBaseReg) |
482 | .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); |
483 | MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); |
484 | } |
485 | } else { |
486 | GlobalBaseReg = |
487 | RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); |
488 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MovePCtoLR)); |
489 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MFLR), GlobalBaseReg); |
490 | } |
491 | } else { |
492 | // We must ensure that this sequence is dominated by the prologue. |
493 | // FIXME: This is a bit of a big hammer since we don't get the benefits |
494 | // of shrink-wrapping whenever we emit this instruction. Considering |
495 | // this is used in any function where we emit a jump table, this may be |
496 | // a significant limitation. We should consider inserting this in the |
497 | // block where it is used and then commoning this sequence up if it |
498 | // appears in multiple places. |
499 | // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of |
500 | // MovePCtoLR8. |
501 | MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true); |
502 | GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); |
503 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MovePCtoLR8)); |
504 | BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::Opcode: MFLR8), GlobalBaseReg); |
505 | } |
506 | } |
507 | return CurDAG->getRegister(Reg: GlobalBaseReg, |
508 | VT: PPCLowering->getPointerTy(DL: CurDAG->getDataLayout())) |
509 | .getNode(); |
510 | } |
511 | |
512 | // Check if a SDValue has the toc-data attribute. |
513 | static bool hasTocDataAttr(SDValue Val) { |
514 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val); |
515 | if (!GA) |
516 | return false; |
517 | |
518 | const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(Val: GA->getGlobal()); |
519 | if (!GV) |
520 | return false; |
521 | |
522 | if (!GV->hasAttribute(Kind: "toc-data" )) |
523 | return false; |
524 | return true; |
525 | } |
526 | |
527 | static CodeModel::Model getCodeModel(const PPCSubtarget &Subtarget, |
528 | const TargetMachine &TM, |
529 | const SDNode *Node) { |
530 | // If there isn't an attribute to override the module code model |
531 | // this will be the effective code model. |
532 | CodeModel::Model ModuleModel = TM.getCodeModel(); |
533 | |
534 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Node->getOperand(Num: 0)); |
535 | if (!GA) |
536 | return ModuleModel; |
537 | |
538 | const GlobalValue *GV = GA->getGlobal(); |
539 | if (!GV) |
540 | return ModuleModel; |
541 | |
542 | return Subtarget.getCodeModel(TM, GV); |
543 | } |
544 | |
545 | /// isInt32Immediate - This method tests to see if the node is a 32-bit constant |
546 | /// operand. If so Imm will receive the 32-bit value. |
547 | static bool isInt32Immediate(SDNode *N, unsigned &Imm) { |
548 | if (N->getOpcode() == ISD::Constant && N->getValueType(ResNo: 0) == MVT::i32) { |
549 | Imm = N->getAsZExtVal(); |
550 | return true; |
551 | } |
552 | return false; |
553 | } |
554 | |
555 | /// isInt64Immediate - This method tests to see if the node is a 64-bit constant |
556 | /// operand. If so Imm will receive the 64-bit value. |
557 | static bool isInt64Immediate(SDNode *N, uint64_t &Imm) { |
558 | if (N->getOpcode() == ISD::Constant && N->getValueType(ResNo: 0) == MVT::i64) { |
559 | Imm = N->getAsZExtVal(); |
560 | return true; |
561 | } |
562 | return false; |
563 | } |
564 | |
565 | // isInt32Immediate - This method tests to see if a constant operand. |
566 | // If so Imm will receive the 32 bit value. |
567 | static bool isInt32Immediate(SDValue N, unsigned &Imm) { |
568 | return isInt32Immediate(N: N.getNode(), Imm); |
569 | } |
570 | |
571 | /// isInt64Immediate - This method tests to see if the value is a 64-bit |
572 | /// constant operand. If so Imm will receive the 64-bit value. |
573 | static bool isInt64Immediate(SDValue N, uint64_t &Imm) { |
574 | return isInt64Immediate(N: N.getNode(), Imm); |
575 | } |
576 | |
577 | static unsigned getBranchHint(unsigned PCC, |
578 | const FunctionLoweringInfo &FuncInfo, |
579 | const SDValue &DestMBB) { |
580 | assert(isa<BasicBlockSDNode>(DestMBB)); |
581 | |
582 | if (!FuncInfo.BPI) return PPC::BR_NO_HINT; |
583 | |
584 | const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); |
585 | const Instruction *BBTerm = BB->getTerminator(); |
586 | |
587 | if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; |
588 | |
589 | const BasicBlock *TBB = BBTerm->getSuccessor(Idx: 0); |
590 | const BasicBlock *FBB = BBTerm->getSuccessor(Idx: 1); |
591 | |
592 | auto TProb = FuncInfo.BPI->getEdgeProbability(Src: BB, Dst: TBB); |
593 | auto FProb = FuncInfo.BPI->getEdgeProbability(Src: BB, Dst: FBB); |
594 | |
595 | // We only want to handle cases which are easy to predict at static time, e.g. |
596 | // C++ throw statement, that is very likely not taken, or calling never |
597 | // returned function, e.g. stdlib exit(). So we set Threshold to filter |
598 | // unwanted cases. |
599 | // |
600 | // Below is LLVM branch weight table, we only want to handle case 1, 2 |
601 | // |
602 | // Case Taken:Nontaken Example |
603 | // 1. Unreachable 1048575:1 C++ throw, stdlib exit(), |
604 | // 2. Invoke-terminating 1:1048575 |
605 | // 3. Coldblock 4:64 __builtin_expect |
606 | // 4. Loop Branch 124:4 For loop |
607 | // 5. PH/ZH/FPH 20:12 |
608 | const uint32_t Threshold = 10000; |
609 | |
610 | if (std::max(a: TProb, b: FProb) / Threshold < std::min(a: TProb, b: FProb)) |
611 | return PPC::BR_NO_HINT; |
612 | |
613 | LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() |
614 | << "::" << BB->getName() << "'\n" |
615 | << " -> " << TBB->getName() << ": " << TProb << "\n" |
616 | << " -> " << FBB->getName() << ": " << FProb << "\n" ); |
617 | |
618 | const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(Val: DestMBB); |
619 | |
620 | // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, |
621 | // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock |
622 | if (BBDN->getBasicBlock()->getBasicBlock() != TBB) |
623 | std::swap(a&: TProb, b&: FProb); |
624 | |
625 | return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; |
626 | } |
627 | |
628 | // isOpcWithIntImmediate - This method tests to see if the node is a specific |
629 | // opcode and that it has a immediate integer right operand. |
630 | // If so Imm will receive the 32 bit value. |
631 | static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { |
632 | return N->getOpcode() == Opc |
633 | && isInt32Immediate(N: N->getOperand(Num: 1).getNode(), Imm); |
634 | } |
635 | |
636 | void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) { |
637 | SDLoc dl(SN); |
638 | int FI = cast<FrameIndexSDNode>(Val: N)->getIndex(); |
639 | SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT: N->getValueType(ResNo: 0)); |
640 | unsigned Opc = N->getValueType(ResNo: 0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; |
641 | if (SN->hasOneUse()) |
642 | CurDAG->SelectNodeTo(N: SN, MachineOpc: Opc, VT: N->getValueType(ResNo: 0), Op1: TFI, |
643 | Op2: getSmallIPtrImm(Imm: Offset, dl)); |
644 | else |
645 | ReplaceNode(F: SN, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Op1: TFI, |
646 | Op2: getSmallIPtrImm(Imm: Offset, dl))); |
647 | } |
648 | |
649 | bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, |
650 | bool isShiftMask, unsigned &SH, |
651 | unsigned &MB, unsigned &ME) { |
652 | // Don't even go down this path for i64, since different logic will be |
653 | // necessary for rldicl/rldicr/rldimi. |
654 | if (N->getValueType(ResNo: 0) != MVT::i32) |
655 | return false; |
656 | |
657 | unsigned Shift = 32; |
658 | unsigned Indeterminant = ~0; // bit mask marking indeterminant results |
659 | unsigned Opcode = N->getOpcode(); |
660 | if (N->getNumOperands() != 2 || |
661 | !isInt32Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Shift) || (Shift > 31)) |
662 | return false; |
663 | |
664 | if (Opcode == ISD::SHL) { |
665 | // apply shift left to mask if it comes first |
666 | if (isShiftMask) Mask = Mask << Shift; |
667 | // determine which bits are made indeterminant by shift |
668 | Indeterminant = ~(0xFFFFFFFFu << Shift); |
669 | } else if (Opcode == ISD::SRL) { |
670 | // apply shift right to mask if it comes first |
671 | if (isShiftMask) Mask = Mask >> Shift; |
672 | // determine which bits are made indeterminant by shift |
673 | Indeterminant = ~(0xFFFFFFFFu >> Shift); |
674 | // adjust for the left rotate |
675 | Shift = 32 - Shift; |
676 | } else if (Opcode == ISD::ROTL) { |
677 | Indeterminant = 0; |
678 | } else { |
679 | return false; |
680 | } |
681 | |
682 | // if the mask doesn't intersect any Indeterminant bits |
683 | if (Mask && !(Mask & Indeterminant)) { |
684 | SH = Shift & 31; |
685 | // make sure the mask is still a mask (wrap arounds may not be) |
686 | return isRunOfOnes(Val: Mask, MB, ME); |
687 | } |
688 | return false; |
689 | } |
690 | |
691 | // isThreadPointerAcquisitionNode - Check if the operands of an ADD_TLS |
692 | // instruction use the thread pointer. |
693 | static bool isThreadPointerAcquisitionNode(SDValue Base, SelectionDAG *CurDAG) { |
694 | assert( |
695 | Base.getOpcode() == PPCISD::ADD_TLS && |
696 | "Only expecting the ADD_TLS instruction to acquire the thread pointer!" ); |
697 | const PPCSubtarget &Subtarget = |
698 | CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>(); |
699 | SDValue ADDTLSOp1 = Base.getOperand(i: 0); |
700 | unsigned ADDTLSOp1Opcode = ADDTLSOp1.getOpcode(); |
701 | |
702 | // Account for when ADD_TLS is used for the initial-exec TLS model on Linux. |
703 | // |
704 | // Although ADD_TLS does not explicitly use the thread pointer |
705 | // register when LD_GOT_TPREL_L is one of it's operands, the LD_GOT_TPREL_L |
706 | // instruction will have a relocation specifier, @got@tprel, that is used to |
707 | // generate a GOT entry. The linker replaces this entry with an offset for a |
708 | // for a thread local variable, which will be relative to the thread pointer. |
709 | if (ADDTLSOp1Opcode == PPCISD::LD_GOT_TPREL_L) |
710 | return true; |
711 | // When using PC-Relative instructions for initial-exec, a MAT_PCREL_ADDR |
712 | // node is produced instead to represent the aforementioned situation. |
713 | LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: ADDTLSOp1); |
714 | if (LD && LD->getBasePtr().getOpcode() == PPCISD::MAT_PCREL_ADDR) |
715 | return true; |
716 | |
717 | // A GET_TPOINTER PPCISD node (only produced on AIX 32-bit mode) as an operand |
718 | // to ADD_TLS represents a call to .__get_tpointer to get the thread pointer, |
719 | // later returning it into R3. |
720 | if (ADDTLSOp1Opcode == PPCISD::GET_TPOINTER) |
721 | return true; |
722 | |
723 | // The ADD_TLS note is explicitly acquiring the thread pointer (X13/R13). |
724 | RegisterSDNode *AddFirstOpReg = |
725 | dyn_cast_or_null<RegisterSDNode>(Val: ADDTLSOp1.getNode()); |
726 | if (AddFirstOpReg && |
727 | AddFirstOpReg->getReg() == Subtarget.getThreadPointerRegister()) |
728 | return true; |
729 | |
730 | return false; |
731 | } |
732 | |
733 | // canOptimizeTLSDFormToXForm - Optimize TLS accesses when an ADD_TLS |
734 | // instruction is present. An ADD_TLS instruction, followed by a D-Form memory |
735 | // operation, can be optimized to use an X-Form load or store, allowing the |
736 | // ADD_TLS node to be removed completely. |
737 | static bool canOptimizeTLSDFormToXForm(SelectionDAG *CurDAG, SDValue Base) { |
738 | |
739 | // Do not do this transformation at -O0. |
740 | if (CurDAG->getTarget().getOptLevel() == CodeGenOptLevel::None) |
741 | return false; |
742 | |
743 | // In order to perform this optimization inside tryTLSXForm[Load|Store], |
744 | // Base is expected to be an ADD_TLS node. |
745 | if (Base.getOpcode() != PPCISD::ADD_TLS) |
746 | return false; |
747 | for (auto *ADDTLSUse : Base.getNode()->uses()) { |
748 | // The optimization to convert the D-Form load/store into its X-Form |
749 | // counterpart should only occur if the source value offset of the load/ |
750 | // store is 0. This also means that The offset should always be undefined. |
751 | if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: ADDTLSUse)) { |
752 | if (LD->getSrcValueOffset() != 0 || !LD->getOffset().isUndef()) |
753 | return false; |
754 | } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: ADDTLSUse)) { |
755 | if (ST->getSrcValueOffset() != 0 || !ST->getOffset().isUndef()) |
756 | return false; |
757 | } else // Don't optimize if there are ADD_TLS users that aren't load/stores. |
758 | return false; |
759 | } |
760 | |
761 | if (Base.getOperand(i: 1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR) |
762 | return false; |
763 | |
764 | // Does the ADD_TLS node of the load/store use the thread pointer? |
765 | // If the thread pointer is not used as one of the operands of ADD_TLS, |
766 | // then this optimization is not valid. |
767 | return isThreadPointerAcquisitionNode(Base, CurDAG); |
768 | } |
769 | |
770 | bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) { |
771 | SDValue Base = ST->getBasePtr(); |
772 | if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
773 | return false; |
774 | |
775 | SDLoc dl(ST); |
776 | EVT MemVT = ST->getMemoryVT(); |
777 | EVT RegVT = ST->getValue().getValueType(); |
778 | |
779 | unsigned Opcode; |
780 | switch (MemVT.getSimpleVT().SimpleTy) { |
781 | default: |
782 | return false; |
783 | case MVT::i8: { |
784 | Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS; |
785 | break; |
786 | } |
787 | case MVT::i16: { |
788 | Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS; |
789 | break; |
790 | } |
791 | case MVT::i32: { |
792 | Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS; |
793 | break; |
794 | } |
795 | case MVT::i64: { |
796 | Opcode = PPC::STDXTLS; |
797 | break; |
798 | } |
799 | case MVT::f32: { |
800 | Opcode = PPC::STFSXTLS; |
801 | break; |
802 | } |
803 | case MVT::f64: { |
804 | Opcode = PPC::STFDXTLS; |
805 | break; |
806 | } |
807 | } |
808 | SDValue Chain = ST->getChain(); |
809 | SDVTList VTs = ST->getVTList(); |
810 | SDValue Ops[] = {ST->getValue(), Base.getOperand(i: 0), Base.getOperand(i: 1), |
811 | Chain}; |
812 | SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
813 | transferMemOperands(N: ST, Result: MN); |
814 | ReplaceNode(F: ST, T: MN); |
815 | return true; |
816 | } |
817 | |
818 | bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) { |
819 | SDValue Base = LD->getBasePtr(); |
820 | if (!canOptimizeTLSDFormToXForm(CurDAG, Base)) |
821 | return false; |
822 | |
823 | SDLoc dl(LD); |
824 | EVT MemVT = LD->getMemoryVT(); |
825 | EVT RegVT = LD->getValueType(ResNo: 0); |
826 | bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
827 | unsigned Opcode; |
828 | switch (MemVT.getSimpleVT().SimpleTy) { |
829 | default: |
830 | return false; |
831 | case MVT::i8: { |
832 | Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS; |
833 | break; |
834 | } |
835 | case MVT::i16: { |
836 | if (RegVT == MVT::i32) |
837 | Opcode = isSExt ? PPC::LHAXTLS_32 : PPC::LHZXTLS_32; |
838 | else |
839 | Opcode = isSExt ? PPC::LHAXTLS : PPC::LHZXTLS; |
840 | break; |
841 | } |
842 | case MVT::i32: { |
843 | if (RegVT == MVT::i32) |
844 | Opcode = isSExt ? PPC::LWAXTLS_32 : PPC::LWZXTLS_32; |
845 | else |
846 | Opcode = isSExt ? PPC::LWAXTLS : PPC::LWZXTLS; |
847 | break; |
848 | } |
849 | case MVT::i64: { |
850 | Opcode = PPC::LDXTLS; |
851 | break; |
852 | } |
853 | case MVT::f32: { |
854 | Opcode = PPC::LFSXTLS; |
855 | break; |
856 | } |
857 | case MVT::f64: { |
858 | Opcode = PPC::LFDXTLS; |
859 | break; |
860 | } |
861 | } |
862 | SDValue Chain = LD->getChain(); |
863 | SDVTList VTs = LD->getVTList(); |
864 | SDValue Ops[] = {Base.getOperand(i: 0), Base.getOperand(i: 1), Chain}; |
865 | SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); |
866 | transferMemOperands(N: LD, Result: MN); |
867 | ReplaceNode(F: LD, T: MN); |
868 | return true; |
869 | } |
870 | |
871 | /// Turn an or of two masked values into the rotate left word immediate then |
872 | /// mask insert (rlwimi) instruction. |
873 | bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) { |
874 | SDValue Op0 = N->getOperand(Num: 0); |
875 | SDValue Op1 = N->getOperand(Num: 1); |
876 | SDLoc dl(N); |
877 | |
878 | KnownBits LKnown = CurDAG->computeKnownBits(Op: Op0); |
879 | KnownBits RKnown = CurDAG->computeKnownBits(Op: Op1); |
880 | |
881 | unsigned TargetMask = LKnown.Zero.getZExtValue(); |
882 | unsigned InsertMask = RKnown.Zero.getZExtValue(); |
883 | |
884 | if ((TargetMask | InsertMask) == 0xFFFFFFFF) { |
885 | unsigned Op0Opc = Op0.getOpcode(); |
886 | unsigned Op1Opc = Op1.getOpcode(); |
887 | unsigned Value, SH = 0; |
888 | TargetMask = ~TargetMask; |
889 | InsertMask = ~InsertMask; |
890 | |
891 | // If the LHS has a foldable shift and the RHS does not, then swap it to the |
892 | // RHS so that we can fold the shift into the insert. |
893 | if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { |
894 | if (Op0.getOperand(i: 0).getOpcode() == ISD::SHL || |
895 | Op0.getOperand(i: 0).getOpcode() == ISD::SRL) { |
896 | if (Op1.getOperand(i: 0).getOpcode() != ISD::SHL && |
897 | Op1.getOperand(i: 0).getOpcode() != ISD::SRL) { |
898 | std::swap(a&: Op0, b&: Op1); |
899 | std::swap(a&: Op0Opc, b&: Op1Opc); |
900 | std::swap(a&: TargetMask, b&: InsertMask); |
901 | } |
902 | } |
903 | } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { |
904 | if (Op1Opc == ISD::AND && Op1.getOperand(i: 0).getOpcode() != ISD::SHL && |
905 | Op1.getOperand(i: 0).getOpcode() != ISD::SRL) { |
906 | std::swap(a&: Op0, b&: Op1); |
907 | std::swap(a&: Op0Opc, b&: Op1Opc); |
908 | std::swap(a&: TargetMask, b&: InsertMask); |
909 | } |
910 | } |
911 | |
912 | unsigned MB, ME; |
913 | if (isRunOfOnes(Val: InsertMask, MB, ME)) { |
914 | if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && |
915 | isInt32Immediate(N: Op1.getOperand(i: 1), Imm&: Value)) { |
916 | Op1 = Op1.getOperand(i: 0); |
917 | SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; |
918 | } |
919 | if (Op1Opc == ISD::AND) { |
920 | // The AND mask might not be a constant, and we need to make sure that |
921 | // if we're going to fold the masking with the insert, all bits not |
922 | // know to be zero in the mask are known to be one. |
923 | KnownBits MKnown = CurDAG->computeKnownBits(Op: Op1.getOperand(i: 1)); |
924 | bool CanFoldMask = InsertMask == MKnown.One.getZExtValue(); |
925 | |
926 | unsigned SHOpc = Op1.getOperand(i: 0).getOpcode(); |
927 | if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && |
928 | isInt32Immediate(N: Op1.getOperand(i: 0).getOperand(i: 1), Imm&: Value)) { |
929 | // Note that Value must be in range here (less than 32) because |
930 | // otherwise there would not be any bits set in InsertMask. |
931 | Op1 = Op1.getOperand(i: 0).getOperand(i: 0); |
932 | SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; |
933 | } |
934 | } |
935 | |
936 | SH &= 31; |
937 | SDValue Ops[] = { Op0, Op1, getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl), |
938 | getI32Imm(Imm: ME, dl) }; |
939 | ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
940 | return true; |
941 | } |
942 | } |
943 | return false; |
944 | } |
945 | |
946 | static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) { |
947 | unsigned MaxTruncation = 0; |
948 | // Cannot use range-based for loop here as we need the actual use (i.e. we |
949 | // need the operand number corresponding to the use). A range-based for |
950 | // will unbox the use and provide an SDNode*. |
951 | for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); |
952 | Use != UseEnd; ++Use) { |
953 | unsigned Opc = |
954 | Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); |
955 | switch (Opc) { |
956 | default: return 0; |
957 | case ISD::TRUNCATE: |
958 | if (Use->isMachineOpcode()) |
959 | return 0; |
960 | MaxTruncation = |
961 | std::max(a: MaxTruncation, b: (unsigned)Use->getValueType(ResNo: 0).getSizeInBits()); |
962 | continue; |
963 | case ISD::STORE: { |
964 | if (Use->isMachineOpcode()) |
965 | return 0; |
966 | StoreSDNode *STN = cast<StoreSDNode>(Val: *Use); |
967 | unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); |
968 | if (MemVTSize == 64 || Use.getOperandNo() != 0) |
969 | return 0; |
970 | MaxTruncation = std::max(a: MaxTruncation, b: MemVTSize); |
971 | continue; |
972 | } |
973 | case PPC::STW8: |
974 | case PPC::STWX8: |
975 | case PPC::STWU8: |
976 | case PPC::STWUX8: |
977 | if (Use.getOperandNo() != 0) |
978 | return 0; |
979 | MaxTruncation = std::max(a: MaxTruncation, b: 32u); |
980 | continue; |
981 | case PPC::STH8: |
982 | case PPC::STHX8: |
983 | case PPC::STHU8: |
984 | case PPC::STHUX8: |
985 | if (Use.getOperandNo() != 0) |
986 | return 0; |
987 | MaxTruncation = std::max(a: MaxTruncation, b: 16u); |
988 | continue; |
989 | case PPC::STB8: |
990 | case PPC::STBX8: |
991 | case PPC::STBU8: |
992 | case PPC::STBUX8: |
993 | if (Use.getOperandNo() != 0) |
994 | return 0; |
995 | MaxTruncation = std::max(a: MaxTruncation, b: 8u); |
996 | continue; |
997 | } |
998 | } |
999 | return MaxTruncation; |
1000 | } |
1001 | |
1002 | // For any 32 < Num < 64, check if the Imm contains at least Num consecutive |
1003 | // zeros and return the number of bits by the left of these consecutive zeros. |
1004 | static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) { |
1005 | unsigned HiTZ = llvm::countr_zero<uint32_t>(Val: Hi_32(Value: Imm)); |
1006 | unsigned LoLZ = llvm::countl_zero<uint32_t>(Val: Lo_32(Value: Imm)); |
1007 | if ((HiTZ + LoLZ) >= Num) |
1008 | return (32 + HiTZ); |
1009 | return 0; |
1010 | } |
1011 | |
1012 | // Direct materialization of 64-bit constants by enumerated patterns. |
1013 | static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl, |
1014 | uint64_t Imm, unsigned &InstCnt) { |
1015 | unsigned TZ = llvm::countr_zero<uint64_t>(Val: Imm); |
1016 | unsigned LZ = llvm::countl_zero<uint64_t>(Val: Imm); |
1017 | unsigned TO = llvm::countr_one<uint64_t>(Value: Imm); |
1018 | unsigned LO = llvm::countl_one<uint64_t>(Value: Imm); |
1019 | unsigned Hi32 = Hi_32(Value: Imm); |
1020 | unsigned Lo32 = Lo_32(Value: Imm); |
1021 | SDNode *Result = nullptr; |
1022 | unsigned Shift = 0; |
1023 | |
1024 | auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1025 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1026 | }; |
1027 | |
1028 | // Following patterns use 1 instructions to materialize the Imm. |
1029 | InstCnt = 1; |
1030 | // 1-1) Patterns : {zeros}{15-bit valve} |
1031 | // {ones}{15-bit valve} |
1032 | if (isInt<16>(x: Imm)) { |
1033 | SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
1034 | return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
1035 | } |
1036 | // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros} |
1037 | // {ones}{15-bit valve}{16 zeros} |
1038 | if (TZ > 15 && (LZ > 32 || LO > 32)) |
1039 | return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1040 | getI32Imm((Imm >> 16) & 0xffff)); |
1041 | |
1042 | // Following patterns use 2 instructions to materialize the Imm. |
1043 | InstCnt = 2; |
1044 | assert(LZ < 64 && "Unexpected leading zeros here." ); |
1045 | // Count of ones follwing the leading zeros. |
1046 | unsigned FO = llvm::countl_one<uint64_t>(Value: Imm << LZ); |
1047 | // 2-1) Patterns : {zeros}{31-bit value} |
1048 | // {ones}{31-bit value} |
1049 | if (isInt<32>(x: Imm)) { |
1050 | uint64_t ImmHi16 = (Imm >> 16) & 0xffff; |
1051 | unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1052 | Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1053 | return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1054 | getI32Imm(Imm & 0xffff)); |
1055 | } |
1056 | // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros} |
1057 | // {zeros}{15-bit value}{zeros} |
1058 | // {zeros}{ones}{15-bit value} |
1059 | // {ones}{15-bit value}{zeros} |
1060 | // We can take advantage of LI's sign-extension semantics to generate leading |
1061 | // ones, and then use RLDIC to mask off the ones in both sides after rotation. |
1062 | if ((LZ + FO + TZ) > 48) { |
1063 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1064 | getI32Imm((Imm >> TZ) & 0xffff)); |
1065 | return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1066 | getI32Imm(TZ), getI32Imm(LZ)); |
1067 | } |
1068 | // 2-3) Pattern : {zeros}{15-bit value}{ones} |
1069 | // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value, |
1070 | // therefore we can take advantage of LI's sign-extension semantics, and then |
1071 | // mask them off after rotation. |
1072 | // |
1073 | // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+ |
1074 | // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
1075 | // +------------------------+ +------------------------+ |
1076 | // 63 0 63 0 |
1077 | // Imm (Imm >> (48 - LZ) & 0xffff) |
1078 | // +----sext-----|--16-bit--+ +clear-|-----------------+ |
1079 | // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
1080 | // +------------------------+ +------------------------+ |
1081 | // 63 0 63 0 |
1082 | // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ |
1083 | if ((LZ + TO) > 48) { |
1084 | // Since the immediates with (LZ > 32) have been handled by previous |
1085 | // patterns, here we have (LZ <= 32) to make sure we will not shift right |
1086 | // the Imm by a negative value. |
1087 | assert(LZ <= 32 && "Unexpected shift value." ); |
1088 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1089 | getI32Imm((Imm >> (48 - LZ) & 0xffff))); |
1090 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1091 | getI32Imm(48 - LZ), getI32Imm(LZ)); |
1092 | } |
1093 | // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones} |
1094 | // {ones}{15-bit value}{ones} |
1095 | // We can take advantage of LI's sign-extension semantics to generate leading |
1096 | // ones, and then use RLDICL to mask off the ones in left sides (if required) |
1097 | // after rotation. |
1098 | // |
1099 | // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+ |
1100 | // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb| |
1101 | // +------------------------+ +------------------------+ |
1102 | // 63 0 63 0 |
1103 | // Imm (Imm >> TO) & 0xffff |
1104 | // +----sext-----|--16-bit--+ +LZ|---------------------+ |
1105 | // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111| |
1106 | // +------------------------+ +------------------------+ |
1107 | // 63 0 63 0 |
1108 | // LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ |
1109 | if ((LZ + FO + TO) > 48) { |
1110 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1111 | getI32Imm((Imm >> TO) & 0xffff)); |
1112 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1113 | getI32Imm(TO), getI32Imm(LZ)); |
1114 | } |
1115 | // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value} |
1116 | // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit |
1117 | // value, we can use LI for Lo16 without generating leading ones then add the |
1118 | // Hi16(in Lo32). |
1119 | if (LZ == 32 && ((Lo32 & 0x8000) == 0)) { |
1120 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1121 | getI32Imm(Lo32 & 0xffff)); |
1122 | return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0), |
1123 | getI32Imm(Lo32 >> 16)); |
1124 | } |
1125 | // 2-6) Patterns : {******}{49 zeros}{******} |
1126 | // {******}{49 ones}{******} |
1127 | // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15 |
1128 | // bits remain on both sides. Rotate right the Imm to construct an int<16> |
1129 | // value, use LI for int<16> value and then use RLDICL without mask to rotate |
1130 | // it back. |
1131 | // |
1132 | // 1) findContiguousZerosAtLeast(Imm, 49) |
1133 | // +------|--zeros-|------+ +---ones--||---15 bit--+ |
1134 | // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb| |
1135 | // +----------------------+ +----------------------+ |
1136 | // 63 0 63 0 |
1137 | // |
1138 | // 2) findContiguousZerosAtLeast(~Imm, 49) |
1139 | // +------|--ones--|------+ +---ones--||---15 bit--+ |
1140 | // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
1141 | // +----------------------+ +----------------------+ |
1142 | // 63 0 63 0 |
1143 | if ((Shift = findContiguousZerosAtLeast(Imm, Num: 49)) || |
1144 | (Shift = findContiguousZerosAtLeast(Imm: ~Imm, Num: 49))) { |
1145 | uint64_t RotImm = APInt(64, Imm).rotr(rotateAmt: Shift).getZExtValue(); |
1146 | Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, |
1147 | getI32Imm(RotImm & 0xffff)); |
1148 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1149 | getI32Imm(Shift), getI32Imm(0)); |
1150 | } |
1151 | // 2-7) Patterns : High word == Low word |
1152 | // This may require 2 to 3 instructions, depending on whether Lo32 can be |
1153 | // materialized in 1 instruction. |
1154 | if (Hi32 == Lo32) { |
1155 | // Handle the first 32 bits. |
1156 | uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff; |
1157 | uint64_t ImmLo16 = Lo32 & 0xffff; |
1158 | if (isInt<16>(x: Lo32)) |
1159 | Result = |
1160 | CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, getI32Imm(ImmLo16)); |
1161 | else if (!ImmLo16) |
1162 | Result = |
1163 | CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16)); |
1164 | else { |
1165 | InstCnt = 3; |
1166 | Result = |
1167 | CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16)); |
1168 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
1169 | SDValue(Result, 0), getI32Imm(ImmLo16)); |
1170 | } |
1171 | // Use rldimi to insert the Low word into High word. |
1172 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1173 | getI32Imm(0)}; |
1174 | return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1175 | } |
1176 | |
1177 | // Following patterns use 3 instructions to materialize the Imm. |
1178 | InstCnt = 3; |
1179 | // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros} |
1180 | // {zeros}{31-bit value}{zeros} |
1181 | // {zeros}{ones}{31-bit value} |
1182 | // {ones}{31-bit value}{zeros} |
1183 | // We can take advantage of LIS's sign-extension semantics to generate leading |
1184 | // ones, add the remaining bits with ORI, and then use RLDIC to mask off the |
1185 | // ones in both sides after rotation. |
1186 | if ((LZ + FO + TZ) > 32) { |
1187 | uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff; |
1188 | unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1189 | Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1190 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1191 | getI32Imm((Imm >> TZ) & 0xffff)); |
1192 | return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1193 | getI32Imm(TZ), getI32Imm(LZ)); |
1194 | } |
1195 | // 3-2) Pattern : {zeros}{31-bit value}{ones} |
1196 | // Shift right the Imm by (32 - LZ) bits to construct a negative 32 bits |
1197 | // value, therefore we can take advantage of LIS's sign-extension semantics, |
1198 | // add the remaining bits with ORI, and then mask them off after rotation. |
1199 | // This is similar to Pattern 2-3, please refer to the diagram there. |
1200 | if ((LZ + TO) > 32) { |
1201 | // Since the immediates with (LZ > 32) have been handled by previous |
1202 | // patterns, here we have (LZ <= 32) to make sure we will not shift right |
1203 | // the Imm by a negative value. |
1204 | assert(LZ <= 32 && "Unexpected shift value." ); |
1205 | Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1206 | getI32Imm((Imm >> (48 - LZ)) & 0xffff)); |
1207 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1208 | getI32Imm((Imm >> (32 - LZ)) & 0xffff)); |
1209 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1210 | getI32Imm(32 - LZ), getI32Imm(LZ)); |
1211 | } |
1212 | // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones} |
1213 | // {ones}{31-bit value}{ones} |
1214 | // We can take advantage of LIS's sign-extension semantics to generate leading |
1215 | // ones, add the remaining bits with ORI, and then use RLDICL to mask off the |
1216 | // ones in left sides (if required) after rotation. |
1217 | // This is similar to Pattern 2-4, please refer to the diagram there. |
1218 | if ((LZ + FO + TO) > 32) { |
1219 | Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, |
1220 | getI32Imm((Imm >> (TO + 16)) & 0xffff)); |
1221 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1222 | getI32Imm((Imm >> TO) & 0xffff)); |
1223 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1224 | getI32Imm(TO), getI32Imm(LZ)); |
1225 | } |
1226 | // 3-4) Patterns : {******}{33 zeros}{******} |
1227 | // {******}{33 ones}{******} |
1228 | // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31 |
1229 | // bits remain on both sides. Rotate right the Imm to construct an int<32> |
1230 | // value, use LIS + ORI for int<32> value and then use RLDICL without mask to |
1231 | // rotate it back. |
1232 | // This is similar to Pattern 2-6, please refer to the diagram there. |
1233 | if ((Shift = findContiguousZerosAtLeast(Imm, Num: 33)) || |
1234 | (Shift = findContiguousZerosAtLeast(Imm: ~Imm, Num: 33))) { |
1235 | uint64_t RotImm = APInt(64, Imm).rotr(rotateAmt: Shift).getZExtValue(); |
1236 | uint64_t ImmHi16 = (RotImm >> 16) & 0xffff; |
1237 | unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; |
1238 | Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); |
1239 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1240 | getI32Imm(RotImm & 0xffff)); |
1241 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1242 | getI32Imm(Shift), getI32Imm(0)); |
1243 | } |
1244 | |
1245 | InstCnt = 0; |
1246 | return nullptr; |
1247 | } |
1248 | |
1249 | // Try to select instructions to generate a 64 bit immediate using prefix as |
1250 | // well as non prefix instructions. The function will return the SDNode |
1251 | // to materialize that constant or it will return nullptr if it does not |
1252 | // find one. The variable InstCnt is set to the number of instructions that |
1253 | // were selected. |
1254 | static SDNode *selectI64ImmDirectPrefix(SelectionDAG *CurDAG, const SDLoc &dl, |
1255 | uint64_t Imm, unsigned &InstCnt) { |
1256 | unsigned TZ = llvm::countr_zero<uint64_t>(Val: Imm); |
1257 | unsigned LZ = llvm::countl_zero<uint64_t>(Val: Imm); |
1258 | unsigned TO = llvm::countr_one<uint64_t>(Value: Imm); |
1259 | unsigned FO = llvm::countl_one<uint64_t>(Value: LZ == 64 ? 0 : (Imm << LZ)); |
1260 | unsigned Hi32 = Hi_32(Value: Imm); |
1261 | unsigned Lo32 = Lo_32(Value: Imm); |
1262 | |
1263 | auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1264 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1265 | }; |
1266 | |
1267 | auto getI64Imm = [CurDAG, dl](uint64_t Imm) { |
1268 | return CurDAG->getTargetConstant(Imm, dl, MVT::i64); |
1269 | }; |
1270 | |
1271 | // Following patterns use 1 instruction to materialize Imm. |
1272 | InstCnt = 1; |
1273 | |
1274 | // The pli instruction can materialize up to 34 bits directly. |
1275 | // If a constant fits within 34-bits, emit the pli instruction here directly. |
1276 | if (isInt<34>(Imm)) |
1277 | return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1278 | CurDAG->getTargetConstant(Imm, dl, MVT::i64)); |
1279 | |
1280 | // Require at least two instructions. |
1281 | InstCnt = 2; |
1282 | SDNode *Result = nullptr; |
1283 | // Patterns : {zeros}{ones}{33-bit value}{zeros} |
1284 | // {zeros}{33-bit value}{zeros} |
1285 | // {zeros}{ones}{33-bit value} |
1286 | // {ones}{33-bit value}{zeros} |
1287 | // We can take advantage of PLI's sign-extension semantics to generate leading |
1288 | // ones, and then use RLDIC to mask off the ones on both sides after rotation. |
1289 | if ((LZ + FO + TZ) > 30) { |
1290 | APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff); |
1291 | APInt Extended = SignedInt34.sext(width: 64); |
1292 | Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1293 | getI64Imm(*Extended.getRawData())); |
1294 | return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), |
1295 | getI32Imm(TZ), getI32Imm(LZ)); |
1296 | } |
1297 | // Pattern : {zeros}{33-bit value}{ones} |
1298 | // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value, |
1299 | // therefore we can take advantage of PLI's sign-extension semantics, and then |
1300 | // mask them off after rotation. |
1301 | // |
1302 | // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+ |
1303 | // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| |
1304 | // +------------------------+ +------------------------+ |
1305 | // 63 0 63 0 |
1306 | // |
1307 | // +----sext-----|--34-bit--+ +clear-|-----------------+ |
1308 | // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| |
1309 | // +------------------------+ +------------------------+ |
1310 | // 63 0 63 0 |
1311 | if ((LZ + TO) > 30) { |
1312 | APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff); |
1313 | APInt Extended = SignedInt34.sext(width: 64); |
1314 | Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1315 | getI64Imm(*Extended.getRawData())); |
1316 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1317 | getI32Imm(30 - LZ), getI32Imm(LZ)); |
1318 | } |
1319 | // Patterns : {zeros}{ones}{33-bit value}{ones} |
1320 | // {ones}{33-bit value}{ones} |
1321 | // Similar to LI we can take advantage of PLI's sign-extension semantics to |
1322 | // generate leading ones, and then use RLDICL to mask off the ones in left |
1323 | // sides (if required) after rotation. |
1324 | if ((LZ + FO + TO) > 30) { |
1325 | APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff); |
1326 | APInt Extended = SignedInt34.sext(width: 64); |
1327 | Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, |
1328 | getI64Imm(*Extended.getRawData())); |
1329 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), |
1330 | getI32Imm(TO), getI32Imm(LZ)); |
1331 | } |
1332 | // Patterns : {******}{31 zeros}{******} |
1333 | // : {******}{31 ones}{******} |
1334 | // If Imm contains 31 consecutive zeros/ones then the remaining bit count |
1335 | // is 33. Rotate right the Imm to construct a int<33> value, we can use PLI |
1336 | // for the int<33> value and then use RLDICL without a mask to rotate it back. |
1337 | // |
1338 | // +------|--ones--|------+ +---ones--||---33 bit--+ |
1339 | // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| |
1340 | // +----------------------+ +----------------------+ |
1341 | // 63 0 63 0 |
1342 | for (unsigned Shift = 0; Shift < 63; ++Shift) { |
1343 | uint64_t RotImm = APInt(64, Imm).rotr(rotateAmt: Shift).getZExtValue(); |
1344 | if (isInt<34>(x: RotImm)) { |
1345 | Result = |
1346 | CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm)); |
1347 | return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
1348 | SDValue(Result, 0), getI32Imm(Shift), |
1349 | getI32Imm(0)); |
1350 | } |
1351 | } |
1352 | |
1353 | // Patterns : High word == Low word |
1354 | // This is basically a splat of a 32 bit immediate. |
1355 | if (Hi32 == Lo32) { |
1356 | Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
1357 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1358 | getI32Imm(0)}; |
1359 | return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1360 | } |
1361 | |
1362 | InstCnt = 3; |
1363 | // Catch-all |
1364 | // This pattern can form any 64 bit immediate in 3 instructions. |
1365 | SDNode *ResultHi = |
1366 | CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); |
1367 | SDNode *ResultLo = |
1368 | CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32)); |
1369 | SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32), |
1370 | getI32Imm(0)}; |
1371 | return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1372 | } |
1373 | |
1374 | static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm, |
1375 | unsigned *InstCnt = nullptr) { |
1376 | unsigned InstCntDirect = 0; |
1377 | // No more than 3 instructions are used if we can select the i64 immediate |
1378 | // directly. |
1379 | SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCnt&: InstCntDirect); |
1380 | |
1381 | const PPCSubtarget &Subtarget = |
1382 | CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>(); |
1383 | |
1384 | // If we have prefixed instructions and there is a chance we can |
1385 | // materialize the constant with fewer prefixed instructions than |
1386 | // non-prefixed, try that. |
1387 | if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) { |
1388 | unsigned InstCntDirectP = 0; |
1389 | SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCnt&: InstCntDirectP); |
1390 | // Use the prefix case in either of two cases: |
1391 | // 1) We have no result from the non-prefix case to use. |
1392 | // 2) The non-prefix case uses more instructions than the prefix case. |
1393 | // If the prefix and non-prefix cases use the same number of instructions |
1394 | // we will prefer the non-prefix case. |
1395 | if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) { |
1396 | if (InstCnt) |
1397 | *InstCnt = InstCntDirectP; |
1398 | return ResultP; |
1399 | } |
1400 | } |
1401 | |
1402 | if (Result) { |
1403 | if (InstCnt) |
1404 | *InstCnt = InstCntDirect; |
1405 | return Result; |
1406 | } |
1407 | auto getI32Imm = [CurDAG, dl](unsigned Imm) { |
1408 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
1409 | }; |
1410 | |
1411 | uint32_t Hi16OfLo32 = (Lo_32(Value: Imm) >> 16) & 0xffff; |
1412 | uint32_t Lo16OfLo32 = Lo_32(Value: Imm) & 0xffff; |
1413 | |
1414 | // Try to use 4 instructions to materialize the immediate which is "almost" a |
1415 | // splat of a 32 bit immediate. |
1416 | if (Hi16OfLo32 && Lo16OfLo32) { |
1417 | uint32_t Hi16OfHi32 = (Hi_32(Value: Imm) >> 16) & 0xffff; |
1418 | uint32_t Lo16OfHi32 = Hi_32(Value: Imm) & 0xffff; |
1419 | bool IsSelected = false; |
1420 | |
1421 | auto getSplat = [CurDAG, dl, getI32Imm](uint32_t Hi16, uint32_t Lo16) { |
1422 | SDNode *Result = |
1423 | CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi16)); |
1424 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
1425 | SDValue(Result, 0), getI32Imm(Lo16)); |
1426 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), |
1427 | getI32Imm(0)}; |
1428 | return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1429 | }; |
1430 | |
1431 | if (Hi16OfHi32 == Lo16OfHi32 && Lo16OfHi32 == Lo16OfLo32) { |
1432 | IsSelected = true; |
1433 | Result = getSplat(Hi16OfLo32, Lo16OfLo32); |
1434 | // Modify Hi16OfHi32. |
1435 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(48), |
1436 | getI32Imm(0)}; |
1437 | Result = CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); |
1438 | } else if (Hi16OfHi32 == Hi16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
1439 | IsSelected = true; |
1440 | Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
1441 | // Modify Lo16OfLo32. |
1442 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
1443 | getI32Imm(16), getI32Imm(31)}; |
1444 | Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
1445 | } else if (Lo16OfHi32 == Lo16OfLo32 && Hi16OfLo32 == Lo16OfLo32) { |
1446 | IsSelected = true; |
1447 | Result = getSplat(Hi16OfHi32, Lo16OfHi32); |
1448 | // Modify Hi16OfLo32. |
1449 | SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16), |
1450 | getI32Imm(0), getI32Imm(15)}; |
1451 | Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); |
1452 | } |
1453 | if (IsSelected == true) { |
1454 | if (InstCnt) |
1455 | *InstCnt = 4; |
1456 | return Result; |
1457 | } |
1458 | } |
1459 | |
1460 | // Handle the upper 32 bit value. |
1461 | Result = |
1462 | selectI64ImmDirect(CurDAG, dl, Imm: Imm & 0xffffffff00000000, InstCnt&: InstCntDirect); |
1463 | // Add in the last bits as required. |
1464 | if (Hi16OfLo32) { |
1465 | Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, |
1466 | SDValue(Result, 0), getI32Imm(Hi16OfLo32)); |
1467 | ++InstCntDirect; |
1468 | } |
1469 | if (Lo16OfLo32) { |
1470 | Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), |
1471 | getI32Imm(Lo16OfLo32)); |
1472 | ++InstCntDirect; |
1473 | } |
1474 | if (InstCnt) |
1475 | *InstCnt = InstCntDirect; |
1476 | return Result; |
1477 | } |
1478 | |
1479 | // Select a 64-bit constant. |
1480 | static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) { |
1481 | SDLoc dl(N); |
1482 | |
1483 | // Get 64 bit value. |
1484 | int64_t Imm = N->getAsZExtVal(); |
1485 | if (unsigned MinSize = allUsesTruncate(CurDAG, N)) { |
1486 | uint64_t SextImm = SignExtend64(X: Imm, B: MinSize); |
1487 | SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
1488 | if (isInt<16>(SextImm)) |
1489 | return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); |
1490 | } |
1491 | return selectI64Imm(CurDAG, dl, Imm); |
1492 | } |
1493 | |
1494 | namespace { |
1495 | |
1496 | class BitPermutationSelector { |
1497 | struct ValueBit { |
1498 | SDValue V; |
1499 | |
1500 | // The bit number in the value, using a convention where bit 0 is the |
1501 | // lowest-order bit. |
1502 | unsigned Idx; |
1503 | |
1504 | // ConstZero means a bit we need to mask off. |
1505 | // Variable is a bit comes from an input variable. |
1506 | // VariableKnownToBeZero is also a bit comes from an input variable, |
1507 | // but it is known to be already zero. So we do not need to mask them. |
1508 | enum Kind { |
1509 | ConstZero, |
1510 | Variable, |
1511 | VariableKnownToBeZero |
1512 | } K; |
1513 | |
1514 | ValueBit(SDValue V, unsigned I, Kind K = Variable) |
1515 | : V(V), Idx(I), K(K) {} |
1516 | ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {} |
1517 | |
1518 | bool isZero() const { |
1519 | return K == ConstZero || K == VariableKnownToBeZero; |
1520 | } |
1521 | |
1522 | bool hasValue() const { |
1523 | return K == Variable || K == VariableKnownToBeZero; |
1524 | } |
1525 | |
1526 | SDValue getValue() const { |
1527 | assert(hasValue() && "Cannot get the value of a constant bit" ); |
1528 | return V; |
1529 | } |
1530 | |
1531 | unsigned getValueBitIndex() const { |
1532 | assert(hasValue() && "Cannot get the value bit index of a constant bit" ); |
1533 | return Idx; |
1534 | } |
1535 | }; |
1536 | |
1537 | // A bit group has the same underlying value and the same rotate factor. |
1538 | struct BitGroup { |
1539 | SDValue V; |
1540 | unsigned RLAmt; |
1541 | unsigned StartIdx, EndIdx; |
1542 | |
1543 | // This rotation amount assumes that the lower 32 bits of the quantity are |
1544 | // replicated in the high 32 bits by the rotation operator (which is done |
1545 | // by rlwinm and friends in 64-bit mode). |
1546 | bool Repl32; |
1547 | // Did converting to Repl32 == true change the rotation factor? If it did, |
1548 | // it decreased it by 32. |
1549 | bool Repl32CR; |
1550 | // Was this group coalesced after setting Repl32 to true? |
1551 | bool Repl32Coalesced; |
1552 | |
1553 | BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) |
1554 | : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), |
1555 | Repl32Coalesced(false) { |
1556 | LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R |
1557 | << " [" << S << ", " << E << "]\n" ); |
1558 | } |
1559 | }; |
1560 | |
1561 | // Information on each (Value, RLAmt) pair (like the number of groups |
1562 | // associated with each) used to choose the lowering method. |
1563 | struct ValueRotInfo { |
1564 | SDValue V; |
1565 | unsigned RLAmt = std::numeric_limits<unsigned>::max(); |
1566 | unsigned NumGroups = 0; |
1567 | unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max(); |
1568 | bool Repl32 = false; |
1569 | |
1570 | ValueRotInfo() = default; |
1571 | |
1572 | // For sorting (in reverse order) by NumGroups, and then by |
1573 | // FirstGroupStartIdx. |
1574 | bool operator < (const ValueRotInfo &Other) const { |
1575 | // We need to sort so that the non-Repl32 come first because, when we're |
1576 | // doing masking, the Repl32 bit groups might be subsumed into the 64-bit |
1577 | // masking operation. |
1578 | if (Repl32 < Other.Repl32) |
1579 | return true; |
1580 | else if (Repl32 > Other.Repl32) |
1581 | return false; |
1582 | else if (NumGroups > Other.NumGroups) |
1583 | return true; |
1584 | else if (NumGroups < Other.NumGroups) |
1585 | return false; |
1586 | else if (RLAmt == 0 && Other.RLAmt != 0) |
1587 | return true; |
1588 | else if (RLAmt != 0 && Other.RLAmt == 0) |
1589 | return false; |
1590 | else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) |
1591 | return true; |
1592 | return false; |
1593 | } |
1594 | }; |
1595 | |
1596 | using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>; |
1597 | using ValueBitsMemoizer = |
1598 | DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>; |
1599 | ValueBitsMemoizer Memoizer; |
1600 | |
1601 | // Return a pair of bool and a SmallVector pointer to a memoization entry. |
1602 | // The bool is true if something interesting was deduced, otherwise if we're |
1603 | // providing only a generic representation of V (or something else likewise |
1604 | // uninteresting for instruction selection) through the SmallVector. |
1605 | std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V, |
1606 | unsigned NumBits) { |
1607 | auto &ValueEntry = Memoizer[V]; |
1608 | if (ValueEntry) |
1609 | return std::make_pair(x&: ValueEntry->first, y: &ValueEntry->second); |
1610 | ValueEntry.reset(p: new ValueBitsMemoizedValue()); |
1611 | bool &Interesting = ValueEntry->first; |
1612 | SmallVector<ValueBit, 64> &Bits = ValueEntry->second; |
1613 | Bits.resize(N: NumBits); |
1614 | |
1615 | switch (V.getOpcode()) { |
1616 | default: break; |
1617 | case ISD::ROTL: |
1618 | if (isa<ConstantSDNode>(Val: V.getOperand(i: 1))) { |
1619 | assert(isPowerOf2_32(NumBits) && "rotl bits should be power of 2!" ); |
1620 | unsigned RotAmt = V.getConstantOperandVal(i: 1) & (NumBits - 1); |
1621 | |
1622 | const auto &LHSBits = *getValueBits(V: V.getOperand(i: 0), NumBits).second; |
1623 | |
1624 | for (unsigned i = 0; i < NumBits; ++i) |
1625 | Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; |
1626 | |
1627 | return std::make_pair(x&: Interesting = true, y: &Bits); |
1628 | } |
1629 | break; |
1630 | case ISD::SHL: |
1631 | case PPCISD::SHL: |
1632 | if (isa<ConstantSDNode>(Val: V.getOperand(i: 1))) { |
1633 | // sld takes 7 bits, slw takes 6. |
1634 | unsigned ShiftAmt = V.getConstantOperandVal(i: 1) & ((NumBits << 1) - 1); |
1635 | |
1636 | const auto &LHSBits = *getValueBits(V: V.getOperand(i: 0), NumBits).second; |
1637 | |
1638 | if (ShiftAmt >= NumBits) { |
1639 | for (unsigned i = 0; i < NumBits; ++i) |
1640 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1641 | } else { |
1642 | for (unsigned i = ShiftAmt; i < NumBits; ++i) |
1643 | Bits[i] = LHSBits[i - ShiftAmt]; |
1644 | for (unsigned i = 0; i < ShiftAmt; ++i) |
1645 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1646 | } |
1647 | |
1648 | return std::make_pair(x&: Interesting = true, y: &Bits); |
1649 | } |
1650 | break; |
1651 | case ISD::SRL: |
1652 | case PPCISD::SRL: |
1653 | if (isa<ConstantSDNode>(Val: V.getOperand(i: 1))) { |
1654 | // srd takes lowest 7 bits, srw takes 6. |
1655 | unsigned ShiftAmt = V.getConstantOperandVal(i: 1) & ((NumBits << 1) - 1); |
1656 | |
1657 | const auto &LHSBits = *getValueBits(V: V.getOperand(i: 0), NumBits).second; |
1658 | |
1659 | if (ShiftAmt >= NumBits) { |
1660 | for (unsigned i = 0; i < NumBits; ++i) |
1661 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1662 | } else { |
1663 | for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) |
1664 | Bits[i] = LHSBits[i + ShiftAmt]; |
1665 | for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) |
1666 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1667 | } |
1668 | |
1669 | return std::make_pair(x&: Interesting = true, y: &Bits); |
1670 | } |
1671 | break; |
1672 | case ISD::AND: |
1673 | if (isa<ConstantSDNode>(Val: V.getOperand(i: 1))) { |
1674 | uint64_t Mask = V.getConstantOperandVal(i: 1); |
1675 | |
1676 | const SmallVector<ValueBit, 64> *LHSBits; |
1677 | // Mark this as interesting, only if the LHS was also interesting. This |
1678 | // prevents the overall procedure from matching a single immediate 'and' |
1679 | // (which is non-optimal because such an and might be folded with other |
1680 | // things if we don't select it here). |
1681 | std::tie(args&: Interesting, args&: LHSBits) = getValueBits(V: V.getOperand(i: 0), NumBits); |
1682 | |
1683 | for (unsigned i = 0; i < NumBits; ++i) |
1684 | if (((Mask >> i) & 1) == 1) |
1685 | Bits[i] = (*LHSBits)[i]; |
1686 | else { |
1687 | // AND instruction masks this bit. If the input is already zero, |
1688 | // we have nothing to do here. Otherwise, make the bit ConstZero. |
1689 | if ((*LHSBits)[i].isZero()) |
1690 | Bits[i] = (*LHSBits)[i]; |
1691 | else |
1692 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1693 | } |
1694 | |
1695 | return std::make_pair(x&: Interesting, y: &Bits); |
1696 | } |
1697 | break; |
1698 | case ISD::OR: { |
1699 | const auto &LHSBits = *getValueBits(V: V.getOperand(i: 0), NumBits).second; |
1700 | const auto &RHSBits = *getValueBits(V: V.getOperand(i: 1), NumBits).second; |
1701 | |
1702 | bool AllDisjoint = true; |
1703 | SDValue LastVal = SDValue(); |
1704 | unsigned LastIdx = 0; |
1705 | for (unsigned i = 0; i < NumBits; ++i) { |
1706 | if (LHSBits[i].isZero() && RHSBits[i].isZero()) { |
1707 | // If both inputs are known to be zero and one is ConstZero and |
1708 | // another is VariableKnownToBeZero, we can select whichever |
1709 | // we like. To minimize the number of bit groups, we select |
1710 | // VariableKnownToBeZero if this bit is the next bit of the same |
1711 | // input variable from the previous bit. Otherwise, we select |
1712 | // ConstZero. |
1713 | if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal && |
1714 | LHSBits[i].getValueBitIndex() == LastIdx + 1) |
1715 | Bits[i] = LHSBits[i]; |
1716 | else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal && |
1717 | RHSBits[i].getValueBitIndex() == LastIdx + 1) |
1718 | Bits[i] = RHSBits[i]; |
1719 | else |
1720 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1721 | } |
1722 | else if (LHSBits[i].isZero()) |
1723 | Bits[i] = RHSBits[i]; |
1724 | else if (RHSBits[i].isZero()) |
1725 | Bits[i] = LHSBits[i]; |
1726 | else { |
1727 | AllDisjoint = false; |
1728 | break; |
1729 | } |
1730 | // We remember the value and bit index of this bit. |
1731 | if (Bits[i].hasValue()) { |
1732 | LastVal = Bits[i].getValue(); |
1733 | LastIdx = Bits[i].getValueBitIndex(); |
1734 | } |
1735 | else { |
1736 | if (LastVal) LastVal = SDValue(); |
1737 | LastIdx = 0; |
1738 | } |
1739 | } |
1740 | |
1741 | if (!AllDisjoint) |
1742 | break; |
1743 | |
1744 | return std::make_pair(x&: Interesting = true, y: &Bits); |
1745 | } |
1746 | case ISD::ZERO_EXTEND: { |
1747 | // We support only the case with zero extension from i32 to i64 so far. |
1748 | if (V.getValueType() != MVT::i64 || |
1749 | V.getOperand(0).getValueType() != MVT::i32) |
1750 | break; |
1751 | |
1752 | const SmallVector<ValueBit, 64> *LHSBits; |
1753 | const unsigned NumOperandBits = 32; |
1754 | std::tie(args&: Interesting, args&: LHSBits) = getValueBits(V: V.getOperand(i: 0), |
1755 | NumBits: NumOperandBits); |
1756 | |
1757 | for (unsigned i = 0; i < NumOperandBits; ++i) |
1758 | Bits[i] = (*LHSBits)[i]; |
1759 | |
1760 | for (unsigned i = NumOperandBits; i < NumBits; ++i) |
1761 | Bits[i] = ValueBit(ValueBit::ConstZero); |
1762 | |
1763 | return std::make_pair(x&: Interesting, y: &Bits); |
1764 | } |
1765 | case ISD::TRUNCATE: { |
1766 | EVT FromType = V.getOperand(i: 0).getValueType(); |
1767 | EVT ToType = V.getValueType(); |
1768 | // We support only the case with truncate from i64 to i32. |
1769 | if (FromType != MVT::i64 || ToType != MVT::i32) |
1770 | break; |
1771 | const unsigned NumAllBits = FromType.getSizeInBits(); |
1772 | SmallVector<ValueBit, 64> *InBits; |
1773 | std::tie(args&: Interesting, args&: InBits) = getValueBits(V: V.getOperand(i: 0), |
1774 | NumBits: NumAllBits); |
1775 | const unsigned NumValidBits = ToType.getSizeInBits(); |
1776 | |
1777 | // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. |
1778 | // So, we cannot include this truncate. |
1779 | bool UseUpper32bit = false; |
1780 | for (unsigned i = 0; i < NumValidBits; ++i) |
1781 | if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) { |
1782 | UseUpper32bit = true; |
1783 | break; |
1784 | } |
1785 | if (UseUpper32bit) |
1786 | break; |
1787 | |
1788 | for (unsigned i = 0; i < NumValidBits; ++i) |
1789 | Bits[i] = (*InBits)[i]; |
1790 | |
1791 | return std::make_pair(x&: Interesting, y: &Bits); |
1792 | } |
1793 | case ISD::AssertZext: { |
1794 | // For AssertZext, we look through the operand and |
1795 | // mark the bits known to be zero. |
1796 | const SmallVector<ValueBit, 64> *LHSBits; |
1797 | std::tie(args&: Interesting, args&: LHSBits) = getValueBits(V: V.getOperand(i: 0), |
1798 | NumBits); |
1799 | |
1800 | EVT FromType = cast<VTSDNode>(Val: V.getOperand(i: 1))->getVT(); |
1801 | const unsigned NumValidBits = FromType.getSizeInBits(); |
1802 | for (unsigned i = 0; i < NumValidBits; ++i) |
1803 | Bits[i] = (*LHSBits)[i]; |
1804 | |
1805 | // These bits are known to be zero but the AssertZext may be from a value |
1806 | // that already has some constant zero bits (i.e. from a masking and). |
1807 | for (unsigned i = NumValidBits; i < NumBits; ++i) |
1808 | Bits[i] = (*LHSBits)[i].hasValue() |
1809 | ? ValueBit((*LHSBits)[i].getValue(), |
1810 | (*LHSBits)[i].getValueBitIndex(), |
1811 | ValueBit::VariableKnownToBeZero) |
1812 | : ValueBit(ValueBit::ConstZero); |
1813 | |
1814 | return std::make_pair(x&: Interesting, y: &Bits); |
1815 | } |
1816 | case ISD::LOAD: |
1817 | LoadSDNode *LD = cast<LoadSDNode>(Val&: V); |
1818 | if (ISD::isZEXTLoad(N: V.getNode()) && V.getResNo() == 0) { |
1819 | EVT VT = LD->getMemoryVT(); |
1820 | const unsigned NumValidBits = VT.getSizeInBits(); |
1821 | |
1822 | for (unsigned i = 0; i < NumValidBits; ++i) |
1823 | Bits[i] = ValueBit(V, i); |
1824 | |
1825 | // These bits are known to be zero. |
1826 | for (unsigned i = NumValidBits; i < NumBits; ++i) |
1827 | Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero); |
1828 | |
1829 | // Zero-extending load itself cannot be optimized. So, it is not |
1830 | // interesting by itself though it gives useful information. |
1831 | return std::make_pair(x&: Interesting = false, y: &Bits); |
1832 | } |
1833 | break; |
1834 | } |
1835 | |
1836 | for (unsigned i = 0; i < NumBits; ++i) |
1837 | Bits[i] = ValueBit(V, i); |
1838 | |
1839 | return std::make_pair(x&: Interesting = false, y: &Bits); |
1840 | } |
1841 | |
1842 | // For each value (except the constant ones), compute the left-rotate amount |
1843 | // to get it from its original to final position. |
1844 | void computeRotationAmounts() { |
1845 | NeedMask = false; |
1846 | RLAmt.resize(N: Bits.size()); |
1847 | for (unsigned i = 0; i < Bits.size(); ++i) |
1848 | if (Bits[i].hasValue()) { |
1849 | unsigned VBI = Bits[i].getValueBitIndex(); |
1850 | if (i >= VBI) |
1851 | RLAmt[i] = i - VBI; |
1852 | else |
1853 | RLAmt[i] = Bits.size() - (VBI - i); |
1854 | } else if (Bits[i].isZero()) { |
1855 | NeedMask = true; |
1856 | RLAmt[i] = UINT32_MAX; |
1857 | } else { |
1858 | llvm_unreachable("Unknown value bit type" ); |
1859 | } |
1860 | } |
1861 | |
1862 | // Collect groups of consecutive bits with the same underlying value and |
1863 | // rotation factor. If we're doing late masking, we ignore zeros, otherwise |
1864 | // they break up groups. |
1865 | void collectBitGroups(bool LateMask) { |
1866 | BitGroups.clear(); |
1867 | |
1868 | unsigned LastRLAmt = RLAmt[0]; |
1869 | SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); |
1870 | unsigned LastGroupStartIdx = 0; |
1871 | bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
1872 | for (unsigned i = 1; i < Bits.size(); ++i) { |
1873 | unsigned ThisRLAmt = RLAmt[i]; |
1874 | SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); |
1875 | if (LateMask && !ThisValue) { |
1876 | ThisValue = LastValue; |
1877 | ThisRLAmt = LastRLAmt; |
1878 | // If we're doing late masking, then the first bit group always starts |
1879 | // at zero (even if the first bits were zero). |
1880 | if (BitGroups.empty()) |
1881 | LastGroupStartIdx = 0; |
1882 | } |
1883 | |
1884 | // If this bit is known to be zero and the current group is a bit group |
1885 | // of zeros, we do not need to terminate the current bit group even the |
1886 | // Value or RLAmt does not match here. Instead, we terminate this group |
1887 | // when the first non-zero bit appears later. |
1888 | if (IsGroupOfZeros && Bits[i].isZero()) |
1889 | continue; |
1890 | |
1891 | // If this bit has the same underlying value and the same rotate factor as |
1892 | // the last one, then they're part of the same group. |
1893 | if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) |
1894 | // We cannot continue the current group if this bits is not known to |
1895 | // be zero in a bit group of zeros. |
1896 | if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero())) |
1897 | continue; |
1898 | |
1899 | if (LastValue.getNode()) |
1900 | BitGroups.push_back(Elt: BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
1901 | i-1)); |
1902 | LastRLAmt = ThisRLAmt; |
1903 | LastValue = ThisValue; |
1904 | LastGroupStartIdx = i; |
1905 | IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); |
1906 | } |
1907 | if (LastValue.getNode()) |
1908 | BitGroups.push_back(Elt: BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, |
1909 | Bits.size()-1)); |
1910 | |
1911 | if (BitGroups.empty()) |
1912 | return; |
1913 | |
1914 | // We might be able to combine the first and last groups. |
1915 | if (BitGroups.size() > 1) { |
1916 | // If the first and last groups are the same, then remove the first group |
1917 | // in favor of the last group, making the ending index of the last group |
1918 | // equal to the ending index of the to-be-removed first group. |
1919 | if (BitGroups[0].StartIdx == 0 && |
1920 | BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && |
1921 | BitGroups[0].V == BitGroups[BitGroups.size()-1].V && |
1922 | BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { |
1923 | LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n" ); |
1924 | BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; |
1925 | BitGroups.erase(CI: BitGroups.begin()); |
1926 | } |
1927 | } |
1928 | } |
1929 | |
1930 | // Take all (SDValue, RLAmt) pairs and sort them by the number of groups |
1931 | // associated with each. If the number of groups are same, we prefer a group |
1932 | // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate |
1933 | // instruction. If there is a degeneracy, pick the one that occurs |
1934 | // first (in the final value). |
1935 | void collectValueRotInfo() { |
1936 | ValueRots.clear(); |
1937 | |
1938 | for (auto &BG : BitGroups) { |
1939 | unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); |
1940 | ValueRotInfo &VRI = ValueRots[std::make_pair(x&: BG.V, y&: RLAmtKey)]; |
1941 | VRI.V = BG.V; |
1942 | VRI.RLAmt = BG.RLAmt; |
1943 | VRI.Repl32 = BG.Repl32; |
1944 | VRI.NumGroups += 1; |
1945 | VRI.FirstGroupStartIdx = std::min(a: VRI.FirstGroupStartIdx, b: BG.StartIdx); |
1946 | } |
1947 | |
1948 | // Now that we've collected the various ValueRotInfo instances, we need to |
1949 | // sort them. |
1950 | ValueRotsVec.clear(); |
1951 | for (auto &I : ValueRots) { |
1952 | ValueRotsVec.push_back(Elt: I.second); |
1953 | } |
1954 | llvm::sort(C&: ValueRotsVec); |
1955 | } |
1956 | |
1957 | // In 64-bit mode, rlwinm and friends have a rotation operator that |
1958 | // replicates the low-order 32 bits into the high-order 32-bits. The mask |
1959 | // indices of these instructions can only be in the lower 32 bits, so they |
1960 | // can only represent some 64-bit bit groups. However, when they can be used, |
1961 | // the 32-bit replication can be used to represent, as a single bit group, |
1962 | // otherwise separate bit groups. We'll convert to replicated-32-bit bit |
1963 | // groups when possible. Returns true if any of the bit groups were |
1964 | // converted. |
1965 | void assignRepl32BitGroups() { |
1966 | // If we have bits like this: |
1967 | // |
1968 | // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 |
1969 | // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 |
1970 | // Groups: | RLAmt = 8 | RLAmt = 40 | |
1971 | // |
1972 | // But, making use of a 32-bit operation that replicates the low-order 32 |
1973 | // bits into the high-order 32 bits, this can be one bit group with a RLAmt |
1974 | // of 8. |
1975 | |
1976 | auto IsAllLow32 = [this](BitGroup & BG) { |
1977 | if (BG.StartIdx <= BG.EndIdx) { |
1978 | for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { |
1979 | if (!Bits[i].hasValue()) |
1980 | continue; |
1981 | if (Bits[i].getValueBitIndex() >= 32) |
1982 | return false; |
1983 | } |
1984 | } else { |
1985 | for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { |
1986 | if (!Bits[i].hasValue()) |
1987 | continue; |
1988 | if (Bits[i].getValueBitIndex() >= 32) |
1989 | return false; |
1990 | } |
1991 | for (unsigned i = 0; i <= BG.EndIdx; ++i) { |
1992 | if (!Bits[i].hasValue()) |
1993 | continue; |
1994 | if (Bits[i].getValueBitIndex() >= 32) |
1995 | return false; |
1996 | } |
1997 | } |
1998 | |
1999 | return true; |
2000 | }; |
2001 | |
2002 | for (auto &BG : BitGroups) { |
2003 | // If this bit group has RLAmt of 0 and will not be merged with |
2004 | // another bit group, we don't benefit from Repl32. We don't mark |
2005 | // such group to give more freedom for later instruction selection. |
2006 | if (BG.RLAmt == 0) { |
2007 | auto PotentiallyMerged = [this](BitGroup & BG) { |
2008 | for (auto &BG2 : BitGroups) |
2009 | if (&BG != &BG2 && BG.V == BG2.V && |
2010 | (BG2.RLAmt == 0 || BG2.RLAmt == 32)) |
2011 | return true; |
2012 | return false; |
2013 | }; |
2014 | if (!PotentiallyMerged(BG)) |
2015 | continue; |
2016 | } |
2017 | if (BG.StartIdx < 32 && BG.EndIdx < 32) { |
2018 | if (IsAllLow32(BG)) { |
2019 | if (BG.RLAmt >= 32) { |
2020 | BG.RLAmt -= 32; |
2021 | BG.Repl32CR = true; |
2022 | } |
2023 | |
2024 | BG.Repl32 = true; |
2025 | |
2026 | LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " |
2027 | << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" |
2028 | << BG.StartIdx << ", " << BG.EndIdx << "]\n" ); |
2029 | } |
2030 | } |
2031 | } |
2032 | |
2033 | // Now walk through the bit groups, consolidating where possible. |
2034 | for (auto I = BitGroups.begin(); I != BitGroups.end();) { |
2035 | // We might want to remove this bit group by merging it with the previous |
2036 | // group (which might be the ending group). |
2037 | auto IP = (I == BitGroups.begin()) ? |
2038 | std::prev(x: BitGroups.end()) : std::prev(x: I); |
2039 | if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && |
2040 | I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { |
2041 | |
2042 | LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " |
2043 | << I->V.getNode() << " RLAmt = " << I->RLAmt << " [" |
2044 | << I->StartIdx << ", " << I->EndIdx |
2045 | << "] with group with range [" << IP->StartIdx << ", " |
2046 | << IP->EndIdx << "]\n" ); |
2047 | |
2048 | IP->EndIdx = I->EndIdx; |
2049 | IP->Repl32CR = IP->Repl32CR || I->Repl32CR; |
2050 | IP->Repl32Coalesced = true; |
2051 | I = BitGroups.erase(CI: I); |
2052 | continue; |
2053 | } else { |
2054 | // There is a special case worth handling: If there is a single group |
2055 | // covering the entire upper 32 bits, and it can be merged with both |
2056 | // the next and previous groups (which might be the same group), then |
2057 | // do so. If it is the same group (so there will be only one group in |
2058 | // total), then we need to reverse the order of the range so that it |
2059 | // covers the entire 64 bits. |
2060 | if (I->StartIdx == 32 && I->EndIdx == 63) { |
2061 | assert(std::next(I) == BitGroups.end() && |
2062 | "bit group ends at index 63 but there is another?" ); |
2063 | auto IN = BitGroups.begin(); |
2064 | |
2065 | if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && |
2066 | (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && |
2067 | IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && |
2068 | IsAllLow32(*I)) { |
2069 | |
2070 | LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() |
2071 | << " RLAmt = " << I->RLAmt << " [" << I->StartIdx |
2072 | << ", " << I->EndIdx |
2073 | << "] with 32-bit replicated groups with ranges [" |
2074 | << IP->StartIdx << ", " << IP->EndIdx << "] and [" |
2075 | << IN->StartIdx << ", " << IN->EndIdx << "]\n" ); |
2076 | |
2077 | if (IP == IN) { |
2078 | // There is only one other group; change it to cover the whole |
2079 | // range (backward, so that it can still be Repl32 but cover the |
2080 | // whole 64-bit range). |
2081 | IP->StartIdx = 31; |
2082 | IP->EndIdx = 30; |
2083 | IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; |
2084 | IP->Repl32Coalesced = true; |
2085 | I = BitGroups.erase(CI: I); |
2086 | } else { |
2087 | // There are two separate groups, one before this group and one |
2088 | // after us (at the beginning). We're going to remove this group, |
2089 | // but also the group at the very beginning. |
2090 | IP->EndIdx = IN->EndIdx; |
2091 | IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; |
2092 | IP->Repl32Coalesced = true; |
2093 | I = BitGroups.erase(CI: I); |
2094 | BitGroups.erase(CI: BitGroups.begin()); |
2095 | } |
2096 | |
2097 | // This must be the last group in the vector (and we might have |
2098 | // just invalidated the iterator above), so break here. |
2099 | break; |
2100 | } |
2101 | } |
2102 | } |
2103 | |
2104 | ++I; |
2105 | } |
2106 | } |
2107 | |
2108 | SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { |
2109 | return CurDAG->getTargetConstant(Imm, dl, MVT::i32); |
2110 | } |
2111 | |
2112 | uint64_t getZerosMask() { |
2113 | uint64_t Mask = 0; |
2114 | for (unsigned i = 0; i < Bits.size(); ++i) { |
2115 | if (Bits[i].hasValue()) |
2116 | continue; |
2117 | Mask |= (UINT64_C(1) << i); |
2118 | } |
2119 | |
2120 | return ~Mask; |
2121 | } |
2122 | |
2123 | // This method extends an input value to 64 bit if input is 32-bit integer. |
2124 | // While selecting instructions in BitPermutationSelector in 64-bit mode, |
2125 | // an input value can be a 32-bit integer if a ZERO_EXTEND node is included. |
2126 | // In such case, we extend it to 64 bit to be consistent with other values. |
2127 | SDValue ExtendToInt64(SDValue V, const SDLoc &dl) { |
2128 | if (V.getValueSizeInBits() == 64) |
2129 | return V; |
2130 | |
2131 | assert(V.getValueSizeInBits() == 32); |
2132 | SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
2133 | SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, |
2134 | MVT::i64), 0); |
2135 | SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, |
2136 | MVT::i64, ImDef, V, |
2137 | SubRegIdx), 0); |
2138 | return ExtVal; |
2139 | } |
2140 | |
2141 | SDValue TruncateToInt32(SDValue V, const SDLoc &dl) { |
2142 | if (V.getValueSizeInBits() == 32) |
2143 | return V; |
2144 | |
2145 | assert(V.getValueSizeInBits() == 64); |
2146 | SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
2147 | SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, |
2148 | MVT::i32, V, SubRegIdx), 0); |
2149 | return SubVal; |
2150 | } |
2151 | |
2152 | // Depending on the number of groups for a particular value, it might be |
2153 | // better to rotate, mask explicitly (using andi/andis), and then or the |
2154 | // result. Select this part of the result first. |
2155 | void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
2156 | if (BPermRewriterNoMasking) |
2157 | return; |
2158 | |
2159 | for (ValueRotInfo &VRI : ValueRotsVec) { |
2160 | unsigned Mask = 0; |
2161 | for (unsigned i = 0; i < Bits.size(); ++i) { |
2162 | if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) |
2163 | continue; |
2164 | if (RLAmt[i] != VRI.RLAmt) |
2165 | continue; |
2166 | Mask |= (1u << i); |
2167 | } |
2168 | |
2169 | // Compute the masks for andi/andis that would be necessary. |
2170 | unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
2171 | assert((ANDIMask != 0 || ANDISMask != 0) && |
2172 | "No set bits in mask for value bit groups" ); |
2173 | bool NeedsRotate = VRI.RLAmt != 0; |
2174 | |
2175 | // We're trying to minimize the number of instructions. If we have one |
2176 | // group, using one of andi/andis can break even. If we have three |
2177 | // groups, we can use both andi and andis and break even (to use both |
2178 | // andi and andis we also need to or the results together). We need four |
2179 | // groups if we also need to rotate. To use andi/andis we need to do more |
2180 | // than break even because rotate-and-mask instructions tend to be easier |
2181 | // to schedule. |
2182 | |
2183 | // FIXME: We've biased here against using andi/andis, which is right for |
2184 | // POWER cores, but not optimal everywhere. For example, on the A2, |
2185 | // andi/andis have single-cycle latency whereas the rotate-and-mask |
2186 | // instructions take two cycles, and it would be better to bias toward |
2187 | // andi/andis in break-even cases. |
2188 | |
2189 | unsigned NumAndInsts = (unsigned) NeedsRotate + |
2190 | (unsigned) (ANDIMask != 0) + |
2191 | (unsigned) (ANDISMask != 0) + |
2192 | (unsigned) (ANDIMask != 0 && ANDISMask != 0) + |
2193 | (unsigned) (bool) Res; |
2194 | |
2195 | LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
2196 | << " RL: " << VRI.RLAmt << ":" |
2197 | << "\n\t\t\tisel using masking: " << NumAndInsts |
2198 | << " using rotates: " << VRI.NumGroups << "\n" ); |
2199 | |
2200 | if (NumAndInsts >= VRI.NumGroups) |
2201 | continue; |
2202 | |
2203 | LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n" ); |
2204 | |
2205 | if (InstCnt) *InstCnt += NumAndInsts; |
2206 | |
2207 | SDValue VRot; |
2208 | if (VRI.RLAmt) { |
2209 | SDValue Ops[] = |
2210 | { TruncateToInt32(V: VRI.V, dl), getI32Imm(Imm: VRI.RLAmt, dl), |
2211 | getI32Imm(Imm: 0, dl), getI32Imm(Imm: 31, dl) }; |
2212 | VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
2213 | Ops), 0); |
2214 | } else { |
2215 | VRot = TruncateToInt32(V: VRI.V, dl); |
2216 | } |
2217 | |
2218 | SDValue ANDIVal, ANDISVal; |
2219 | if (ANDIMask != 0) |
2220 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
2221 | VRot, getI32Imm(ANDIMask, dl)), |
2222 | 0); |
2223 | if (ANDISMask != 0) |
2224 | ANDISVal = |
2225 | SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, |
2226 | getI32Imm(ANDISMask, dl)), |
2227 | 0); |
2228 | |
2229 | SDValue TotalVal; |
2230 | if (!ANDIVal) |
2231 | TotalVal = ANDISVal; |
2232 | else if (!ANDISVal) |
2233 | TotalVal = ANDIVal; |
2234 | else |
2235 | TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2236 | ANDIVal, ANDISVal), 0); |
2237 | |
2238 | if (!Res) |
2239 | Res = TotalVal; |
2240 | else |
2241 | Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2242 | Res, TotalVal), 0); |
2243 | |
2244 | // Now, remove all groups with this underlying value and rotation |
2245 | // factor. |
2246 | eraseMatchingBitGroups(F: [VRI](const BitGroup &BG) { |
2247 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
2248 | }); |
2249 | } |
2250 | } |
2251 | |
2252 | // Instruction selection for the 32-bit case. |
2253 | SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { |
2254 | SDLoc dl(N); |
2255 | SDValue Res; |
2256 | |
2257 | if (InstCnt) *InstCnt = 0; |
2258 | |
2259 | // Take care of cases that should use andi/andis first. |
2260 | SelectAndParts32(dl, Res, InstCnt); |
2261 | |
2262 | // If we've not yet selected a 'starting' instruction, and we have no zeros |
2263 | // to fill in, select the (Value, RLAmt) with the highest priority (largest |
2264 | // number of groups), and start with this rotated value. |
2265 | if ((!NeedMask || LateMask) && !Res) { |
2266 | ValueRotInfo &VRI = ValueRotsVec[0]; |
2267 | if (VRI.RLAmt) { |
2268 | if (InstCnt) *InstCnt += 1; |
2269 | SDValue Ops[] = |
2270 | { TruncateToInt32(V: VRI.V, dl), getI32Imm(Imm: VRI.RLAmt, dl), |
2271 | getI32Imm(Imm: 0, dl), getI32Imm(Imm: 31, dl) }; |
2272 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), |
2273 | 0); |
2274 | } else { |
2275 | Res = TruncateToInt32(V: VRI.V, dl); |
2276 | } |
2277 | |
2278 | // Now, remove all groups with this underlying value and rotation factor. |
2279 | eraseMatchingBitGroups(F: [VRI](const BitGroup &BG) { |
2280 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; |
2281 | }); |
2282 | } |
2283 | |
2284 | if (InstCnt) *InstCnt += BitGroups.size(); |
2285 | |
2286 | // Insert the other groups (one at a time). |
2287 | for (auto &BG : BitGroups) { |
2288 | if (!Res) { |
2289 | SDValue Ops[] = |
2290 | { TruncateToInt32(V: BG.V, dl), getI32Imm(Imm: BG.RLAmt, dl), |
2291 | getI32Imm(Imm: Bits.size() - BG.EndIdx - 1, dl), |
2292 | getI32Imm(Imm: Bits.size() - BG.StartIdx - 1, dl) }; |
2293 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
2294 | } else { |
2295 | SDValue Ops[] = |
2296 | { Res, TruncateToInt32(V: BG.V, dl), getI32Imm(Imm: BG.RLAmt, dl), |
2297 | getI32Imm(Imm: Bits.size() - BG.EndIdx - 1, dl), |
2298 | getI32Imm(Imm: Bits.size() - BG.StartIdx - 1, dl) }; |
2299 | Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); |
2300 | } |
2301 | } |
2302 | |
2303 | if (LateMask) { |
2304 | unsigned Mask = (unsigned) getZerosMask(); |
2305 | |
2306 | unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; |
2307 | assert((ANDIMask != 0 || ANDISMask != 0) && |
2308 | "No set bits in zeros mask?" ); |
2309 | |
2310 | if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
2311 | (unsigned) (ANDISMask != 0) + |
2312 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2313 | |
2314 | SDValue ANDIVal, ANDISVal; |
2315 | if (ANDIMask != 0) |
2316 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, |
2317 | Res, getI32Imm(ANDIMask, dl)), |
2318 | 0); |
2319 | if (ANDISMask != 0) |
2320 | ANDISVal = |
2321 | SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, |
2322 | getI32Imm(ANDISMask, dl)), |
2323 | 0); |
2324 | |
2325 | if (!ANDIVal) |
2326 | Res = ANDISVal; |
2327 | else if (!ANDISVal) |
2328 | Res = ANDIVal; |
2329 | else |
2330 | Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, |
2331 | ANDIVal, ANDISVal), 0); |
2332 | } |
2333 | |
2334 | return Res.getNode(); |
2335 | } |
2336 | |
2337 | unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, |
2338 | unsigned MaskStart, unsigned MaskEnd, |
2339 | bool IsIns) { |
2340 | // In the notation used by the instructions, 'start' and 'end' are reversed |
2341 | // because bits are counted from high to low order. |
2342 | unsigned InstMaskStart = 64 - MaskEnd - 1, |
2343 | InstMaskEnd = 64 - MaskStart - 1; |
2344 | |
2345 | if (Repl32) |
2346 | return 1; |
2347 | |
2348 | if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || |
2349 | InstMaskEnd == 63 - RLAmt) |
2350 | return 1; |
2351 | |
2352 | return 2; |
2353 | } |
2354 | |
2355 | // For 64-bit values, not all combinations of rotates and masks are |
2356 | // available. Produce one if it is available. |
2357 | SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt, |
2358 | bool Repl32, unsigned MaskStart, unsigned MaskEnd, |
2359 | unsigned *InstCnt = nullptr) { |
2360 | // In the notation used by the instructions, 'start' and 'end' are reversed |
2361 | // because bits are counted from high to low order. |
2362 | unsigned InstMaskStart = 64 - MaskEnd - 1, |
2363 | InstMaskEnd = 64 - MaskStart - 1; |
2364 | |
2365 | if (InstCnt) *InstCnt += 1; |
2366 | |
2367 | if (Repl32) { |
2368 | // This rotation amount assumes that the lower 32 bits of the quantity |
2369 | // are replicated in the high 32 bits by the rotation operator (which is |
2370 | // done by rlwinm and friends). |
2371 | assert(InstMaskStart >= 32 && "Mask cannot start out of range" ); |
2372 | assert(InstMaskEnd >= 32 && "Mask cannot end out of range" ); |
2373 | SDValue Ops[] = |
2374 | { ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2375 | getI32Imm(Imm: InstMaskStart - 32, dl), getI32Imm(Imm: InstMaskEnd - 32, dl) }; |
2376 | return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, |
2377 | Ops), 0); |
2378 | } |
2379 | |
2380 | if (InstMaskEnd == 63) { |
2381 | SDValue Ops[] = |
2382 | { ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2383 | getI32Imm(Imm: InstMaskStart, dl) }; |
2384 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); |
2385 | } |
2386 | |
2387 | if (InstMaskStart == 0) { |
2388 | SDValue Ops[] = |
2389 | { ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2390 | getI32Imm(Imm: InstMaskEnd, dl) }; |
2391 | return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); |
2392 | } |
2393 | |
2394 | if (InstMaskEnd == 63 - RLAmt) { |
2395 | SDValue Ops[] = |
2396 | { ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2397 | getI32Imm(Imm: InstMaskStart, dl) }; |
2398 | return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); |
2399 | } |
2400 | |
2401 | // We cannot do this with a single instruction, so we'll use two. The |
2402 | // problem is that we're not free to choose both a rotation amount and mask |
2403 | // start and end independently. We can choose an arbitrary mask start and |
2404 | // end, but then the rotation amount is fixed. Rotation, however, can be |
2405 | // inverted, and so by applying an "inverse" rotation first, we can get the |
2406 | // desired result. |
2407 | if (InstCnt) *InstCnt += 1; |
2408 | |
2409 | // The rotation mask for the second instruction must be MaskStart. |
2410 | unsigned RLAmt2 = MaskStart; |
2411 | // The first instruction must rotate V so that the overall rotation amount |
2412 | // is RLAmt. |
2413 | unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
2414 | if (RLAmt1) |
2415 | V = SelectRotMask64(V, dl, RLAmt: RLAmt1, Repl32: false, MaskStart: 0, MaskEnd: 63); |
2416 | return SelectRotMask64(V, dl, RLAmt: RLAmt2, Repl32: false, MaskStart, MaskEnd); |
2417 | } |
2418 | |
2419 | // For 64-bit values, not all combinations of rotates and masks are |
2420 | // available. Produce a rotate-mask-and-insert if one is available. |
2421 | SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl, |
2422 | unsigned RLAmt, bool Repl32, unsigned MaskStart, |
2423 | unsigned MaskEnd, unsigned *InstCnt = nullptr) { |
2424 | // In the notation used by the instructions, 'start' and 'end' are reversed |
2425 | // because bits are counted from high to low order. |
2426 | unsigned InstMaskStart = 64 - MaskEnd - 1, |
2427 | InstMaskEnd = 64 - MaskStart - 1; |
2428 | |
2429 | if (InstCnt) *InstCnt += 1; |
2430 | |
2431 | if (Repl32) { |
2432 | // This rotation amount assumes that the lower 32 bits of the quantity |
2433 | // are replicated in the high 32 bits by the rotation operator (which is |
2434 | // done by rlwinm and friends). |
2435 | assert(InstMaskStart >= 32 && "Mask cannot start out of range" ); |
2436 | assert(InstMaskEnd >= 32 && "Mask cannot end out of range" ); |
2437 | SDValue Ops[] = |
2438 | { ExtendToInt64(V: Base, dl), ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2439 | getI32Imm(Imm: InstMaskStart - 32, dl), getI32Imm(Imm: InstMaskEnd - 32, dl) }; |
2440 | return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, |
2441 | Ops), 0); |
2442 | } |
2443 | |
2444 | if (InstMaskEnd == 63 - RLAmt) { |
2445 | SDValue Ops[] = |
2446 | { ExtendToInt64(V: Base, dl), ExtendToInt64(V, dl), getI32Imm(Imm: RLAmt, dl), |
2447 | getI32Imm(Imm: InstMaskStart, dl) }; |
2448 | return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); |
2449 | } |
2450 | |
2451 | // We cannot do this with a single instruction, so we'll use two. The |
2452 | // problem is that we're not free to choose both a rotation amount and mask |
2453 | // start and end independently. We can choose an arbitrary mask start and |
2454 | // end, but then the rotation amount is fixed. Rotation, however, can be |
2455 | // inverted, and so by applying an "inverse" rotation first, we can get the |
2456 | // desired result. |
2457 | if (InstCnt) *InstCnt += 1; |
2458 | |
2459 | // The rotation mask for the second instruction must be MaskStart. |
2460 | unsigned RLAmt2 = MaskStart; |
2461 | // The first instruction must rotate V so that the overall rotation amount |
2462 | // is RLAmt. |
2463 | unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; |
2464 | if (RLAmt1) |
2465 | V = SelectRotMask64(V, dl, RLAmt: RLAmt1, Repl32: false, MaskStart: 0, MaskEnd: 63); |
2466 | return SelectRotMaskIns64(Base, V, dl, RLAmt: RLAmt2, Repl32: false, MaskStart, MaskEnd); |
2467 | } |
2468 | |
2469 | void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { |
2470 | if (BPermRewriterNoMasking) |
2471 | return; |
2472 | |
2473 | // The idea here is the same as in the 32-bit version, but with additional |
2474 | // complications from the fact that Repl32 might be true. Because we |
2475 | // aggressively convert bit groups to Repl32 form (which, for small |
2476 | // rotation factors, involves no other change), and then coalesce, it might |
2477 | // be the case that a single 64-bit masking operation could handle both |
2478 | // some Repl32 groups and some non-Repl32 groups. If converting to Repl32 |
2479 | // form allowed coalescing, then we must use a 32-bit rotaton in order to |
2480 | // completely capture the new combined bit group. |
2481 | |
2482 | for (ValueRotInfo &VRI : ValueRotsVec) { |
2483 | uint64_t Mask = 0; |
2484 | |
2485 | // We need to add to the mask all bits from the associated bit groups. |
2486 | // If Repl32 is false, we need to add bits from bit groups that have |
2487 | // Repl32 true, but are trivially convertable to Repl32 false. Such a |
2488 | // group is trivially convertable if it overlaps only with the lower 32 |
2489 | // bits, and the group has not been coalesced. |
2490 | auto MatchingBG = [VRI](const BitGroup &BG) { |
2491 | if (VRI.V != BG.V) |
2492 | return false; |
2493 | |
2494 | unsigned EffRLAmt = BG.RLAmt; |
2495 | if (!VRI.Repl32 && BG.Repl32) { |
2496 | if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && |
2497 | !BG.Repl32Coalesced) { |
2498 | if (BG.Repl32CR) |
2499 | EffRLAmt += 32; |
2500 | } else { |
2501 | return false; |
2502 | } |
2503 | } else if (VRI.Repl32 != BG.Repl32) { |
2504 | return false; |
2505 | } |
2506 | |
2507 | return VRI.RLAmt == EffRLAmt; |
2508 | }; |
2509 | |
2510 | for (auto &BG : BitGroups) { |
2511 | if (!MatchingBG(BG)) |
2512 | continue; |
2513 | |
2514 | if (BG.StartIdx <= BG.EndIdx) { |
2515 | for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) |
2516 | Mask |= (UINT64_C(1) << i); |
2517 | } else { |
2518 | for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) |
2519 | Mask |= (UINT64_C(1) << i); |
2520 | for (unsigned i = 0; i <= BG.EndIdx; ++i) |
2521 | Mask |= (UINT64_C(1) << i); |
2522 | } |
2523 | } |
2524 | |
2525 | // We can use the 32-bit andi/andis technique if the mask does not |
2526 | // require any higher-order bits. This can save an instruction compared |
2527 | // to always using the general 64-bit technique. |
2528 | bool Use32BitInsts = isUInt<32>(x: Mask); |
2529 | // Compute the masks for andi/andis that would be necessary. |
2530 | unsigned ANDIMask = (Mask & UINT16_MAX), |
2531 | ANDISMask = (Mask >> 16) & UINT16_MAX; |
2532 | |
2533 | bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(x: Mask)); |
2534 | |
2535 | unsigned NumAndInsts = (unsigned) NeedsRotate + |
2536 | (unsigned) (bool) Res; |
2537 | unsigned NumOfSelectInsts = 0; |
2538 | selectI64Imm(CurDAG, dl, Imm: Mask, InstCnt: &NumOfSelectInsts); |
2539 | assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant." ); |
2540 | if (Use32BitInsts) |
2541 | NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + |
2542 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2543 | else |
2544 | NumAndInsts += NumOfSelectInsts + /* and */ 1; |
2545 | |
2546 | unsigned NumRLInsts = 0; |
2547 | bool FirstBG = true; |
2548 | bool MoreBG = false; |
2549 | for (auto &BG : BitGroups) { |
2550 | if (!MatchingBG(BG)) { |
2551 | MoreBG = true; |
2552 | continue; |
2553 | } |
2554 | NumRLInsts += |
2555 | SelectRotMask64Count(RLAmt: BG.RLAmt, Repl32: BG.Repl32, MaskStart: BG.StartIdx, MaskEnd: BG.EndIdx, |
2556 | IsIns: !FirstBG); |
2557 | FirstBG = false; |
2558 | } |
2559 | |
2560 | LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() |
2561 | << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":" ) |
2562 | << "\n\t\t\tisel using masking: " << NumAndInsts |
2563 | << " using rotates: " << NumRLInsts << "\n" ); |
2564 | |
2565 | // When we'd use andi/andis, we bias toward using the rotates (andi only |
2566 | // has a record form, and is cracked on POWER cores). However, when using |
2567 | // general 64-bit constant formation, bias toward the constant form, |
2568 | // because that exposes more opportunities for CSE. |
2569 | if (NumAndInsts > NumRLInsts) |
2570 | continue; |
2571 | // When merging multiple bit groups, instruction or is used. |
2572 | // But when rotate is used, rldimi can inert the rotated value into any |
2573 | // register, so instruction or can be avoided. |
2574 | if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts) |
2575 | continue; |
2576 | |
2577 | LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n" ); |
2578 | |
2579 | if (InstCnt) *InstCnt += NumAndInsts; |
2580 | |
2581 | SDValue VRot; |
2582 | // We actually need to generate a rotation if we have a non-zero rotation |
2583 | // factor or, in the Repl32 case, if we care about any of the |
2584 | // higher-order replicated bits. In the latter case, we generate a mask |
2585 | // backward so that it actually includes the entire 64 bits. |
2586 | if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(x: Mask))) |
2587 | VRot = SelectRotMask64(V: VRI.V, dl, RLAmt: VRI.RLAmt, Repl32: VRI.Repl32, |
2588 | MaskStart: VRI.Repl32 ? 31 : 0, MaskEnd: VRI.Repl32 ? 30 : 63); |
2589 | else |
2590 | VRot = VRI.V; |
2591 | |
2592 | SDValue TotalVal; |
2593 | if (Use32BitInsts) { |
2594 | assert((ANDIMask != 0 || ANDISMask != 0) && |
2595 | "No set bits in mask when using 32-bit ands for 64-bit value" ); |
2596 | |
2597 | SDValue ANDIVal, ANDISVal; |
2598 | if (ANDIMask != 0) |
2599 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
2600 | ExtendToInt64(VRot, dl), |
2601 | getI32Imm(ANDIMask, dl)), |
2602 | 0); |
2603 | if (ANDISMask != 0) |
2604 | ANDISVal = |
2605 | SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
2606 | ExtendToInt64(VRot, dl), |
2607 | getI32Imm(ANDISMask, dl)), |
2608 | 0); |
2609 | |
2610 | if (!ANDIVal) |
2611 | TotalVal = ANDISVal; |
2612 | else if (!ANDISVal) |
2613 | TotalVal = ANDIVal; |
2614 | else |
2615 | TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2616 | ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
2617 | } else { |
2618 | TotalVal = SDValue(selectI64Imm(CurDAG, dl, Imm: Mask), 0); |
2619 | TotalVal = |
2620 | SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
2621 | ExtendToInt64(VRot, dl), TotalVal), |
2622 | 0); |
2623 | } |
2624 | |
2625 | if (!Res) |
2626 | Res = TotalVal; |
2627 | else |
2628 | Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2629 | ExtendToInt64(Res, dl), TotalVal), |
2630 | 0); |
2631 | |
2632 | // Now, remove all groups with this underlying value and rotation |
2633 | // factor. |
2634 | eraseMatchingBitGroups(F: MatchingBG); |
2635 | } |
2636 | } |
2637 | |
2638 | // Instruction selection for the 64-bit case. |
2639 | SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { |
2640 | SDLoc dl(N); |
2641 | SDValue Res; |
2642 | |
2643 | if (InstCnt) *InstCnt = 0; |
2644 | |
2645 | // Take care of cases that should use andi/andis first. |
2646 | SelectAndParts64(dl, Res, InstCnt); |
2647 | |
2648 | // If we've not yet selected a 'starting' instruction, and we have no zeros |
2649 | // to fill in, select the (Value, RLAmt) with the highest priority (largest |
2650 | // number of groups), and start with this rotated value. |
2651 | if ((!NeedMask || LateMask) && !Res) { |
2652 | // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 |
2653 | // groups will come first, and so the VRI representing the largest number |
2654 | // of groups might not be first (it might be the first Repl32 groups). |
2655 | unsigned MaxGroupsIdx = 0; |
2656 | if (!ValueRotsVec[0].Repl32) { |
2657 | for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) |
2658 | if (ValueRotsVec[i].Repl32) { |
2659 | if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) |
2660 | MaxGroupsIdx = i; |
2661 | break; |
2662 | } |
2663 | } |
2664 | |
2665 | ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; |
2666 | bool NeedsRotate = false; |
2667 | if (VRI.RLAmt) { |
2668 | NeedsRotate = true; |
2669 | } else if (VRI.Repl32) { |
2670 | for (auto &BG : BitGroups) { |
2671 | if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || |
2672 | BG.Repl32 != VRI.Repl32) |
2673 | continue; |
2674 | |
2675 | // We don't need a rotate if the bit group is confined to the lower |
2676 | // 32 bits. |
2677 | if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) |
2678 | continue; |
2679 | |
2680 | NeedsRotate = true; |
2681 | break; |
2682 | } |
2683 | } |
2684 | |
2685 | if (NeedsRotate) |
2686 | Res = SelectRotMask64(V: VRI.V, dl, RLAmt: VRI.RLAmt, Repl32: VRI.Repl32, |
2687 | MaskStart: VRI.Repl32 ? 31 : 0, MaskEnd: VRI.Repl32 ? 30 : 63, |
2688 | InstCnt); |
2689 | else |
2690 | Res = VRI.V; |
2691 | |
2692 | // Now, remove all groups with this underlying value and rotation factor. |
2693 | if (Res) |
2694 | eraseMatchingBitGroups(F: [VRI](const BitGroup &BG) { |
2695 | return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && |
2696 | BG.Repl32 == VRI.Repl32; |
2697 | }); |
2698 | } |
2699 | |
2700 | // Because 64-bit rotates are more flexible than inserts, we might have a |
2701 | // preference regarding which one we do first (to save one instruction). |
2702 | if (!Res) |
2703 | for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { |
2704 | if (SelectRotMask64Count(RLAmt: I->RLAmt, Repl32: I->Repl32, MaskStart: I->StartIdx, MaskEnd: I->EndIdx, |
2705 | IsIns: false) < |
2706 | SelectRotMask64Count(RLAmt: I->RLAmt, Repl32: I->Repl32, MaskStart: I->StartIdx, MaskEnd: I->EndIdx, |
2707 | IsIns: true)) { |
2708 | if (I != BitGroups.begin()) { |
2709 | BitGroup BG = *I; |
2710 | BitGroups.erase(CI: I); |
2711 | BitGroups.insert(I: BitGroups.begin(), Elt: BG); |
2712 | } |
2713 | |
2714 | break; |
2715 | } |
2716 | } |
2717 | |
2718 | // Insert the other groups (one at a time). |
2719 | for (auto &BG : BitGroups) { |
2720 | if (!Res) |
2721 | Res = SelectRotMask64(V: BG.V, dl, RLAmt: BG.RLAmt, Repl32: BG.Repl32, MaskStart: BG.StartIdx, |
2722 | MaskEnd: BG.EndIdx, InstCnt); |
2723 | else |
2724 | Res = SelectRotMaskIns64(Base: Res, V: BG.V, dl, RLAmt: BG.RLAmt, Repl32: BG.Repl32, |
2725 | MaskStart: BG.StartIdx, MaskEnd: BG.EndIdx, InstCnt); |
2726 | } |
2727 | |
2728 | if (LateMask) { |
2729 | uint64_t Mask = getZerosMask(); |
2730 | |
2731 | // We can use the 32-bit andi/andis technique if the mask does not |
2732 | // require any higher-order bits. This can save an instruction compared |
2733 | // to always using the general 64-bit technique. |
2734 | bool Use32BitInsts = isUInt<32>(x: Mask); |
2735 | // Compute the masks for andi/andis that would be necessary. |
2736 | unsigned ANDIMask = (Mask & UINT16_MAX), |
2737 | ANDISMask = (Mask >> 16) & UINT16_MAX; |
2738 | |
2739 | if (Use32BitInsts) { |
2740 | assert((ANDIMask != 0 || ANDISMask != 0) && |
2741 | "No set bits in mask when using 32-bit ands for 64-bit value" ); |
2742 | |
2743 | if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + |
2744 | (unsigned) (ANDISMask != 0) + |
2745 | (unsigned) (ANDIMask != 0 && ANDISMask != 0); |
2746 | |
2747 | SDValue ANDIVal, ANDISVal; |
2748 | if (ANDIMask != 0) |
2749 | ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, |
2750 | ExtendToInt64(Res, dl), |
2751 | getI32Imm(ANDIMask, dl)), |
2752 | 0); |
2753 | if (ANDISMask != 0) |
2754 | ANDISVal = |
2755 | SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, |
2756 | ExtendToInt64(Res, dl), |
2757 | getI32Imm(ANDISMask, dl)), |
2758 | 0); |
2759 | |
2760 | if (!ANDIVal) |
2761 | Res = ANDISVal; |
2762 | else if (!ANDISVal) |
2763 | Res = ANDIVal; |
2764 | else |
2765 | Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
2766 | ExtendToInt64(ANDIVal, dl), ANDISVal), 0); |
2767 | } else { |
2768 | unsigned NumOfSelectInsts = 0; |
2769 | SDValue MaskVal = |
2770 | SDValue(selectI64Imm(CurDAG, dl, Imm: Mask, InstCnt: &NumOfSelectInsts), 0); |
2771 | Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, |
2772 | ExtendToInt64(Res, dl), MaskVal), |
2773 | 0); |
2774 | if (InstCnt) |
2775 | *InstCnt += NumOfSelectInsts + /* and */ 1; |
2776 | } |
2777 | } |
2778 | |
2779 | return Res.getNode(); |
2780 | } |
2781 | |
2782 | SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { |
2783 | // Fill in BitGroups. |
2784 | collectBitGroups(LateMask); |
2785 | if (BitGroups.empty()) |
2786 | return nullptr; |
2787 | |
2788 | // For 64-bit values, figure out when we can use 32-bit instructions. |
2789 | if (Bits.size() == 64) |
2790 | assignRepl32BitGroups(); |
2791 | |
2792 | // Fill in ValueRotsVec. |
2793 | collectValueRotInfo(); |
2794 | |
2795 | if (Bits.size() == 32) { |
2796 | return Select32(N, LateMask, InstCnt); |
2797 | } else { |
2798 | assert(Bits.size() == 64 && "Not 64 bits here?" ); |
2799 | return Select64(N, LateMask, InstCnt); |
2800 | } |
2801 | |
2802 | return nullptr; |
2803 | } |
2804 | |
2805 | void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) { |
2806 | erase_if(C&: BitGroups, P: F); |
2807 | } |
2808 | |
2809 | SmallVector<ValueBit, 64> Bits; |
2810 | |
2811 | bool NeedMask = false; |
2812 | SmallVector<unsigned, 64> RLAmt; |
2813 | |
2814 | SmallVector<BitGroup, 16> BitGroups; |
2815 | |
2816 | DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots; |
2817 | SmallVector<ValueRotInfo, 16> ValueRotsVec; |
2818 | |
2819 | SelectionDAG *CurDAG = nullptr; |
2820 | |
2821 | public: |
2822 | BitPermutationSelector(SelectionDAG *DAG) |
2823 | : CurDAG(DAG) {} |
2824 | |
2825 | // Here we try to match complex bit permutations into a set of |
2826 | // rotate-and-shift/shift/and/or instructions, using a set of heuristics |
2827 | // known to produce optimal code for common cases (like i32 byte swapping). |
2828 | SDNode *Select(SDNode *N) { |
2829 | Memoizer.clear(); |
2830 | auto Result = |
2831 | getValueBits(V: SDValue(N, 0), NumBits: N->getValueType(ResNo: 0).getSizeInBits()); |
2832 | if (!Result.first) |
2833 | return nullptr; |
2834 | Bits = std::move(*Result.second); |
2835 | |
2836 | LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" |
2837 | " selection for: " ); |
2838 | LLVM_DEBUG(N->dump(CurDAG)); |
2839 | |
2840 | // Fill it RLAmt and set NeedMask. |
2841 | computeRotationAmounts(); |
2842 | |
2843 | if (!NeedMask) |
2844 | return Select(N, LateMask: false); |
2845 | |
2846 | // We currently have two techniques for handling results with zeros: early |
2847 | // masking (the default) and late masking. Late masking is sometimes more |
2848 | // efficient, but because the structure of the bit groups is different, it |
2849 | // is hard to tell without generating both and comparing the results. With |
2850 | // late masking, we ignore zeros in the resulting value when inserting each |
2851 | // set of bit groups, and then mask in the zeros at the end. With early |
2852 | // masking, we only insert the non-zero parts of the result at every step. |
2853 | |
2854 | unsigned InstCnt = 0, InstCntLateMask = 0; |
2855 | LLVM_DEBUG(dbgs() << "\tEarly masking:\n" ); |
2856 | SDNode *RN = Select(N, LateMask: false, InstCnt: &InstCnt); |
2857 | LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n" ); |
2858 | |
2859 | LLVM_DEBUG(dbgs() << "\tLate masking:\n" ); |
2860 | SDNode *RNLM = Select(N, LateMask: true, InstCnt: &InstCntLateMask); |
2861 | LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask |
2862 | << " instructions\n" ); |
2863 | |
2864 | if (InstCnt <= InstCntLateMask) { |
2865 | LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n" ); |
2866 | return RN; |
2867 | } |
2868 | |
2869 | LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n" ); |
2870 | return RNLM; |
2871 | } |
2872 | }; |
2873 | |
2874 | class IntegerCompareEliminator { |
2875 | SelectionDAG *CurDAG; |
2876 | PPCDAGToDAGISel *S; |
2877 | // Conversion type for interpreting results of a 32-bit instruction as |
2878 | // a 64-bit value or vice versa. |
2879 | enum ExtOrTruncConversion { Ext, Trunc }; |
2880 | |
2881 | // Modifiers to guide how an ISD::SETCC node's result is to be computed |
2882 | // in a GPR. |
2883 | // ZExtOrig - use the original condition code, zero-extend value |
2884 | // ZExtInvert - invert the condition code, zero-extend value |
2885 | // SExtOrig - use the original condition code, sign-extend value |
2886 | // SExtInvert - invert the condition code, sign-extend value |
2887 | enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert }; |
2888 | |
2889 | // Comparisons against zero to emit GPR code sequences for. Each of these |
2890 | // sequences may need to be emitted for two or more equivalent patterns. |
2891 | // For example (a >= 0) == (a > -1). The direction of the comparison (</>) |
2892 | // matters as well as the extension type: sext (-1/0), zext (1/0). |
2893 | // GEZExt - (zext (LHS >= 0)) |
2894 | // GESExt - (sext (LHS >= 0)) |
2895 | // LEZExt - (zext (LHS <= 0)) |
2896 | // LESExt - (sext (LHS <= 0)) |
2897 | enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt }; |
2898 | |
2899 | SDNode *tryEXTEND(SDNode *N); |
2900 | SDNode *tryLogicOpOfCompares(SDNode *N); |
2901 | SDValue computeLogicOpInGPR(SDValue LogicOp); |
2902 | SDValue signExtendInputIfNeeded(SDValue Input); |
2903 | SDValue zeroExtendInputIfNeeded(SDValue Input); |
2904 | SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv); |
2905 | SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
2906 | ZeroCompare CmpTy); |
2907 | SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2908 | int64_t RHSValue, SDLoc dl); |
2909 | SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2910 | int64_t RHSValue, SDLoc dl); |
2911 | SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2912 | int64_t RHSValue, SDLoc dl); |
2913 | SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
2914 | int64_t RHSValue, SDLoc dl); |
2915 | SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts); |
2916 | |
2917 | public: |
2918 | IntegerCompareEliminator(SelectionDAG *DAG, |
2919 | PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) { |
2920 | assert(CurDAG->getTargetLoweringInfo() |
2921 | .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && |
2922 | "Only expecting to use this on 64 bit targets." ); |
2923 | } |
2924 | SDNode *Select(SDNode *N) { |
2925 | if (CmpInGPR == ICGPR_None) |
2926 | return nullptr; |
2927 | switch (N->getOpcode()) { |
2928 | default: break; |
2929 | case ISD::ZERO_EXTEND: |
2930 | if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 || |
2931 | CmpInGPR == ICGPR_SextI64) |
2932 | return nullptr; |
2933 | [[fallthrough]]; |
2934 | case ISD::SIGN_EXTEND: |
2935 | if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 || |
2936 | CmpInGPR == ICGPR_ZextI64) |
2937 | return nullptr; |
2938 | return tryEXTEND(N); |
2939 | case ISD::AND: |
2940 | case ISD::OR: |
2941 | case ISD::XOR: |
2942 | return tryLogicOpOfCompares(N); |
2943 | } |
2944 | return nullptr; |
2945 | } |
2946 | }; |
2947 | |
2948 | // The obvious case for wanting to keep the value in a GPR. Namely, the |
2949 | // result of the comparison is actually needed in a GPR. |
2950 | SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) { |
2951 | assert((N->getOpcode() == ISD::ZERO_EXTEND || |
2952 | N->getOpcode() == ISD::SIGN_EXTEND) && |
2953 | "Expecting a zero/sign extend node!" ); |
2954 | SDValue WideRes; |
2955 | // If we are zero-extending the result of a logical operation on i1 |
2956 | // values, we can keep the values in GPRs. |
2957 | if (ISD::isBitwiseLogicOp(N->getOperand(0).getOpcode()) && |
2958 | N->getOperand(0).getValueType() == MVT::i1 && |
2959 | N->getOpcode() == ISD::ZERO_EXTEND) |
2960 | WideRes = computeLogicOpInGPR(LogicOp: N->getOperand(Num: 0)); |
2961 | else if (N->getOperand(Num: 0).getOpcode() != ISD::SETCC) |
2962 | return nullptr; |
2963 | else |
2964 | WideRes = |
2965 | getSETCCInGPR(Compare: N->getOperand(Num: 0), |
2966 | ConvOpts: N->getOpcode() == ISD::SIGN_EXTEND ? |
2967 | SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig); |
2968 | |
2969 | if (!WideRes) |
2970 | return nullptr; |
2971 | |
2972 | SDLoc dl(N); |
2973 | bool Input32Bit = WideRes.getValueType() == MVT::i32; |
2974 | bool Output32Bit = N->getValueType(0) == MVT::i32; |
2975 | |
2976 | NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; |
2977 | NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; |
2978 | |
2979 | SDValue ConvOp = WideRes; |
2980 | if (Input32Bit != Output32Bit) |
2981 | ConvOp = addExtOrTrunc(NatWidthRes: WideRes, Conv: Input32Bit ? ExtOrTruncConversion::Ext : |
2982 | ExtOrTruncConversion::Trunc); |
2983 | return ConvOp.getNode(); |
2984 | } |
2985 | |
2986 | // Attempt to perform logical operations on the results of comparisons while |
2987 | // keeping the values in GPRs. Without doing so, these would end up being |
2988 | // lowered to CR-logical operations which suffer from significant latency and |
2989 | // low ILP. |
2990 | SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) { |
2991 | if (N->getValueType(0) != MVT::i1) |
2992 | return nullptr; |
2993 | assert(ISD::isBitwiseLogicOp(N->getOpcode()) && |
2994 | "Expected a logic operation on setcc results." ); |
2995 | SDValue LoweredLogical = computeLogicOpInGPR(LogicOp: SDValue(N, 0)); |
2996 | if (!LoweredLogical) |
2997 | return nullptr; |
2998 | |
2999 | SDLoc dl(N); |
3000 | bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8; |
3001 | unsigned = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt; |
3002 | SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
3003 | SDValue LHS = LoweredLogical.getOperand(i: 0); |
3004 | SDValue RHS = LoweredLogical.getOperand(i: 1); |
3005 | SDValue WideOp; |
3006 | SDValue OpToConvToRecForm; |
3007 | |
3008 | // Look through any 32-bit to 64-bit implicit extend nodes to find the |
3009 | // opcode that is input to the XORI. |
3010 | if (IsBitwiseNegate && |
3011 | LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG) |
3012 | OpToConvToRecForm = LoweredLogical.getOperand(i: 0).getOperand(i: 1); |
3013 | else if (IsBitwiseNegate) |
3014 | // If the input to the XORI isn't an extension, that's what we're after. |
3015 | OpToConvToRecForm = LoweredLogical.getOperand(i: 0); |
3016 | else |
3017 | // If this is not an XORI, it is a reg-reg logical op and we can convert |
3018 | // it to record-form. |
3019 | OpToConvToRecForm = LoweredLogical; |
3020 | |
3021 | // Get the record-form version of the node we're looking to use to get the |
3022 | // CR result from. |
3023 | uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode(); |
3024 | int NewOpc = PPCInstrInfo::getRecordFormOpcode(Opcode: NonRecOpc); |
3025 | |
3026 | // Convert the right node to record-form. This is either the logical we're |
3027 | // looking at or it is the input node to the negation (if we're looking at |
3028 | // a bitwise negation). |
3029 | if (NewOpc != -1 && IsBitwiseNegate) { |
3030 | // The input to the XORI has a record-form. Use it. |
3031 | assert(LoweredLogical.getConstantOperandVal(1) == 1 && |
3032 | "Expected a PPC::XORI8 only for bitwise negation." ); |
3033 | // Emit the record-form instruction. |
3034 | std::vector<SDValue> Ops; |
3035 | for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++) |
3036 | Ops.push_back(x: OpToConvToRecForm.getOperand(i)); |
3037 | |
3038 | WideOp = |
3039 | SDValue(CurDAG->getMachineNode(NewOpc, dl, |
3040 | OpToConvToRecForm.getValueType(), |
3041 | MVT::Glue, Ops), 0); |
3042 | } else { |
3043 | assert((NewOpc != -1 || !IsBitwiseNegate) && |
3044 | "No record form available for AND8/OR8/XOR8?" ); |
3045 | WideOp = |
3046 | SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, |
3047 | dl, MVT::i64, MVT::Glue, LHS, RHS), |
3048 | 0); |
3049 | } |
3050 | |
3051 | // Select this node to a single bit from CR0 set by the record-form node |
3052 | // just created. For bitwise negation, use the EQ bit which is the equivalent |
3053 | // of negating the result (i.e. it is a bit set when the result of the |
3054 | // operation is zero). |
3055 | SDValue SRIdxVal = |
3056 | CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); |
3057 | SDValue CRBit = |
3058 | SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, |
3059 | MVT::i1, CR0Reg, SRIdxVal, |
3060 | WideOp.getValue(1)), 0); |
3061 | return CRBit.getNode(); |
3062 | } |
3063 | |
3064 | // Lower a logical operation on i1 values into a GPR sequence if possible. |
3065 | // The result can be kept in a GPR if requested. |
3066 | // Three types of inputs can be handled: |
3067 | // - SETCC |
3068 | // - TRUNCATE |
3069 | // - Logical operation (AND/OR/XOR) |
3070 | // There is also a special case that is handled (namely a complement operation |
3071 | // achieved with xor %a, -1). |
3072 | SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) { |
3073 | assert(ISD::isBitwiseLogicOp(LogicOp.getOpcode()) && |
3074 | "Can only handle logic operations here." ); |
3075 | assert(LogicOp.getValueType() == MVT::i1 && |
3076 | "Can only handle logic operations on i1 values here." ); |
3077 | SDLoc dl(LogicOp); |
3078 | SDValue LHS, RHS; |
3079 | |
3080 | // Special case: xor %a, -1 |
3081 | bool IsBitwiseNegation = isBitwiseNot(V: LogicOp); |
3082 | |
3083 | // Produces a GPR sequence for each operand of the binary logic operation. |
3084 | // For SETCC, it produces the respective comparison, for TRUNCATE it truncates |
3085 | // the value in a GPR and for logic operations, it will recursively produce |
3086 | // a GPR sequence for the operation. |
3087 | auto getLogicOperand = [&] (SDValue Operand) -> SDValue { |
3088 | unsigned OperandOpcode = Operand.getOpcode(); |
3089 | if (OperandOpcode == ISD::SETCC) |
3090 | return getSETCCInGPR(Compare: Operand, ConvOpts: SetccInGPROpts::ZExtOrig); |
3091 | else if (OperandOpcode == ISD::TRUNCATE) { |
3092 | SDValue InputOp = Operand.getOperand(i: 0); |
3093 | EVT InVT = InputOp.getValueType(); |
3094 | return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : |
3095 | PPC::RLDICL, dl, InVT, InputOp, |
3096 | S->getI64Imm(0, dl), |
3097 | S->getI64Imm(63, dl)), 0); |
3098 | } else if (ISD::isBitwiseLogicOp(Opcode: OperandOpcode)) |
3099 | return computeLogicOpInGPR(LogicOp: Operand); |
3100 | return SDValue(); |
3101 | }; |
3102 | LHS = getLogicOperand(LogicOp.getOperand(i: 0)); |
3103 | RHS = getLogicOperand(LogicOp.getOperand(i: 1)); |
3104 | |
3105 | // If a GPR sequence can't be produced for the LHS we can't proceed. |
3106 | // Not producing a GPR sequence for the RHS is only a problem if this isn't |
3107 | // a bitwise negation operation. |
3108 | if (!LHS || (!RHS && !IsBitwiseNegation)) |
3109 | return SDValue(); |
3110 | |
3111 | NumLogicOpsOnComparison++; |
3112 | |
3113 | // We will use the inputs as 64-bit values. |
3114 | if (LHS.getValueType() == MVT::i32) |
3115 | LHS = addExtOrTrunc(NatWidthRes: LHS, Conv: ExtOrTruncConversion::Ext); |
3116 | if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32) |
3117 | RHS = addExtOrTrunc(NatWidthRes: RHS, Conv: ExtOrTruncConversion::Ext); |
3118 | |
3119 | unsigned NewOpc; |
3120 | switch (LogicOp.getOpcode()) { |
3121 | default: llvm_unreachable("Unknown logic operation." ); |
3122 | case ISD::AND: NewOpc = PPC::AND8; break; |
3123 | case ISD::OR: NewOpc = PPC::OR8; break; |
3124 | case ISD::XOR: NewOpc = PPC::XOR8; break; |
3125 | } |
3126 | |
3127 | if (IsBitwiseNegation) { |
3128 | RHS = S->getI64Imm(Imm: 1, dl); |
3129 | NewOpc = PPC::XORI8; |
3130 | } |
3131 | |
3132 | return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); |
3133 | |
3134 | } |
3135 | |
3136 | /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it. |
3137 | /// Otherwise just reinterpret it as a 64-bit value. |
3138 | /// Useful when emitting comparison code for 32-bit values without using |
3139 | /// the compare instruction (which only considers the lower 32-bits). |
3140 | SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) { |
3141 | assert(Input.getValueType() == MVT::i32 && |
3142 | "Can only sign-extend 32-bit values here." ); |
3143 | unsigned Opc = Input.getOpcode(); |
3144 | |
3145 | // The value was sign extended and then truncated to 32-bits. No need to |
3146 | // sign extend it again. |
3147 | if (Opc == ISD::TRUNCATE && |
3148 | (Input.getOperand(i: 0).getOpcode() == ISD::AssertSext || |
3149 | Input.getOperand(i: 0).getOpcode() == ISD::SIGN_EXTEND)) |
3150 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3151 | |
3152 | LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Val&: Input); |
3153 | // The input is a sign-extending load. All ppc sign-extending loads |
3154 | // sign-extend to the full 64-bits. |
3155 | if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) |
3156 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3157 | |
3158 | ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Val&: Input); |
3159 | // We don't sign-extend constants. |
3160 | if (InputConst) |
3161 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3162 | |
3163 | SDLoc dl(Input); |
3164 | SignExtensionsAdded++; |
3165 | return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, |
3166 | MVT::i64, Input), 0); |
3167 | } |
3168 | |
3169 | /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it. |
3170 | /// Otherwise just reinterpret it as a 64-bit value. |
3171 | /// Useful when emitting comparison code for 32-bit values without using |
3172 | /// the compare instruction (which only considers the lower 32-bits). |
3173 | SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) { |
3174 | assert(Input.getValueType() == MVT::i32 && |
3175 | "Can only zero-extend 32-bit values here." ); |
3176 | unsigned Opc = Input.getOpcode(); |
3177 | |
3178 | // The only condition under which we can omit the actual extend instruction: |
3179 | // - The value is a positive constant |
3180 | // - The value comes from a load that isn't a sign-extending load |
3181 | // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. |
3182 | bool IsTruncateOfZExt = Opc == ISD::TRUNCATE && |
3183 | (Input.getOperand(i: 0).getOpcode() == ISD::AssertZext || |
3184 | Input.getOperand(i: 0).getOpcode() == ISD::ZERO_EXTEND); |
3185 | if (IsTruncateOfZExt) |
3186 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3187 | |
3188 | ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Val&: Input); |
3189 | if (InputConst && InputConst->getSExtValue() >= 0) |
3190 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3191 | |
3192 | LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Val&: Input); |
3193 | // The input is a load that doesn't sign-extend (it will be zero-extended). |
3194 | if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) |
3195 | return addExtOrTrunc(NatWidthRes: Input, Conv: ExtOrTruncConversion::Ext); |
3196 | |
3197 | // None of the above, need to zero-extend. |
3198 | SDLoc dl(Input); |
3199 | ZeroExtensionsAdded++; |
3200 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, |
3201 | S->getI64Imm(0, dl), |
3202 | S->getI64Imm(32, dl)), 0); |
3203 | } |
3204 | |
3205 | // Handle a 32-bit value in a 64-bit register and vice-versa. These are of |
3206 | // course not actual zero/sign extensions that will generate machine code, |
3207 | // they're just a way to reinterpret a 32 bit value in a register as a |
3208 | // 64 bit value and vice-versa. |
3209 | SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes, |
3210 | ExtOrTruncConversion Conv) { |
3211 | SDLoc dl(NatWidthRes); |
3212 | |
3213 | // For reinterpreting 32-bit values as 64 bit values, we generate |
3214 | // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1> |
3215 | if (Conv == ExtOrTruncConversion::Ext) { |
3216 | SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); |
3217 | SDValue SubRegIdx = |
3218 | CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
3219 | return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, |
3220 | ImDef, NatWidthRes, SubRegIdx), 0); |
3221 | } |
3222 | |
3223 | assert(Conv == ExtOrTruncConversion::Trunc && |
3224 | "Unknown convertion between 32 and 64 bit values." ); |
3225 | // For reinterpreting 64-bit values as 32-bit values, we just need to |
3226 | // EXTRACT_SUBREG (i.e. extract the low word). |
3227 | SDValue SubRegIdx = |
3228 | CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); |
3229 | return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, |
3230 | NatWidthRes, SubRegIdx), 0); |
3231 | } |
3232 | |
3233 | // Produce a GPR sequence for compound comparisons (<=, >=) against zero. |
3234 | // Handle both zero-extensions and sign-extensions. |
3235 | SDValue |
3236 | IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, |
3237 | ZeroCompare CmpTy) { |
3238 | EVT InVT = LHS.getValueType(); |
3239 | bool Is32Bit = InVT == MVT::i32; |
3240 | SDValue ToExtend; |
3241 | |
3242 | // Produce the value that needs to be either zero or sign extended. |
3243 | switch (CmpTy) { |
3244 | case ZeroCompare::GEZExt: |
3245 | case ZeroCompare::GESExt: |
3246 | ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, |
3247 | dl, InVT, LHS, LHS), 0); |
3248 | break; |
3249 | case ZeroCompare::LEZExt: |
3250 | case ZeroCompare::LESExt: { |
3251 | if (Is32Bit) { |
3252 | // Upper 32 bits cannot be undefined for this sequence. |
3253 | LHS = signExtendInputIfNeeded(Input: LHS); |
3254 | SDValue Neg = |
3255 | SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3256 | ToExtend = |
3257 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3258 | Neg, S->getI64Imm(1, dl), |
3259 | S->getI64Imm(63, dl)), 0); |
3260 | } else { |
3261 | SDValue Addi = |
3262 | SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3263 | S->getI64Imm(~0ULL, dl)), 0); |
3264 | ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, |
3265 | Addi, LHS), 0); |
3266 | } |
3267 | break; |
3268 | } |
3269 | } |
3270 | |
3271 | // For 64-bit sequences, the extensions are the same for the GE/LE cases. |
3272 | if (!Is32Bit && |
3273 | (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt)) |
3274 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3275 | ToExtend, S->getI64Imm(1, dl), |
3276 | S->getI64Imm(63, dl)), 0); |
3277 | if (!Is32Bit && |
3278 | (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt)) |
3279 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, |
3280 | S->getI64Imm(63, dl)), 0); |
3281 | |
3282 | assert(Is32Bit && "Should have handled the 32-bit sequences above." ); |
3283 | // For 32-bit sequences, the extensions differ between GE/LE cases. |
3284 | switch (CmpTy) { |
3285 | case ZeroCompare::GEZExt: { |
3286 | SDValue ShiftOps[] = { ToExtend, S->getI32Imm(Imm: 1, dl), S->getI32Imm(Imm: 31, dl), |
3287 | S->getI32Imm(Imm: 31, dl) }; |
3288 | return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3289 | ShiftOps), 0); |
3290 | } |
3291 | case ZeroCompare::GESExt: |
3292 | return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, |
3293 | S->getI32Imm(31, dl)), 0); |
3294 | case ZeroCompare::LEZExt: |
3295 | return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, |
3296 | S->getI32Imm(1, dl)), 0); |
3297 | case ZeroCompare::LESExt: |
3298 | return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, |
3299 | S->getI32Imm(-1, dl)), 0); |
3300 | } |
3301 | |
3302 | // The above case covers all the enumerators so it can't have a default clause |
3303 | // to avoid compiler warnings. |
3304 | llvm_unreachable("Unknown zero-comparison type." ); |
3305 | } |
3306 | |
3307 | /// Produces a zero-extended result of comparing two 32-bit values according to |
3308 | /// the passed condition code. |
3309 | SDValue |
3310 | IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS, |
3311 | ISD::CondCode CC, |
3312 | int64_t RHSValue, SDLoc dl) { |
3313 | if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
3314 | CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext) |
3315 | return SDValue(); |
3316 | bool IsRHSZero = RHSValue == 0; |
3317 | bool IsRHSOne = RHSValue == 1; |
3318 | bool IsRHSNegOne = RHSValue == -1LL; |
3319 | switch (CC) { |
3320 | default: return SDValue(); |
3321 | case ISD::SETEQ: { |
3322 | // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) |
3323 | // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) |
3324 | SDValue Xor = IsRHSZero ? LHS : |
3325 | SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3326 | SDValue Clz = |
3327 | SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3328 | SDValue ShiftOps[] = { Clz, S->getI32Imm(Imm: 27, dl), S->getI32Imm(Imm: 5, dl), |
3329 | S->getI32Imm(Imm: 31, dl) }; |
3330 | return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3331 | ShiftOps), 0); |
3332 | } |
3333 | case ISD::SETNE: { |
3334 | // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) |
3335 | // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) |
3336 | SDValue Xor = IsRHSZero ? LHS : |
3337 | SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3338 | SDValue Clz = |
3339 | SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3340 | SDValue ShiftOps[] = { Clz, S->getI32Imm(Imm: 27, dl), S->getI32Imm(Imm: 5, dl), |
3341 | S->getI32Imm(Imm: 31, dl) }; |
3342 | SDValue Shift = |
3343 | SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
3344 | return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
3345 | S->getI32Imm(1, dl)), 0); |
3346 | } |
3347 | case ISD::SETGE: { |
3348 | // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) |
3349 | // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) |
3350 | if(IsRHSZero) |
3351 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GEZExt); |
3352 | |
3353 | // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
3354 | // by swapping inputs and falling through. |
3355 | std::swap(a&: LHS, b&: RHS); |
3356 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3357 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3358 | [[fallthrough]]; |
3359 | } |
3360 | case ISD::SETLE: { |
3361 | if (CmpInGPR == ICGPR_NonExtIn) |
3362 | return SDValue(); |
3363 | // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) |
3364 | // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) |
3365 | if(IsRHSZero) { |
3366 | if (CmpInGPR == ICGPR_NonExtIn) |
3367 | return SDValue(); |
3368 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LEZExt); |
3369 | } |
3370 | |
3371 | // The upper 32-bits of the register can't be undefined for this sequence. |
3372 | LHS = signExtendInputIfNeeded(Input: LHS); |
3373 | RHS = signExtendInputIfNeeded(Input: RHS); |
3374 | SDValue Sub = |
3375 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3376 | SDValue Shift = |
3377 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, |
3378 | S->getI64Imm(1, dl), S->getI64Imm(63, dl)), |
3379 | 0); |
3380 | return |
3381 | SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, |
3382 | MVT::i64, Shift, S->getI32Imm(1, dl)), 0); |
3383 | } |
3384 | case ISD::SETGT: { |
3385 | // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) |
3386 | // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) |
3387 | // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) |
3388 | // Handle SETLT -1 (which is equivalent to SETGE 0). |
3389 | if (IsRHSNegOne) |
3390 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GEZExt); |
3391 | |
3392 | if (IsRHSZero) { |
3393 | if (CmpInGPR == ICGPR_NonExtIn) |
3394 | return SDValue(); |
3395 | // The upper 32-bits of the register can't be undefined for this sequence. |
3396 | LHS = signExtendInputIfNeeded(Input: LHS); |
3397 | RHS = signExtendInputIfNeeded(Input: RHS); |
3398 | SDValue Neg = |
3399 | SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3400 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3401 | Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); |
3402 | } |
3403 | // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
3404 | // (%b < %a) by swapping inputs and falling through. |
3405 | std::swap(a&: LHS, b&: RHS); |
3406 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3407 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3408 | IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3409 | [[fallthrough]]; |
3410 | } |
3411 | case ISD::SETLT: { |
3412 | // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) |
3413 | // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) |
3414 | // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) |
3415 | // Handle SETLT 1 (which is equivalent to SETLE 0). |
3416 | if (IsRHSOne) { |
3417 | if (CmpInGPR == ICGPR_NonExtIn) |
3418 | return SDValue(); |
3419 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LEZExt); |
3420 | } |
3421 | |
3422 | if (IsRHSZero) { |
3423 | SDValue ShiftOps[] = { LHS, S->getI32Imm(Imm: 1, dl), S->getI32Imm(Imm: 31, dl), |
3424 | S->getI32Imm(Imm: 31, dl) }; |
3425 | return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, |
3426 | ShiftOps), 0); |
3427 | } |
3428 | |
3429 | if (CmpInGPR == ICGPR_NonExtIn) |
3430 | return SDValue(); |
3431 | // The upper 32-bits of the register can't be undefined for this sequence. |
3432 | LHS = signExtendInputIfNeeded(Input: LHS); |
3433 | RHS = signExtendInputIfNeeded(Input: RHS); |
3434 | SDValue SUBFNode = |
3435 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3436 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3437 | SUBFNode, S->getI64Imm(1, dl), |
3438 | S->getI64Imm(63, dl)), 0); |
3439 | } |
3440 | case ISD::SETUGE: |
3441 | // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) |
3442 | // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) |
3443 | std::swap(a&: LHS, b&: RHS); |
3444 | [[fallthrough]]; |
3445 | case ISD::SETULE: { |
3446 | if (CmpInGPR == ICGPR_NonExtIn) |
3447 | return SDValue(); |
3448 | // The upper 32-bits of the register can't be undefined for this sequence. |
3449 | LHS = zeroExtendInputIfNeeded(Input: LHS); |
3450 | RHS = zeroExtendInputIfNeeded(Input: RHS); |
3451 | SDValue Subtract = |
3452 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3453 | SDValue SrdiNode = |
3454 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3455 | Subtract, S->getI64Imm(1, dl), |
3456 | S->getI64Imm(63, dl)), 0); |
3457 | return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, |
3458 | S->getI32Imm(1, dl)), 0); |
3459 | } |
3460 | case ISD::SETUGT: |
3461 | // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) |
3462 | // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) |
3463 | std::swap(a&: LHS, b&: RHS); |
3464 | [[fallthrough]]; |
3465 | case ISD::SETULT: { |
3466 | if (CmpInGPR == ICGPR_NonExtIn) |
3467 | return SDValue(); |
3468 | // The upper 32-bits of the register can't be undefined for this sequence. |
3469 | LHS = zeroExtendInputIfNeeded(Input: LHS); |
3470 | RHS = zeroExtendInputIfNeeded(Input: RHS); |
3471 | SDValue Subtract = |
3472 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3473 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3474 | Subtract, S->getI64Imm(1, dl), |
3475 | S->getI64Imm(63, dl)), 0); |
3476 | } |
3477 | } |
3478 | } |
3479 | |
3480 | /// Produces a sign-extended result of comparing two 32-bit values according to |
3481 | /// the passed condition code. |
3482 | SDValue |
3483 | IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS, |
3484 | ISD::CondCode CC, |
3485 | int64_t RHSValue, SDLoc dl) { |
3486 | if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || |
3487 | CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext) |
3488 | return SDValue(); |
3489 | bool IsRHSZero = RHSValue == 0; |
3490 | bool IsRHSOne = RHSValue == 1; |
3491 | bool IsRHSNegOne = RHSValue == -1LL; |
3492 | |
3493 | switch (CC) { |
3494 | default: return SDValue(); |
3495 | case ISD::SETEQ: { |
3496 | // (sext (setcc %a, %b, seteq)) -> |
3497 | // (ashr (shl (ctlz (xor %a, %b)), 58), 63) |
3498 | // (sext (setcc %a, 0, seteq)) -> |
3499 | // (ashr (shl (ctlz %a), 58), 63) |
3500 | SDValue CountInput = IsRHSZero ? LHS : |
3501 | SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3502 | SDValue Cntlzw = |
3503 | SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); |
3504 | SDValue SHLOps[] = { Cntlzw, S->getI32Imm(Imm: 27, dl), |
3505 | S->getI32Imm(Imm: 5, dl), S->getI32Imm(Imm: 31, dl) }; |
3506 | SDValue Slwi = |
3507 | SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); |
3508 | return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); |
3509 | } |
3510 | case ISD::SETNE: { |
3511 | // Bitwise xor the operands, count leading zeros, shift right by 5 bits and |
3512 | // flip the bit, finally take 2's complement. |
3513 | // (sext (setcc %a, %b, setne)) -> |
3514 | // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1)) |
3515 | // Same as above, but the first xor is not needed. |
3516 | // (sext (setcc %a, 0, setne)) -> |
3517 | // (neg (xor (lshr (ctlz %a), 5), 1)) |
3518 | SDValue Xor = IsRHSZero ? LHS : |
3519 | SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); |
3520 | SDValue Clz = |
3521 | SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); |
3522 | SDValue ShiftOps[] = |
3523 | { Clz, S->getI32Imm(Imm: 27, dl), S->getI32Imm(Imm: 5, dl), S->getI32Imm(Imm: 31, dl) }; |
3524 | SDValue Shift = |
3525 | SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); |
3526 | SDValue Xori = |
3527 | SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, |
3528 | S->getI32Imm(1, dl)), 0); |
3529 | return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); |
3530 | } |
3531 | case ISD::SETGE: { |
3532 | // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) |
3533 | // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) |
3534 | if (IsRHSZero) |
3535 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GESExt); |
3536 | |
3537 | // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) |
3538 | // by swapping inputs and falling through. |
3539 | std::swap(a&: LHS, b&: RHS); |
3540 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3541 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3542 | [[fallthrough]]; |
3543 | } |
3544 | case ISD::SETLE: { |
3545 | if (CmpInGPR == ICGPR_NonExtIn) |
3546 | return SDValue(); |
3547 | // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) |
3548 | // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) |
3549 | if (IsRHSZero) |
3550 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LESExt); |
3551 | |
3552 | // The upper 32-bits of the register can't be undefined for this sequence. |
3553 | LHS = signExtendInputIfNeeded(Input: LHS); |
3554 | RHS = signExtendInputIfNeeded(Input: RHS); |
3555 | SDValue SUBFNode = |
3556 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, |
3557 | LHS, RHS), 0); |
3558 | SDValue Srdi = |
3559 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3560 | SUBFNode, S->getI64Imm(1, dl), |
3561 | S->getI64Imm(63, dl)), 0); |
3562 | return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, |
3563 | S->getI32Imm(-1, dl)), 0); |
3564 | } |
3565 | case ISD::SETGT: { |
3566 | // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) |
3567 | // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) |
3568 | // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) |
3569 | if (IsRHSNegOne) |
3570 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GESExt); |
3571 | if (IsRHSZero) { |
3572 | if (CmpInGPR == ICGPR_NonExtIn) |
3573 | return SDValue(); |
3574 | // The upper 32-bits of the register can't be undefined for this sequence. |
3575 | LHS = signExtendInputIfNeeded(Input: LHS); |
3576 | RHS = signExtendInputIfNeeded(Input: RHS); |
3577 | SDValue Neg = |
3578 | SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); |
3579 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, |
3580 | S->getI64Imm(63, dl)), 0); |
3581 | } |
3582 | // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as |
3583 | // (%b < %a) by swapping inputs and falling through. |
3584 | std::swap(a&: LHS, b&: RHS); |
3585 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3586 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3587 | IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3588 | [[fallthrough]]; |
3589 | } |
3590 | case ISD::SETLT: { |
3591 | // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) |
3592 | // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) |
3593 | // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) |
3594 | if (IsRHSOne) { |
3595 | if (CmpInGPR == ICGPR_NonExtIn) |
3596 | return SDValue(); |
3597 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LESExt); |
3598 | } |
3599 | if (IsRHSZero) |
3600 | return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, |
3601 | S->getI32Imm(31, dl)), 0); |
3602 | |
3603 | if (CmpInGPR == ICGPR_NonExtIn) |
3604 | return SDValue(); |
3605 | // The upper 32-bits of the register can't be undefined for this sequence. |
3606 | LHS = signExtendInputIfNeeded(Input: LHS); |
3607 | RHS = signExtendInputIfNeeded(Input: RHS); |
3608 | SDValue SUBFNode = |
3609 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3610 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3611 | SUBFNode, S->getI64Imm(63, dl)), 0); |
3612 | } |
3613 | case ISD::SETUGE: |
3614 | // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) |
3615 | // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) |
3616 | std::swap(a&: LHS, b&: RHS); |
3617 | [[fallthrough]]; |
3618 | case ISD::SETULE: { |
3619 | if (CmpInGPR == ICGPR_NonExtIn) |
3620 | return SDValue(); |
3621 | // The upper 32-bits of the register can't be undefined for this sequence. |
3622 | LHS = zeroExtendInputIfNeeded(Input: LHS); |
3623 | RHS = zeroExtendInputIfNeeded(Input: RHS); |
3624 | SDValue Subtract = |
3625 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); |
3626 | SDValue Shift = |
3627 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, |
3628 | S->getI32Imm(1, dl), S->getI32Imm(63,dl)), |
3629 | 0); |
3630 | return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, |
3631 | S->getI32Imm(-1, dl)), 0); |
3632 | } |
3633 | case ISD::SETUGT: |
3634 | // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) |
3635 | // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) |
3636 | std::swap(a&: LHS, b&: RHS); |
3637 | [[fallthrough]]; |
3638 | case ISD::SETULT: { |
3639 | if (CmpInGPR == ICGPR_NonExtIn) |
3640 | return SDValue(); |
3641 | // The upper 32-bits of the register can't be undefined for this sequence. |
3642 | LHS = zeroExtendInputIfNeeded(Input: LHS); |
3643 | RHS = zeroExtendInputIfNeeded(Input: RHS); |
3644 | SDValue Subtract = |
3645 | SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); |
3646 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3647 | Subtract, S->getI64Imm(63, dl)), 0); |
3648 | } |
3649 | } |
3650 | } |
3651 | |
3652 | /// Produces a zero-extended result of comparing two 64-bit values according to |
3653 | /// the passed condition code. |
3654 | SDValue |
3655 | IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS, |
3656 | ISD::CondCode CC, |
3657 | int64_t RHSValue, SDLoc dl) { |
3658 | if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
3659 | CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext) |
3660 | return SDValue(); |
3661 | bool IsRHSZero = RHSValue == 0; |
3662 | bool IsRHSOne = RHSValue == 1; |
3663 | bool IsRHSNegOne = RHSValue == -1LL; |
3664 | switch (CC) { |
3665 | default: return SDValue(); |
3666 | case ISD::SETEQ: { |
3667 | // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) |
3668 | // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) |
3669 | SDValue Xor = IsRHSZero ? LHS : |
3670 | SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3671 | SDValue Clz = |
3672 | SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); |
3673 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, |
3674 | S->getI64Imm(58, dl), |
3675 | S->getI64Imm(63, dl)), 0); |
3676 | } |
3677 | case ISD::SETNE: { |
3678 | // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
3679 | // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) |
3680 | // {addcz.reg, addcz.CA} = (addcarry %a, -1) |
3681 | // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
3682 | SDValue Xor = IsRHSZero ? LHS : |
3683 | SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3684 | SDValue AC = |
3685 | SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
3686 | Xor, S->getI32Imm(~0U, dl)), 0); |
3687 | return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, |
3688 | Xor, AC.getValue(1)), 0); |
3689 | } |
3690 | case ISD::SETGE: { |
3691 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3692 | // (zext (setcc %a, %b, setge)) -> |
3693 | // (adde (lshr %b, 63), (ashr %a, 63), subc.CA) |
3694 | // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) |
3695 | if (IsRHSZero) |
3696 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GEZExt); |
3697 | std::swap(a&: LHS, b&: RHS); |
3698 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3699 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3700 | [[fallthrough]]; |
3701 | } |
3702 | case ISD::SETLE: { |
3703 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3704 | // (zext (setcc %a, %b, setge)) -> |
3705 | // (adde (lshr %a, 63), (ashr %b, 63), subc.CA) |
3706 | // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) |
3707 | if (IsRHSZero) |
3708 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LEZExt); |
3709 | SDValue ShiftL = |
3710 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3711 | S->getI64Imm(1, dl), |
3712 | S->getI64Imm(63, dl)), 0); |
3713 | SDValue ShiftR = |
3714 | SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
3715 | S->getI64Imm(63, dl)), 0); |
3716 | SDValue SubtractCarry = |
3717 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3718 | LHS, RHS), 1); |
3719 | return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3720 | ShiftR, ShiftL, SubtractCarry), 0); |
3721 | } |
3722 | case ISD::SETGT: { |
3723 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3724 | // (zext (setcc %a, %b, setgt)) -> |
3725 | // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
3726 | // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) |
3727 | if (IsRHSNegOne) |
3728 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GEZExt); |
3729 | if (IsRHSZero) { |
3730 | SDValue Addi = |
3731 | SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3732 | S->getI64Imm(~0ULL, dl)), 0); |
3733 | SDValue Nor = |
3734 | SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); |
3735 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, |
3736 | S->getI64Imm(1, dl), |
3737 | S->getI64Imm(63, dl)), 0); |
3738 | } |
3739 | std::swap(a&: LHS, b&: RHS); |
3740 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3741 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3742 | IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3743 | [[fallthrough]]; |
3744 | } |
3745 | case ISD::SETLT: { |
3746 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3747 | // (zext (setcc %a, %b, setlt)) -> |
3748 | // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
3749 | // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) |
3750 | if (IsRHSOne) |
3751 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LEZExt); |
3752 | if (IsRHSZero) |
3753 | return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3754 | S->getI64Imm(1, dl), |
3755 | S->getI64Imm(63, dl)), 0); |
3756 | SDValue SRADINode = |
3757 | SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3758 | LHS, S->getI64Imm(63, dl)), 0); |
3759 | SDValue SRDINode = |
3760 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3761 | RHS, S->getI64Imm(1, dl), |
3762 | S->getI64Imm(63, dl)), 0); |
3763 | SDValue SUBFC8Carry = |
3764 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3765 | RHS, LHS), 1); |
3766 | SDValue ADDE8Node = |
3767 | SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3768 | SRDINode, SRADINode, SUBFC8Carry), 0); |
3769 | return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
3770 | ADDE8Node, S->getI64Imm(1, dl)), 0); |
3771 | } |
3772 | case ISD::SETUGE: |
3773 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3774 | // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) |
3775 | std::swap(a&: LHS, b&: RHS); |
3776 | [[fallthrough]]; |
3777 | case ISD::SETULE: { |
3778 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3779 | // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) |
3780 | SDValue SUBFC8Carry = |
3781 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3782 | LHS, RHS), 1); |
3783 | SDValue SUBFE8Node = |
3784 | SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, |
3785 | LHS, LHS, SUBFC8Carry), 0); |
3786 | return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, |
3787 | SUBFE8Node, S->getI64Imm(1, dl)), 0); |
3788 | } |
3789 | case ISD::SETUGT: |
3790 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3791 | // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) |
3792 | std::swap(a&: LHS, b&: RHS); |
3793 | [[fallthrough]]; |
3794 | case ISD::SETULT: { |
3795 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3796 | // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) |
3797 | SDValue SubtractCarry = |
3798 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3799 | RHS, LHS), 1); |
3800 | SDValue ExtSub = |
3801 | SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
3802 | LHS, LHS, SubtractCarry), 0); |
3803 | return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
3804 | ExtSub), 0); |
3805 | } |
3806 | } |
3807 | } |
3808 | |
3809 | /// Produces a sign-extended result of comparing two 64-bit values according to |
3810 | /// the passed condition code. |
3811 | SDValue |
3812 | IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS, |
3813 | ISD::CondCode CC, |
3814 | int64_t RHSValue, SDLoc dl) { |
3815 | if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || |
3816 | CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext) |
3817 | return SDValue(); |
3818 | bool IsRHSZero = RHSValue == 0; |
3819 | bool IsRHSOne = RHSValue == 1; |
3820 | bool IsRHSNegOne = RHSValue == -1LL; |
3821 | switch (CC) { |
3822 | default: return SDValue(); |
3823 | case ISD::SETEQ: { |
3824 | // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) |
3825 | // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) |
3826 | // {addcz.reg, addcz.CA} = (addcarry %a, -1) |
3827 | // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) |
3828 | SDValue AddInput = IsRHSZero ? LHS : |
3829 | SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3830 | SDValue Addic = |
3831 | SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, |
3832 | AddInput, S->getI32Imm(~0U, dl)), 0); |
3833 | return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, |
3834 | Addic, Addic.getValue(1)), 0); |
3835 | } |
3836 | case ISD::SETNE: { |
3837 | // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b)) |
3838 | // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) |
3839 | // {subfcz.reg, subfcz.CA} = (subcarry 0, %a) |
3840 | // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) |
3841 | SDValue Xor = IsRHSZero ? LHS : |
3842 | SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); |
3843 | SDValue SC = |
3844 | SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, |
3845 | Xor, S->getI32Imm(0, dl)), 0); |
3846 | return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, |
3847 | SC, SC.getValue(1)), 0); |
3848 | } |
3849 | case ISD::SETGE: { |
3850 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3851 | // (zext (setcc %a, %b, setge)) -> |
3852 | // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) |
3853 | // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) |
3854 | if (IsRHSZero) |
3855 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GESExt); |
3856 | std::swap(a&: LHS, b&: RHS); |
3857 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3858 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3859 | [[fallthrough]]; |
3860 | } |
3861 | case ISD::SETLE: { |
3862 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3863 | // (zext (setcc %a, %b, setge)) -> |
3864 | // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) |
3865 | // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) |
3866 | if (IsRHSZero) |
3867 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LESExt); |
3868 | SDValue ShiftR = |
3869 | SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, |
3870 | S->getI64Imm(63, dl)), 0); |
3871 | SDValue ShiftL = |
3872 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, |
3873 | S->getI64Imm(1, dl), |
3874 | S->getI64Imm(63, dl)), 0); |
3875 | SDValue SubtractCarry = |
3876 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3877 | LHS, RHS), 1); |
3878 | SDValue Adde = |
3879 | SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, |
3880 | ShiftR, ShiftL, SubtractCarry), 0); |
3881 | return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); |
3882 | } |
3883 | case ISD::SETGT: { |
3884 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3885 | // (zext (setcc %a, %b, setgt)) -> |
3886 | // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) |
3887 | // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) |
3888 | if (IsRHSNegOne) |
3889 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::GESExt); |
3890 | if (IsRHSZero) { |
3891 | SDValue Add = |
3892 | SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, |
3893 | S->getI64Imm(-1, dl)), 0); |
3894 | SDValue Nor = |
3895 | SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); |
3896 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, |
3897 | S->getI64Imm(63, dl)), 0); |
3898 | } |
3899 | std::swap(a&: LHS, b&: RHS); |
3900 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
3901 | IsRHSZero = RHSConst && RHSConst->isZero(); |
3902 | IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; |
3903 | [[fallthrough]]; |
3904 | } |
3905 | case ISD::SETLT: { |
3906 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3907 | // (zext (setcc %a, %b, setlt)) -> |
3908 | // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) |
3909 | // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) |
3910 | if (IsRHSOne) |
3911 | return getCompoundZeroComparisonInGPR(LHS, dl, CmpTy: ZeroCompare::LESExt); |
3912 | if (IsRHSZero) { |
3913 | return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, |
3914 | S->getI64Imm(63, dl)), 0); |
3915 | } |
3916 | SDValue SRADINode = |
3917 | SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, |
3918 | LHS, S->getI64Imm(63, dl)), 0); |
3919 | SDValue SRDINode = |
3920 | SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, |
3921 | RHS, S->getI64Imm(1, dl), |
3922 | S->getI64Imm(63, dl)), 0); |
3923 | SDValue SUBFC8Carry = |
3924 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3925 | RHS, LHS), 1); |
3926 | SDValue ADDE8Node = |
3927 | SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, |
3928 | SRDINode, SRADINode, SUBFC8Carry), 0); |
3929 | SDValue XORI8Node = |
3930 | SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
3931 | ADDE8Node, S->getI64Imm(1, dl)), 0); |
3932 | return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, |
3933 | XORI8Node), 0); |
3934 | } |
3935 | case ISD::SETUGE: |
3936 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3937 | // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) |
3938 | std::swap(a&: LHS, b&: RHS); |
3939 | [[fallthrough]]; |
3940 | case ISD::SETULE: { |
3941 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3942 | // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) |
3943 | SDValue SubtractCarry = |
3944 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3945 | LHS, RHS), 1); |
3946 | SDValue ExtSub = |
3947 | SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, |
3948 | LHS, SubtractCarry), 0); |
3949 | return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, |
3950 | ExtSub, ExtSub), 0); |
3951 | } |
3952 | case ISD::SETUGT: |
3953 | // {subc.reg, subc.CA} = (subcarry %b, %a) |
3954 | // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) |
3955 | std::swap(a&: LHS, b&: RHS); |
3956 | [[fallthrough]]; |
3957 | case ISD::SETULT: { |
3958 | // {subc.reg, subc.CA} = (subcarry %a, %b) |
3959 | // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) |
3960 | SDValue SubCarry = |
3961 | SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, |
3962 | RHS, LHS), 1); |
3963 | return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, |
3964 | LHS, LHS, SubCarry), 0); |
3965 | } |
3966 | } |
3967 | } |
3968 | |
3969 | /// Do all uses of this SDValue need the result in a GPR? |
3970 | /// This is meant to be used on values that have type i1 since |
3971 | /// it is somewhat meaningless to ask if values of other types |
3972 | /// should be kept in GPR's. |
3973 | static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) { |
3974 | assert(Compare.getOpcode() == ISD::SETCC && |
3975 | "An ISD::SETCC node required here." ); |
3976 | |
3977 | // For values that have a single use, the caller should obviously already have |
3978 | // checked if that use is an extending use. We check the other uses here. |
3979 | if (Compare.hasOneUse()) |
3980 | return true; |
3981 | // We want the value in a GPR if it is being extended, used for a select, or |
3982 | // used in logical operations. |
3983 | for (auto *CompareUse : Compare.getNode()->uses()) |
3984 | if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && |
3985 | CompareUse->getOpcode() != ISD::ZERO_EXTEND && |
3986 | CompareUse->getOpcode() != ISD::SELECT && |
3987 | !ISD::isBitwiseLogicOp(Opcode: CompareUse->getOpcode())) { |
3988 | OmittedForNonExtendUses++; |
3989 | return false; |
3990 | } |
3991 | return true; |
3992 | } |
3993 | |
3994 | /// Returns an equivalent of a SETCC node but with the result the same width as |
3995 | /// the inputs. This can also be used for SELECT_CC if either the true or false |
3996 | /// values is a power of two while the other is zero. |
3997 | SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare, |
3998 | SetccInGPROpts ConvOpts) { |
3999 | assert((Compare.getOpcode() == ISD::SETCC || |
4000 | Compare.getOpcode() == ISD::SELECT_CC) && |
4001 | "An ISD::SETCC node required here." ); |
4002 | |
4003 | // Don't convert this comparison to a GPR sequence because there are uses |
4004 | // of the i1 result (i.e. uses that require the result in the CR). |
4005 | if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG)) |
4006 | return SDValue(); |
4007 | |
4008 | SDValue LHS = Compare.getOperand(i: 0); |
4009 | SDValue RHS = Compare.getOperand(i: 1); |
4010 | |
4011 | // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC. |
4012 | int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2; |
4013 | ISD::CondCode CC = |
4014 | cast<CondCodeSDNode>(Val: Compare.getOperand(i: CCOpNum))->get(); |
4015 | EVT InputVT = LHS.getValueType(); |
4016 | if (InputVT != MVT::i32 && InputVT != MVT::i64) |
4017 | return SDValue(); |
4018 | |
4019 | if (ConvOpts == SetccInGPROpts::ZExtInvert || |
4020 | ConvOpts == SetccInGPROpts::SExtInvert) |
4021 | CC = ISD::getSetCCInverse(Operation: CC, Type: InputVT); |
4022 | |
4023 | bool Inputs32Bit = InputVT == MVT::i32; |
4024 | |
4025 | SDLoc dl(Compare); |
4026 | ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(Val&: RHS); |
4027 | int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; |
4028 | bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig || |
4029 | ConvOpts == SetccInGPROpts::SExtInvert; |
4030 | |
4031 | if (IsSext && Inputs32Bit) |
4032 | return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
4033 | else if (Inputs32Bit) |
4034 | return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
4035 | else if (IsSext) |
4036 | return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl); |
4037 | return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl); |
4038 | } |
4039 | |
4040 | } // end anonymous namespace |
4041 | |
4042 | bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) { |
4043 | if (N->getValueType(0) != MVT::i32 && |
4044 | N->getValueType(0) != MVT::i64) |
4045 | return false; |
4046 | |
4047 | // This optimization will emit code that assumes 64-bit registers |
4048 | // so we don't want to run it in 32-bit mode. Also don't run it |
4049 | // on functions that are not to be optimized. |
4050 | if (TM.getOptLevel() == CodeGenOptLevel::None || !TM.isPPC64()) |
4051 | return false; |
4052 | |
4053 | // For POWER10, it is more profitable to use the set boolean extension |
4054 | // instructions rather than the integer compare elimination codegen. |
4055 | // Users can override this via the command line option, `--ppc-gpr-icmps`. |
4056 | if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1()) |
4057 | return false; |
4058 | |
4059 | switch (N->getOpcode()) { |
4060 | default: break; |
4061 | case ISD::ZERO_EXTEND: |
4062 | case ISD::SIGN_EXTEND: |
4063 | case ISD::AND: |
4064 | case ISD::OR: |
4065 | case ISD::XOR: { |
4066 | IntegerCompareEliminator ICmpElim(CurDAG, this); |
4067 | if (SDNode *New = ICmpElim.Select(N)) { |
4068 | ReplaceNode(F: N, T: New); |
4069 | return true; |
4070 | } |
4071 | } |
4072 | } |
4073 | return false; |
4074 | } |
4075 | |
4076 | bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) { |
4077 | if (N->getValueType(0) != MVT::i32 && |
4078 | N->getValueType(0) != MVT::i64) |
4079 | return false; |
4080 | |
4081 | if (!UseBitPermRewriter) |
4082 | return false; |
4083 | |
4084 | switch (N->getOpcode()) { |
4085 | default: break; |
4086 | case ISD::SRL: |
4087 | // If we are on P10, we have a pattern for 32-bit (srl (bswap r), 16) that |
4088 | // uses the BRH instruction. |
4089 | if (Subtarget->isISA3_1() && N->getValueType(0) == MVT::i32 && |
4090 | N->getOperand(0).getOpcode() == ISD::BSWAP) { |
4091 | auto &OpRight = N->getOperand(Num: 1); |
4092 | ConstantSDNode *SRLConst = dyn_cast<ConstantSDNode>(Val: OpRight); |
4093 | if (SRLConst && SRLConst->getSExtValue() == 16) |
4094 | return false; |
4095 | } |
4096 | [[fallthrough]]; |
4097 | case ISD::ROTL: |
4098 | case ISD::SHL: |
4099 | case ISD::AND: |
4100 | case ISD::OR: { |
4101 | BitPermutationSelector BPS(CurDAG); |
4102 | if (SDNode *New = BPS.Select(N)) { |
4103 | ReplaceNode(F: N, T: New); |
4104 | return true; |
4105 | } |
4106 | return false; |
4107 | } |
4108 | } |
4109 | |
4110 | return false; |
4111 | } |
4112 | |
4113 | /// SelectCC - Select a comparison of the specified values with the specified |
4114 | /// condition code, returning the CR# of the expression. |
4115 | SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, |
4116 | const SDLoc &dl, SDValue Chain) { |
4117 | // Always select the LHS. |
4118 | unsigned Opc; |
4119 | |
4120 | if (LHS.getValueType() == MVT::i32) { |
4121 | unsigned Imm; |
4122 | if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
4123 | if (isInt32Immediate(N: RHS, Imm)) { |
4124 | // SETEQ/SETNE comparison with 16-bit immediate, fold it. |
4125 | if (isUInt<16>(Imm)) |
4126 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
4127 | getI32Imm(Imm & 0xFFFF, dl)), |
4128 | 0); |
4129 | // If this is a 16-bit signed immediate, fold it. |
4130 | if (isInt<16>((int)Imm)) |
4131 | return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
4132 | getI32Imm(Imm & 0xFFFF, dl)), |
4133 | 0); |
4134 | |
4135 | // For non-equality comparisons, the default code would materialize the |
4136 | // constant, then compare against it, like this: |
4137 | // lis r2, 4660 |
4138 | // ori r2, r2, 22136 |
4139 | // cmpw cr0, r3, r2 |
4140 | // Since we are just comparing for equality, we can emit this instead: |
4141 | // xoris r0,r3,0x1234 |
4142 | // cmplwi cr0,r0,0x5678 |
4143 | // beq cr0,L6 |
4144 | SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, |
4145 | getI32Imm(Imm >> 16, dl)), 0); |
4146 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, |
4147 | getI32Imm(Imm & 0xFFFF, dl)), 0); |
4148 | } |
4149 | Opc = PPC::CMPLW; |
4150 | } else if (ISD::isUnsignedIntSetCC(Code: CC)) { |
4151 | if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) |
4152 | return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, |
4153 | getI32Imm(Imm & 0xFFFF, dl)), 0); |
4154 | Opc = PPC::CMPLW; |
4155 | } else { |
4156 | int16_t SImm; |
4157 | if (isIntS16Immediate(RHS, SImm)) |
4158 | return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, |
4159 | getI32Imm((int)SImm & 0xFFFF, |
4160 | dl)), |
4161 | 0); |
4162 | Opc = PPC::CMPW; |
4163 | } |
4164 | } else if (LHS.getValueType() == MVT::i64) { |
4165 | uint64_t Imm; |
4166 | if (CC == ISD::SETEQ || CC == ISD::SETNE) { |
4167 | if (isInt64Immediate(N: RHS.getNode(), Imm)) { |
4168 | // SETEQ/SETNE comparison with 16-bit immediate, fold it. |
4169 | if (isUInt<16>(Imm)) |
4170 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
4171 | getI32Imm(Imm & 0xFFFF, dl)), |
4172 | 0); |
4173 | // If this is a 16-bit signed immediate, fold it. |
4174 | if (isInt<16>(Imm)) |
4175 | return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
4176 | getI32Imm(Imm & 0xFFFF, dl)), |
4177 | 0); |
4178 | |
4179 | // For non-equality comparisons, the default code would materialize the |
4180 | // constant, then compare against it, like this: |
4181 | // lis r2, 4660 |
4182 | // ori r2, r2, 22136 |
4183 | // cmpd cr0, r3, r2 |
4184 | // Since we are just comparing for equality, we can emit this instead: |
4185 | // xoris r0,r3,0x1234 |
4186 | // cmpldi cr0,r0,0x5678 |
4187 | // beq cr0,L6 |
4188 | if (isUInt<32>(x: Imm)) { |
4189 | SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, |
4190 | getI64Imm(Imm >> 16, dl)), 0); |
4191 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, |
4192 | getI64Imm(Imm & 0xFFFF, dl)), |
4193 | 0); |
4194 | } |
4195 | } |
4196 | Opc = PPC::CMPLD; |
4197 | } else if (ISD::isUnsignedIntSetCC(Code: CC)) { |
4198 | if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) |
4199 | return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, |
4200 | getI64Imm(Imm & 0xFFFF, dl)), 0); |
4201 | Opc = PPC::CMPLD; |
4202 | } else { |
4203 | int16_t SImm; |
4204 | if (isIntS16Immediate(RHS, SImm)) |
4205 | return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, |
4206 | getI64Imm(SImm & 0xFFFF, dl)), |
4207 | 0); |
4208 | Opc = PPC::CMPD; |
4209 | } |
4210 | } else if (LHS.getValueType() == MVT::f32) { |
4211 | if (Subtarget->hasSPE()) { |
4212 | switch (CC) { |
4213 | default: |
4214 | case ISD::SETEQ: |
4215 | case ISD::SETNE: |
4216 | Opc = PPC::EFSCMPEQ; |
4217 | break; |
4218 | case ISD::SETLT: |
4219 | case ISD::SETGE: |
4220 | case ISD::SETOLT: |
4221 | case ISD::SETOGE: |
4222 | case ISD::SETULT: |
4223 | case ISD::SETUGE: |
4224 | Opc = PPC::EFSCMPLT; |
4225 | break; |
4226 | case ISD::SETGT: |
4227 | case ISD::SETLE: |
4228 | case ISD::SETOGT: |
4229 | case ISD::SETOLE: |
4230 | case ISD::SETUGT: |
4231 | case ISD::SETULE: |
4232 | Opc = PPC::EFSCMPGT; |
4233 | break; |
4234 | } |
4235 | } else |
4236 | Opc = PPC::FCMPUS; |
4237 | } else if (LHS.getValueType() == MVT::f64) { |
4238 | if (Subtarget->hasSPE()) { |
4239 | switch (CC) { |
4240 | default: |
4241 | case ISD::SETEQ: |
4242 | case ISD::SETNE: |
4243 | Opc = PPC::EFDCMPEQ; |
4244 | break; |
4245 | case ISD::SETLT: |
4246 | case ISD::SETGE: |
4247 | case ISD::SETOLT: |
4248 | case ISD::SETOGE: |
4249 | case ISD::SETULT: |
4250 | case ISD::SETUGE: |
4251 | Opc = PPC::EFDCMPLT; |
4252 | break; |
4253 | case ISD::SETGT: |
4254 | case ISD::SETLE: |
4255 | case ISD::SETOGT: |
4256 | case ISD::SETOLE: |
4257 | case ISD::SETUGT: |
4258 | case ISD::SETULE: |
4259 | Opc = PPC::EFDCMPGT; |
4260 | break; |
4261 | } |
4262 | } else |
4263 | Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; |
4264 | } else { |
4265 | assert(LHS.getValueType() == MVT::f128 && "Unknown vt!" ); |
4266 | assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector" ); |
4267 | Opc = PPC::XSCMPUQP; |
4268 | } |
4269 | if (Chain) |
4270 | return SDValue( |
4271 | CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain), |
4272 | 0); |
4273 | else |
4274 | return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); |
4275 | } |
4276 | |
4277 | static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT, |
4278 | const PPCSubtarget *Subtarget) { |
4279 | // For SPE instructions, the result is in GT bit of the CR |
4280 | bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); |
4281 | |
4282 | switch (CC) { |
4283 | case ISD::SETUEQ: |
4284 | case ISD::SETONE: |
4285 | case ISD::SETOLE: |
4286 | case ISD::SETOGE: |
4287 | llvm_unreachable("Should be lowered by legalize!" ); |
4288 | default: llvm_unreachable("Unknown condition!" ); |
4289 | case ISD::SETOEQ: |
4290 | case ISD::SETEQ: |
4291 | return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ; |
4292 | case ISD::SETUNE: |
4293 | case ISD::SETNE: |
4294 | return UseSPE ? PPC::PRED_LE : PPC::PRED_NE; |
4295 | case ISD::SETOLT: |
4296 | case ISD::SETLT: |
4297 | return UseSPE ? PPC::PRED_GT : PPC::PRED_LT; |
4298 | case ISD::SETULE: |
4299 | case ISD::SETLE: |
4300 | return PPC::PRED_LE; |
4301 | case ISD::SETOGT: |
4302 | case ISD::SETGT: |
4303 | return PPC::PRED_GT; |
4304 | case ISD::SETUGE: |
4305 | case ISD::SETGE: |
4306 | return UseSPE ? PPC::PRED_LE : PPC::PRED_GE; |
4307 | case ISD::SETO: return PPC::PRED_NU; |
4308 | case ISD::SETUO: return PPC::PRED_UN; |
4309 | // These two are invalid for floating point. Assume we have int. |
4310 | case ISD::SETULT: return PPC::PRED_LT; |
4311 | case ISD::SETUGT: return PPC::PRED_GT; |
4312 | } |
4313 | } |
4314 | |
4315 | /// getCRIdxForSetCC - Return the index of the condition register field |
4316 | /// associated with the SetCC condition, and whether or not the field is |
4317 | /// treated as inverted. That is, lt = 0; ge = 0 inverted. |
4318 | static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { |
4319 | Invert = false; |
4320 | switch (CC) { |
4321 | default: llvm_unreachable("Unknown condition!" ); |
4322 | case ISD::SETOLT: |
4323 | case ISD::SETLT: return 0; // Bit #0 = SETOLT |
4324 | case ISD::SETOGT: |
4325 | case ISD::SETGT: return 1; // Bit #1 = SETOGT |
4326 | case ISD::SETOEQ: |
4327 | case ISD::SETEQ: return 2; // Bit #2 = SETOEQ |
4328 | case ISD::SETUO: return 3; // Bit #3 = SETUO |
4329 | case ISD::SETUGE: |
4330 | case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE |
4331 | case ISD::SETULE: |
4332 | case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE |
4333 | case ISD::SETUNE: |
4334 | case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE |
4335 | case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO |
4336 | case ISD::SETUEQ: |
4337 | case ISD::SETOGE: |
4338 | case ISD::SETOLE: |
4339 | case ISD::SETONE: |
4340 | llvm_unreachable("Invalid branch code: should be expanded by legalize" ); |
4341 | // These are invalid for floating point. Assume integer. |
4342 | case ISD::SETULT: return 0; |
4343 | case ISD::SETUGT: return 1; |
4344 | } |
4345 | } |
4346 | |
4347 | // getVCmpInst: return the vector compare instruction for the specified |
4348 | // vector type and condition code. Since this is for altivec specific code, |
4349 | // only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128, |
4350 | // and v4f32). |
4351 | static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, |
4352 | bool HasVSX, bool &Swap, bool &Negate) { |
4353 | Swap = false; |
4354 | Negate = false; |
4355 | |
4356 | if (VecVT.isFloatingPoint()) { |
4357 | /* Handle some cases by swapping input operands. */ |
4358 | switch (CC) { |
4359 | case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; |
4360 | case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
4361 | case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; |
4362 | case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; |
4363 | case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
4364 | case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; |
4365 | default: break; |
4366 | } |
4367 | /* Handle some cases by negating the result. */ |
4368 | switch (CC) { |
4369 | case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
4370 | case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; |
4371 | case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; |
4372 | case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; |
4373 | default: break; |
4374 | } |
4375 | /* We have instructions implementing the remaining cases. */ |
4376 | switch (CC) { |
4377 | case ISD::SETEQ: |
4378 | case ISD::SETOEQ: |
4379 | if (VecVT == MVT::v4f32) |
4380 | return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; |
4381 | else if (VecVT == MVT::v2f64) |
4382 | return PPC::XVCMPEQDP; |
4383 | break; |
4384 | case ISD::SETGT: |
4385 | case ISD::SETOGT: |
4386 | if (VecVT == MVT::v4f32) |
4387 | return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; |
4388 | else if (VecVT == MVT::v2f64) |
4389 | return PPC::XVCMPGTDP; |
4390 | break; |
4391 | case ISD::SETGE: |
4392 | case ISD::SETOGE: |
4393 | if (VecVT == MVT::v4f32) |
4394 | return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; |
4395 | else if (VecVT == MVT::v2f64) |
4396 | return PPC::XVCMPGEDP; |
4397 | break; |
4398 | default: |
4399 | break; |
4400 | } |
4401 | llvm_unreachable("Invalid floating-point vector compare condition" ); |
4402 | } else { |
4403 | /* Handle some cases by swapping input operands. */ |
4404 | switch (CC) { |
4405 | case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; |
4406 | case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; |
4407 | case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; |
4408 | case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; |
4409 | default: break; |
4410 | } |
4411 | /* Handle some cases by negating the result. */ |
4412 | switch (CC) { |
4413 | case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; |
4414 | case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; |
4415 | case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; |
4416 | case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; |
4417 | default: break; |
4418 | } |
4419 | /* We have instructions implementing the remaining cases. */ |
4420 | switch (CC) { |
4421 | case ISD::SETEQ: |
4422 | case ISD::SETUEQ: |
4423 | if (VecVT == MVT::v16i8) |
4424 | return PPC::VCMPEQUB; |
4425 | else if (VecVT == MVT::v8i16) |
4426 | return PPC::VCMPEQUH; |
4427 | else if (VecVT == MVT::v4i32) |
4428 | return PPC::VCMPEQUW; |
4429 | else if (VecVT == MVT::v2i64) |
4430 | return PPC::VCMPEQUD; |
4431 | else if (VecVT == MVT::v1i128) |
4432 | return PPC::VCMPEQUQ; |
4433 | break; |
4434 | case ISD::SETGT: |
4435 | if (VecVT == MVT::v16i8) |
4436 | return PPC::VCMPGTSB; |
4437 | else if (VecVT == MVT::v8i16) |
4438 | return PPC::VCMPGTSH; |
4439 | else if (VecVT == MVT::v4i32) |
4440 | return PPC::VCMPGTSW; |
4441 | else if (VecVT == MVT::v2i64) |
4442 | return PPC::VCMPGTSD; |
4443 | else if (VecVT == MVT::v1i128) |
4444 | return PPC::VCMPGTSQ; |
4445 | break; |
4446 | case ISD::SETUGT: |
4447 | if (VecVT == MVT::v16i8) |
4448 | return PPC::VCMPGTUB; |
4449 | else if (VecVT == MVT::v8i16) |
4450 | return PPC::VCMPGTUH; |
4451 | else if (VecVT == MVT::v4i32) |
4452 | return PPC::VCMPGTUW; |
4453 | else if (VecVT == MVT::v2i64) |
4454 | return PPC::VCMPGTUD; |
4455 | else if (VecVT == MVT::v1i128) |
4456 | return PPC::VCMPGTUQ; |
4457 | break; |
4458 | default: |
4459 | break; |
4460 | } |
4461 | llvm_unreachable("Invalid integer vector compare condition" ); |
4462 | } |
4463 | } |
4464 | |
4465 | bool PPCDAGToDAGISel::trySETCC(SDNode *N) { |
4466 | SDLoc dl(N); |
4467 | unsigned Imm; |
4468 | bool IsStrict = N->isStrictFPOpcode(); |
4469 | ISD::CondCode CC = |
4470 | cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get(); |
4471 | EVT PtrVT = |
4472 | CurDAG->getTargetLoweringInfo().getPointerTy(DL: CurDAG->getDataLayout()); |
4473 | bool isPPC64 = (PtrVT == MVT::i64); |
4474 | SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue(); |
4475 | |
4476 | SDValue LHS = N->getOperand(Num: IsStrict ? 1 : 0); |
4477 | SDValue RHS = N->getOperand(Num: IsStrict ? 2 : 1); |
4478 | |
4479 | if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(N: RHS, Imm)) { |
4480 | // We can codegen setcc op, imm very efficiently compared to a brcond. |
4481 | // Check for those cases here. |
4482 | // setcc op, 0 |
4483 | if (Imm == 0) { |
4484 | SDValue Op = LHS; |
4485 | switch (CC) { |
4486 | default: break; |
4487 | case ISD::SETEQ: { |
4488 | Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); |
4489 | SDValue Ops[] = { Op, getI32Imm(Imm: 27, dl), getI32Imm(Imm: 5, dl), |
4490 | getI32Imm(Imm: 31, dl) }; |
4491 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4492 | return true; |
4493 | } |
4494 | case ISD::SETNE: { |
4495 | if (isPPC64) break; |
4496 | SDValue AD = |
4497 | SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4498 | Op, getI32Imm(~0U, dl)), 0); |
4499 | CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); |
4500 | return true; |
4501 | } |
4502 | case ISD::SETLT: { |
4503 | SDValue Ops[] = { Op, getI32Imm(Imm: 1, dl), getI32Imm(Imm: 31, dl), |
4504 | getI32Imm(Imm: 31, dl) }; |
4505 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4506 | return true; |
4507 | } |
4508 | case ISD::SETGT: { |
4509 | SDValue T = |
4510 | SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); |
4511 | T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); |
4512 | SDValue Ops[] = { T, getI32Imm(Imm: 1, dl), getI32Imm(Imm: 31, dl), |
4513 | getI32Imm(Imm: 31, dl) }; |
4514 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4515 | return true; |
4516 | } |
4517 | } |
4518 | } else if (Imm == ~0U) { // setcc op, -1 |
4519 | SDValue Op = LHS; |
4520 | switch (CC) { |
4521 | default: break; |
4522 | case ISD::SETEQ: |
4523 | if (isPPC64) break; |
4524 | Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4525 | Op, getI32Imm(1, dl)), 0); |
4526 | CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, |
4527 | SDValue(CurDAG->getMachineNode(PPC::LI, dl, |
4528 | MVT::i32, |
4529 | getI32Imm(0, dl)), |
4530 | 0), Op.getValue(1)); |
4531 | return true; |
4532 | case ISD::SETNE: { |
4533 | if (isPPC64) break; |
4534 | Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); |
4535 | SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
4536 | Op, getI32Imm(~0U, dl)); |
4537 | CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, |
4538 | SDValue(AD, 1)); |
4539 | return true; |
4540 | } |
4541 | case ISD::SETLT: { |
4542 | SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, |
4543 | getI32Imm(1, dl)), 0); |
4544 | SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, |
4545 | Op), 0); |
4546 | SDValue Ops[] = { AN, getI32Imm(Imm: 1, dl), getI32Imm(Imm: 31, dl), |
4547 | getI32Imm(Imm: 31, dl) }; |
4548 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4549 | return true; |
4550 | } |
4551 | case ISD::SETGT: { |
4552 | SDValue Ops[] = { Op, getI32Imm(Imm: 1, dl), getI32Imm(Imm: 31, dl), |
4553 | getI32Imm(Imm: 31, dl) }; |
4554 | Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
4555 | CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); |
4556 | return true; |
4557 | } |
4558 | } |
4559 | } |
4560 | } |
4561 | |
4562 | // Altivec Vector compare instructions do not set any CR register by default and |
4563 | // vector compare operations return the same type as the operands. |
4564 | if (!IsStrict && LHS.getValueType().isVector()) { |
4565 | if (Subtarget->hasSPE()) |
4566 | return false; |
4567 | |
4568 | EVT VecVT = LHS.getValueType(); |
4569 | bool Swap, Negate; |
4570 | unsigned int VCmpInst = |
4571 | getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate); |
4572 | if (Swap) |
4573 | std::swap(a&: LHS, b&: RHS); |
4574 | |
4575 | EVT ResVT = VecVT.changeVectorElementTypeToInteger(); |
4576 | if (Negate) { |
4577 | SDValue VCmp(CurDAG->getMachineNode(Opcode: VCmpInst, dl, VT: ResVT, Op1: LHS, Op2: RHS), 0); |
4578 | CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, |
4579 | ResVT, VCmp, VCmp); |
4580 | return true; |
4581 | } |
4582 | |
4583 | CurDAG->SelectNodeTo(N, MachineOpc: VCmpInst, VT: ResVT, Op1: LHS, Op2: RHS); |
4584 | return true; |
4585 | } |
4586 | |
4587 | if (Subtarget->useCRBits()) |
4588 | return false; |
4589 | |
4590 | bool Inv; |
4591 | unsigned Idx = getCRIdxForSetCC(CC, Invert&: Inv); |
4592 | SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain); |
4593 | if (IsStrict) |
4594 | CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: CCReg.getValue(R: 1)); |
4595 | SDValue IntCR; |
4596 | |
4597 | // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that |
4598 | // The correct compare instruction is already set by SelectCC() |
4599 | if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { |
4600 | Idx = 1; |
4601 | } |
4602 | |
4603 | // Force the ccreg into CR7. |
4604 | SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); |
4605 | |
4606 | SDValue InGlue; // Null incoming flag value. |
4607 | CCReg = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: CR7Reg, N: CCReg, |
4608 | Glue: InGlue).getValue(R: 1); |
4609 | |
4610 | IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, |
4611 | CCReg), 0); |
4612 | |
4613 | SDValue Ops[] = { IntCR, getI32Imm(Imm: (32 - (3 - Idx)) & 31, dl), |
4614 | getI32Imm(Imm: 31, dl), getI32Imm(Imm: 31, dl) }; |
4615 | if (!Inv) { |
4616 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4617 | return true; |
4618 | } |
4619 | |
4620 | // Get the specified bit. |
4621 | SDValue Tmp = |
4622 | SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
4623 | CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); |
4624 | return true; |
4625 | } |
4626 | |
4627 | /// Does this node represent a load/store node whose address can be represented |
4628 | /// with a register plus an immediate that's a multiple of \p Val: |
4629 | bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { |
4630 | LoadSDNode *LDN = dyn_cast<LoadSDNode>(Val: N); |
4631 | StoreSDNode *STN = dyn_cast<StoreSDNode>(Val: N); |
4632 | MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(Val: N); |
4633 | SDValue AddrOp; |
4634 | if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) |
4635 | AddrOp = N->getOperand(Num: 1); |
4636 | else if (STN) |
4637 | AddrOp = STN->getOperand(Num: 2); |
4638 | |
4639 | // If the address points a frame object or a frame object with an offset, |
4640 | // we need to check the object alignment. |
4641 | short Imm = 0; |
4642 | if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>( |
4643 | Val: AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(i: 0) : |
4644 | AddrOp)) { |
4645 | // If op0 is a frame index that is under aligned, we can't do it either, |
4646 | // because it is translated to r31 or r1 + slot + offset. We won't know the |
4647 | // slot number until the stack frame is finalized. |
4648 | const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); |
4649 | unsigned SlotAlign = MFI.getObjectAlign(ObjectIdx: FI->getIndex()).value(); |
4650 | if ((SlotAlign % Val) != 0) |
4651 | return false; |
4652 | |
4653 | // If we have an offset, we need further check on the offset. |
4654 | if (AddrOp.getOpcode() != ISD::ADD) |
4655 | return true; |
4656 | } |
4657 | |
4658 | if (AddrOp.getOpcode() == ISD::ADD) |
4659 | return isIntS16Immediate(Op: AddrOp.getOperand(i: 1), Imm) && !(Imm % Val); |
4660 | |
4661 | // If the address comes from the outside, the offset will be zero. |
4662 | return AddrOp.getOpcode() == ISD::CopyFromReg; |
4663 | } |
4664 | |
4665 | void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { |
4666 | // Transfer memoperands. |
4667 | MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand(); |
4668 | CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Result), NewMemRefs: {MemOp}); |
4669 | } |
4670 | |
4671 | static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG, |
4672 | bool &NeedSwapOps, bool &IsUnCmp) { |
4673 | |
4674 | assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here." ); |
4675 | |
4676 | SDValue LHS = N->getOperand(Num: 0); |
4677 | SDValue RHS = N->getOperand(Num: 1); |
4678 | SDValue TrueRes = N->getOperand(Num: 2); |
4679 | SDValue FalseRes = N->getOperand(Num: 3); |
4680 | ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(Val&: TrueRes); |
4681 | if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 && |
4682 | N->getSimpleValueType(0) != MVT::i32)) |
4683 | return false; |
4684 | |
4685 | // We are looking for any of: |
4686 | // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
4687 | // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) |
4688 | // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) |
4689 | // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) |
4690 | int64_t TrueResVal = TrueConst->getSExtValue(); |
4691 | if ((TrueResVal < -1 || TrueResVal > 1) || |
4692 | (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || |
4693 | (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) || |
4694 | (TrueResVal == 0 && |
4695 | (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ))) |
4696 | return false; |
4697 | |
4698 | SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC |
4699 | ? FalseRes |
4700 | : FalseRes.getOperand(i: 0); |
4701 | bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC; |
4702 | if (SetOrSelCC.getOpcode() != ISD::SETCC && |
4703 | SetOrSelCC.getOpcode() != ISD::SELECT_CC) |
4704 | return false; |
4705 | |
4706 | // Without this setb optimization, the outer SELECT_CC will be manually |
4707 | // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass |
4708 | // transforms pseudo instruction to isel instruction. When there are more than |
4709 | // one use for result like zext/sext, with current optimization we only see |
4710 | // isel is replaced by setb but can't see any significant gain. Since |
4711 | // setb has longer latency than original isel, we should avoid this. Another |
4712 | // point is that setb requires comparison always kept, it can break the |
4713 | // opportunity to get the comparison away if we have in future. |
4714 | if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse())) |
4715 | return false; |
4716 | |
4717 | SDValue InnerLHS = SetOrSelCC.getOperand(i: 0); |
4718 | SDValue InnerRHS = SetOrSelCC.getOperand(i: 1); |
4719 | ISD::CondCode InnerCC = |
4720 | cast<CondCodeSDNode>(Val: SetOrSelCC.getOperand(i: InnerIsSel ? 4 : 2))->get(); |
4721 | // If the inner comparison is a select_cc, make sure the true/false values are |
4722 | // 1/-1 and canonicalize it if needed. |
4723 | if (InnerIsSel) { |
4724 | ConstantSDNode *SelCCTrueConst = |
4725 | dyn_cast<ConstantSDNode>(Val: SetOrSelCC.getOperand(i: 2)); |
4726 | ConstantSDNode *SelCCFalseConst = |
4727 | dyn_cast<ConstantSDNode>(Val: SetOrSelCC.getOperand(i: 3)); |
4728 | if (!SelCCTrueConst || !SelCCFalseConst) |
4729 | return false; |
4730 | int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); |
4731 | int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); |
4732 | // The values must be -1/1 (requiring a swap) or 1/-1. |
4733 | if (SelCCTVal == -1 && SelCCFVal == 1) { |
4734 | std::swap(a&: InnerLHS, b&: InnerRHS); |
4735 | } else if (SelCCTVal != 1 || SelCCFVal != -1) |
4736 | return false; |
4737 | } |
4738 | |
4739 | // Canonicalize unsigned case |
4740 | if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) { |
4741 | IsUnCmp = true; |
4742 | InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT; |
4743 | } |
4744 | |
4745 | bool InnerSwapped = false; |
4746 | if (LHS == InnerRHS && RHS == InnerLHS) |
4747 | InnerSwapped = true; |
4748 | else if (LHS != InnerLHS || RHS != InnerRHS) |
4749 | return false; |
4750 | |
4751 | switch (CC) { |
4752 | // (select_cc lhs, rhs, 0, \ |
4753 | // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) |
4754 | case ISD::SETEQ: |
4755 | if (!InnerIsSel) |
4756 | return false; |
4757 | if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT) |
4758 | return false; |
4759 | NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped; |
4760 | break; |
4761 | |
4762 | // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
4763 | // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) |
4764 | // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) |
4765 | // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt) |
4766 | // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt) |
4767 | // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt) |
4768 | case ISD::SETULT: |
4769 | if (!IsUnCmp && InnerCC != ISD::SETNE) |
4770 | return false; |
4771 | IsUnCmp = true; |
4772 | [[fallthrough]]; |
4773 | case ISD::SETLT: |
4774 | if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) || |
4775 | (InnerCC == ISD::SETLT && InnerSwapped)) |
4776 | NeedSwapOps = (TrueResVal == 1); |
4777 | else |
4778 | return false; |
4779 | break; |
4780 | |
4781 | // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
4782 | // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt) |
4783 | // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt) |
4784 | // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) |
4785 | // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) |
4786 | // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) |
4787 | case ISD::SETUGT: |
4788 | if (!IsUnCmp && InnerCC != ISD::SETNE) |
4789 | return false; |
4790 | IsUnCmp = true; |
4791 | [[fallthrough]]; |
4792 | case ISD::SETGT: |
4793 | if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) || |
4794 | (InnerCC == ISD::SETGT && InnerSwapped)) |
4795 | NeedSwapOps = (TrueResVal == -1); |
4796 | else |
4797 | return false; |
4798 | break; |
4799 | |
4800 | default: |
4801 | return false; |
4802 | } |
4803 | |
4804 | LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: " ); |
4805 | LLVM_DEBUG(N->dump()); |
4806 | |
4807 | return true; |
4808 | } |
4809 | |
4810 | // Return true if it's a software square-root/divide operand. |
4811 | static bool isSWTestOp(SDValue N) { |
4812 | if (N.getOpcode() == PPCISD::FTSQRT) |
4813 | return true; |
4814 | if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(Val: N.getOperand(i: 0)) || |
4815 | N.getOpcode() != ISD::INTRINSIC_WO_CHAIN) |
4816 | return false; |
4817 | switch (N.getConstantOperandVal(i: 0)) { |
4818 | case Intrinsic::ppc_vsx_xvtdivdp: |
4819 | case Intrinsic::ppc_vsx_xvtdivsp: |
4820 | case Intrinsic::ppc_vsx_xvtsqrtdp: |
4821 | case Intrinsic::ppc_vsx_xvtsqrtsp: |
4822 | return true; |
4823 | } |
4824 | return false; |
4825 | } |
4826 | |
4827 | bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) { |
4828 | assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected." ); |
4829 | // We are looking for following patterns, where `truncate to i1` actually has |
4830 | // the same semantic with `and 1`. |
4831 | // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp) |
4832 | // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp) |
4833 | // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp) |
4834 | // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp) |
4835 | // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp) |
4836 | // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp) |
4837 | // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp) |
4838 | // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp) |
4839 | ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
4840 | if (CC != ISD::SETEQ && CC != ISD::SETNE) |
4841 | return false; |
4842 | |
4843 | SDValue CmpRHS = N->getOperand(Num: 3); |
4844 | if (!isNullConstant(V: CmpRHS)) |
4845 | return false; |
4846 | |
4847 | SDValue CmpLHS = N->getOperand(Num: 2); |
4848 | if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(N: CmpLHS.getOperand(i: 0))) |
4849 | return false; |
4850 | |
4851 | unsigned PCC = 0; |
4852 | bool IsCCNE = CC == ISD::SETNE; |
4853 | if (CmpLHS.getOpcode() == ISD::AND && |
4854 | isa<ConstantSDNode>(Val: CmpLHS.getOperand(i: 1))) |
4855 | switch (CmpLHS.getConstantOperandVal(i: 1)) { |
4856 | case 1: |
4857 | PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
4858 | break; |
4859 | case 2: |
4860 | PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE; |
4861 | break; |
4862 | case 4: |
4863 | PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE; |
4864 | break; |
4865 | case 8: |
4866 | PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE; |
4867 | break; |
4868 | default: |
4869 | return false; |
4870 | } |
4871 | else if (CmpLHS.getOpcode() == ISD::TRUNCATE && |
4872 | CmpLHS.getValueType() == MVT::i1) |
4873 | PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; |
4874 | |
4875 | if (PCC) { |
4876 | SDLoc dl(N); |
4877 | SDValue Ops[] = {getI32Imm(Imm: PCC, dl), CmpLHS.getOperand(i: 0), N->getOperand(Num: 4), |
4878 | N->getOperand(Num: 0)}; |
4879 | CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
4880 | return true; |
4881 | } |
4882 | return false; |
4883 | } |
4884 | |
4885 | bool PPCDAGToDAGISel::trySelectLoopCountIntrinsic(SDNode *N) { |
4886 | // Sometimes the promoted value of the intrinsic is ANDed by some non-zero |
4887 | // value, for example when crbits is disabled. If so, select the |
4888 | // loop_decrement intrinsics now. |
4889 | ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
4890 | SDValue LHS = N->getOperand(Num: 2), RHS = N->getOperand(Num: 3); |
4891 | |
4892 | if (LHS.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) || |
4893 | isNullConstant(V: LHS.getOperand(i: 1))) |
4894 | return false; |
4895 | |
4896 | if (LHS.getOperand(0).getOpcode() != ISD::INTRINSIC_W_CHAIN || |
4897 | LHS.getOperand(0).getConstantOperandVal(1) != Intrinsic::loop_decrement) |
4898 | return false; |
4899 | |
4900 | if (!isa<ConstantSDNode>(Val: RHS)) |
4901 | return false; |
4902 | |
4903 | assert((CC == ISD::SETEQ || CC == ISD::SETNE) && |
4904 | "Counter decrement comparison is not EQ or NE" ); |
4905 | |
4906 | SDValue OldDecrement = LHS.getOperand(i: 0); |
4907 | assert(OldDecrement.hasOneUse() && "loop decrement has more than one use!" ); |
4908 | |
4909 | SDLoc DecrementLoc(OldDecrement); |
4910 | SDValue ChainInput = OldDecrement.getOperand(i: 0); |
4911 | SDValue DecrementOps[] = {Subtarget->isPPC64() ? getI64Imm(Imm: 1, dl: DecrementLoc) |
4912 | : getI32Imm(Imm: 1, dl: DecrementLoc)}; |
4913 | unsigned DecrementOpcode = |
4914 | Subtarget->isPPC64() ? PPC::DecreaseCTR8loop : PPC::DecreaseCTRloop; |
4915 | SDNode *NewDecrement = CurDAG->getMachineNode(DecrementOpcode, DecrementLoc, |
4916 | MVT::i1, DecrementOps); |
4917 | |
4918 | unsigned Val = RHS->getAsZExtVal(); |
4919 | bool IsBranchOnTrue = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val); |
4920 | unsigned Opcode = IsBranchOnTrue ? PPC::BC : PPC::BCn; |
4921 | |
4922 | ReplaceUses(F: LHS.getValue(R: 0), T: LHS.getOperand(i: 1)); |
4923 | CurDAG->RemoveDeadNode(N: LHS.getNode()); |
4924 | |
4925 | // Mark the old loop_decrement intrinsic as dead. |
4926 | ReplaceUses(F: OldDecrement.getValue(R: 1), T: ChainInput); |
4927 | CurDAG->RemoveDeadNode(N: OldDecrement.getNode()); |
4928 | |
4929 | SDValue Chain = CurDAG->getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, |
4930 | ChainInput, N->getOperand(0)); |
4931 | |
4932 | CurDAG->SelectNodeTo(N, Opcode, MVT::Other, SDValue(NewDecrement, 0), |
4933 | N->getOperand(4), Chain); |
4934 | return true; |
4935 | } |
4936 | |
4937 | bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) { |
4938 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
4939 | unsigned Imm; |
4940 | if (!isInt32Immediate(N: N->getOperand(Num: 1), Imm)) |
4941 | return false; |
4942 | |
4943 | SDLoc dl(N); |
4944 | SDValue Val = N->getOperand(Num: 0); |
4945 | unsigned SH, MB, ME; |
4946 | // If this is an and of a value rotated between 0 and 31 bits and then and'd |
4947 | // with a mask, emit rlwinm |
4948 | if (isRotateAndMask(N: Val.getNode(), Mask: Imm, isShiftMask: false, SH, MB, ME)) { |
4949 | Val = Val.getOperand(i: 0); |
4950 | SDValue Ops[] = {Val, getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl), |
4951 | getI32Imm(Imm: ME, dl)}; |
4952 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4953 | return true; |
4954 | } |
4955 | |
4956 | // If this is just a masked value where the input is not handled, and |
4957 | // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm |
4958 | if (isRunOfOnes(Val: Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) { |
4959 | SDValue Ops[] = {Val, getI32Imm(Imm: 0, dl), getI32Imm(Imm: MB, dl), |
4960 | getI32Imm(Imm: ME, dl)}; |
4961 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
4962 | return true; |
4963 | } |
4964 | |
4965 | // AND X, 0 -> 0, not "rlwinm 32". |
4966 | if (Imm == 0) { |
4967 | ReplaceUses(F: SDValue(N, 0), T: N->getOperand(Num: 1)); |
4968 | return true; |
4969 | } |
4970 | |
4971 | return false; |
4972 | } |
4973 | |
4974 | bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) { |
4975 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
4976 | uint64_t Imm64; |
4977 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64)) |
4978 | return false; |
4979 | |
4980 | unsigned MB, ME; |
4981 | if (isRunOfOnes64(Val: Imm64, MB, ME) && MB >= 32 && MB <= ME) { |
4982 | // MB ME |
4983 | // +----------------------+ |
4984 | // |xxxxxxxxxxx00011111000| |
4985 | // +----------------------+ |
4986 | // 0 32 64 |
4987 | // We can only do it if the MB is larger than 32 and MB <= ME |
4988 | // as RLWINM will replace the contents of [0 - 32) with [32 - 64) even |
4989 | // we didn't rotate it. |
4990 | SDLoc dl(N); |
4991 | SDValue Ops[] = {N->getOperand(Num: 0), getI64Imm(Imm: 0, dl), getI64Imm(Imm: MB - 32, dl), |
4992 | getI64Imm(Imm: ME - 32, dl)}; |
4993 | CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); |
4994 | return true; |
4995 | } |
4996 | |
4997 | return false; |
4998 | } |
4999 | |
5000 | bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) { |
5001 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
5002 | uint64_t Imm64; |
5003 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64)) |
5004 | return false; |
5005 | |
5006 | // Do nothing if it is 16-bit imm as the pattern in the .td file handle |
5007 | // it well with "andi.". |
5008 | if (isUInt<16>(x: Imm64)) |
5009 | return false; |
5010 | |
5011 | SDLoc Loc(N); |
5012 | SDValue Val = N->getOperand(Num: 0); |
5013 | |
5014 | // Optimized with two rldicl's as follows: |
5015 | // Add missing bits on left to the mask and check that the mask is a |
5016 | // wrapped run of ones, i.e. |
5017 | // Change pattern |0001111100000011111111| |
5018 | // to |1111111100000011111111|. |
5019 | unsigned NumOfLeadingZeros = llvm::countl_zero(Val: Imm64); |
5020 | if (NumOfLeadingZeros != 0) |
5021 | Imm64 |= maskLeadingOnes<uint64_t>(N: NumOfLeadingZeros); |
5022 | |
5023 | unsigned MB, ME; |
5024 | if (!isRunOfOnes64(Val: Imm64, MB, ME)) |
5025 | return false; |
5026 | |
5027 | // ME MB MB-ME+63 |
5028 | // +----------------------+ +----------------------+ |
5029 | // |1111111100000011111111| -> |0000001111111111111111| |
5030 | // +----------------------+ +----------------------+ |
5031 | // 0 63 0 63 |
5032 | // There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between. |
5033 | unsigned OnesOnLeft = ME + 1; |
5034 | unsigned ZerosInBetween = (MB - ME + 63) & 63; |
5035 | // Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear |
5036 | // on the left the bits that are already zeros in the mask. |
5037 | Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val, |
5038 | getI64Imm(OnesOnLeft, Loc), |
5039 | getI64Imm(ZerosInBetween, Loc)), |
5040 | 0); |
5041 | // MB-ME+63 ME MB |
5042 | // +----------------------+ +----------------------+ |
5043 | // |0000001111111111111111| -> |0001111100000011111111| |
5044 | // +----------------------+ +----------------------+ |
5045 | // 0 63 0 63 |
5046 | // Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the |
5047 | // left the number of ones we previously added. |
5048 | SDValue Ops[] = {Val, getI64Imm(Imm: 64 - OnesOnLeft, dl: Loc), |
5049 | getI64Imm(Imm: NumOfLeadingZeros, dl: Loc)}; |
5050 | CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
5051 | return true; |
5052 | } |
5053 | |
5054 | bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) { |
5055 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
5056 | unsigned Imm; |
5057 | if (!isInt32Immediate(N: N->getOperand(Num: 1), Imm)) |
5058 | return false; |
5059 | |
5060 | SDValue Val = N->getOperand(Num: 0); |
5061 | unsigned Imm2; |
5062 | // ISD::OR doesn't get all the bitfield insertion fun. |
5063 | // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a |
5064 | // bitfield insert. |
5065 | if (Val.getOpcode() != ISD::OR || !isInt32Immediate(N: Val.getOperand(i: 1), Imm&: Imm2)) |
5066 | return false; |
5067 | |
5068 | // The idea here is to check whether this is equivalent to: |
5069 | // (c1 & m) | (x & ~m) |
5070 | // where m is a run-of-ones mask. The logic here is that, for each bit in |
5071 | // c1 and c2: |
5072 | // - if both are 1, then the output will be 1. |
5073 | // - if both are 0, then the output will be 0. |
5074 | // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will |
5075 | // come from x. |
5076 | // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will |
5077 | // be 0. |
5078 | // If that last condition is never the case, then we can form m from the |
5079 | // bits that are the same between c1 and c2. |
5080 | unsigned MB, ME; |
5081 | if (isRunOfOnes(Val: ~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) { |
5082 | SDLoc dl(N); |
5083 | SDValue Ops[] = {Val.getOperand(i: 0), Val.getOperand(i: 1), getI32Imm(Imm: 0, dl), |
5084 | getI32Imm(Imm: MB, dl), getI32Imm(Imm: ME, dl)}; |
5085 | ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); |
5086 | return true; |
5087 | } |
5088 | |
5089 | return false; |
5090 | } |
5091 | |
5092 | bool PPCDAGToDAGISel::tryAsSingleRLDCL(SDNode *N) { |
5093 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
5094 | |
5095 | uint64_t Imm64; |
5096 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64) || !isMask_64(Value: Imm64)) |
5097 | return false; |
5098 | |
5099 | SDValue Val = N->getOperand(Num: 0); |
5100 | |
5101 | if (Val.getOpcode() != ISD::ROTL) |
5102 | return false; |
5103 | |
5104 | // Looking to try to avoid a situation like this one: |
5105 | // %2 = tail call i64 @llvm.fshl.i64(i64 %word, i64 %word, i64 23) |
5106 | // %and1 = and i64 %2, 9223372036854775807 |
5107 | // In this function we are looking to try to match RLDCL. However, the above |
5108 | // DAG would better match RLDICL instead which is not what we are looking |
5109 | // for here. |
5110 | SDValue RotateAmt = Val.getOperand(i: 1); |
5111 | if (RotateAmt.getOpcode() == ISD::Constant) |
5112 | return false; |
5113 | |
5114 | unsigned MB = 64 - llvm::countr_one(Value: Imm64); |
5115 | SDLoc dl(N); |
5116 | SDValue Ops[] = {Val.getOperand(i: 0), RotateAmt, getI32Imm(Imm: MB, dl)}; |
5117 | CurDAG->SelectNodeTo(N, PPC::RLDCL, MVT::i64, Ops); |
5118 | return true; |
5119 | } |
5120 | |
5121 | bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) { |
5122 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
5123 | uint64_t Imm64; |
5124 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64) || !isMask_64(Value: Imm64)) |
5125 | return false; |
5126 | |
5127 | // If this is a 64-bit zero-extension mask, emit rldicl. |
5128 | unsigned MB = 64 - llvm::countr_one(Value: Imm64); |
5129 | unsigned SH = 0; |
5130 | unsigned Imm; |
5131 | SDValue Val = N->getOperand(Num: 0); |
5132 | SDLoc dl(N); |
5133 | |
5134 | if (Val.getOpcode() == ISD::ANY_EXTEND) { |
5135 | auto Op0 = Val.getOperand(i: 0); |
5136 | if (Op0.getOpcode() == ISD::SRL && |
5137 | isInt32Immediate(N: Op0.getOperand(i: 1).getNode(), Imm) && Imm <= MB) { |
5138 | |
5139 | auto ResultType = Val.getNode()->getValueType(ResNo: 0); |
5140 | auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType); |
5141 | SDValue IDVal(ImDef, 0); |
5142 | |
5143 | Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType, |
5144 | IDVal, Op0.getOperand(0), |
5145 | getI32Imm(1, dl)), |
5146 | 0); |
5147 | SH = 64 - Imm; |
5148 | } |
5149 | } |
5150 | |
5151 | // If the operand is a logical right shift, we can fold it into this |
5152 | // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) |
5153 | // for n <= mb. The right shift is really a left rotate followed by a |
5154 | // mask, and this mask is a more-restrictive sub-mask of the mask implied |
5155 | // by the shift. |
5156 | if (Val.getOpcode() == ISD::SRL && |
5157 | isInt32Immediate(N: Val.getOperand(i: 1).getNode(), Imm) && Imm <= MB) { |
5158 | assert(Imm < 64 && "Illegal shift amount" ); |
5159 | Val = Val.getOperand(i: 0); |
5160 | SH = 64 - Imm; |
5161 | } |
5162 | |
5163 | SDValue Ops[] = {Val, getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl)}; |
5164 | CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); |
5165 | return true; |
5166 | } |
5167 | |
5168 | bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) { |
5169 | assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected" ); |
5170 | uint64_t Imm64; |
5171 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64) || |
5172 | !isMask_64(Value: ~Imm64)) |
5173 | return false; |
5174 | |
5175 | // If this is a negated 64-bit zero-extension mask, |
5176 | // i.e. the immediate is a sequence of ones from most significant side |
5177 | // and all zero for reminder, we should use rldicr. |
5178 | unsigned MB = 63 - llvm::countr_one(Value: ~Imm64); |
5179 | unsigned SH = 0; |
5180 | SDLoc dl(N); |
5181 | SDValue Ops[] = {N->getOperand(Num: 0), getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl)}; |
5182 | CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
5183 | return true; |
5184 | } |
5185 | |
5186 | bool PPCDAGToDAGISel::tryAsSingleRLDIMI(SDNode *N) { |
5187 | assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected" ); |
5188 | uint64_t Imm64; |
5189 | unsigned MB, ME; |
5190 | SDValue N0 = N->getOperand(Num: 0); |
5191 | |
5192 | // We won't get fewer instructions if the imm is 32-bit integer. |
5193 | // rldimi requires the imm to have consecutive ones with both sides zero. |
5194 | // Also, make sure the first Op has only one use, otherwise this may increase |
5195 | // register pressure since rldimi is destructive. |
5196 | if (!isInt64Immediate(N: N->getOperand(Num: 1).getNode(), Imm&: Imm64) || |
5197 | isUInt<32>(x: Imm64) || !isRunOfOnes64(Val: Imm64, MB, ME) || !N0.hasOneUse()) |
5198 | return false; |
5199 | |
5200 | unsigned SH = 63 - ME; |
5201 | SDLoc Dl(N); |
5202 | // Use select64Imm for making LI instr instead of directly putting Imm64 |
5203 | SDValue Ops[] = { |
5204 | N->getOperand(Num: 0), |
5205 | SDValue(selectI64Imm(CurDAG, N: getI64Imm(Imm: -1, dl: Dl).getNode()), 0), |
5206 | getI32Imm(Imm: SH, dl: Dl), getI32Imm(Imm: MB, dl: Dl)}; |
5207 | CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops); |
5208 | return true; |
5209 | } |
5210 | |
5211 | // Select - Convert the specified operand from a target-independent to a |
5212 | // target-specific node if it hasn't already been changed. |
5213 | void PPCDAGToDAGISel::Select(SDNode *N) { |
5214 | SDLoc dl(N); |
5215 | if (N->isMachineOpcode()) { |
5216 | N->setNodeId(-1); |
5217 | return; // Already selected. |
5218 | } |
5219 | |
5220 | // In case any misguided DAG-level optimizations form an ADD with a |
5221 | // TargetConstant operand, crash here instead of miscompiling (by selecting |
5222 | // an r+r add instead of some kind of r+i add). |
5223 | if (N->getOpcode() == ISD::ADD && |
5224 | N->getOperand(Num: 1).getOpcode() == ISD::TargetConstant) |
5225 | llvm_unreachable("Invalid ADD with TargetConstant operand" ); |
5226 | |
5227 | // Try matching complex bit permutations before doing anything else. |
5228 | if (tryBitPermutation(N)) |
5229 | return; |
5230 | |
5231 | // Try to emit integer compares as GPR-only sequences (i.e. no use of CR). |
5232 | if (tryIntCompareInGPR(N)) |
5233 | return; |
5234 | |
5235 | switch (N->getOpcode()) { |
5236 | default: break; |
5237 | |
5238 | case ISD::Constant: |
5239 | if (N->getValueType(0) == MVT::i64) { |
5240 | ReplaceNode(F: N, T: selectI64Imm(CurDAG, N)); |
5241 | return; |
5242 | } |
5243 | break; |
5244 | |
5245 | case ISD::INTRINSIC_VOID: { |
5246 | auto IntrinsicID = N->getConstantOperandVal(Num: 1); |
5247 | if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw && |
5248 | IntrinsicID != Intrinsic::ppc_trapd && |
5249 | IntrinsicID != Intrinsic::ppc_trap) |
5250 | break; |
5251 | unsigned Opcode = (IntrinsicID == Intrinsic::ppc_tdw || |
5252 | IntrinsicID == Intrinsic::ppc_trapd) |
5253 | ? PPC::TDI |
5254 | : PPC::TWI; |
5255 | SmallVector<SDValue, 4> OpsWithMD; |
5256 | unsigned MDIndex; |
5257 | if (IntrinsicID == Intrinsic::ppc_tdw || |
5258 | IntrinsicID == Intrinsic::ppc_tw) { |
5259 | SDValue Ops[] = {N->getOperand(Num: 4), N->getOperand(Num: 2), N->getOperand(Num: 3)}; |
5260 | int16_t SImmOperand2; |
5261 | int16_t SImmOperand3; |
5262 | int16_t SImmOperand4; |
5263 | bool isOperand2IntS16Immediate = |
5264 | isIntS16Immediate(Op: N->getOperand(Num: 2), Imm&: SImmOperand2); |
5265 | bool isOperand3IntS16Immediate = |
5266 | isIntS16Immediate(Op: N->getOperand(Num: 3), Imm&: SImmOperand3); |
5267 | // We will emit PPC::TD or PPC::TW if the 2nd and 3rd operands are reg + |
5268 | // reg or imm + imm. The imm + imm form will be optimized to either an |
5269 | // unconditional trap or a nop in a later pass. |
5270 | if (isOperand2IntS16Immediate == isOperand3IntS16Immediate) |
5271 | Opcode = IntrinsicID == Intrinsic::ppc_tdw ? PPC::TD : PPC::TW; |
5272 | else if (isOperand3IntS16Immediate) |
5273 | // The 2nd and 3rd operands are reg + imm. |
5274 | Ops[2] = getI32Imm(Imm: int(SImmOperand3) & 0xFFFF, dl); |
5275 | else { |
5276 | // The 2nd and 3rd operands are imm + reg. |
5277 | bool isOperand4IntS16Immediate = |
5278 | isIntS16Immediate(Op: N->getOperand(Num: 4), Imm&: SImmOperand4); |
5279 | (void)isOperand4IntS16Immediate; |
5280 | assert(isOperand4IntS16Immediate && |
5281 | "The 4th operand is not an Immediate" ); |
5282 | // We need to flip the condition immediate TO. |
5283 | int16_t TO = int(SImmOperand4) & 0x1F; |
5284 | // We swap the first and second bit of TO if they are not same. |
5285 | if ((TO & 0x1) != ((TO & 0x2) >> 1)) |
5286 | TO = (TO & 0x1) ? TO + 1 : TO - 1; |
5287 | // We swap the fourth and fifth bit of TO if they are not same. |
5288 | if ((TO & 0x8) != ((TO & 0x10) >> 1)) |
5289 | TO = (TO & 0x8) ? TO + 8 : TO - 8; |
5290 | Ops[0] = getI32Imm(Imm: TO, dl); |
5291 | Ops[1] = N->getOperand(Num: 3); |
5292 | Ops[2] = getI32Imm(Imm: int(SImmOperand2) & 0xFFFF, dl); |
5293 | } |
5294 | OpsWithMD = {Ops[0], Ops[1], Ops[2]}; |
5295 | MDIndex = 5; |
5296 | } else { |
5297 | OpsWithMD = {getI32Imm(Imm: 24, dl), N->getOperand(Num: 2), getI32Imm(Imm: 0, dl)}; |
5298 | MDIndex = 3; |
5299 | } |
5300 | |
5301 | if (N->getNumOperands() > MDIndex) { |
5302 | SDValue MDV = N->getOperand(Num: MDIndex); |
5303 | const MDNode *MD = cast<MDNodeSDNode>(Val&: MDV)->getMD(); |
5304 | assert(MD->getNumOperands() != 0 && "Empty MDNode in operands!" ); |
5305 | assert((isa<MDString>(MD->getOperand(0)) && cast<MDString>( |
5306 | MD->getOperand(0))->getString().equals("ppc-trap-reason" )) |
5307 | && "Unsupported annotation data type!" ); |
5308 | for (unsigned i = 1; i < MD->getNumOperands(); i++) { |
5309 | assert(isa<MDString>(MD->getOperand(i)) && |
5310 | "Invalid data type for annotation ppc-trap-reason!" ); |
5311 | OpsWithMD.push_back( |
5312 | Elt: getI32Imm(Imm: std::stoi(str: cast<MDString>( |
5313 | Val: MD->getOperand(I: i))->getString().str()), dl)); |
5314 | } |
5315 | } |
5316 | OpsWithMD.push_back(Elt: N->getOperand(Num: 0)); // chain |
5317 | CurDAG->SelectNodeTo(N, Opcode, MVT::Other, OpsWithMD); |
5318 | return; |
5319 | } |
5320 | |
5321 | case ISD::INTRINSIC_WO_CHAIN: { |
5322 | // We emit the PPC::FSELS instruction here because of type conflicts with |
5323 | // the comparison operand. The FSELS instruction is defined to use an 8-byte |
5324 | // comparison like the FSELD version. The fsels intrinsic takes a 4-byte |
5325 | // value for the comparison. When selecting through a .td file, a type |
5326 | // error is raised. Must check this first so we never break on the |
5327 | // !Subtarget->isISA3_1() check. |
5328 | auto IntID = N->getConstantOperandVal(Num: 0); |
5329 | if (IntID == Intrinsic::ppc_fsels) { |
5330 | SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3)}; |
5331 | CurDAG->SelectNodeTo(N, PPC::FSELS, MVT::f32, Ops); |
5332 | return; |
5333 | } |
5334 | |
5335 | if (IntID == Intrinsic::ppc_bcdadd_p || IntID == Intrinsic::ppc_bcdsub_p) { |
5336 | auto Pred = N->getConstantOperandVal(Num: 1); |
5337 | unsigned Opcode = |
5338 | IntID == Intrinsic::ppc_bcdadd_p ? PPC::BCDADD_rec : PPC::BCDSUB_rec; |
5339 | unsigned SubReg = 0; |
5340 | unsigned ShiftVal = 0; |
5341 | bool Reverse = false; |
5342 | switch (Pred) { |
5343 | case 0: |
5344 | SubReg = PPC::sub_eq; |
5345 | ShiftVal = 1; |
5346 | break; |
5347 | case 1: |
5348 | SubReg = PPC::sub_eq; |
5349 | ShiftVal = 1; |
5350 | Reverse = true; |
5351 | break; |
5352 | case 2: |
5353 | SubReg = PPC::sub_lt; |
5354 | ShiftVal = 3; |
5355 | break; |
5356 | case 3: |
5357 | SubReg = PPC::sub_lt; |
5358 | ShiftVal = 3; |
5359 | Reverse = true; |
5360 | break; |
5361 | case 4: |
5362 | SubReg = PPC::sub_gt; |
5363 | ShiftVal = 2; |
5364 | break; |
5365 | case 5: |
5366 | SubReg = PPC::sub_gt; |
5367 | ShiftVal = 2; |
5368 | Reverse = true; |
5369 | break; |
5370 | case 6: |
5371 | SubReg = PPC::sub_un; |
5372 | break; |
5373 | case 7: |
5374 | SubReg = PPC::sub_un; |
5375 | Reverse = true; |
5376 | break; |
5377 | } |
5378 | |
5379 | EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
5380 | SDValue Ops[] = {N->getOperand(2), N->getOperand(3), |
5381 | CurDAG->getTargetConstant(0, dl, MVT::i32)}; |
5382 | SDValue BCDOp = SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, Ops), 0); |
5383 | SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
5384 | // On Power10, we can use SETBC[R]. On prior architectures, we have to use |
5385 | // MFOCRF and shift/negate the value. |
5386 | if (Subtarget->isISA3_1()) { |
5387 | SDValue SubRegIdx = CurDAG->getTargetConstant(SubReg, dl, MVT::i32); |
5388 | SDValue CRBit = SDValue( |
5389 | CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
5390 | CR6Reg, SubRegIdx, BCDOp.getValue(1)), |
5391 | 0); |
5392 | CurDAG->SelectNodeTo(N, Reverse ? PPC::SETBCR : PPC::SETBC, MVT::i32, |
5393 | CRBit); |
5394 | } else { |
5395 | SDValue Move = |
5396 | SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR6Reg, |
5397 | BCDOp.getValue(1)), |
5398 | 0); |
5399 | SDValue Ops[] = {Move, getI32Imm(Imm: (32 - (4 + ShiftVal)) & 31, dl), |
5400 | getI32Imm(Imm: 31, dl), getI32Imm(Imm: 31, dl)}; |
5401 | if (!Reverse) |
5402 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5403 | else { |
5404 | SDValue Shift = SDValue( |
5405 | CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); |
5406 | CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Shift, getI32Imm(1, dl)); |
5407 | } |
5408 | } |
5409 | return; |
5410 | } |
5411 | |
5412 | if (!Subtarget->isISA3_1()) |
5413 | break; |
5414 | unsigned Opcode = 0; |
5415 | switch (IntID) { |
5416 | default: |
5417 | break; |
5418 | case Intrinsic::ppc_altivec_vstribr_p: |
5419 | Opcode = PPC::VSTRIBR_rec; |
5420 | break; |
5421 | case Intrinsic::ppc_altivec_vstribl_p: |
5422 | Opcode = PPC::VSTRIBL_rec; |
5423 | break; |
5424 | case Intrinsic::ppc_altivec_vstrihr_p: |
5425 | Opcode = PPC::VSTRIHR_rec; |
5426 | break; |
5427 | case Intrinsic::ppc_altivec_vstrihl_p: |
5428 | Opcode = PPC::VSTRIHL_rec; |
5429 | break; |
5430 | } |
5431 | if (!Opcode) |
5432 | break; |
5433 | |
5434 | // Generate the appropriate vector string isolate intrinsic to match. |
5435 | EVT VTs[] = {MVT::v16i8, MVT::Glue}; |
5436 | SDValue VecStrOp = |
5437 | SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(Num: 2)), 0); |
5438 | // Vector string isolate instructions update the EQ bit of CR6. |
5439 | // Generate a SETBC instruction to extract the bit and place it in a GPR. |
5440 | SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32); |
5441 | SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); |
5442 | SDValue CRBit = SDValue( |
5443 | CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, |
5444 | CR6Reg, SubRegIdx, VecStrOp.getValue(1)), |
5445 | 0); |
5446 | CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit); |
5447 | return; |
5448 | } |
5449 | |
5450 | case ISD::SETCC: |
5451 | case ISD::STRICT_FSETCC: |
5452 | case ISD::STRICT_FSETCCS: |
5453 | if (trySETCC(N)) |
5454 | return; |
5455 | break; |
5456 | // These nodes will be transformed into GETtlsADDR32 node, which |
5457 | // later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT |
5458 | case PPCISD::ADDI_TLSLD_L_ADDR: |
5459 | case PPCISD::ADDI_TLSGD_L_ADDR: { |
5460 | const Module *Mod = MF->getFunction().getParent(); |
5461 | if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
5462 | !Subtarget->isSecurePlt() || !Subtarget->isTargetELF() || |
5463 | Mod->getPICLevel() == PICLevel::SmallPIC) |
5464 | break; |
5465 | // Attach global base pointer on GETtlsADDR32 node in order to |
5466 | // generate secure plt code for TLS symbols. |
5467 | getGlobalBaseReg(); |
5468 | } break; |
5469 | case PPCISD::CALL: { |
5470 | if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || |
5471 | !TM.isPositionIndependent() || !Subtarget->isSecurePlt() || |
5472 | !Subtarget->isTargetELF()) |
5473 | break; |
5474 | |
5475 | SDValue Op = N->getOperand(Num: 1); |
5476 | |
5477 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) { |
5478 | if (GA->getTargetFlags() == PPCII::MO_PLT) |
5479 | getGlobalBaseReg(); |
5480 | } |
5481 | else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Val&: Op)) { |
5482 | if (ES->getTargetFlags() == PPCII::MO_PLT) |
5483 | getGlobalBaseReg(); |
5484 | } |
5485 | } |
5486 | break; |
5487 | |
5488 | case PPCISD::GlobalBaseReg: |
5489 | ReplaceNode(F: N, T: getGlobalBaseReg()); |
5490 | return; |
5491 | |
5492 | case ISD::FrameIndex: |
5493 | selectFrameIndex(SN: N, N); |
5494 | return; |
5495 | |
5496 | case PPCISD::MFOCRF: { |
5497 | SDValue InGlue = N->getOperand(Num: 1); |
5498 | ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, |
5499 | N->getOperand(0), InGlue)); |
5500 | return; |
5501 | } |
5502 | |
5503 | case PPCISD::READ_TIME_BASE: |
5504 | ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, |
5505 | MVT::Other, N->getOperand(0))); |
5506 | return; |
5507 | |
5508 | case PPCISD::SRA_ADDZE: { |
5509 | SDValue N0 = N->getOperand(Num: 0); |
5510 | SDValue ShiftAmt = |
5511 | CurDAG->getTargetConstant(Val: *cast<ConstantSDNode>(Val: N->getOperand(Num: 1))-> |
5512 | getConstantIntValue(), DL: dl, |
5513 | VT: N->getValueType(ResNo: 0)); |
5514 | if (N->getValueType(0) == MVT::i64) { |
5515 | SDNode *Op = |
5516 | CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, |
5517 | N0, ShiftAmt); |
5518 | CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), |
5519 | SDValue(Op, 1)); |
5520 | return; |
5521 | } else { |
5522 | assert(N->getValueType(0) == MVT::i32 && |
5523 | "Expecting i64 or i32 in PPCISD::SRA_ADDZE" ); |
5524 | SDNode *Op = |
5525 | CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, |
5526 | N0, ShiftAmt); |
5527 | CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), |
5528 | SDValue(Op, 1)); |
5529 | return; |
5530 | } |
5531 | } |
5532 | |
5533 | case ISD::STORE: { |
5534 | // Change TLS initial-exec (or TLS local-exec on AIX) D-form stores to |
5535 | // X-form stores. |
5536 | StoreSDNode *ST = cast<StoreSDNode>(Val: N); |
5537 | if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI()) && |
5538 | ST->getAddressingMode() != ISD::PRE_INC) |
5539 | if (tryTLSXFormStore(ST)) |
5540 | return; |
5541 | break; |
5542 | } |
5543 | case ISD::LOAD: { |
5544 | // Handle preincrement loads. |
5545 | LoadSDNode *LD = cast<LoadSDNode>(Val: N); |
5546 | EVT LoadedVT = LD->getMemoryVT(); |
5547 | |
5548 | // Normal loads are handled by code generated from the .td file. |
5549 | if (LD->getAddressingMode() != ISD::PRE_INC) { |
5550 | // Change TLS initial-exec (or TLS local-exec on AIX) D-form loads to |
5551 | // X-form loads. |
5552 | if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI())) |
5553 | if (tryTLSXFormLoad(LD)) |
5554 | return; |
5555 | break; |
5556 | } |
5557 | |
5558 | SDValue Offset = LD->getOffset(); |
5559 | if (Offset.getOpcode() == ISD::TargetConstant || |
5560 | Offset.getOpcode() == ISD::TargetGlobalAddress) { |
5561 | |
5562 | unsigned Opcode; |
5563 | bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
5564 | if (LD->getValueType(0) != MVT::i64) { |
5565 | // Handle PPC32 integer and normal FP loads. |
5566 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ); |
5567 | switch (LoadedVT.getSimpleVT().SimpleTy) { |
5568 | default: llvm_unreachable("Invalid PPC load type!" ); |
5569 | case MVT::f64: Opcode = PPC::LFDU; break; |
5570 | case MVT::f32: Opcode = PPC::LFSU; break; |
5571 | case MVT::i32: Opcode = PPC::LWZU; break; |
5572 | case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; |
5573 | case MVT::i1: |
5574 | case MVT::i8: Opcode = PPC::LBZU; break; |
5575 | } |
5576 | } else { |
5577 | assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!" ); |
5578 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ); |
5579 | switch (LoadedVT.getSimpleVT().SimpleTy) { |
5580 | default: llvm_unreachable("Invalid PPC load type!" ); |
5581 | case MVT::i64: Opcode = PPC::LDU; break; |
5582 | case MVT::i32: Opcode = PPC::LWZU8; break; |
5583 | case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; |
5584 | case MVT::i1: |
5585 | case MVT::i8: Opcode = PPC::LBZU8; break; |
5586 | } |
5587 | } |
5588 | |
5589 | SDValue Chain = LD->getChain(); |
5590 | SDValue Base = LD->getBasePtr(); |
5591 | SDValue Ops[] = { Offset, Base, Chain }; |
5592 | SDNode *MN = CurDAG->getMachineNode( |
5593 | Opcode, dl, LD->getValueType(0), |
5594 | PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
5595 | transferMemOperands(N, Result: MN); |
5596 | ReplaceNode(F: N, T: MN); |
5597 | return; |
5598 | } else { |
5599 | unsigned Opcode; |
5600 | bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; |
5601 | if (LD->getValueType(0) != MVT::i64) { |
5602 | // Handle PPC32 integer and normal FP loads. |
5603 | assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load" ); |
5604 | switch (LoadedVT.getSimpleVT().SimpleTy) { |
5605 | default: llvm_unreachable("Invalid PPC load type!" ); |
5606 | case MVT::f64: Opcode = PPC::LFDUX; break; |
5607 | case MVT::f32: Opcode = PPC::LFSUX; break; |
5608 | case MVT::i32: Opcode = PPC::LWZUX; break; |
5609 | case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; |
5610 | case MVT::i1: |
5611 | case MVT::i8: Opcode = PPC::LBZUX; break; |
5612 | } |
5613 | } else { |
5614 | assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!" ); |
5615 | assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && |
5616 | "Invalid sext update load" ); |
5617 | switch (LoadedVT.getSimpleVT().SimpleTy) { |
5618 | default: llvm_unreachable("Invalid PPC load type!" ); |
5619 | case MVT::i64: Opcode = PPC::LDUX; break; |
5620 | case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; |
5621 | case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; |
5622 | case MVT::i1: |
5623 | case MVT::i8: Opcode = PPC::LBZUX8; break; |
5624 | } |
5625 | } |
5626 | |
5627 | SDValue Chain = LD->getChain(); |
5628 | SDValue Base = LD->getBasePtr(); |
5629 | SDValue Ops[] = { Base, Offset, Chain }; |
5630 | SDNode *MN = CurDAG->getMachineNode( |
5631 | Opcode, dl, LD->getValueType(0), |
5632 | PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); |
5633 | transferMemOperands(N, Result: MN); |
5634 | ReplaceNode(F: N, T: MN); |
5635 | return; |
5636 | } |
5637 | } |
5638 | |
5639 | case ISD::AND: |
5640 | // If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr |
5641 | if (tryAsSingleRLWINM(N) || tryAsSingleRLWIMI(N) || tryAsSingleRLDCL(N) || |
5642 | tryAsSingleRLDICL(N) || tryAsSingleRLDICR(N) || tryAsSingleRLWINM8(N) || |
5643 | tryAsPairOfRLDICL(N)) |
5644 | return; |
5645 | |
5646 | // Other cases are autogenerated. |
5647 | break; |
5648 | case ISD::OR: { |
5649 | if (N->getValueType(0) == MVT::i32) |
5650 | if (tryBitfieldInsert(N)) |
5651 | return; |
5652 | |
5653 | int16_t Imm; |
5654 | if (N->getOperand(Num: 0)->getOpcode() == ISD::FrameIndex && |
5655 | isIntS16Immediate(Op: N->getOperand(Num: 1), Imm)) { |
5656 | KnownBits LHSKnown = CurDAG->computeKnownBits(Op: N->getOperand(Num: 0)); |
5657 | |
5658 | // If this is equivalent to an add, then we can fold it with the |
5659 | // FrameIndex calculation. |
5660 | if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) { |
5661 | selectFrameIndex(SN: N, N: N->getOperand(Num: 0).getNode(), Offset: (int64_t)Imm); |
5662 | return; |
5663 | } |
5664 | } |
5665 | |
5666 | // If this is 'or' against an imm with consecutive ones and both sides zero, |
5667 | // try to emit rldimi |
5668 | if (tryAsSingleRLDIMI(N)) |
5669 | return; |
5670 | |
5671 | // OR with a 32-bit immediate can be handled by ori + oris |
5672 | // without creating an immediate in a GPR. |
5673 | uint64_t Imm64 = 0; |
5674 | bool IsPPC64 = Subtarget->isPPC64(); |
5675 | if (IsPPC64 && isInt64Immediate(N: N->getOperand(Num: 1), Imm&: Imm64) && |
5676 | (Imm64 & ~0xFFFFFFFFuLL) == 0) { |
5677 | // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later. |
5678 | uint64_t ImmHi = Imm64 >> 16; |
5679 | uint64_t ImmLo = Imm64 & 0xFFFF; |
5680 | if (ImmHi != 0 && ImmLo != 0) { |
5681 | SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, |
5682 | N->getOperand(0), |
5683 | getI16Imm(ImmLo, dl)); |
5684 | SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(Imm: ImmHi, dl)}; |
5685 | CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); |
5686 | return; |
5687 | } |
5688 | } |
5689 | |
5690 | // Other cases are autogenerated. |
5691 | break; |
5692 | } |
5693 | case ISD::XOR: { |
5694 | // XOR with a 32-bit immediate can be handled by xori + xoris |
5695 | // without creating an immediate in a GPR. |
5696 | uint64_t Imm64 = 0; |
5697 | bool IsPPC64 = Subtarget->isPPC64(); |
5698 | if (IsPPC64 && isInt64Immediate(N: N->getOperand(Num: 1), Imm&: Imm64) && |
5699 | (Imm64 & ~0xFFFFFFFFuLL) == 0) { |
5700 | // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later. |
5701 | uint64_t ImmHi = Imm64 >> 16; |
5702 | uint64_t ImmLo = Imm64 & 0xFFFF; |
5703 | if (ImmHi != 0 && ImmLo != 0) { |
5704 | SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, |
5705 | N->getOperand(0), |
5706 | getI16Imm(ImmLo, dl)); |
5707 | SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(Imm: ImmHi, dl)}; |
5708 | CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); |
5709 | return; |
5710 | } |
5711 | } |
5712 | |
5713 | break; |
5714 | } |
5715 | case ISD::ADD: { |
5716 | int16_t Imm; |
5717 | if (N->getOperand(Num: 0)->getOpcode() == ISD::FrameIndex && |
5718 | isIntS16Immediate(Op: N->getOperand(Num: 1), Imm)) { |
5719 | selectFrameIndex(SN: N, N: N->getOperand(Num: 0).getNode(), Offset: (int64_t)Imm); |
5720 | return; |
5721 | } |
5722 | |
5723 | break; |
5724 | } |
5725 | case ISD::SHL: { |
5726 | unsigned Imm, SH, MB, ME; |
5727 | if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm) && |
5728 | isRotateAndMask(N, Mask: Imm, isShiftMask: true, SH, MB, ME)) { |
5729 | SDValue Ops[] = { N->getOperand(Num: 0).getOperand(i: 0), |
5730 | getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl), |
5731 | getI32Imm(Imm: ME, dl) }; |
5732 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5733 | return; |
5734 | } |
5735 | |
5736 | // Other cases are autogenerated. |
5737 | break; |
5738 | } |
5739 | case ISD::SRL: { |
5740 | unsigned Imm, SH, MB, ME; |
5741 | if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm) && |
5742 | isRotateAndMask(N, Mask: Imm, isShiftMask: true, SH, MB, ME)) { |
5743 | SDValue Ops[] = { N->getOperand(Num: 0).getOperand(i: 0), |
5744 | getI32Imm(Imm: SH, dl), getI32Imm(Imm: MB, dl), |
5745 | getI32Imm(Imm: ME, dl) }; |
5746 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5747 | return; |
5748 | } |
5749 | |
5750 | // Other cases are autogenerated. |
5751 | break; |
5752 | } |
5753 | case ISD::MUL: { |
5754 | SDValue Op1 = N->getOperand(Num: 1); |
5755 | if (Op1.getOpcode() != ISD::Constant || |
5756 | (Op1.getValueType() != MVT::i64 && Op1.getValueType() != MVT::i32)) |
5757 | break; |
5758 | |
5759 | // If the multiplier fits int16, we can handle it with mulli. |
5760 | int64_t Imm = Op1->getAsZExtVal(); |
5761 | unsigned Shift = llvm::countr_zero<uint64_t>(Val: Imm); |
5762 | if (isInt<16>(x: Imm) || !Shift) |
5763 | break; |
5764 | |
5765 | // If the shifted value fits int16, we can do this transformation: |
5766 | // (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to |
5767 | // DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2). |
5768 | uint64_t ImmSh = Imm >> Shift; |
5769 | if (!isInt<16>(x: ImmSh)) |
5770 | break; |
5771 | |
5772 | uint64_t SextImm = SignExtend64(X: ImmSh & 0xFFFF, B: 16); |
5773 | if (Op1.getValueType() == MVT::i64) { |
5774 | SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); |
5775 | SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64, |
5776 | N->getOperand(0), SDImm); |
5777 | |
5778 | SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Imm: Shift, dl), |
5779 | getI32Imm(Imm: 63 - Shift, dl)}; |
5780 | CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); |
5781 | return; |
5782 | } else { |
5783 | SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i32); |
5784 | SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI, dl, MVT::i32, |
5785 | N->getOperand(0), SDImm); |
5786 | |
5787 | SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Imm: Shift, dl), |
5788 | getI32Imm(Imm: 0, dl), getI32Imm(Imm: 31 - Shift, dl)}; |
5789 | CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); |
5790 | return; |
5791 | } |
5792 | break; |
5793 | } |
5794 | // FIXME: Remove this once the ANDI glue bug is fixed: |
5795 | case PPCISD::ANDI_rec_1_EQ_BIT: |
5796 | case PPCISD::ANDI_rec_1_GT_BIT: { |
5797 | if (!ANDIGlueBug) |
5798 | break; |
5799 | |
5800 | EVT InVT = N->getOperand(Num: 0).getValueType(); |
5801 | assert((InVT == MVT::i64 || InVT == MVT::i32) && |
5802 | "Invalid input type for ANDI_rec_1_EQ_BIT" ); |
5803 | |
5804 | unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec; |
5805 | SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, |
5806 | N->getOperand(0), |
5807 | CurDAG->getTargetConstant(1, dl, InVT)), |
5808 | 0); |
5809 | SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); |
5810 | SDValue SRIdxVal = CurDAG->getTargetConstant( |
5811 | N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, |
5812 | dl, MVT::i32); |
5813 | |
5814 | CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, |
5815 | SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */); |
5816 | return; |
5817 | } |
5818 | case ISD::SELECT_CC: { |
5819 | ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get(); |
5820 | EVT PtrVT = |
5821 | CurDAG->getTargetLoweringInfo().getPointerTy(DL: CurDAG->getDataLayout()); |
5822 | bool isPPC64 = (PtrVT == MVT::i64); |
5823 | |
5824 | // If this is a select of i1 operands, we'll pattern match it. |
5825 | if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1) |
5826 | break; |
5827 | |
5828 | if (Subtarget->isISA3_0() && Subtarget->isPPC64()) { |
5829 | bool NeedSwapOps = false; |
5830 | bool IsUnCmp = false; |
5831 | if (mayUseP9Setb(N, CC, DAG: CurDAG, NeedSwapOps, IsUnCmp)) { |
5832 | SDValue LHS = N->getOperand(Num: 0); |
5833 | SDValue RHS = N->getOperand(Num: 1); |
5834 | if (NeedSwapOps) |
5835 | std::swap(a&: LHS, b&: RHS); |
5836 | |
5837 | // Make use of SelectCC to generate the comparison to set CR bits, for |
5838 | // equality comparisons having one literal operand, SelectCC probably |
5839 | // doesn't need to materialize the whole literal and just use xoris to |
5840 | // check it first, it leads the following comparison result can't |
5841 | // exactly represent GT/LT relationship. So to avoid this we specify |
5842 | // SETGT/SETUGT here instead of SETEQ. |
5843 | SDValue GenCC = |
5844 | SelectCC(LHS, RHS, CC: IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl); |
5845 | CurDAG->SelectNodeTo( |
5846 | N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, |
5847 | N->getValueType(0), GenCC); |
5848 | NumP9Setb++; |
5849 | return; |
5850 | } |
5851 | } |
5852 | |
5853 | // Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc |
5854 | if (!isPPC64 && isNullConstant(N->getOperand(1)) && |
5855 | isOneConstant(N->getOperand(2)) && isNullConstant(N->getOperand(3)) && |
5856 | CC == ISD::SETNE && |
5857 | // FIXME: Implement this optzn for PPC64. |
5858 | N->getValueType(0) == MVT::i32) { |
5859 | SDNode *Tmp = |
5860 | CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, |
5861 | N->getOperand(0), getI32Imm(~0U, dl)); |
5862 | CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), |
5863 | N->getOperand(0), SDValue(Tmp, 1)); |
5864 | return; |
5865 | } |
5866 | |
5867 | SDValue CCReg = SelectCC(LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), CC, dl); |
5868 | |
5869 | if (N->getValueType(0) == MVT::i1) { |
5870 | // An i1 select is: (c & t) | (!c & f). |
5871 | bool Inv; |
5872 | unsigned Idx = getCRIdxForSetCC(CC, Invert&: Inv); |
5873 | |
5874 | unsigned SRI; |
5875 | switch (Idx) { |
5876 | default: llvm_unreachable("Invalid CC index" ); |
5877 | case 0: SRI = PPC::sub_lt; break; |
5878 | case 1: SRI = PPC::sub_gt; break; |
5879 | case 2: SRI = PPC::sub_eq; break; |
5880 | case 3: SRI = PPC::sub_un; break; |
5881 | } |
5882 | |
5883 | SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); |
5884 | |
5885 | SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, |
5886 | CCBit, CCBit), 0); |
5887 | SDValue C = Inv ? NotCCBit : CCBit, |
5888 | NotC = Inv ? CCBit : NotCCBit; |
5889 | |
5890 | SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
5891 | C, N->getOperand(2)), 0); |
5892 | SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, |
5893 | NotC, N->getOperand(3)), 0); |
5894 | |
5895 | CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); |
5896 | return; |
5897 | } |
5898 | |
5899 | unsigned BROpc = |
5900 | getPredicateForSetCC(CC, VT: N->getOperand(Num: 0).getValueType(), Subtarget); |
5901 | |
5902 | unsigned SelectCCOp; |
5903 | if (N->getValueType(0) == MVT::i32) |
5904 | SelectCCOp = PPC::SELECT_CC_I4; |
5905 | else if (N->getValueType(0) == MVT::i64) |
5906 | SelectCCOp = PPC::SELECT_CC_I8; |
5907 | else if (N->getValueType(0) == MVT::f32) { |
5908 | if (Subtarget->hasP8Vector()) |
5909 | SelectCCOp = PPC::SELECT_CC_VSSRC; |
5910 | else if (Subtarget->hasSPE()) |
5911 | SelectCCOp = PPC::SELECT_CC_SPE4; |
5912 | else |
5913 | SelectCCOp = PPC::SELECT_CC_F4; |
5914 | } else if (N->getValueType(0) == MVT::f64) { |
5915 | if (Subtarget->hasVSX()) |
5916 | SelectCCOp = PPC::SELECT_CC_VSFRC; |
5917 | else if (Subtarget->hasSPE()) |
5918 | SelectCCOp = PPC::SELECT_CC_SPE; |
5919 | else |
5920 | SelectCCOp = PPC::SELECT_CC_F8; |
5921 | } else if (N->getValueType(0) == MVT::f128) |
5922 | SelectCCOp = PPC::SELECT_CC_F16; |
5923 | else if (Subtarget->hasSPE()) |
5924 | SelectCCOp = PPC::SELECT_CC_SPE; |
5925 | else if (N->getValueType(0) == MVT::v2f64 || |
5926 | N->getValueType(0) == MVT::v2i64) |
5927 | SelectCCOp = PPC::SELECT_CC_VSRC; |
5928 | else |
5929 | SelectCCOp = PPC::SELECT_CC_VRRC; |
5930 | |
5931 | SDValue Ops[] = { CCReg, N->getOperand(Num: 2), N->getOperand(Num: 3), |
5932 | getI32Imm(Imm: BROpc, dl) }; |
5933 | CurDAG->SelectNodeTo(N, MachineOpc: SelectCCOp, VT: N->getValueType(ResNo: 0), Ops); |
5934 | return; |
5935 | } |
5936 | case ISD::VECTOR_SHUFFLE: |
5937 | if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || |
5938 | N->getValueType(0) == MVT::v2i64)) { |
5939 | ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N); |
5940 | |
5941 | SDValue Op1 = N->getOperand(Num: SVN->getMaskElt(Idx: 0) < 2 ? 0 : 1), |
5942 | Op2 = N->getOperand(Num: SVN->getMaskElt(Idx: 1) < 2 ? 0 : 1); |
5943 | unsigned DM[2]; |
5944 | |
5945 | for (int i = 0; i < 2; ++i) |
5946 | if (SVN->getMaskElt(Idx: i) <= 0 || SVN->getMaskElt(Idx: i) == 2) |
5947 | DM[i] = 0; |
5948 | else |
5949 | DM[i] = 1; |
5950 | |
5951 | if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && |
5952 | Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && |
5953 | isa<LoadSDNode>(Val: Op1.getOperand(i: 0))) { |
5954 | LoadSDNode *LD = cast<LoadSDNode>(Val: Op1.getOperand(i: 0)); |
5955 | SDValue Base, Offset; |
5956 | |
5957 | if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && |
5958 | (LD->getMemoryVT() == MVT::f64 || |
5959 | LD->getMemoryVT() == MVT::i64) && |
5960 | SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { |
5961 | SDValue Chain = LD->getChain(); |
5962 | SDValue Ops[] = { Base, Offset, Chain }; |
5963 | MachineMemOperand *MemOp = LD->getMemOperand(); |
5964 | SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, |
5965 | N->getValueType(0), Ops); |
5966 | CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: NewN), NewMemRefs: {MemOp}); |
5967 | return; |
5968 | } |
5969 | } |
5970 | |
5971 | // For little endian, we must swap the input operands and adjust |
5972 | // the mask elements (reverse and invert them). |
5973 | if (Subtarget->isLittleEndian()) { |
5974 | std::swap(a&: Op1, b&: Op2); |
5975 | unsigned tmp = DM[0]; |
5976 | DM[0] = 1 - DM[1]; |
5977 | DM[1] = 1 - tmp; |
5978 | } |
5979 | |
5980 | SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, |
5981 | MVT::i32); |
5982 | SDValue Ops[] = { Op1, Op2, DMV }; |
5983 | CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); |
5984 | return; |
5985 | } |
5986 | |
5987 | break; |
5988 | case PPCISD::BDNZ: |
5989 | case PPCISD::BDZ: { |
5990 | bool IsPPC64 = Subtarget->isPPC64(); |
5991 | SDValue Ops[] = { N->getOperand(Num: 1), N->getOperand(Num: 0) }; |
5992 | CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ |
5993 | ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) |
5994 | : (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), |
5995 | MVT::Other, Ops); |
5996 | return; |
5997 | } |
5998 | case PPCISD::COND_BRANCH: { |
5999 | // Op #0 is the Chain. |
6000 | // Op #1 is the PPC::PRED_* number. |
6001 | // Op #2 is the CR# |
6002 | // Op #3 is the Dest MBB |
6003 | // Op #4 is the Flag. |
6004 | // Prevent PPC::PRED_* from being selected into LI. |
6005 | unsigned PCC = N->getConstantOperandVal(Num: 1); |
6006 | if (EnableBranchHint) |
6007 | PCC |= getBranchHint(PCC, FuncInfo: *FuncInfo, DestMBB: N->getOperand(Num: 3)); |
6008 | |
6009 | SDValue Pred = getI32Imm(Imm: PCC, dl); |
6010 | SDValue Ops[] = { Pred, N->getOperand(Num: 2), N->getOperand(Num: 3), |
6011 | N->getOperand(Num: 0), N->getOperand(Num: 4) }; |
6012 | CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
6013 | return; |
6014 | } |
6015 | case ISD::BR_CC: { |
6016 | if (tryFoldSWTestBRCC(N)) |
6017 | return; |
6018 | if (trySelectLoopCountIntrinsic(N)) |
6019 | return; |
6020 | ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get(); |
6021 | unsigned PCC = |
6022 | getPredicateForSetCC(CC, VT: N->getOperand(Num: 2).getValueType(), Subtarget); |
6023 | |
6024 | if (N->getOperand(2).getValueType() == MVT::i1) { |
6025 | unsigned Opc; |
6026 | bool Swap; |
6027 | switch (PCC) { |
6028 | default: llvm_unreachable("Unexpected Boolean-operand predicate" ); |
6029 | case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; |
6030 | case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; |
6031 | case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; |
6032 | case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; |
6033 | case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; |
6034 | case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; |
6035 | } |
6036 | |
6037 | // A signed comparison of i1 values produces the opposite result to an |
6038 | // unsigned one if the condition code includes less-than or greater-than. |
6039 | // This is because 1 is the most negative signed i1 number and the most |
6040 | // positive unsigned i1 number. The CR-logical operations used for such |
6041 | // comparisons are non-commutative so for signed comparisons vs. unsigned |
6042 | // ones, the input operands just need to be swapped. |
6043 | if (ISD::isSignedIntSetCC(Code: CC)) |
6044 | Swap = !Swap; |
6045 | |
6046 | SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, |
6047 | N->getOperand(Swap ? 3 : 2), |
6048 | N->getOperand(Swap ? 2 : 3)), 0); |
6049 | CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), |
6050 | N->getOperand(0)); |
6051 | return; |
6052 | } |
6053 | |
6054 | if (EnableBranchHint) |
6055 | PCC |= getBranchHint(PCC, FuncInfo: *FuncInfo, DestMBB: N->getOperand(Num: 4)); |
6056 | |
6057 | SDValue CondCode = SelectCC(LHS: N->getOperand(Num: 2), RHS: N->getOperand(Num: 3), CC, dl); |
6058 | SDValue Ops[] = { getI32Imm(Imm: PCC, dl), CondCode, |
6059 | N->getOperand(Num: 4), N->getOperand(Num: 0) }; |
6060 | CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); |
6061 | return; |
6062 | } |
6063 | case ISD::BRIND: { |
6064 | // FIXME: Should custom lower this. |
6065 | SDValue Chain = N->getOperand(Num: 0); |
6066 | SDValue Target = N->getOperand(Num: 1); |
6067 | unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; |
6068 | unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; |
6069 | Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, |
6070 | Chain), 0); |
6071 | CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); |
6072 | return; |
6073 | } |
6074 | case PPCISD::TOC_ENTRY: { |
6075 | const bool isPPC64 = Subtarget->isPPC64(); |
6076 | const bool isELFABI = Subtarget->isSVR4ABI(); |
6077 | const bool isAIXABI = Subtarget->isAIXABI(); |
6078 | |
6079 | // PowerPC only support small, medium and large code model. |
6080 | const CodeModel::Model CModel = getCodeModel(Subtarget: *Subtarget, TM, Node: N); |
6081 | |
6082 | assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) && |
6083 | "PowerPC doesn't support tiny or kernel code models." ); |
6084 | |
6085 | if (isAIXABI && CModel == CodeModel::Medium) |
6086 | report_fatal_error(reason: "Medium code model is not supported on AIX." ); |
6087 | |
6088 | // For 64-bit ELF small code model, we allow SelectCodeCommon to handle |
6089 | // this, selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. For AIX |
6090 | // small code model, we need to check for a toc-data attribute. |
6091 | if (isPPC64 && !isAIXABI && CModel == CodeModel::Small) |
6092 | break; |
6093 | |
6094 | auto replaceWith = [this, &dl](unsigned OpCode, SDNode *TocEntry, |
6095 | EVT OperandTy) { |
6096 | SDValue GA = TocEntry->getOperand(Num: 0); |
6097 | SDValue TocBase = TocEntry->getOperand(Num: 1); |
6098 | SDNode *MN = CurDAG->getMachineNode(Opcode: OpCode, dl, VT: OperandTy, Op1: GA, Op2: TocBase); |
6099 | transferMemOperands(N: TocEntry, Result: MN); |
6100 | ReplaceNode(F: TocEntry, T: MN); |
6101 | }; |
6102 | |
6103 | // Handle 32-bit small code model. |
6104 | if (!isPPC64 && CModel == CodeModel::Small) { |
6105 | // Transforms the ISD::TOC_ENTRY node to passed in Opcode, either |
6106 | // PPC::ADDItoc, or PPC::LWZtoc |
6107 | if (isELFABI) { |
6108 | assert(TM.isPositionIndependent() && |
6109 | "32-bit ELF can only have TOC entries in position independent" |
6110 | " code." ); |
6111 | // 32-bit ELF always uses a small code model toc access. |
6112 | replaceWith(PPC::LWZtoc, N, MVT::i32); |
6113 | return; |
6114 | } |
6115 | |
6116 | assert(isAIXABI && "ELF ABI already handled" ); |
6117 | |
6118 | if (hasTocDataAttr(Val: N->getOperand(Num: 0))) { |
6119 | replaceWith(PPC::ADDItoc, N, MVT::i32); |
6120 | return; |
6121 | } |
6122 | |
6123 | replaceWith(PPC::LWZtoc, N, MVT::i32); |
6124 | return; |
6125 | } |
6126 | |
6127 | if (isPPC64 && CModel == CodeModel::Small) { |
6128 | assert(isAIXABI && "ELF ABI handled in common SelectCode" ); |
6129 | |
6130 | if (hasTocDataAttr(Val: N->getOperand(Num: 0))) { |
6131 | replaceWith(PPC::ADDItoc8, N, MVT::i64); |
6132 | return; |
6133 | } |
6134 | // Break if it doesn't have toc data attribute. Proceed with common |
6135 | // SelectCode. |
6136 | break; |
6137 | } |
6138 | |
6139 | assert(CModel != CodeModel::Small && "All small code models handled." ); |
6140 | |
6141 | assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" |
6142 | " ELF/AIX or 32-bit AIX in the following." ); |
6143 | |
6144 | // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode |
6145 | // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code non |
6146 | // toc-data symbols. |
6147 | // We generate two instructions as described below. The first source |
6148 | // operand is a symbol reference. If it must be toc-referenced according to |
6149 | // Subtarget, we generate: |
6150 | // [32-bit AIX] |
6151 | // LWZtocL(@sym, ADDIStocHA(%r2, @sym)) |
6152 | // [64-bit ELF/AIX] |
6153 | // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) |
6154 | // Otherwise we generate: |
6155 | // ADDItocL8(ADDIStocHA8(%x2, @sym), @sym) |
6156 | |
6157 | // For large code model toc-data symbols we generate: |
6158 | // [32-bit AIX] |
6159 | // ADDItocL(ADDIStocHA(%x2, @sym), @sym) |
6160 | // [64-bit AIX] |
6161 | // Currently not supported. |
6162 | |
6163 | SDValue GA = N->getOperand(Num: 0); |
6164 | SDValue TOCbase = N->getOperand(Num: 1); |
6165 | |
6166 | EVT VT = isPPC64 ? MVT::i64 : MVT::i32; |
6167 | SDNode *Tmp = CurDAG->getMachineNode( |
6168 | isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); |
6169 | |
6170 | // On AIX if the symbol has the toc-data attribute it will be defined |
6171 | // in the TOC entry, so we use an ADDItocL similar to the medium code |
6172 | // model ELF abi. |
6173 | if (isAIXABI && hasTocDataAttr(Val: GA)) { |
6174 | if (isPPC64) |
6175 | report_fatal_error( |
6176 | reason: "64-bit large code model toc-data not yet supported" ); |
6177 | |
6178 | ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, VT, |
6179 | SDValue(Tmp, 0), GA)); |
6180 | return; |
6181 | } |
6182 | |
6183 | if (PPCLowering->isAccessedAsGotIndirect(N: GA)) { |
6184 | // If it is accessed as got-indirect, we need an extra LWZ/LD to load |
6185 | // the address. |
6186 | SDNode *MN = CurDAG->getMachineNode( |
6187 | isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0)); |
6188 | |
6189 | transferMemOperands(N, Result: MN); |
6190 | ReplaceNode(F: N, T: MN); |
6191 | return; |
6192 | } |
6193 | |
6194 | // Build the address relative to the TOC-pointer. |
6195 | ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL8, dl, MVT::i64, |
6196 | SDValue(Tmp, 0), GA)); |
6197 | return; |
6198 | } |
6199 | case PPCISD::PPC32_PICGOT: |
6200 | // Generate a PIC-safe GOT reference. |
6201 | assert(Subtarget->is32BitELFABI() && |
6202 | "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4" ); |
6203 | CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, |
6204 | PPCLowering->getPointerTy(CurDAG->getDataLayout()), |
6205 | MVT::i32); |
6206 | return; |
6207 | |
6208 | case PPCISD::VADD_SPLAT: { |
6209 | // This expands into one of three sequences, depending on whether |
6210 | // the first operand is odd or even, positive or negative. |
6211 | assert(isa<ConstantSDNode>(N->getOperand(0)) && |
6212 | isa<ConstantSDNode>(N->getOperand(1)) && |
6213 | "Invalid operand on VADD_SPLAT!" ); |
6214 | |
6215 | int Elt = N->getConstantOperandVal(Num: 0); |
6216 | int EltSize = N->getConstantOperandVal(Num: 1); |
6217 | unsigned Opc1, Opc2, Opc3; |
6218 | EVT VT; |
6219 | |
6220 | if (EltSize == 1) { |
6221 | Opc1 = PPC::VSPLTISB; |
6222 | Opc2 = PPC::VADDUBM; |
6223 | Opc3 = PPC::VSUBUBM; |
6224 | VT = MVT::v16i8; |
6225 | } else if (EltSize == 2) { |
6226 | Opc1 = PPC::VSPLTISH; |
6227 | Opc2 = PPC::VADDUHM; |
6228 | Opc3 = PPC::VSUBUHM; |
6229 | VT = MVT::v8i16; |
6230 | } else { |
6231 | assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!" ); |
6232 | Opc1 = PPC::VSPLTISW; |
6233 | Opc2 = PPC::VADDUWM; |
6234 | Opc3 = PPC::VSUBUWM; |
6235 | VT = MVT::v4i32; |
6236 | } |
6237 | |
6238 | if ((Elt & 1) == 0) { |
6239 | // Elt is even, in the range [-32,-18] + [16,30]. |
6240 | // |
6241 | // Convert: VADD_SPLAT elt, size |
6242 | // Into: tmp = VSPLTIS[BHW] elt |
6243 | // VADDU[BHW]M tmp, tmp |
6244 | // Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 |
6245 | SDValue EltVal = getI32Imm(Imm: Elt >> 1, dl); |
6246 | SDNode *Tmp = CurDAG->getMachineNode(Opcode: Opc1, dl, VT, Op1: EltVal); |
6247 | SDValue TmpVal = SDValue(Tmp, 0); |
6248 | ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc2, dl, VT, Op1: TmpVal, Op2: TmpVal)); |
6249 | return; |
6250 | } else if (Elt > 0) { |
6251 | // Elt is odd and positive, in the range [17,31]. |
6252 | // |
6253 | // Convert: VADD_SPLAT elt, size |
6254 | // Into: tmp1 = VSPLTIS[BHW] elt-16 |
6255 | // tmp2 = VSPLTIS[BHW] -16 |
6256 | // VSUBU[BHW]M tmp1, tmp2 |
6257 | SDValue EltVal = getI32Imm(Imm: Elt - 16, dl); |
6258 | SDNode *Tmp1 = CurDAG->getMachineNode(Opcode: Opc1, dl, VT, Op1: EltVal); |
6259 | EltVal = getI32Imm(Imm: -16, dl); |
6260 | SDNode *Tmp2 = CurDAG->getMachineNode(Opcode: Opc1, dl, VT, Op1: EltVal); |
6261 | ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc3, dl, VT, Op1: SDValue(Tmp1, 0), |
6262 | Op2: SDValue(Tmp2, 0))); |
6263 | return; |
6264 | } else { |
6265 | // Elt is odd and negative, in the range [-31,-17]. |
6266 | // |
6267 | // Convert: VADD_SPLAT elt, size |
6268 | // Into: tmp1 = VSPLTIS[BHW] elt+16 |
6269 | // tmp2 = VSPLTIS[BHW] -16 |
6270 | // VADDU[BHW]M tmp1, tmp2 |
6271 | SDValue EltVal = getI32Imm(Imm: Elt + 16, dl); |
6272 | SDNode *Tmp1 = CurDAG->getMachineNode(Opcode: Opc1, dl, VT, Op1: EltVal); |
6273 | EltVal = getI32Imm(Imm: -16, dl); |
6274 | SDNode *Tmp2 = CurDAG->getMachineNode(Opcode: Opc1, dl, VT, Op1: EltVal); |
6275 | ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc2, dl, VT, Op1: SDValue(Tmp1, 0), |
6276 | Op2: SDValue(Tmp2, 0))); |
6277 | return; |
6278 | } |
6279 | } |
6280 | case PPCISD::LD_SPLAT: { |
6281 | // Here we want to handle splat load for type v16i8 and v8i16 when there is |
6282 | // no direct move, we don't need to use stack for this case. If target has |
6283 | // direct move, we should be able to get the best selection in the .td file. |
6284 | if (!Subtarget->hasAltivec() || Subtarget->hasDirectMove()) |
6285 | break; |
6286 | |
6287 | EVT Type = N->getValueType(ResNo: 0); |
6288 | if (Type != MVT::v16i8 && Type != MVT::v8i16) |
6289 | break; |
6290 | |
6291 | // If the alignment for the load is 16 or bigger, we don't need the |
6292 | // permutated mask to get the required value. The value must be the 0 |
6293 | // element in big endian target or 7/15 in little endian target in the |
6294 | // result vsx register of lvx instruction. |
6295 | // Select the instruction in the .td file. |
6296 | if (cast<MemIntrinsicSDNode>(Val: N)->getAlign() >= Align(16) && |
6297 | isOffsetMultipleOf(N, Val: 16)) |
6298 | break; |
6299 | |
6300 | SDValue ZeroReg = |
6301 | CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, |
6302 | Subtarget->isPPC64() ? MVT::i64 : MVT::i32); |
6303 | unsigned LIOpcode = Subtarget->isPPC64() ? PPC::LI8 : PPC::LI; |
6304 | // v16i8 LD_SPLAT addr |
6305 | // ======> |
6306 | // Mask = LVSR/LVSL 0, addr |
6307 | // LoadLow = LVX 0, addr |
6308 | // Perm = VPERM LoadLow, LoadLow, Mask |
6309 | // Splat = VSPLTB 15/0, Perm |
6310 | // |
6311 | // v8i16 LD_SPLAT addr |
6312 | // ======> |
6313 | // Mask = LVSR/LVSL 0, addr |
6314 | // LoadLow = LVX 0, addr |
6315 | // LoadHigh = LVX (LI, 1), addr |
6316 | // Perm = VPERM LoadLow, LoadHigh, Mask |
6317 | // Splat = VSPLTH 7/0, Perm |
6318 | unsigned SplatOp = (Type == MVT::v16i8) ? PPC::VSPLTB : PPC::VSPLTH; |
6319 | unsigned SplatElemIndex = |
6320 | Subtarget->isLittleEndian() ? ((Type == MVT::v16i8) ? 15 : 7) : 0; |
6321 | |
6322 | SDNode *Mask = CurDAG->getMachineNode( |
6323 | Subtarget->isLittleEndian() ? PPC::LVSR : PPC::LVSL, dl, Type, ZeroReg, |
6324 | N->getOperand(1)); |
6325 | |
6326 | SDNode *LoadLow = |
6327 | CurDAG->getMachineNode(PPC::LVX, dl, MVT::v16i8, MVT::Other, |
6328 | {ZeroReg, N->getOperand(1), N->getOperand(0)}); |
6329 | |
6330 | SDNode *LoadHigh = LoadLow; |
6331 | if (Type == MVT::v8i16) { |
6332 | LoadHigh = CurDAG->getMachineNode( |
6333 | PPC::LVX, dl, MVT::v16i8, MVT::Other, |
6334 | {SDValue(CurDAG->getMachineNode( |
6335 | LIOpcode, dl, MVT::i32, |
6336 | CurDAG->getTargetConstant(1, dl, MVT::i8)), |
6337 | 0), |
6338 | N->getOperand(1), SDValue(LoadLow, 1)}); |
6339 | } |
6340 | |
6341 | CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: SDValue(LoadHigh, 1)); |
6342 | transferMemOperands(N, Result: LoadHigh); |
6343 | |
6344 | SDNode *Perm = |
6345 | CurDAG->getMachineNode(PPC::VPERM, dl, Type, SDValue(LoadLow, 0), |
6346 | SDValue(LoadHigh, 0), SDValue(Mask, 0)); |
6347 | CurDAG->SelectNodeTo(N, SplatOp, Type, |
6348 | CurDAG->getTargetConstant(SplatElemIndex, dl, MVT::i8), |
6349 | SDValue(Perm, 0)); |
6350 | return; |
6351 | } |
6352 | } |
6353 | |
6354 | SelectCode(N); |
6355 | } |
6356 | |
6357 | // If the target supports the cmpb instruction, do the idiom recognition here. |
6358 | // We don't do this as a DAG combine because we don't want to do it as nodes |
6359 | // are being combined (because we might miss part of the eventual idiom). We |
6360 | // don't want to do it during instruction selection because we want to reuse |
6361 | // the logic for lowering the masking operations already part of the |
6362 | // instruction selector. |
6363 | SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { |
6364 | SDLoc dl(N); |
6365 | |
6366 | assert(N->getOpcode() == ISD::OR && |
6367 | "Only OR nodes are supported for CMPB" ); |
6368 | |
6369 | SDValue Res; |
6370 | if (!Subtarget->hasCMPB()) |
6371 | return Res; |
6372 | |
6373 | if (N->getValueType(0) != MVT::i32 && |
6374 | N->getValueType(0) != MVT::i64) |
6375 | return Res; |
6376 | |
6377 | EVT VT = N->getValueType(ResNo: 0); |
6378 | |
6379 | SDValue RHS, LHS; |
6380 | bool BytesFound[8] = {false, false, false, false, false, false, false, false}; |
6381 | uint64_t Mask = 0, Alt = 0; |
6382 | |
6383 | auto IsByteSelectCC = [this](SDValue O, unsigned &b, |
6384 | uint64_t &Mask, uint64_t &Alt, |
6385 | SDValue &LHS, SDValue &RHS) { |
6386 | if (O.getOpcode() != ISD::SELECT_CC) |
6387 | return false; |
6388 | ISD::CondCode CC = cast<CondCodeSDNode>(Val: O.getOperand(i: 4))->get(); |
6389 | |
6390 | if (!isa<ConstantSDNode>(Val: O.getOperand(i: 2)) || |
6391 | !isa<ConstantSDNode>(Val: O.getOperand(i: 3))) |
6392 | return false; |
6393 | |
6394 | uint64_t PM = O.getConstantOperandVal(i: 2); |
6395 | uint64_t PAlt = O.getConstantOperandVal(i: 3); |
6396 | for (b = 0; b < 8; ++b) { |
6397 | uint64_t Mask = UINT64_C(0xFF) << (8*b); |
6398 | if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) |
6399 | break; |
6400 | } |
6401 | |
6402 | if (b == 8) |
6403 | return false; |
6404 | Mask |= PM; |
6405 | Alt |= PAlt; |
6406 | |
6407 | if (!isa<ConstantSDNode>(Val: O.getOperand(i: 1)) || |
6408 | O.getConstantOperandVal(i: 1) != 0) { |
6409 | SDValue Op0 = O.getOperand(i: 0), Op1 = O.getOperand(i: 1); |
6410 | if (Op0.getOpcode() == ISD::TRUNCATE) |
6411 | Op0 = Op0.getOperand(i: 0); |
6412 | if (Op1.getOpcode() == ISD::TRUNCATE) |
6413 | Op1 = Op1.getOperand(i: 0); |
6414 | |
6415 | if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && |
6416 | Op0.getOperand(i: 1) == Op1.getOperand(i: 1) && CC == ISD::SETEQ && |
6417 | isa<ConstantSDNode>(Val: Op0.getOperand(i: 1))) { |
6418 | |
6419 | unsigned Bits = Op0.getValueSizeInBits(); |
6420 | if (b != Bits/8-1) |
6421 | return false; |
6422 | if (Op0.getConstantOperandVal(i: 1) != Bits-8) |
6423 | return false; |
6424 | |
6425 | LHS = Op0.getOperand(i: 0); |
6426 | RHS = Op1.getOperand(i: 0); |
6427 | return true; |
6428 | } |
6429 | |
6430 | // When we have small integers (i16 to be specific), the form present |
6431 | // post-legalization uses SETULT in the SELECT_CC for the |
6432 | // higher-order byte, depending on the fact that the |
6433 | // even-higher-order bytes are known to all be zero, for example: |
6434 | // select_cc (xor $lhs, $rhs), 256, 65280, 0, setult |
6435 | // (so when the second byte is the same, because all higher-order |
6436 | // bits from bytes 3 and 4 are known to be zero, the result of the |
6437 | // xor can be at most 255) |
6438 | if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && |
6439 | isa<ConstantSDNode>(Val: O.getOperand(i: 1))) { |
6440 | |
6441 | uint64_t ULim = O.getConstantOperandVal(i: 1); |
6442 | if (ULim != (UINT64_C(1) << b*8)) |
6443 | return false; |
6444 | |
6445 | // Now we need to make sure that the upper bytes are known to be |
6446 | // zero. |
6447 | unsigned Bits = Op0.getValueSizeInBits(); |
6448 | if (!CurDAG->MaskedValueIsZero( |
6449 | Op: Op0, Mask: APInt::getHighBitsSet(numBits: Bits, hiBitsSet: Bits - (b + 1) * 8))) |
6450 | return false; |
6451 | |
6452 | LHS = Op0.getOperand(i: 0); |
6453 | RHS = Op0.getOperand(i: 1); |
6454 | return true; |
6455 | } |
6456 | |
6457 | return false; |
6458 | } |
6459 | |
6460 | if (CC != ISD::SETEQ) |
6461 | return false; |
6462 | |
6463 | SDValue Op = O.getOperand(i: 0); |
6464 | if (Op.getOpcode() == ISD::AND) { |
6465 | if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) |
6466 | return false; |
6467 | if (Op.getConstantOperandVal(i: 1) != (UINT64_C(0xFF) << (8*b))) |
6468 | return false; |
6469 | |
6470 | SDValue XOR = Op.getOperand(i: 0); |
6471 | if (XOR.getOpcode() == ISD::TRUNCATE) |
6472 | XOR = XOR.getOperand(i: 0); |
6473 | if (XOR.getOpcode() != ISD::XOR) |
6474 | return false; |
6475 | |
6476 | LHS = XOR.getOperand(i: 0); |
6477 | RHS = XOR.getOperand(i: 1); |
6478 | return true; |
6479 | } else if (Op.getOpcode() == ISD::SRL) { |
6480 | if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1))) |
6481 | return false; |
6482 | unsigned Bits = Op.getValueSizeInBits(); |
6483 | if (b != Bits/8-1) |
6484 | return false; |
6485 | if (Op.getConstantOperandVal(i: 1) != Bits-8) |
6486 | return false; |
6487 | |
6488 | SDValue XOR = Op.getOperand(i: 0); |
6489 | if (XOR.getOpcode() == ISD::TRUNCATE) |
6490 | XOR = XOR.getOperand(i: 0); |
6491 | if (XOR.getOpcode() != ISD::XOR) |
6492 | return false; |
6493 | |
6494 | LHS = XOR.getOperand(i: 0); |
6495 | RHS = XOR.getOperand(i: 1); |
6496 | return true; |
6497 | } |
6498 | |
6499 | return false; |
6500 | }; |
6501 | |
6502 | SmallVector<SDValue, 8> Queue(1, SDValue(N, 0)); |
6503 | while (!Queue.empty()) { |
6504 | SDValue V = Queue.pop_back_val(); |
6505 | |
6506 | for (const SDValue &O : V.getNode()->ops()) { |
6507 | unsigned b = 0; |
6508 | uint64_t M = 0, A = 0; |
6509 | SDValue OLHS, ORHS; |
6510 | if (O.getOpcode() == ISD::OR) { |
6511 | Queue.push_back(Elt: O); |
6512 | } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { |
6513 | if (!LHS) { |
6514 | LHS = OLHS; |
6515 | RHS = ORHS; |
6516 | BytesFound[b] = true; |
6517 | Mask |= M; |
6518 | Alt |= A; |
6519 | } else if ((LHS == ORHS && RHS == OLHS) || |
6520 | (RHS == ORHS && LHS == OLHS)) { |
6521 | BytesFound[b] = true; |
6522 | Mask |= M; |
6523 | Alt |= A; |
6524 | } else { |
6525 | return Res; |
6526 | } |
6527 | } else { |
6528 | return Res; |
6529 | } |
6530 | } |
6531 | } |
6532 | |
6533 | unsigned LastB = 0, BCnt = 0; |
6534 | for (unsigned i = 0; i < 8; ++i) |
6535 | if (BytesFound[LastB]) { |
6536 | ++BCnt; |
6537 | LastB = i; |
6538 | } |
6539 | |
6540 | if (!LastB || BCnt < 2) |
6541 | return Res; |
6542 | |
6543 | // Because we'll be zero-extending the output anyway if don't have a specific |
6544 | // value for each input byte (via the Mask), we can 'anyext' the inputs. |
6545 | if (LHS.getValueType() != VT) { |
6546 | LHS = CurDAG->getAnyExtOrTrunc(Op: LHS, DL: dl, VT); |
6547 | RHS = CurDAG->getAnyExtOrTrunc(Op: RHS, DL: dl, VT); |
6548 | } |
6549 | |
6550 | Res = CurDAG->getNode(Opcode: PPCISD::CMPB, DL: dl, VT, N1: LHS, N2: RHS); |
6551 | |
6552 | bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); |
6553 | if (NonTrivialMask && !Alt) { |
6554 | // Res = Mask & CMPB |
6555 | Res = CurDAG->getNode(Opcode: ISD::AND, DL: dl, VT, N1: Res, |
6556 | N2: CurDAG->getConstant(Val: Mask, DL: dl, VT)); |
6557 | } else if (Alt) { |
6558 | // Res = (CMPB & Mask) | (~CMPB & Alt) |
6559 | // Which, as suggested here: |
6560 | // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge |
6561 | // can be written as: |
6562 | // Res = Alt ^ ((Alt ^ Mask) & CMPB) |
6563 | // useful because the (Alt ^ Mask) can be pre-computed. |
6564 | Res = CurDAG->getNode(Opcode: ISD::AND, DL: dl, VT, N1: Res, |
6565 | N2: CurDAG->getConstant(Val: Mask ^ Alt, DL: dl, VT)); |
6566 | Res = CurDAG->getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Res, |
6567 | N2: CurDAG->getConstant(Val: Alt, DL: dl, VT)); |
6568 | } |
6569 | |
6570 | return Res; |
6571 | } |
6572 | |
6573 | // When CR bit registers are enabled, an extension of an i1 variable to a i32 |
6574 | // or i64 value is lowered in terms of a SELECT_I[48] operation, and thus |
6575 | // involves constant materialization of a 0 or a 1 or both. If the result of |
6576 | // the extension is then operated upon by some operator that can be constant |
6577 | // folded with a constant 0 or 1, and that constant can be materialized using |
6578 | // only one instruction (like a zero or one), then we should fold in those |
6579 | // operations with the select. |
6580 | void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { |
6581 | if (!Subtarget->useCRBits()) |
6582 | return; |
6583 | |
6584 | if (N->getOpcode() != ISD::ZERO_EXTEND && |
6585 | N->getOpcode() != ISD::SIGN_EXTEND && |
6586 | N->getOpcode() != ISD::ANY_EXTEND) |
6587 | return; |
6588 | |
6589 | if (N->getOperand(0).getValueType() != MVT::i1) |
6590 | return; |
6591 | |
6592 | if (!N->hasOneUse()) |
6593 | return; |
6594 | |
6595 | SDLoc dl(N); |
6596 | EVT VT = N->getValueType(ResNo: 0); |
6597 | SDValue Cond = N->getOperand(Num: 0); |
6598 | SDValue ConstTrue = |
6599 | CurDAG->getConstant(Val: N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, DL: dl, VT); |
6600 | SDValue ConstFalse = CurDAG->getConstant(Val: 0, DL: dl, VT); |
6601 | |
6602 | do { |
6603 | SDNode *User = *N->use_begin(); |
6604 | if (User->getNumOperands() != 2) |
6605 | break; |
6606 | |
6607 | auto TryFold = [this, N, User, dl](SDValue Val) { |
6608 | SDValue UserO0 = User->getOperand(Num: 0), UserO1 = User->getOperand(Num: 1); |
6609 | SDValue O0 = UserO0.getNode() == N ? Val : UserO0; |
6610 | SDValue O1 = UserO1.getNode() == N ? Val : UserO1; |
6611 | |
6612 | return CurDAG->FoldConstantArithmetic(Opcode: User->getOpcode(), DL: dl, |
6613 | VT: User->getValueType(ResNo: 0), Ops: {O0, O1}); |
6614 | }; |
6615 | |
6616 | // FIXME: When the semantics of the interaction between select and undef |
6617 | // are clearly defined, it may turn out to be unnecessary to break here. |
6618 | SDValue TrueRes = TryFold(ConstTrue); |
6619 | if (!TrueRes || TrueRes.isUndef()) |
6620 | break; |
6621 | SDValue FalseRes = TryFold(ConstFalse); |
6622 | if (!FalseRes || FalseRes.isUndef()) |
6623 | break; |
6624 | |
6625 | // For us to materialize these using one instruction, we must be able to |
6626 | // represent them as signed 16-bit integers. |
6627 | uint64_t True = TrueRes->getAsZExtVal(), False = FalseRes->getAsZExtVal(); |
6628 | if (!isInt<16>(x: True) || !isInt<16>(x: False)) |
6629 | break; |
6630 | |
6631 | // We can replace User with a new SELECT node, and try again to see if we |
6632 | // can fold the select with its user. |
6633 | Res = CurDAG->getSelect(DL: dl, VT: User->getValueType(ResNo: 0), Cond, LHS: TrueRes, RHS: FalseRes); |
6634 | N = User; |
6635 | ConstTrue = TrueRes; |
6636 | ConstFalse = FalseRes; |
6637 | } while (N->hasOneUse()); |
6638 | } |
6639 | |
6640 | void PPCDAGToDAGISel::PreprocessISelDAG() { |
6641 | SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
6642 | |
6643 | bool MadeChange = false; |
6644 | while (Position != CurDAG->allnodes_begin()) { |
6645 | SDNode *N = &*--Position; |
6646 | if (N->use_empty()) |
6647 | continue; |
6648 | |
6649 | SDValue Res; |
6650 | switch (N->getOpcode()) { |
6651 | default: break; |
6652 | case ISD::OR: |
6653 | Res = combineToCMPB(N); |
6654 | break; |
6655 | } |
6656 | |
6657 | if (!Res) |
6658 | foldBoolExts(Res, N); |
6659 | |
6660 | if (Res) { |
6661 | LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: " ); |
6662 | LLVM_DEBUG(N->dump(CurDAG)); |
6663 | LLVM_DEBUG(dbgs() << "\nNew: " ); |
6664 | LLVM_DEBUG(Res.getNode()->dump(CurDAG)); |
6665 | LLVM_DEBUG(dbgs() << "\n" ); |
6666 | |
6667 | CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res); |
6668 | MadeChange = true; |
6669 | } |
6670 | } |
6671 | |
6672 | if (MadeChange) |
6673 | CurDAG->RemoveDeadNodes(); |
6674 | } |
6675 | |
6676 | /// PostprocessISelDAG - Perform some late peephole optimizations |
6677 | /// on the DAG representation. |
6678 | void PPCDAGToDAGISel::PostprocessISelDAG() { |
6679 | // Skip peepholes at -O0. |
6680 | if (TM.getOptLevel() == CodeGenOptLevel::None) |
6681 | return; |
6682 | |
6683 | PeepholePPC64(); |
6684 | PeepholeCROps(); |
6685 | PeepholePPC64ZExt(); |
6686 | } |
6687 | |
6688 | // Check if all users of this node will become isel where the second operand |
6689 | // is the constant zero. If this is so, and if we can negate the condition, |
6690 | // then we can flip the true and false operands. This will allow the zero to |
6691 | // be folded with the isel so that we don't need to materialize a register |
6692 | // containing zero. |
6693 | bool PPCDAGToDAGISel::(SDNode *N) { |
6694 | for (const SDNode *User : N->uses()) { |
6695 | if (!User->isMachineOpcode()) |
6696 | return false; |
6697 | if (User->getMachineOpcode() != PPC::SELECT_I4 && |
6698 | User->getMachineOpcode() != PPC::SELECT_I8) |
6699 | return false; |
6700 | |
6701 | SDNode *Op1 = User->getOperand(Num: 1).getNode(); |
6702 | SDNode *Op2 = User->getOperand(Num: 2).getNode(); |
6703 | // If we have a degenerate select with two equal operands, swapping will |
6704 | // not do anything, and we may run into an infinite loop. |
6705 | if (Op1 == Op2) |
6706 | return false; |
6707 | |
6708 | if (!Op2->isMachineOpcode()) |
6709 | return false; |
6710 | |
6711 | if (Op2->getMachineOpcode() != PPC::LI && |
6712 | Op2->getMachineOpcode() != PPC::LI8) |
6713 | return false; |
6714 | |
6715 | if (!isNullConstant(V: Op2->getOperand(Num: 0))) |
6716 | return false; |
6717 | } |
6718 | |
6719 | return true; |
6720 | } |
6721 | |
6722 | void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { |
6723 | SmallVector<SDNode *, 4> ToReplace; |
6724 | for (SDNode *User : N->uses()) { |
6725 | assert((User->getMachineOpcode() == PPC::SELECT_I4 || |
6726 | User->getMachineOpcode() == PPC::SELECT_I8) && |
6727 | "Must have all select users" ); |
6728 | ToReplace.push_back(Elt: User); |
6729 | } |
6730 | |
6731 | for (SDNode *User : ToReplace) { |
6732 | SDNode *ResNode = |
6733 | CurDAG->getMachineNode(Opcode: User->getMachineOpcode(), dl: SDLoc(User), |
6734 | VT: User->getValueType(ResNo: 0), Op1: User->getOperand(Num: 0), |
6735 | Op2: User->getOperand(Num: 2), |
6736 | Op3: User->getOperand(Num: 1)); |
6737 | |
6738 | LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: " ); |
6739 | LLVM_DEBUG(User->dump(CurDAG)); |
6740 | LLVM_DEBUG(dbgs() << "\nNew: " ); |
6741 | LLVM_DEBUG(ResNode->dump(CurDAG)); |
6742 | LLVM_DEBUG(dbgs() << "\n" ); |
6743 | |
6744 | ReplaceUses(F: User, T: ResNode); |
6745 | } |
6746 | } |
6747 | |
6748 | void PPCDAGToDAGISel::PeepholeCROps() { |
6749 | bool IsModified; |
6750 | do { |
6751 | IsModified = false; |
6752 | for (SDNode &Node : CurDAG->allnodes()) { |
6753 | MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Val: &Node); |
6754 | if (!MachineNode || MachineNode->use_empty()) |
6755 | continue; |
6756 | SDNode *ResNode = MachineNode; |
6757 | |
6758 | bool Op1Set = false, Op1Unset = false, |
6759 | Op1Not = false, |
6760 | Op2Set = false, Op2Unset = false, |
6761 | Op2Not = false; |
6762 | |
6763 | unsigned Opcode = MachineNode->getMachineOpcode(); |
6764 | switch (Opcode) { |
6765 | default: break; |
6766 | case PPC::CRAND: |
6767 | case PPC::CRNAND: |
6768 | case PPC::CROR: |
6769 | case PPC::CRXOR: |
6770 | case PPC::CRNOR: |
6771 | case PPC::CREQV: |
6772 | case PPC::CRANDC: |
6773 | case PPC::CRORC: { |
6774 | SDValue Op = MachineNode->getOperand(Num: 1); |
6775 | if (Op.isMachineOpcode()) { |
6776 | if (Op.getMachineOpcode() == PPC::CRSET) |
6777 | Op2Set = true; |
6778 | else if (Op.getMachineOpcode() == PPC::CRUNSET) |
6779 | Op2Unset = true; |
6780 | else if ((Op.getMachineOpcode() == PPC::CRNOR && |
6781 | Op.getOperand(0) == Op.getOperand(1)) || |
6782 | Op.getMachineOpcode() == PPC::CRNOT) |
6783 | Op2Not = true; |
6784 | } |
6785 | [[fallthrough]]; |
6786 | } |
6787 | case PPC::BC: |
6788 | case PPC::BCn: |
6789 | case PPC::SELECT_I4: |
6790 | case PPC::SELECT_I8: |
6791 | case PPC::SELECT_F4: |
6792 | case PPC::SELECT_F8: |
6793 | case PPC::SELECT_SPE: |
6794 | case PPC::SELECT_SPE4: |
6795 | case PPC::SELECT_VRRC: |
6796 | case PPC::SELECT_VSFRC: |
6797 | case PPC::SELECT_VSSRC: |
6798 | case PPC::SELECT_VSRC: { |
6799 | SDValue Op = MachineNode->getOperand(Num: 0); |
6800 | if (Op.isMachineOpcode()) { |
6801 | if (Op.getMachineOpcode() == PPC::CRSET) |
6802 | Op1Set = true; |
6803 | else if (Op.getMachineOpcode() == PPC::CRUNSET) |
6804 | Op1Unset = true; |
6805 | else if ((Op.getMachineOpcode() == PPC::CRNOR && |
6806 | Op.getOperand(0) == Op.getOperand(1)) || |
6807 | Op.getMachineOpcode() == PPC::CRNOT) |
6808 | Op1Not = true; |
6809 | } |
6810 | } |
6811 | break; |
6812 | } |
6813 | |
6814 | bool SelectSwap = false; |
6815 | switch (Opcode) { |
6816 | default: break; |
6817 | case PPC::CRAND: |
6818 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
6819 | // x & x = x |
6820 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
6821 | else if (Op1Set) |
6822 | // 1 & y = y |
6823 | ResNode = MachineNode->getOperand(Num: 1).getNode(); |
6824 | else if (Op2Set) |
6825 | // x & 1 = x |
6826 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
6827 | else if (Op1Unset || Op2Unset) |
6828 | // x & 0 = 0 & y = 0 |
6829 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6830 | MVT::i1); |
6831 | else if (Op1Not) |
6832 | // ~x & y = andc(y, x) |
6833 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6834 | MVT::i1, MachineNode->getOperand(1), |
6835 | MachineNode->getOperand(0). |
6836 | getOperand(0)); |
6837 | else if (Op2Not) |
6838 | // x & ~y = andc(x, y) |
6839 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6840 | MVT::i1, MachineNode->getOperand(0), |
6841 | MachineNode->getOperand(1). |
6842 | getOperand(0)); |
6843 | else if (AllUsersSelectZero(N: MachineNode)) { |
6844 | ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
6845 | MVT::i1, MachineNode->getOperand(0), |
6846 | MachineNode->getOperand(1)); |
6847 | SelectSwap = true; |
6848 | } |
6849 | break; |
6850 | case PPC::CRNAND: |
6851 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
6852 | // nand(x, x) -> nor(x, x) |
6853 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6854 | MVT::i1, MachineNode->getOperand(0), |
6855 | MachineNode->getOperand(0)); |
6856 | else if (Op1Set) |
6857 | // nand(1, y) -> nor(y, y) |
6858 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6859 | MVT::i1, MachineNode->getOperand(1), |
6860 | MachineNode->getOperand(1)); |
6861 | else if (Op2Set) |
6862 | // nand(x, 1) -> nor(x, x) |
6863 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6864 | MVT::i1, MachineNode->getOperand(0), |
6865 | MachineNode->getOperand(0)); |
6866 | else if (Op1Unset || Op2Unset) |
6867 | // nand(x, 0) = nand(0, y) = 1 |
6868 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
6869 | MVT::i1); |
6870 | else if (Op1Not) |
6871 | // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) |
6872 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6873 | MVT::i1, MachineNode->getOperand(0). |
6874 | getOperand(0), |
6875 | MachineNode->getOperand(1)); |
6876 | else if (Op2Not) |
6877 | // nand(x, ~y) = ~x | y = orc(y, x) |
6878 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6879 | MVT::i1, MachineNode->getOperand(1). |
6880 | getOperand(0), |
6881 | MachineNode->getOperand(0)); |
6882 | else if (AllUsersSelectZero(N: MachineNode)) { |
6883 | ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
6884 | MVT::i1, MachineNode->getOperand(0), |
6885 | MachineNode->getOperand(1)); |
6886 | SelectSwap = true; |
6887 | } |
6888 | break; |
6889 | case PPC::CROR: |
6890 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
6891 | // x | x = x |
6892 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
6893 | else if (Op1Set || Op2Set) |
6894 | // x | 1 = 1 | y = 1 |
6895 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
6896 | MVT::i1); |
6897 | else if (Op1Unset) |
6898 | // 0 | y = y |
6899 | ResNode = MachineNode->getOperand(Num: 1).getNode(); |
6900 | else if (Op2Unset) |
6901 | // x | 0 = x |
6902 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
6903 | else if (Op1Not) |
6904 | // ~x | y = orc(y, x) |
6905 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6906 | MVT::i1, MachineNode->getOperand(1), |
6907 | MachineNode->getOperand(0). |
6908 | getOperand(0)); |
6909 | else if (Op2Not) |
6910 | // x | ~y = orc(x, y) |
6911 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
6912 | MVT::i1, MachineNode->getOperand(0), |
6913 | MachineNode->getOperand(1). |
6914 | getOperand(0)); |
6915 | else if (AllUsersSelectZero(N: MachineNode)) { |
6916 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6917 | MVT::i1, MachineNode->getOperand(0), |
6918 | MachineNode->getOperand(1)); |
6919 | SelectSwap = true; |
6920 | } |
6921 | break; |
6922 | case PPC::CRXOR: |
6923 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
6924 | // xor(x, x) = 0 |
6925 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6926 | MVT::i1); |
6927 | else if (Op1Set) |
6928 | // xor(1, y) -> nor(y, y) |
6929 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6930 | MVT::i1, MachineNode->getOperand(1), |
6931 | MachineNode->getOperand(1)); |
6932 | else if (Op2Set) |
6933 | // xor(x, 1) -> nor(x, x) |
6934 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6935 | MVT::i1, MachineNode->getOperand(0), |
6936 | MachineNode->getOperand(0)); |
6937 | else if (Op1Unset) |
6938 | // xor(0, y) = y |
6939 | ResNode = MachineNode->getOperand(Num: 1).getNode(); |
6940 | else if (Op2Unset) |
6941 | // xor(x, 0) = x |
6942 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
6943 | else if (Op1Not) |
6944 | // xor(~x, y) = eqv(x, y) |
6945 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6946 | MVT::i1, MachineNode->getOperand(0). |
6947 | getOperand(0), |
6948 | MachineNode->getOperand(1)); |
6949 | else if (Op2Not) |
6950 | // xor(x, ~y) = eqv(x, y) |
6951 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6952 | MVT::i1, MachineNode->getOperand(0), |
6953 | MachineNode->getOperand(1). |
6954 | getOperand(0)); |
6955 | else if (AllUsersSelectZero(N: MachineNode)) { |
6956 | ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), |
6957 | MVT::i1, MachineNode->getOperand(0), |
6958 | MachineNode->getOperand(1)); |
6959 | SelectSwap = true; |
6960 | } |
6961 | break; |
6962 | case PPC::CRNOR: |
6963 | if (Op1Set || Op2Set) |
6964 | // nor(1, y) -> 0 |
6965 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
6966 | MVT::i1); |
6967 | else if (Op1Unset) |
6968 | // nor(0, y) = ~y -> nor(y, y) |
6969 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6970 | MVT::i1, MachineNode->getOperand(1), |
6971 | MachineNode->getOperand(1)); |
6972 | else if (Op2Unset) |
6973 | // nor(x, 0) = ~x |
6974 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
6975 | MVT::i1, MachineNode->getOperand(0), |
6976 | MachineNode->getOperand(0)); |
6977 | else if (Op1Not) |
6978 | // nor(~x, y) = andc(x, y) |
6979 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6980 | MVT::i1, MachineNode->getOperand(0). |
6981 | getOperand(0), |
6982 | MachineNode->getOperand(1)); |
6983 | else if (Op2Not) |
6984 | // nor(x, ~y) = andc(y, x) |
6985 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
6986 | MVT::i1, MachineNode->getOperand(1). |
6987 | getOperand(0), |
6988 | MachineNode->getOperand(0)); |
6989 | else if (AllUsersSelectZero(N: MachineNode)) { |
6990 | ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
6991 | MVT::i1, MachineNode->getOperand(0), |
6992 | MachineNode->getOperand(1)); |
6993 | SelectSwap = true; |
6994 | } |
6995 | break; |
6996 | case PPC::CREQV: |
6997 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
6998 | // eqv(x, x) = 1 |
6999 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
7000 | MVT::i1); |
7001 | else if (Op1Set) |
7002 | // eqv(1, y) = y |
7003 | ResNode = MachineNode->getOperand(Num: 1).getNode(); |
7004 | else if (Op2Set) |
7005 | // eqv(x, 1) = x |
7006 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
7007 | else if (Op1Unset) |
7008 | // eqv(0, y) = ~y -> nor(y, y) |
7009 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7010 | MVT::i1, MachineNode->getOperand(1), |
7011 | MachineNode->getOperand(1)); |
7012 | else if (Op2Unset) |
7013 | // eqv(x, 0) = ~x |
7014 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7015 | MVT::i1, MachineNode->getOperand(0), |
7016 | MachineNode->getOperand(0)); |
7017 | else if (Op1Not) |
7018 | // eqv(~x, y) = xor(x, y) |
7019 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
7020 | MVT::i1, MachineNode->getOperand(0). |
7021 | getOperand(0), |
7022 | MachineNode->getOperand(1)); |
7023 | else if (Op2Not) |
7024 | // eqv(x, ~y) = xor(x, y) |
7025 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
7026 | MVT::i1, MachineNode->getOperand(0), |
7027 | MachineNode->getOperand(1). |
7028 | getOperand(0)); |
7029 | else if (AllUsersSelectZero(N: MachineNode)) { |
7030 | ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), |
7031 | MVT::i1, MachineNode->getOperand(0), |
7032 | MachineNode->getOperand(1)); |
7033 | SelectSwap = true; |
7034 | } |
7035 | break; |
7036 | case PPC::CRANDC: |
7037 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
7038 | // andc(x, x) = 0 |
7039 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
7040 | MVT::i1); |
7041 | else if (Op1Set) |
7042 | // andc(1, y) = ~y |
7043 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7044 | MVT::i1, MachineNode->getOperand(1), |
7045 | MachineNode->getOperand(1)); |
7046 | else if (Op1Unset || Op2Set) |
7047 | // andc(0, y) = andc(x, 1) = 0 |
7048 | ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), |
7049 | MVT::i1); |
7050 | else if (Op2Unset) |
7051 | // andc(x, 0) = x |
7052 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
7053 | else if (Op1Not) |
7054 | // andc(~x, y) = ~(x | y) = nor(x, y) |
7055 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7056 | MVT::i1, MachineNode->getOperand(0). |
7057 | getOperand(0), |
7058 | MachineNode->getOperand(1)); |
7059 | else if (Op2Not) |
7060 | // andc(x, ~y) = x & y |
7061 | ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), |
7062 | MVT::i1, MachineNode->getOperand(0), |
7063 | MachineNode->getOperand(1). |
7064 | getOperand(0)); |
7065 | else if (AllUsersSelectZero(N: MachineNode)) { |
7066 | ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), |
7067 | MVT::i1, MachineNode->getOperand(1), |
7068 | MachineNode->getOperand(0)); |
7069 | SelectSwap = true; |
7070 | } |
7071 | break; |
7072 | case PPC::CRORC: |
7073 | if (MachineNode->getOperand(Num: 0) == MachineNode->getOperand(Num: 1)) |
7074 | // orc(x, x) = 1 |
7075 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
7076 | MVT::i1); |
7077 | else if (Op1Set || Op2Unset) |
7078 | // orc(1, y) = orc(x, 0) = 1 |
7079 | ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), |
7080 | MVT::i1); |
7081 | else if (Op2Set) |
7082 | // orc(x, 1) = x |
7083 | ResNode = MachineNode->getOperand(Num: 0).getNode(); |
7084 | else if (Op1Unset) |
7085 | // orc(0, y) = ~y |
7086 | ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), |
7087 | MVT::i1, MachineNode->getOperand(1), |
7088 | MachineNode->getOperand(1)); |
7089 | else if (Op1Not) |
7090 | // orc(~x, y) = ~(x & y) = nand(x, y) |
7091 | ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), |
7092 | MVT::i1, MachineNode->getOperand(0). |
7093 | getOperand(0), |
7094 | MachineNode->getOperand(1)); |
7095 | else if (Op2Not) |
7096 | // orc(x, ~y) = x | y |
7097 | ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), |
7098 | MVT::i1, MachineNode->getOperand(0), |
7099 | MachineNode->getOperand(1). |
7100 | getOperand(0)); |
7101 | else if (AllUsersSelectZero(N: MachineNode)) { |
7102 | ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), |
7103 | MVT::i1, MachineNode->getOperand(1), |
7104 | MachineNode->getOperand(0)); |
7105 | SelectSwap = true; |
7106 | } |
7107 | break; |
7108 | case PPC::SELECT_I4: |
7109 | case PPC::SELECT_I8: |
7110 | case PPC::SELECT_F4: |
7111 | case PPC::SELECT_F8: |
7112 | case PPC::SELECT_SPE: |
7113 | case PPC::SELECT_SPE4: |
7114 | case PPC::SELECT_VRRC: |
7115 | case PPC::SELECT_VSFRC: |
7116 | case PPC::SELECT_VSSRC: |
7117 | case PPC::SELECT_VSRC: |
7118 | if (Op1Set) |
7119 | ResNode = MachineNode->getOperand(Num: 1).getNode(); |
7120 | else if (Op1Unset) |
7121 | ResNode = MachineNode->getOperand(Num: 2).getNode(); |
7122 | else if (Op1Not) |
7123 | ResNode = CurDAG->getMachineNode(Opcode: MachineNode->getMachineOpcode(), |
7124 | dl: SDLoc(MachineNode), |
7125 | VT: MachineNode->getValueType(ResNo: 0), |
7126 | Op1: MachineNode->getOperand(Num: 0). |
7127 | getOperand(i: 0), |
7128 | Op2: MachineNode->getOperand(Num: 2), |
7129 | Op3: MachineNode->getOperand(Num: 1)); |
7130 | break; |
7131 | case PPC::BC: |
7132 | case PPC::BCn: |
7133 | if (Op1Not) |
7134 | ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : |
7135 | PPC::BC, |
7136 | SDLoc(MachineNode), |
7137 | MVT::Other, |
7138 | MachineNode->getOperand(0). |
7139 | getOperand(0), |
7140 | MachineNode->getOperand(1), |
7141 | MachineNode->getOperand(2)); |
7142 | // FIXME: Handle Op1Set, Op1Unset here too. |
7143 | break; |
7144 | } |
7145 | |
7146 | // If we're inverting this node because it is used only by selects that |
7147 | // we'd like to swap, then swap the selects before the node replacement. |
7148 | if (SelectSwap) |
7149 | SwapAllSelectUsers(N: MachineNode); |
7150 | |
7151 | if (ResNode != MachineNode) { |
7152 | LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: " ); |
7153 | LLVM_DEBUG(MachineNode->dump(CurDAG)); |
7154 | LLVM_DEBUG(dbgs() << "\nNew: " ); |
7155 | LLVM_DEBUG(ResNode->dump(CurDAG)); |
7156 | LLVM_DEBUG(dbgs() << "\n" ); |
7157 | |
7158 | ReplaceUses(F: MachineNode, T: ResNode); |
7159 | IsModified = true; |
7160 | } |
7161 | } |
7162 | if (IsModified) |
7163 | CurDAG->RemoveDeadNodes(); |
7164 | } while (IsModified); |
7165 | } |
7166 | |
7167 | // Gather the set of 32-bit operations that are known to have their |
7168 | // higher-order 32 bits zero, where ToPromote contains all such operations. |
7169 | static bool PeepholePPC64ZExtGather(SDValue Op32, |
7170 | SmallPtrSetImpl<SDNode *> &ToPromote) { |
7171 | if (!Op32.isMachineOpcode()) |
7172 | return false; |
7173 | |
7174 | // First, check for the "frontier" instructions (those that will clear the |
7175 | // higher-order 32 bits. |
7176 | |
7177 | // For RLWINM and RLWNM, we need to make sure that the mask does not wrap |
7178 | // around. If it does not, then these instructions will clear the |
7179 | // higher-order bits. |
7180 | if ((Op32.getMachineOpcode() == PPC::RLWINM || |
7181 | Op32.getMachineOpcode() == PPC::RLWNM) && |
7182 | Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { |
7183 | ToPromote.insert(Ptr: Op32.getNode()); |
7184 | return true; |
7185 | } |
7186 | |
7187 | // SLW and SRW always clear the higher-order bits. |
7188 | if (Op32.getMachineOpcode() == PPC::SLW || |
7189 | Op32.getMachineOpcode() == PPC::SRW) { |
7190 | ToPromote.insert(Ptr: Op32.getNode()); |
7191 | return true; |
7192 | } |
7193 | |
7194 | // For LI and LIS, we need the immediate to be positive (so that it is not |
7195 | // sign extended). |
7196 | if (Op32.getMachineOpcode() == PPC::LI || |
7197 | Op32.getMachineOpcode() == PPC::LIS) { |
7198 | if (!isUInt<15>(x: Op32.getConstantOperandVal(i: 0))) |
7199 | return false; |
7200 | |
7201 | ToPromote.insert(Ptr: Op32.getNode()); |
7202 | return true; |
7203 | } |
7204 | |
7205 | // LHBRX and LWBRX always clear the higher-order bits. |
7206 | if (Op32.getMachineOpcode() == PPC::LHBRX || |
7207 | Op32.getMachineOpcode() == PPC::LWBRX) { |
7208 | ToPromote.insert(Ptr: Op32.getNode()); |
7209 | return true; |
7210 | } |
7211 | |
7212 | // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. |
7213 | if (Op32.getMachineOpcode() == PPC::CNTLZW || |
7214 | Op32.getMachineOpcode() == PPC::CNTTZW) { |
7215 | ToPromote.insert(Ptr: Op32.getNode()); |
7216 | return true; |
7217 | } |
7218 | |
7219 | // Next, check for those instructions we can look through. |
7220 | |
7221 | // Assuming the mask does not wrap around, then the higher-order bits are |
7222 | // taken directly from the first operand. |
7223 | if (Op32.getMachineOpcode() == PPC::RLWIMI && |
7224 | Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { |
7225 | SmallPtrSet<SDNode *, 16> ToPromote1; |
7226 | if (!PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: 0), ToPromote&: ToPromote1)) |
7227 | return false; |
7228 | |
7229 | ToPromote.insert(Ptr: Op32.getNode()); |
7230 | ToPromote.insert(I: ToPromote1.begin(), E: ToPromote1.end()); |
7231 | return true; |
7232 | } |
7233 | |
7234 | // For OR, the higher-order bits are zero if that is true for both operands. |
7235 | // For SELECT_I4, the same is true (but the relevant operand numbers are |
7236 | // shifted by 1). |
7237 | if (Op32.getMachineOpcode() == PPC::OR || |
7238 | Op32.getMachineOpcode() == PPC::SELECT_I4) { |
7239 | unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; |
7240 | SmallPtrSet<SDNode *, 16> ToPromote1; |
7241 | if (!PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: B+0), ToPromote&: ToPromote1)) |
7242 | return false; |
7243 | if (!PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: B+1), ToPromote&: ToPromote1)) |
7244 | return false; |
7245 | |
7246 | ToPromote.insert(Ptr: Op32.getNode()); |
7247 | ToPromote.insert(I: ToPromote1.begin(), E: ToPromote1.end()); |
7248 | return true; |
7249 | } |
7250 | |
7251 | // For ORI and ORIS, we need the higher-order bits of the first operand to be |
7252 | // zero, and also for the constant to be positive (so that it is not sign |
7253 | // extended). |
7254 | if (Op32.getMachineOpcode() == PPC::ORI || |
7255 | Op32.getMachineOpcode() == PPC::ORIS) { |
7256 | SmallPtrSet<SDNode *, 16> ToPromote1; |
7257 | if (!PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: 0), ToPromote&: ToPromote1)) |
7258 | return false; |
7259 | if (!isUInt<15>(x: Op32.getConstantOperandVal(i: 1))) |
7260 | return false; |
7261 | |
7262 | ToPromote.insert(Ptr: Op32.getNode()); |
7263 | ToPromote.insert(I: ToPromote1.begin(), E: ToPromote1.end()); |
7264 | return true; |
7265 | } |
7266 | |
7267 | // The higher-order bits of AND are zero if that is true for at least one of |
7268 | // the operands. |
7269 | if (Op32.getMachineOpcode() == PPC::AND) { |
7270 | SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2; |
7271 | bool Op0OK = |
7272 | PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: 0), ToPromote&: ToPromote1); |
7273 | bool Op1OK = |
7274 | PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: 1), ToPromote&: ToPromote2); |
7275 | if (!Op0OK && !Op1OK) |
7276 | return false; |
7277 | |
7278 | ToPromote.insert(Ptr: Op32.getNode()); |
7279 | |
7280 | if (Op0OK) |
7281 | ToPromote.insert(I: ToPromote1.begin(), E: ToPromote1.end()); |
7282 | |
7283 | if (Op1OK) |
7284 | ToPromote.insert(I: ToPromote2.begin(), E: ToPromote2.end()); |
7285 | |
7286 | return true; |
7287 | } |
7288 | |
7289 | // For ANDI and ANDIS, the higher-order bits are zero if either that is true |
7290 | // of the first operand, or if the second operand is positive (so that it is |
7291 | // not sign extended). |
7292 | if (Op32.getMachineOpcode() == PPC::ANDI_rec || |
7293 | Op32.getMachineOpcode() == PPC::ANDIS_rec) { |
7294 | SmallPtrSet<SDNode *, 16> ToPromote1; |
7295 | bool Op0OK = |
7296 | PeepholePPC64ZExtGather(Op32: Op32.getOperand(i: 0), ToPromote&: ToPromote1); |
7297 | bool Op1OK = isUInt<15>(x: Op32.getConstantOperandVal(i: 1)); |
7298 | if (!Op0OK && !Op1OK) |
7299 | return false; |
7300 | |
7301 | ToPromote.insert(Ptr: Op32.getNode()); |
7302 | |
7303 | if (Op0OK) |
7304 | ToPromote.insert(I: ToPromote1.begin(), E: ToPromote1.end()); |
7305 | |
7306 | return true; |
7307 | } |
7308 | |
7309 | return false; |
7310 | } |
7311 | |
7312 | void PPCDAGToDAGISel::PeepholePPC64ZExt() { |
7313 | if (!Subtarget->isPPC64()) |
7314 | return; |
7315 | |
7316 | // When we zero-extend from i32 to i64, we use a pattern like this: |
7317 | // def : Pat<(i64 (zext i32:$in)), |
7318 | // (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), |
7319 | // 0, 32)>; |
7320 | // There are several 32-bit shift/rotate instructions, however, that will |
7321 | // clear the higher-order bits of their output, rendering the RLDICL |
7322 | // unnecessary. When that happens, we remove it here, and redefine the |
7323 | // relevant 32-bit operation to be a 64-bit operation. |
7324 | |
7325 | SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
7326 | |
7327 | bool MadeChange = false; |
7328 | while (Position != CurDAG->allnodes_begin()) { |
7329 | SDNode *N = &*--Position; |
7330 | // Skip dead nodes and any non-machine opcodes. |
7331 | if (N->use_empty() || !N->isMachineOpcode()) |
7332 | continue; |
7333 | |
7334 | if (N->getMachineOpcode() != PPC::RLDICL) |
7335 | continue; |
7336 | |
7337 | if (N->getConstantOperandVal(Num: 1) != 0 || |
7338 | N->getConstantOperandVal(Num: 2) != 32) |
7339 | continue; |
7340 | |
7341 | SDValue ISR = N->getOperand(Num: 0); |
7342 | if (!ISR.isMachineOpcode() || |
7343 | ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) |
7344 | continue; |
7345 | |
7346 | if (!ISR.hasOneUse()) |
7347 | continue; |
7348 | |
7349 | if (ISR.getConstantOperandVal(2) != PPC::sub_32) |
7350 | continue; |
7351 | |
7352 | SDValue IDef = ISR.getOperand(i: 0); |
7353 | if (!IDef.isMachineOpcode() || |
7354 | IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) |
7355 | continue; |
7356 | |
7357 | // We now know that we're looking at a canonical i32 -> i64 zext. See if we |
7358 | // can get rid of it. |
7359 | |
7360 | SDValue Op32 = ISR->getOperand(Num: 1); |
7361 | if (!Op32.isMachineOpcode()) |
7362 | continue; |
7363 | |
7364 | // There are some 32-bit instructions that always clear the high-order 32 |
7365 | // bits, there are also some instructions (like AND) that we can look |
7366 | // through. |
7367 | SmallPtrSet<SDNode *, 16> ToPromote; |
7368 | if (!PeepholePPC64ZExtGather(Op32, ToPromote)) |
7369 | continue; |
7370 | |
7371 | // If the ToPromote set contains nodes that have uses outside of the set |
7372 | // (except for the original INSERT_SUBREG), then abort the transformation. |
7373 | bool OutsideUse = false; |
7374 | for (SDNode *PN : ToPromote) { |
7375 | for (SDNode *UN : PN->uses()) { |
7376 | if (!ToPromote.count(Ptr: UN) && UN != ISR.getNode()) { |
7377 | OutsideUse = true; |
7378 | break; |
7379 | } |
7380 | } |
7381 | |
7382 | if (OutsideUse) |
7383 | break; |
7384 | } |
7385 | if (OutsideUse) |
7386 | continue; |
7387 | |
7388 | MadeChange = true; |
7389 | |
7390 | // We now know that this zero extension can be removed by promoting to |
7391 | // nodes in ToPromote to 64-bit operations, where for operations in the |
7392 | // frontier of the set, we need to insert INSERT_SUBREGs for their |
7393 | // operands. |
7394 | for (SDNode *PN : ToPromote) { |
7395 | unsigned NewOpcode; |
7396 | switch (PN->getMachineOpcode()) { |
7397 | default: |
7398 | llvm_unreachable("Don't know the 64-bit variant of this instruction" ); |
7399 | case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; |
7400 | case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; |
7401 | case PPC::SLW: NewOpcode = PPC::SLW8; break; |
7402 | case PPC::SRW: NewOpcode = PPC::SRW8; break; |
7403 | case PPC::LI: NewOpcode = PPC::LI8; break; |
7404 | case PPC::LIS: NewOpcode = PPC::LIS8; break; |
7405 | case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; |
7406 | case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; |
7407 | case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; |
7408 | case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break; |
7409 | case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; |
7410 | case PPC::OR: NewOpcode = PPC::OR8; break; |
7411 | case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; |
7412 | case PPC::ORI: NewOpcode = PPC::ORI8; break; |
7413 | case PPC::ORIS: NewOpcode = PPC::ORIS8; break; |
7414 | case PPC::AND: NewOpcode = PPC::AND8; break; |
7415 | case PPC::ANDI_rec: |
7416 | NewOpcode = PPC::ANDI8_rec; |
7417 | break; |
7418 | case PPC::ANDIS_rec: |
7419 | NewOpcode = PPC::ANDIS8_rec; |
7420 | break; |
7421 | } |
7422 | |
7423 | // Note: During the replacement process, the nodes will be in an |
7424 | // inconsistent state (some instructions will have operands with values |
7425 | // of the wrong type). Once done, however, everything should be right |
7426 | // again. |
7427 | |
7428 | SmallVector<SDValue, 4> Ops; |
7429 | for (const SDValue &V : PN->ops()) { |
7430 | if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && |
7431 | !isa<ConstantSDNode>(V)) { |
7432 | SDValue ReplOpOps[] = { ISR.getOperand(i: 0), V, ISR.getOperand(i: 2) }; |
7433 | SDNode *ReplOp = |
7434 | CurDAG->getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: SDLoc(V), |
7435 | VTs: ISR.getNode()->getVTList(), Ops: ReplOpOps); |
7436 | Ops.push_back(Elt: SDValue(ReplOp, 0)); |
7437 | } else { |
7438 | Ops.push_back(Elt: V); |
7439 | } |
7440 | } |
7441 | |
7442 | // Because all to-be-promoted nodes only have users that are other |
7443 | // promoted nodes (or the original INSERT_SUBREG), we can safely replace |
7444 | // the i32 result value type with i64. |
7445 | |
7446 | SmallVector<EVT, 2> NewVTs; |
7447 | SDVTList VTs = PN->getVTList(); |
7448 | for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) |
7449 | if (VTs.VTs[i] == MVT::i32) |
7450 | NewVTs.push_back(MVT::i64); |
7451 | else |
7452 | NewVTs.push_back(Elt: VTs.VTs[i]); |
7453 | |
7454 | LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: " ); |
7455 | LLVM_DEBUG(PN->dump(CurDAG)); |
7456 | |
7457 | CurDAG->SelectNodeTo(N: PN, MachineOpc: NewOpcode, VTs: CurDAG->getVTList(VTs: NewVTs), Ops); |
7458 | |
7459 | LLVM_DEBUG(dbgs() << "\nNew: " ); |
7460 | LLVM_DEBUG(PN->dump(CurDAG)); |
7461 | LLVM_DEBUG(dbgs() << "\n" ); |
7462 | } |
7463 | |
7464 | // Now we replace the original zero extend and its associated INSERT_SUBREG |
7465 | // with the value feeding the INSERT_SUBREG (which has now been promoted to |
7466 | // return an i64). |
7467 | |
7468 | LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: " ); |
7469 | LLVM_DEBUG(N->dump(CurDAG)); |
7470 | LLVM_DEBUG(dbgs() << "\nNew: " ); |
7471 | LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); |
7472 | LLVM_DEBUG(dbgs() << "\n" ); |
7473 | |
7474 | ReplaceUses(F: N, T: Op32.getNode()); |
7475 | } |
7476 | |
7477 | if (MadeChange) |
7478 | CurDAG->RemoveDeadNodes(); |
7479 | } |
7480 | |
7481 | static bool isVSXSwap(SDValue N) { |
7482 | if (!N->isMachineOpcode()) |
7483 | return false; |
7484 | unsigned Opc = N->getMachineOpcode(); |
7485 | |
7486 | // Single-operand XXPERMDI or the regular XXPERMDI/XXSLDWI where the immediate |
7487 | // operand is 2. |
7488 | if (Opc == PPC::XXPERMDIs) { |
7489 | return isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) && |
7490 | N->getConstantOperandVal(Num: 1) == 2; |
7491 | } else if (Opc == PPC::XXPERMDI || Opc == PPC::XXSLDWI) { |
7492 | return N->getOperand(Num: 0) == N->getOperand(Num: 1) && |
7493 | isa<ConstantSDNode>(Val: N->getOperand(Num: 2)) && |
7494 | N->getConstantOperandVal(Num: 2) == 2; |
7495 | } |
7496 | |
7497 | return false; |
7498 | } |
7499 | |
7500 | // TODO: Make this complete and replace with a table-gen bit. |
7501 | static bool isLaneInsensitive(SDValue N) { |
7502 | if (!N->isMachineOpcode()) |
7503 | return false; |
7504 | unsigned Opc = N->getMachineOpcode(); |
7505 | |
7506 | switch (Opc) { |
7507 | default: |
7508 | return false; |
7509 | case PPC::VAVGSB: |
7510 | case PPC::VAVGUB: |
7511 | case PPC::VAVGSH: |
7512 | case PPC::VAVGUH: |
7513 | case PPC::VAVGSW: |
7514 | case PPC::VAVGUW: |
7515 | case PPC::VMAXFP: |
7516 | case PPC::VMAXSB: |
7517 | case PPC::VMAXUB: |
7518 | case PPC::VMAXSH: |
7519 | case PPC::VMAXUH: |
7520 | case PPC::VMAXSW: |
7521 | case PPC::VMAXUW: |
7522 | case PPC::VMINFP: |
7523 | case PPC::VMINSB: |
7524 | case PPC::VMINUB: |
7525 | case PPC::VMINSH: |
7526 | case PPC::VMINUH: |
7527 | case PPC::VMINSW: |
7528 | case PPC::VMINUW: |
7529 | case PPC::VADDFP: |
7530 | case PPC::VADDUBM: |
7531 | case PPC::VADDUHM: |
7532 | case PPC::VADDUWM: |
7533 | case PPC::VSUBFP: |
7534 | case PPC::VSUBUBM: |
7535 | case PPC::VSUBUHM: |
7536 | case PPC::VSUBUWM: |
7537 | case PPC::VAND: |
7538 | case PPC::VANDC: |
7539 | case PPC::VOR: |
7540 | case PPC::VORC: |
7541 | case PPC::VXOR: |
7542 | case PPC::VNOR: |
7543 | case PPC::VMULUWM: |
7544 | return true; |
7545 | } |
7546 | } |
7547 | |
7548 | // Try to simplify (xxswap (vec-op (xxswap) (xxswap))) where vec-op is |
7549 | // lane-insensitive. |
7550 | static void reduceVSXSwap(SDNode *N, SelectionDAG *DAG) { |
7551 | // Our desired xxswap might be source of COPY_TO_REGCLASS. |
7552 | // TODO: Can we put this a common method for DAG? |
7553 | auto SkipRCCopy = [](SDValue V) { |
7554 | while (V->isMachineOpcode() && |
7555 | V->getMachineOpcode() == TargetOpcode::COPY_TO_REGCLASS) { |
7556 | // All values in the chain should have single use. |
7557 | if (V->use_empty() || !V->use_begin()->isOnlyUserOf(N: V.getNode())) |
7558 | return SDValue(); |
7559 | V = V->getOperand(Num: 0); |
7560 | } |
7561 | return V.hasOneUse() ? V : SDValue(); |
7562 | }; |
7563 | |
7564 | SDValue VecOp = SkipRCCopy(N->getOperand(Num: 0)); |
7565 | if (!VecOp || !isLaneInsensitive(N: VecOp)) |
7566 | return; |
7567 | |
7568 | SDValue LHS = SkipRCCopy(VecOp.getOperand(i: 0)), |
7569 | RHS = SkipRCCopy(VecOp.getOperand(i: 1)); |
7570 | if (!LHS || !RHS || !isVSXSwap(N: LHS) || !isVSXSwap(N: RHS)) |
7571 | return; |
7572 | |
7573 | // These swaps may still have chain-uses here, count on dead code elimination |
7574 | // in following passes to remove them. |
7575 | DAG->ReplaceAllUsesOfValueWith(From: LHS, To: LHS.getOperand(i: 0)); |
7576 | DAG->ReplaceAllUsesOfValueWith(From: RHS, To: RHS.getOperand(i: 0)); |
7577 | DAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: N->getOperand(Num: 0)); |
7578 | } |
7579 | |
7580 | // Check if an SDValue has the 'aix-small-tls' global variable attribute. |
7581 | static bool hasAIXSmallTLSAttr(SDValue Val) { |
7582 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val)) |
7583 | if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal())) |
7584 | if (GV->hasAttribute(Kind: "aix-small-tls" )) |
7585 | return true; |
7586 | |
7587 | return false; |
7588 | } |
7589 | |
7590 | // Is an ADDI eligible for folding for non-TOC-based local-exec accesses? |
7591 | static bool isEligibleToFoldADDIForLocalExecAccesses(SelectionDAG *DAG, |
7592 | SDValue ADDIToFold) { |
7593 | // Check if ADDIToFold (the ADDI that we want to fold into local-exec |
7594 | // accesses), is truly an ADDI. |
7595 | if (!ADDIToFold.isMachineOpcode() || |
7596 | (ADDIToFold.getMachineOpcode() != PPC::ADDI8)) |
7597 | return false; |
7598 | |
7599 | // Folding is only allowed for the AIX small-local-exec TLS target attribute |
7600 | // or when the 'aix-small-tls' global variable attribute is present. |
7601 | const PPCSubtarget &Subtarget = |
7602 | DAG->getMachineFunction().getSubtarget<PPCSubtarget>(); |
7603 | SDValue TLSVarNode = ADDIToFold.getOperand(i: 1); |
7604 | if (!(Subtarget.hasAIXSmallLocalExecTLS() || hasAIXSmallTLSAttr(Val: TLSVarNode))) |
7605 | return false; |
7606 | |
7607 | // The first operand of the ADDIToFold should be the thread pointer. |
7608 | // This transformation is only performed if the first operand of the |
7609 | // addi is the thread pointer. |
7610 | SDValue TPRegNode = ADDIToFold.getOperand(i: 0); |
7611 | RegisterSDNode *TPReg = dyn_cast<RegisterSDNode>(Val: TPRegNode.getNode()); |
7612 | if (!TPReg || (TPReg->getReg() != Subtarget.getThreadPointerRegister())) |
7613 | return false; |
7614 | |
7615 | // The second operand of the ADDIToFold should be the global TLS address |
7616 | // (the local-exec TLS variable). We only perform the folding if the TLS |
7617 | // variable is the second operand. |
7618 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: TLSVarNode); |
7619 | if (!GA) |
7620 | return false; |
7621 | |
7622 | // The local-exec TLS variable should only have the MO_TPREL_FLAG target flag, |
7623 | // so this optimization is not performed otherwise if the flag is not set. |
7624 | unsigned TargetFlags = GA->getTargetFlags(); |
7625 | if (TargetFlags != PPCII::MO_TPREL_FLAG) |
7626 | return false; |
7627 | |
7628 | // If all conditions are satisfied, the ADDI is valid for folding. |
7629 | return true; |
7630 | } |
7631 | |
7632 | // For non-TOC-based local-exec access where an addi is feeding into another |
7633 | // addi, fold this sequence into a single addi if possible. |
7634 | // Before this optimization, the sequence appears as: |
7635 | // addi rN, r13, sym@le |
7636 | // addi rM, rN, imm |
7637 | // After this optimization, we can fold the two addi into a single one: |
7638 | // addi rM, r13, sym@le + imm |
7639 | static void foldADDIForLocalExecAccesses(SDNode *N, SelectionDAG *DAG) { |
7640 | if (N->getMachineOpcode() != PPC::ADDI8) |
7641 | return; |
7642 | |
7643 | // InitialADDI is the addi feeding into N (also an addi), and the addi that |
7644 | // we want optimized out. |
7645 | SDValue InitialADDI = N->getOperand(Num: 0); |
7646 | |
7647 | if (!isEligibleToFoldADDIForLocalExecAccesses(DAG, ADDIToFold: InitialADDI)) |
7648 | return; |
7649 | |
7650 | // At this point, InitialADDI can be folded into a non-TOC-based local-exec |
7651 | // access. The first operand of InitialADDI should be the thread pointer, |
7652 | // which has been checked in isEligibleToFoldADDIForLocalExecAccesses(). |
7653 | SDValue TPRegNode = InitialADDI.getOperand(i: 0); |
7654 | [[maybe_unused]] RegisterSDNode *TPReg = dyn_cast<RegisterSDNode>(Val: TPRegNode.getNode()); |
7655 | [[maybe_unused]] const PPCSubtarget &Subtarget = |
7656 | DAG->getMachineFunction().getSubtarget<PPCSubtarget>(); |
7657 | assert((TPReg && (TPReg->getReg() == Subtarget.getThreadPointerRegister())) && |
7658 | "Expecting the first operand to be a thread pointer for folding addi " |
7659 | "in local-exec accesses!" ); |
7660 | |
7661 | // The second operand of the InitialADDI should be the global TLS address |
7662 | // (the local-exec TLS variable), with the MO_TPREL_FLAG target flag. |
7663 | // This has been checked in isEligibleToFoldADDIForLocalExecAccesses(). |
7664 | SDValue TLSVarNode = InitialADDI.getOperand(i: 1); |
7665 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: TLSVarNode); |
7666 | assert(GA && "Expecting a valid GlobalAddressSDNode when folding addi into " |
7667 | "local-exec accesses!" ); |
7668 | unsigned TargetFlags = GA->getTargetFlags(); |
7669 | |
7670 | // The second operand of the addi that we want to preserve will be an |
7671 | // immediate. We add this immediate, together with the address of the TLS |
7672 | // variable found in InitialADDI, in order to preserve the correct TLS address |
7673 | // information during assembly printing. The offset is likely to be non-zero |
7674 | // when we end up in this case. |
7675 | int Offset = N->getConstantOperandVal(Num: 1); |
7676 | TLSVarNode = DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA), MVT::i64, |
7677 | Offset, TargetFlags); |
7678 | |
7679 | (void)DAG->UpdateNodeOperands(N, Op1: TPRegNode, Op2: TLSVarNode); |
7680 | if (InitialADDI.getNode()->use_empty()) |
7681 | DAG->RemoveDeadNode(N: InitialADDI.getNode()); |
7682 | } |
7683 | |
7684 | void PPCDAGToDAGISel::PeepholePPC64() { |
7685 | SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
7686 | |
7687 | while (Position != CurDAG->allnodes_begin()) { |
7688 | SDNode *N = &*--Position; |
7689 | // Skip dead nodes and any non-machine opcodes. |
7690 | if (N->use_empty() || !N->isMachineOpcode()) |
7691 | continue; |
7692 | |
7693 | if (isVSXSwap(N: SDValue(N, 0))) |
7694 | reduceVSXSwap(N, DAG: CurDAG); |
7695 | |
7696 | // This optimization is performed for non-TOC-based local-exec accesses. |
7697 | foldADDIForLocalExecAccesses(N, DAG: CurDAG); |
7698 | |
7699 | unsigned FirstOp; |
7700 | unsigned StorageOpcode = N->getMachineOpcode(); |
7701 | bool RequiresMod4Offset = false; |
7702 | |
7703 | switch (StorageOpcode) { |
7704 | default: continue; |
7705 | |
7706 | case PPC::LWA: |
7707 | case PPC::LD: |
7708 | case PPC::DFLOADf64: |
7709 | case PPC::DFLOADf32: |
7710 | RequiresMod4Offset = true; |
7711 | [[fallthrough]]; |
7712 | case PPC::LBZ: |
7713 | case PPC::LBZ8: |
7714 | case PPC::LFD: |
7715 | case PPC::LFS: |
7716 | case PPC::LHA: |
7717 | case PPC::LHA8: |
7718 | case PPC::LHZ: |
7719 | case PPC::LHZ8: |
7720 | case PPC::LWZ: |
7721 | case PPC::LWZ8: |
7722 | FirstOp = 0; |
7723 | break; |
7724 | |
7725 | case PPC::STD: |
7726 | case PPC::DFSTOREf64: |
7727 | case PPC::DFSTOREf32: |
7728 | RequiresMod4Offset = true; |
7729 | [[fallthrough]]; |
7730 | case PPC::STB: |
7731 | case PPC::STB8: |
7732 | case PPC::STFD: |
7733 | case PPC::STFS: |
7734 | case PPC::STH: |
7735 | case PPC::STH8: |
7736 | case PPC::STW: |
7737 | case PPC::STW8: |
7738 | FirstOp = 1; |
7739 | break; |
7740 | } |
7741 | |
7742 | // If this is a load or store with a zero offset, or within the alignment, |
7743 | // we may be able to fold an add-immediate into the memory operation. |
7744 | // The check against alignment is below, as it can't occur until we check |
7745 | // the arguments to N |
7746 | if (!isa<ConstantSDNode>(Val: N->getOperand(Num: FirstOp))) |
7747 | continue; |
7748 | |
7749 | SDValue Base = N->getOperand(Num: FirstOp + 1); |
7750 | if (!Base.isMachineOpcode()) |
7751 | continue; |
7752 | |
7753 | unsigned Flags = 0; |
7754 | bool ReplaceFlags = true; |
7755 | |
7756 | // When the feeding operation is an add-immediate of some sort, |
7757 | // determine whether we need to add relocation information to the |
7758 | // target flags on the immediate operand when we fold it into the |
7759 | // load instruction. |
7760 | // |
7761 | // For something like ADDItocL8, the relocation information is |
7762 | // inferred from the opcode; when we process it in the AsmPrinter, |
7763 | // we add the necessary relocation there. A load, though, can receive |
7764 | // relocation from various flavors of ADDIxxx, so we need to carry |
7765 | // the relocation information in the target flags. |
7766 | switch (Base.getMachineOpcode()) { |
7767 | default: continue; |
7768 | |
7769 | case PPC::ADDI8: |
7770 | case PPC::ADDI: |
7771 | // In some cases (such as TLS) the relocation information |
7772 | // is already in place on the operand, so copying the operand |
7773 | // is sufficient. |
7774 | ReplaceFlags = false; |
7775 | break; |
7776 | case PPC::ADDIdtprelL: |
7777 | Flags = PPCII::MO_DTPREL_LO; |
7778 | break; |
7779 | case PPC::ADDItlsldL: |
7780 | Flags = PPCII::MO_TLSLD_LO; |
7781 | break; |
7782 | case PPC::ADDItocL8: |
7783 | Flags = PPCII::MO_TOC_LO; |
7784 | break; |
7785 | } |
7786 | |
7787 | SDValue ImmOpnd = Base.getOperand(i: 1); |
7788 | |
7789 | // On PPC64, the TOC base pointer is guaranteed by the ABI only to have |
7790 | // 8-byte alignment, and so we can only use offsets less than 8 (otherwise, |
7791 | // we might have needed different @ha relocation values for the offset |
7792 | // pointers). |
7793 | int MaxDisplacement = 7; |
7794 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: ImmOpnd)) { |
7795 | const GlobalValue *GV = GA->getGlobal(); |
7796 | Align Alignment = GV->getPointerAlignment(DL: CurDAG->getDataLayout()); |
7797 | MaxDisplacement = std::min(a: (int)Alignment.value() - 1, b: MaxDisplacement); |
7798 | } |
7799 | |
7800 | bool UpdateHBase = false; |
7801 | SDValue HBase = Base.getOperand(i: 0); |
7802 | |
7803 | int Offset = N->getConstantOperandVal(Num: FirstOp); |
7804 | if (ReplaceFlags) { |
7805 | if (Offset < 0 || Offset > MaxDisplacement) { |
7806 | // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only |
7807 | // one use, then we can do this for any offset, we just need to also |
7808 | // update the offset (i.e. the symbol addend) on the addis also. |
7809 | if (Base.getMachineOpcode() != PPC::ADDItocL8) |
7810 | continue; |
7811 | |
7812 | if (!HBase.isMachineOpcode() || |
7813 | HBase.getMachineOpcode() != PPC::ADDIStocHA8) |
7814 | continue; |
7815 | |
7816 | if (!Base.hasOneUse() || !HBase.hasOneUse()) |
7817 | continue; |
7818 | |
7819 | SDValue HImmOpnd = HBase.getOperand(i: 1); |
7820 | if (HImmOpnd != ImmOpnd) |
7821 | continue; |
7822 | |
7823 | UpdateHBase = true; |
7824 | } |
7825 | } else { |
7826 | // Global addresses can be folded, but only if they are sufficiently |
7827 | // aligned. |
7828 | if (RequiresMod4Offset) { |
7829 | if (GlobalAddressSDNode *GA = |
7830 | dyn_cast<GlobalAddressSDNode>(Val&: ImmOpnd)) { |
7831 | const GlobalValue *GV = GA->getGlobal(); |
7832 | Align Alignment = GV->getPointerAlignment(DL: CurDAG->getDataLayout()); |
7833 | if (Alignment < 4) |
7834 | continue; |
7835 | } |
7836 | } |
7837 | |
7838 | // If we're directly folding the addend from an addi instruction, then: |
7839 | // 1. In general, the offset on the memory access must be zero. |
7840 | // 2. If the addend is a constant, then it can be combined with a |
7841 | // non-zero offset, but only if the result meets the encoding |
7842 | // requirements. |
7843 | if (auto *C = dyn_cast<ConstantSDNode>(Val&: ImmOpnd)) { |
7844 | Offset += C->getSExtValue(); |
7845 | |
7846 | if (RequiresMod4Offset && (Offset % 4) != 0) |
7847 | continue; |
7848 | |
7849 | if (!isInt<16>(x: Offset)) |
7850 | continue; |
7851 | |
7852 | ImmOpnd = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(ImmOpnd), |
7853 | VT: ImmOpnd.getValueType()); |
7854 | } else if (Offset != 0) { |
7855 | // This optimization is performed for non-TOC-based local-exec accesses. |
7856 | if (isEligibleToFoldADDIForLocalExecAccesses(DAG: CurDAG, ADDIToFold: Base)) { |
7857 | // Add the non-zero offset information into the load or store |
7858 | // instruction to be used for non-TOC-based local-exec accesses. |
7859 | GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: ImmOpnd); |
7860 | assert(GA && "Expecting a valid GlobalAddressSDNode when folding " |
7861 | "addi into local-exec accesses!" ); |
7862 | ImmOpnd = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA), |
7863 | MVT::i64, Offset, |
7864 | GA->getTargetFlags()); |
7865 | } else |
7866 | continue; |
7867 | } |
7868 | } |
7869 | |
7870 | // We found an opportunity. Reverse the operands from the add |
7871 | // immediate and substitute them into the load or store. If |
7872 | // needed, update the target flags for the immediate operand to |
7873 | // reflect the necessary relocation information. |
7874 | LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: " ); |
7875 | LLVM_DEBUG(Base->dump(CurDAG)); |
7876 | LLVM_DEBUG(dbgs() << "\nN: " ); |
7877 | LLVM_DEBUG(N->dump(CurDAG)); |
7878 | LLVM_DEBUG(dbgs() << "\n" ); |
7879 | |
7880 | // If the relocation information isn't already present on the |
7881 | // immediate operand, add it now. |
7882 | if (ReplaceFlags) { |
7883 | if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: ImmOpnd)) { |
7884 | SDLoc dl(GA); |
7885 | const GlobalValue *GV = GA->getGlobal(); |
7886 | Align Alignment = GV->getPointerAlignment(DL: CurDAG->getDataLayout()); |
7887 | // We can't perform this optimization for data whose alignment |
7888 | // is insufficient for the instruction encoding. |
7889 | if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) { |
7890 | LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n" ); |
7891 | continue; |
7892 | } |
7893 | ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); |
7894 | } else if (ConstantPoolSDNode *CP = |
7895 | dyn_cast<ConstantPoolSDNode>(Val&: ImmOpnd)) { |
7896 | const Constant *C = CP->getConstVal(); |
7897 | ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(), |
7898 | Offset, Flags); |
7899 | } |
7900 | } |
7901 | |
7902 | if (FirstOp == 1) // Store |
7903 | (void)CurDAG->UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: ImmOpnd, |
7904 | Op3: Base.getOperand(i: 0), Op4: N->getOperand(Num: 3)); |
7905 | else // Load |
7906 | (void)CurDAG->UpdateNodeOperands(N, Op1: ImmOpnd, Op2: Base.getOperand(i: 0), |
7907 | Op3: N->getOperand(Num: 2)); |
7908 | |
7909 | if (UpdateHBase) |
7910 | (void)CurDAG->UpdateNodeOperands(N: HBase.getNode(), Op1: HBase.getOperand(i: 0), |
7911 | Op2: ImmOpnd); |
7912 | |
7913 | // The add-immediate may now be dead, in which case remove it. |
7914 | if (Base.getNode()->use_empty()) |
7915 | CurDAG->RemoveDeadNode(N: Base.getNode()); |
7916 | } |
7917 | } |
7918 | |
7919 | /// createPPCISelDag - This pass converts a legalized DAG into a |
7920 | /// PowerPC-specific DAG, ready for instruction scheduling. |
7921 | /// |
7922 | FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM, |
7923 | CodeGenOptLevel OptLevel) { |
7924 | return new PPCDAGToDAGISel(TM, OptLevel); |
7925 | } |
7926 | |