1//==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the Emit routines for the SelectionDAG class, which creates
10// MachineInstrs based on the decisions of the SelectionDAG instruction
11// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "InstrEmitter.h"
16#include "SDNodeDbgValue.h"
17#include "llvm/BinaryFormat/Dwarf.h"
18#include "llvm/CodeGen/MachineConstantPool.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/StackMaps.h"
23#include "llvm/CodeGen/TargetInstrInfo.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/TargetSubtargetInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/PseudoProbe.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Target/TargetMachine.h"
30using namespace llvm;
31
32#define DEBUG_TYPE "instr-emitter"
33
34/// MinRCSize - Smallest register class we allow when constraining virtual
35/// registers. If satisfying all register class constraints would require
36/// using a smaller register class, emit a COPY to a new virtual register
37/// instead.
38const unsigned MinRCSize = 4;
39
40/// CountResults - The results of target nodes have register or immediate
41/// operands first, then an optional chain, and optional glue operands (which do
42/// not go into the resulting MachineInstr).
43unsigned InstrEmitter::CountResults(SDNode *Node) {
44 unsigned N = Node->getNumValues();
45 while (N && Node->getValueType(ResNo: N - 1) == MVT::Glue)
46 --N;
47 if (N && Node->getValueType(ResNo: N - 1) == MVT::Other)
48 --N; // Skip over chain result.
49 return N;
50}
51
52/// countOperands - The inputs to target nodes have any actual inputs first,
53/// followed by an optional chain operand, then an optional glue operand.
54/// Compute the number of actual operands that will go into the resulting
55/// MachineInstr.
56///
57/// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
58/// the chain and glue. These operands may be implicit on the machine instr.
59static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
60 unsigned &NumImpUses) {
61 unsigned N = Node->getNumOperands();
62 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
63 --N;
64 if (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Other)
65 --N; // Ignore chain if it exists.
66
67 // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
68 NumImpUses = N - NumExpUses;
69 for (unsigned I = N; I > NumExpUses; --I) {
70 if (isa<RegisterMaskSDNode>(Val: Node->getOperand(Num: I - 1)))
71 continue;
72 if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: I - 1)))
73 if (RN->getReg().isPhysical())
74 continue;
75 NumImpUses = N - I;
76 break;
77 }
78
79 return N;
80}
81
82/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
83/// implicit physical register output.
84void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
85 Register SrcReg,
86 DenseMap<SDValue, Register> &VRBaseMap) {
87 Register VRBase;
88 if (SrcReg.isVirtual()) {
89 // Just use the input register directly!
90 SDValue Op(Node, ResNo);
91 if (IsClone)
92 VRBaseMap.erase(Val: Op);
93 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: SrcReg)).second;
94 (void)isNew; // Silence compiler warning.
95 assert(isNew && "Node emitted out of order - early");
96 return;
97 }
98
99 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
100 // the CopyToReg'd destination register instead of creating a new vreg.
101 bool MatchReg = true;
102 const TargetRegisterClass *UseRC = nullptr;
103 MVT VT = Node->getSimpleValueType(ResNo);
104
105 // Stick to the preferred register classes for legal types.
106 if (TLI->isTypeLegal(VT))
107 UseRC = TLI->getRegClassFor(VT, isDivergent: Node->isDivergent());
108
109 for (SDNode *User : Node->uses()) {
110 bool Match = true;
111 if (User->getOpcode() == ISD::CopyToReg &&
112 User->getOperand(Num: 2).getNode() == Node &&
113 User->getOperand(Num: 2).getResNo() == ResNo) {
114 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
115 if (DestReg.isVirtual()) {
116 VRBase = DestReg;
117 Match = false;
118 } else if (DestReg != SrcReg)
119 Match = false;
120 } else {
121 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
122 SDValue Op = User->getOperand(Num: i);
123 if (Op.getNode() != Node || Op.getResNo() != ResNo)
124 continue;
125 MVT VT = Node->getSimpleValueType(ResNo: Op.getResNo());
126 if (VT == MVT::Other || VT == MVT::Glue)
127 continue;
128 Match = false;
129 if (User->isMachineOpcode()) {
130 const MCInstrDesc &II = TII->get(Opcode: User->getMachineOpcode());
131 const TargetRegisterClass *RC = nullptr;
132 if (i + II.getNumDefs() < II.getNumOperands()) {
133 RC = TRI->getAllocatableClass(
134 RC: TII->getRegClass(MCID: II, OpNum: i + II.getNumDefs(), TRI, MF: *MF));
135 }
136 if (!UseRC)
137 UseRC = RC;
138 else if (RC) {
139 const TargetRegisterClass *ComRC =
140 TRI->getCommonSubClass(A: UseRC, B: RC);
141 // If multiple uses expect disjoint register classes, we emit
142 // copies in AddRegisterOperand.
143 if (ComRC)
144 UseRC = ComRC;
145 }
146 }
147 }
148 }
149 MatchReg &= Match;
150 if (VRBase)
151 break;
152 }
153
154 const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
155 SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg, VT);
156
157 // Figure out the register class to create for the destreg.
158 if (VRBase) {
159 DstRC = MRI->getRegClass(Reg: VRBase);
160 } else if (UseRC) {
161 assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
162 "Incompatible phys register def and uses!");
163 DstRC = UseRC;
164 } else
165 DstRC = SrcRC;
166
167 // If all uses are reading from the src physical register and copying the
168 // register is either impossible or very expensive, then don't create a copy.
169 if (MatchReg && SrcRC->getCopyCost() < 0) {
170 VRBase = SrcReg;
171 } else {
172 // Create the reg, emit the copy.
173 VRBase = MRI->createVirtualRegister(RegClass: DstRC);
174 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
175 DestReg: VRBase).addReg(RegNo: SrcReg);
176 }
177
178 SDValue Op(Node, ResNo);
179 if (IsClone)
180 VRBaseMap.erase(Val: Op);
181 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
182 (void)isNew; // Silence compiler warning.
183 assert(isNew && "Node emitted out of order - early");
184}
185
186void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
187 MachineInstrBuilder &MIB,
188 const MCInstrDesc &II,
189 bool IsClone, bool IsCloned,
190 DenseMap<SDValue, Register> &VRBaseMap) {
191 assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
192 "IMPLICIT_DEF should have been handled as a special case elsewhere!");
193
194 unsigned NumResults = CountResults(Node);
195 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
196 II.isVariadic() && II.variadicOpsAreDefs();
197 unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
198 if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
199 NumVRegs = NumResults;
200 for (unsigned i = 0; i < NumVRegs; ++i) {
201 // If the specific node value is only used by a CopyToReg and the dest reg
202 // is a vreg in the same register class, use the CopyToReg'd destination
203 // register instead of creating a new vreg.
204 Register VRBase;
205 const TargetRegisterClass *RC =
206 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: II, OpNum: i, TRI, MF: *MF));
207 // Always let the value type influence the used register class. The
208 // constraints on the instruction may be too lax to represent the value
209 // type correctly. For example, a 64-bit float (X86::FR64) can't live in
210 // the 32-bit float super-class (X86::FR32).
211 if (i < NumResults && TLI->isTypeLegal(VT: Node->getSimpleValueType(ResNo: i))) {
212 const TargetRegisterClass *VTRC = TLI->getRegClassFor(
213 VT: Node->getSimpleValueType(ResNo: i),
214 isDivergent: (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC))));
215 if (RC)
216 VTRC = TRI->getCommonSubClass(A: RC, B: VTRC);
217 if (VTRC)
218 RC = VTRC;
219 }
220
221 if (!II.operands().empty() && II.operands()[i].isOptionalDef()) {
222 // Optional def must be a physical register.
223 VRBase = cast<RegisterSDNode>(Val: Node->getOperand(Num: i-NumResults))->getReg();
224 assert(VRBase.isPhysical());
225 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
226 }
227
228 if (!VRBase && !IsClone && !IsCloned)
229 for (SDNode *User : Node->uses()) {
230 if (User->getOpcode() == ISD::CopyToReg &&
231 User->getOperand(Num: 2).getNode() == Node &&
232 User->getOperand(Num: 2).getResNo() == i) {
233 Register Reg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
234 if (Reg.isVirtual()) {
235 const TargetRegisterClass *RegRC = MRI->getRegClass(Reg);
236 if (RegRC == RC) {
237 VRBase = Reg;
238 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
239 break;
240 }
241 }
242 }
243 }
244
245 // Create the result registers for this node and add the result regs to
246 // the machine instruction.
247 if (VRBase == 0) {
248 assert(RC && "Isn't a register operand!");
249 VRBase = MRI->createVirtualRegister(RegClass: RC);
250 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
251 }
252
253 // If this def corresponds to a result of the SDNode insert the VRBase into
254 // the lookup map.
255 if (i < NumResults) {
256 SDValue Op(Node, i);
257 if (IsClone)
258 VRBaseMap.erase(Val: Op);
259 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
260 (void)isNew; // Silence compiler warning.
261 assert(isNew && "Node emitted out of order - early");
262 }
263 }
264}
265
266/// getVR - Return the virtual register corresponding to the specified result
267/// of the specified node.
268Register InstrEmitter::getVR(SDValue Op,
269 DenseMap<SDValue, Register> &VRBaseMap) {
270 if (Op.isMachineOpcode() &&
271 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
272 // Add an IMPLICIT_DEF instruction before every use.
273 // IMPLICIT_DEF can produce any type of result so its MCInstrDesc
274 // does not include operand register class info.
275 const TargetRegisterClass *RC = TLI->getRegClassFor(
276 VT: Op.getSimpleValueType(), isDivergent: Op.getNode()->isDivergent());
277 Register VReg = MRI->createVirtualRegister(RegClass: RC);
278 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getDebugLoc(),
279 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: VReg);
280 return VReg;
281 }
282
283 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Val: Op);
284 assert(I != VRBaseMap.end() && "Node emitted out of order - late");
285 return I->second;
286}
287
288
289/// AddRegisterOperand - Add the specified register as an operand to the
290/// specified machine instr. Insert register copies if the register is
291/// not in the required register class.
292void
293InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
294 SDValue Op,
295 unsigned IIOpNum,
296 const MCInstrDesc *II,
297 DenseMap<SDValue, Register> &VRBaseMap,
298 bool IsDebug, bool IsClone, bool IsCloned) {
299 assert(Op.getValueType() != MVT::Other &&
300 Op.getValueType() != MVT::Glue &&
301 "Chain and glue operands should occur at end of operand list!");
302 // Get/emit the operand.
303 Register VReg = getVR(Op, VRBaseMap);
304
305 const MCInstrDesc &MCID = MIB->getDesc();
306 bool isOptDef = IIOpNum < MCID.getNumOperands() &&
307 MCID.operands()[IIOpNum].isOptionalDef();
308
309 // If the instruction requires a register in a different class, create
310 // a new virtual register and copy the value into it, but first attempt to
311 // shrink VReg's register class within reason. For example, if VReg == GR32
312 // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
313 if (II) {
314 const TargetRegisterClass *OpRC = nullptr;
315 if (IIOpNum < II->getNumOperands())
316 OpRC = TII->getRegClass(MCID: *II, OpNum: IIOpNum, TRI, MF: *MF);
317
318 if (OpRC) {
319 unsigned MinNumRegs = MinRCSize;
320 // Don't apply any RC size limit for IMPLICIT_DEF. Each use has a unique
321 // virtual register.
322 if (Op.isMachineOpcode() &&
323 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF)
324 MinNumRegs = 0;
325
326 const TargetRegisterClass *ConstrainedRC
327 = MRI->constrainRegClass(Reg: VReg, RC: OpRC, MinNumRegs);
328 if (!ConstrainedRC) {
329 OpRC = TRI->getAllocatableClass(RC: OpRC);
330 assert(OpRC && "Constraints cannot be fulfilled for allocation");
331 Register NewVReg = MRI->createVirtualRegister(RegClass: OpRC);
332 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getNode()->getDebugLoc(),
333 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg).addReg(RegNo: VReg);
334 VReg = NewVReg;
335 } else {
336 assert(ConstrainedRC->isAllocatable() &&
337 "Constraining an allocatable VReg produced an unallocatable class?");
338 }
339 }
340 }
341
342 // If this value has only one use, that use is a kill. This is a
343 // conservative approximation. InstrEmitter does trivial coalescing
344 // with CopyFromReg nodes, so don't emit kill flags for them.
345 // Avoid kill flags on Schedule cloned nodes, since there will be
346 // multiple uses.
347 // Tied operands are never killed, so we need to check that. And that
348 // means we need to determine the index of the operand.
349 bool isKill = Op.hasOneUse() &&
350 Op.getNode()->getOpcode() != ISD::CopyFromReg &&
351 !IsDebug &&
352 !(IsClone || IsCloned);
353 if (isKill) {
354 unsigned Idx = MIB->getNumOperands();
355 while (Idx > 0 &&
356 MIB->getOperand(i: Idx-1).isReg() &&
357 MIB->getOperand(i: Idx-1).isImplicit())
358 --Idx;
359 bool isTied = MCID.getOperandConstraint(OpNum: Idx, Constraint: MCOI::TIED_TO) != -1;
360 if (isTied)
361 isKill = false;
362 }
363
364 MIB.addReg(RegNo: VReg, flags: getDefRegState(B: isOptDef) | getKillRegState(B: isKill) |
365 getDebugRegState(B: IsDebug));
366}
367
368/// AddOperand - Add the specified operand to the specified machine instr. II
369/// specifies the instruction information for the node, and IIOpNum is the
370/// operand number (in the II) that we are adding.
371void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
372 SDValue Op,
373 unsigned IIOpNum,
374 const MCInstrDesc *II,
375 DenseMap<SDValue, Register> &VRBaseMap,
376 bool IsDebug, bool IsClone, bool IsCloned) {
377 if (Op.isMachineOpcode()) {
378 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
379 IsDebug, IsClone, IsCloned);
380 } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
381 MIB.addImm(Val: C->getSExtValue());
382 } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
383 MIB.addFPImm(Val: F->getConstantFPValue());
384 } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: Op)) {
385 Register VReg = R->getReg();
386 MVT OpVT = Op.getSimpleValueType();
387 const TargetRegisterClass *IIRC =
388 II ? TRI->getAllocatableClass(RC: TII->getRegClass(MCID: *II, OpNum: IIOpNum, TRI, MF: *MF))
389 : nullptr;
390 const TargetRegisterClass *OpRC =
391 TLI->isTypeLegal(VT: OpVT)
392 ? TLI->getRegClassFor(VT: OpVT,
393 isDivergent: Op.getNode()->isDivergent() ||
394 (IIRC && TRI->isDivergentRegClass(RC: IIRC)))
395 : nullptr;
396
397 if (OpRC && IIRC && OpRC != IIRC && VReg.isVirtual()) {
398 Register NewVReg = MRI->createVirtualRegister(RegClass: IIRC);
399 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getNode()->getDebugLoc(),
400 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg).addReg(RegNo: VReg);
401 VReg = NewVReg;
402 }
403 // Turn additional physreg operands into implicit uses on non-variadic
404 // instructions. This is used by call and return instructions passing
405 // arguments in registers.
406 bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
407 MIB.addReg(RegNo: VReg, flags: getImplRegState(B: Imp));
408 } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Val&: Op)) {
409 MIB.addRegMask(Mask: RM->getRegMask());
410 } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
411 MIB.addGlobalAddress(GV: TGA->getGlobal(), Offset: TGA->getOffset(),
412 TargetFlags: TGA->getTargetFlags());
413 } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Val&: Op)) {
414 MIB.addMBB(MBB: BBNode->getBasicBlock());
415 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
416 MIB.addFrameIndex(Idx: FI->getIndex());
417 } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Val&: Op)) {
418 MIB.addJumpTableIndex(Idx: JT->getIndex(), TargetFlags: JT->getTargetFlags());
419 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: Op)) {
420 int Offset = CP->getOffset();
421 Align Alignment = CP->getAlign();
422
423 unsigned Idx;
424 MachineConstantPool *MCP = MF->getConstantPool();
425 if (CP->isMachineConstantPoolEntry())
426 Idx = MCP->getConstantPoolIndex(V: CP->getMachineCPVal(), Alignment);
427 else
428 Idx = MCP->getConstantPoolIndex(C: CP->getConstVal(), Alignment);
429 MIB.addConstantPoolIndex(Idx, Offset, TargetFlags: CP->getTargetFlags());
430 } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Val&: Op)) {
431 MIB.addExternalSymbol(FnName: ES->getSymbol(), TargetFlags: ES->getTargetFlags());
432 } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Val&: Op)) {
433 MIB.addSym(Sym: SymNode->getMCSymbol());
434 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
435 MIB.addBlockAddress(BA: BA->getBlockAddress(),
436 Offset: BA->getOffset(),
437 TargetFlags: BA->getTargetFlags());
438 } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Val&: Op)) {
439 MIB.addTargetIndex(Idx: TI->getIndex(), Offset: TI->getOffset(), TargetFlags: TI->getTargetFlags());
440 } else {
441 assert(Op.getValueType() != MVT::Other &&
442 Op.getValueType() != MVT::Glue &&
443 "Chain and glue operands should occur at end of operand list!");
444 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
445 IsDebug, IsClone, IsCloned);
446 }
447}
448
449Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
450 MVT VT, bool isDivergent, const DebugLoc &DL) {
451 const TargetRegisterClass *VRC = MRI->getRegClass(Reg: VReg);
452 const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(RC: VRC, Idx: SubIdx);
453
454 // RC is a sub-class of VRC that supports SubIdx. Try to constrain VReg
455 // within reason.
456 if (RC && RC != VRC)
457 RC = MRI->constrainRegClass(Reg: VReg, RC, MinNumRegs: MinRCSize);
458
459 // VReg has been adjusted. It can be used with SubIdx operands now.
460 if (RC)
461 return VReg;
462
463 // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual
464 // register instead.
465 RC = TRI->getSubClassWithSubReg(RC: TLI->getRegClassFor(VT, isDivergent), Idx: SubIdx);
466 assert(RC && "No legal register class for VT supports that SubIdx");
467 Register NewReg = MRI->createVirtualRegister(RegClass: RC);
468 BuildMI(BB&: *MBB, I: InsertPos, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewReg)
469 .addReg(RegNo: VReg);
470 return NewReg;
471}
472
473/// EmitSubregNode - Generate machine code for subreg nodes.
474///
475void InstrEmitter::EmitSubregNode(SDNode *Node,
476 DenseMap<SDValue, Register> &VRBaseMap,
477 bool IsClone, bool IsCloned) {
478 Register VRBase;
479 unsigned Opc = Node->getMachineOpcode();
480
481 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
482 // the CopyToReg'd destination register instead of creating a new vreg.
483 for (SDNode *User : Node->uses()) {
484 if (User->getOpcode() == ISD::CopyToReg &&
485 User->getOperand(Num: 2).getNode() == Node) {
486 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
487 if (DestReg.isVirtual()) {
488 VRBase = DestReg;
489 break;
490 }
491 }
492 }
493
494 if (Opc == TargetOpcode::EXTRACT_SUBREG) {
495 // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no
496 // constraints on the %dst register, COPY can target all legal register
497 // classes.
498 unsigned SubIdx = Node->getConstantOperandVal(Num: 1);
499 const TargetRegisterClass *TRC =
500 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
501
502 Register Reg;
503 MachineInstr *DefMI;
504 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: 0));
505 if (R && R->getReg().isPhysical()) {
506 Reg = R->getReg();
507 DefMI = nullptr;
508 } else {
509 Reg = R ? R->getReg() : getVR(Op: Node->getOperand(Num: 0), VRBaseMap);
510 DefMI = MRI->getVRegDef(Reg);
511 }
512
513 Register SrcReg, DstReg;
514 unsigned DefSubIdx;
515 if (DefMI &&
516 TII->isCoalescableExtInstr(MI: *DefMI, SrcReg, DstReg, SubIdx&: DefSubIdx) &&
517 SubIdx == DefSubIdx &&
518 TRC == MRI->getRegClass(Reg: SrcReg)) {
519 // Optimize these:
520 // r1025 = s/zext r1024, 4
521 // r1026 = extract_subreg r1025, 4
522 // to a copy
523 // r1026 = copy r1024
524 VRBase = MRI->createVirtualRegister(RegClass: TRC);
525 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
526 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase).addReg(RegNo: SrcReg);
527 MRI->clearKillFlags(Reg: SrcReg);
528 } else {
529 // Reg may not support a SubIdx sub-register, and we may need to
530 // constrain its register class or issue a COPY to a compatible register
531 // class.
532 if (Reg.isVirtual())
533 Reg = ConstrainForSubReg(VReg: Reg, SubIdx,
534 VT: Node->getOperand(Num: 0).getSimpleValueType(),
535 isDivergent: Node->isDivergent(), DL: Node->getDebugLoc());
536 // Create the destreg if it is missing.
537 if (!VRBase)
538 VRBase = MRI->createVirtualRegister(RegClass: TRC);
539
540 // Create the extract_subreg machine instruction.
541 MachineInstrBuilder CopyMI =
542 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
543 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase);
544 if (Reg.isVirtual())
545 CopyMI.addReg(RegNo: Reg, flags: 0, SubReg: SubIdx);
546 else
547 CopyMI.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx));
548 }
549 } else if (Opc == TargetOpcode::INSERT_SUBREG ||
550 Opc == TargetOpcode::SUBREG_TO_REG) {
551 SDValue N0 = Node->getOperand(Num: 0);
552 SDValue N1 = Node->getOperand(Num: 1);
553 SDValue N2 = Node->getOperand(Num: 2);
554 unsigned SubIdx = N2->getAsZExtVal();
555
556 // Figure out the register class to create for the destreg. It should be
557 // the largest legal register class supporting SubIdx sub-registers.
558 // RegisterCoalescer will constrain it further if it decides to eliminate
559 // the INSERT_SUBREG instruction.
560 //
561 // %dst = INSERT_SUBREG %src, %sub, SubIdx
562 //
563 // is lowered by TwoAddressInstructionPass to:
564 //
565 // %dst = COPY %src
566 // %dst:SubIdx = COPY %sub
567 //
568 // There is no constraint on the %src register class.
569 //
570 const TargetRegisterClass *SRC =
571 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
572 SRC = TRI->getSubClassWithSubReg(RC: SRC, Idx: SubIdx);
573 assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
574
575 if (VRBase == 0 || !SRC->hasSubClassEq(RC: MRI->getRegClass(Reg: VRBase)))
576 VRBase = MRI->createVirtualRegister(RegClass: SRC);
577
578 // Create the insert_subreg or subreg_to_reg machine instruction.
579 MachineInstrBuilder MIB =
580 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: VRBase);
581
582 // If creating a subreg_to_reg, then the first input operand
583 // is an implicit value immediate, otherwise it's a register
584 if (Opc == TargetOpcode::SUBREG_TO_REG) {
585 const ConstantSDNode *SD = cast<ConstantSDNode>(Val&: N0);
586 MIB.addImm(Val: SD->getZExtValue());
587 } else
588 AddOperand(MIB, Op: N0, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false,
589 IsClone, IsCloned);
590 // Add the subregister being inserted
591 AddOperand(MIB, Op: N1, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false,
592 IsClone, IsCloned);
593 MIB.addImm(Val: SubIdx);
594 MBB->insert(I: InsertPos, MI: MIB);
595 } else
596 llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
597
598 SDValue Op(Node, 0);
599 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
600 (void)isNew; // Silence compiler warning.
601 assert(isNew && "Node emitted out of order - early");
602}
603
604/// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
605/// COPY_TO_REGCLASS is just a normal copy, except that the destination
606/// register is constrained to be in a particular register class.
607///
608void
609InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
610 DenseMap<SDValue, Register> &VRBaseMap) {
611 unsigned VReg = getVR(Op: Node->getOperand(Num: 0), VRBaseMap);
612
613 // Create the new VReg in the destination class and emit a copy.
614 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 1);
615 const TargetRegisterClass *DstRC =
616 TRI->getAllocatableClass(RC: TRI->getRegClass(i: DstRCIdx));
617 Register NewVReg = MRI->createVirtualRegister(RegClass: DstRC);
618 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
619 DestReg: NewVReg).addReg(RegNo: VReg);
620
621 SDValue Op(Node, 0);
622 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
623 (void)isNew; // Silence compiler warning.
624 assert(isNew && "Node emitted out of order - early");
625}
626
627/// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
628///
629void InstrEmitter::EmitRegSequence(SDNode *Node,
630 DenseMap<SDValue, Register> &VRBaseMap,
631 bool IsClone, bool IsCloned) {
632 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 0);
633 const TargetRegisterClass *RC = TRI->getRegClass(i: DstRCIdx);
634 Register NewVReg = MRI->createVirtualRegister(RegClass: TRI->getAllocatableClass(RC));
635 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::REG_SEQUENCE);
636 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II, DestReg: NewVReg);
637 unsigned NumOps = Node->getNumOperands();
638 // If the input pattern has a chain, then the root of the corresponding
639 // output pattern will get a chain as well. This can happen to be a
640 // REG_SEQUENCE (which is not "guarded" by countOperands/CountResults).
641 if (NumOps && Node->getOperand(Num: NumOps-1).getValueType() == MVT::Other)
642 --NumOps; // Ignore chain if it exists.
643
644 assert((NumOps & 1) == 1 &&
645 "REG_SEQUENCE must have an odd number of operands!");
646 for (unsigned i = 1; i != NumOps; ++i) {
647 SDValue Op = Node->getOperand(Num: i);
648 if ((i & 1) == 0) {
649 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: i-1));
650 // Skip physical registers as they don't have a vreg to get and we'll
651 // insert copies for them in TwoAddressInstructionPass anyway.
652 if (!R || !R->getReg().isPhysical()) {
653 unsigned SubIdx = Op->getAsZExtVal();
654 unsigned SubReg = getVR(Op: Node->getOperand(Num: i-1), VRBaseMap);
655 const TargetRegisterClass *TRC = MRI->getRegClass(Reg: SubReg);
656 const TargetRegisterClass *SRC =
657 TRI->getMatchingSuperRegClass(A: RC, B: TRC, Idx: SubIdx);
658 if (SRC && SRC != RC) {
659 MRI->setRegClass(Reg: NewVReg, RC: SRC);
660 RC = SRC;
661 }
662 }
663 }
664 AddOperand(MIB, Op, IIOpNum: i+1, II: &II, VRBaseMap, /*IsDebug=*/false,
665 IsClone, IsCloned);
666 }
667
668 MBB->insert(I: InsertPos, MI: MIB);
669 SDValue Op(Node, 0);
670 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
671 (void)isNew; // Silence compiler warning.
672 assert(isNew && "Node emitted out of order - early");
673}
674
675/// EmitDbgValue - Generate machine instruction for a dbg_value node.
676///
677MachineInstr *
678InstrEmitter::EmitDbgValue(SDDbgValue *SD,
679 DenseMap<SDValue, Register> &VRBaseMap) {
680 DebugLoc DL = SD->getDebugLoc();
681 assert(cast<DILocalVariable>(SD->getVariable())
682 ->isValidLocationForIntrinsic(DL) &&
683 "Expected inlined-at fields to agree");
684
685 SD->setIsEmitted();
686
687 assert(!SD->getLocationOps().empty() &&
688 "dbg_value with no location operands?");
689
690 if (SD->isInvalidated())
691 return EmitDbgNoLocation(SD);
692
693 // Attempt to produce a DBG_INSTR_REF if we've been asked to.
694 if (EmitDebugInstrRefs)
695 if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
696 return InstrRef;
697
698 // Emit variadic dbg_value nodes as DBG_VALUE_LIST if they have not been
699 // emitted as instruction references.
700 if (SD->isVariadic())
701 return EmitDbgValueList(SD, VRBaseMap);
702
703 // Emit single-location dbg_value nodes as DBG_VALUE if they have not been
704 // emitted as instruction references.
705 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
706}
707
708MachineOperand GetMOForConstDbgOp(const SDDbgOperand &Op) {
709 const Value *V = Op.getConst();
710 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V)) {
711 if (CI->getBitWidth() > 64)
712 return MachineOperand::CreateCImm(CI);
713 return MachineOperand::CreateImm(Val: CI->getSExtValue());
714 }
715 if (const ConstantFP *CF = dyn_cast<ConstantFP>(Val: V))
716 return MachineOperand::CreateFPImm(CFP: CF);
717 // Note: This assumes that all nullptr constants are zero-valued.
718 if (isa<ConstantPointerNull>(Val: V))
719 return MachineOperand::CreateImm(Val: 0);
720 // Undef or unhandled value type, so return an undef operand.
721 return MachineOperand::CreateReg(
722 /* Reg */ 0U, /* isDef */ false, /* isImp */ false,
723 /* isKill */ false, /* isDead */ false,
724 /* isUndef */ false, /* isEarlyClobber */ false,
725 /* SubReg */ 0, /* isDebug */ true);
726}
727
728void InstrEmitter::AddDbgValueLocationOps(
729 MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
730 ArrayRef<SDDbgOperand> LocationOps,
731 DenseMap<SDValue, Register> &VRBaseMap) {
732 for (const SDDbgOperand &Op : LocationOps) {
733 switch (Op.getKind()) {
734 case SDDbgOperand::FRAMEIX:
735 MIB.addFrameIndex(Idx: Op.getFrameIx());
736 break;
737 case SDDbgOperand::VREG:
738 MIB.addReg(RegNo: Op.getVReg());
739 break;
740 case SDDbgOperand::SDNODE: {
741 SDValue V = SDValue(Op.getSDNode(), Op.getResNo());
742 // It's possible we replaced this SDNode with other(s) and therefore
743 // didn't generate code for it. It's better to catch these cases where
744 // they happen and transfer the debug info, but trying to guarantee that
745 // in all cases would be very fragile; this is a safeguard for any
746 // that were missed.
747 if (VRBaseMap.count(Val: V) == 0)
748 MIB.addReg(RegNo: 0U); // undef
749 else
750 AddOperand(MIB, Op: V, IIOpNum: (*MIB).getNumOperands(), II: &DbgValDesc, VRBaseMap,
751 /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false);
752 } break;
753 case SDDbgOperand::CONST:
754 MIB.add(MO: GetMOForConstDbgOp(Op));
755 break;
756 }
757 }
758}
759
760MachineInstr *
761InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
762 DenseMap<SDValue, Register> &VRBaseMap) {
763 MDNode *Var = SD->getVariable();
764 const DIExpression *Expr = (DIExpression *)SD->getExpression();
765 DebugLoc DL = SD->getDebugLoc();
766 const MCInstrDesc &RefII = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
767
768 // Returns true if the given operand is not a legal debug operand for a
769 // DBG_INSTR_REF.
770 auto IsInvalidOp = [](SDDbgOperand DbgOp) {
771 return DbgOp.getKind() == SDDbgOperand::FRAMEIX;
772 };
773 // Returns true if the given operand is not itself an instruction reference
774 // but is a legal debug operand for a DBG_INSTR_REF.
775 auto IsNonInstrRefOp = [](SDDbgOperand DbgOp) {
776 return DbgOp.getKind() == SDDbgOperand::CONST;
777 };
778
779 // If this variable location does not depend on any instructions or contains
780 // any stack locations, produce it as a standard debug value instead.
781 if (any_of(Range: SD->getLocationOps(), P: IsInvalidOp) ||
782 all_of(Range: SD->getLocationOps(), P: IsNonInstrRefOp)) {
783 if (SD->isVariadic())
784 return EmitDbgValueList(SD, VRBaseMap);
785 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
786 }
787
788 // Immediately fold any indirectness from the LLVM-IR intrinsic into the
789 // expression:
790 if (SD->isIndirect())
791 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
792 // If this is not already a variadic expression, it must be modified to become
793 // one.
794 if (!SD->isVariadic())
795 Expr = DIExpression::convertToVariadicExpression(Expr);
796
797 SmallVector<MachineOperand> MOs;
798
799 // It may not be immediately possible to identify the MachineInstr that
800 // defines a VReg, it can depend for example on the order blocks are
801 // emitted in. When this happens, or when further analysis is needed later,
802 // produce an instruction like this:
803 //
804 // DBG_INSTR_REF !123, !456, %0:gr64
805 //
806 // i.e., point the instruction at the vreg, and patch it up later in
807 // MachineFunction::finalizeDebugInstrRefs.
808 auto AddVRegOp = [&](unsigned VReg) {
809 MOs.push_back(Elt: MachineOperand::CreateReg(
810 /* Reg */ VReg, /* isDef */ false, /* isImp */ false,
811 /* isKill */ false, /* isDead */ false,
812 /* isUndef */ false, /* isEarlyClobber */ false,
813 /* SubReg */ 0, /* isDebug */ true));
814 };
815 unsigned OpCount = SD->getLocationOps().size();
816 for (unsigned OpIdx = 0; OpIdx < OpCount; ++OpIdx) {
817 SDDbgOperand DbgOperand = SD->getLocationOps()[OpIdx];
818
819 // Try to find both the defined register and the instruction defining it.
820 MachineInstr *DefMI = nullptr;
821 unsigned VReg;
822
823 if (DbgOperand.getKind() == SDDbgOperand::VREG) {
824 VReg = DbgOperand.getVReg();
825
826 // No definition means that block hasn't been emitted yet. Leave a vreg
827 // reference to be fixed later.
828 if (!MRI->hasOneDef(RegNo: VReg)) {
829 AddVRegOp(VReg);
830 continue;
831 }
832
833 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
834 } else if (DbgOperand.getKind() == SDDbgOperand::SDNODE) {
835 // Look up the corresponding VReg for the given SDNode, if any.
836 SDNode *Node = DbgOperand.getSDNode();
837 SDValue Op = SDValue(Node, DbgOperand.getResNo());
838 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Val: Op);
839 // No VReg -> produce a DBG_VALUE $noreg instead.
840 if (I == VRBaseMap.end())
841 break;
842
843 // Try to pick out a defining instruction at this point.
844 VReg = getVR(Op, VRBaseMap);
845
846 // Again, if there's no instruction defining the VReg right now, fix it up
847 // later.
848 if (!MRI->hasOneDef(RegNo: VReg)) {
849 AddVRegOp(VReg);
850 continue;
851 }
852
853 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
854 } else {
855 assert(DbgOperand.getKind() == SDDbgOperand::CONST);
856 MOs.push_back(Elt: GetMOForConstDbgOp(Op: DbgOperand));
857 continue;
858 }
859
860 // Avoid copy like instructions: they don't define values, only move them.
861 // Leave a virtual-register reference until it can be fixed up later, to
862 // find the underlying value definition.
863 if (DefMI->isCopyLike() || TII->isCopyInstr(MI: *DefMI)) {
864 AddVRegOp(VReg);
865 continue;
866 }
867
868 // Find the operand number which defines the specified VReg.
869 unsigned OperandIdx = 0;
870 for (const auto &MO : DefMI->operands()) {
871 if (MO.isReg() && MO.isDef() && MO.getReg() == VReg)
872 break;
873 ++OperandIdx;
874 }
875 assert(OperandIdx < DefMI->getNumOperands());
876
877 // Make the DBG_INSTR_REF refer to that instruction, and that operand.
878 unsigned InstrNum = DefMI->getDebugInstrNum();
879 MOs.push_back(Elt: MachineOperand::CreateDbgInstrRef(InstrIdx: InstrNum, OpIdx: OperandIdx));
880 }
881
882 // If we haven't created a valid MachineOperand for every DbgOp, abort and
883 // produce an undef DBG_VALUE.
884 if (MOs.size() != OpCount)
885 return EmitDbgNoLocation(SD);
886
887 return BuildMI(MF&: *MF, DL, MCID: RefII, IsIndirect: false, MOs, Variable: Var, Expr);
888}
889
890MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
891 // An invalidated SDNode must generate an undef DBG_VALUE: although the
892 // original value is no longer computed, earlier DBG_VALUEs live ranges
893 // must not leak into later code.
894 DIVariable *Var = SD->getVariable();
895 const DIExpression *Expr =
896 DIExpression::convertToUndefExpression(Expr: SD->getExpression());
897 DebugLoc DL = SD->getDebugLoc();
898 const MCInstrDesc &Desc = TII->get(Opcode: TargetOpcode::DBG_VALUE);
899 return BuildMI(MF&: *MF, DL, MCID: Desc, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
900}
901
902MachineInstr *
903InstrEmitter::EmitDbgValueList(SDDbgValue *SD,
904 DenseMap<SDValue, Register> &VRBaseMap) {
905 MDNode *Var = SD->getVariable();
906 DIExpression *Expr = SD->getExpression();
907 DebugLoc DL = SD->getDebugLoc();
908 // DBG_VALUE_LIST := "DBG_VALUE_LIST" var, expression, loc (, loc)*
909 const MCInstrDesc &DbgValDesc = TII->get(Opcode: TargetOpcode::DBG_VALUE_LIST);
910 // Build the DBG_VALUE_LIST instruction base.
911 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: DbgValDesc);
912 MIB.addMetadata(MD: Var);
913 MIB.addMetadata(MD: Expr);
914 AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps: SD->getLocationOps(), VRBaseMap);
915 return &*MIB;
916}
917
918MachineInstr *
919InstrEmitter::EmitDbgValueFromSingleOp(SDDbgValue *SD,
920 DenseMap<SDValue, Register> &VRBaseMap) {
921 MDNode *Var = SD->getVariable();
922 DIExpression *Expr = SD->getExpression();
923 DebugLoc DL = SD->getDebugLoc();
924 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_VALUE);
925
926 assert(SD->getLocationOps().size() == 1 &&
927 "Non variadic dbg_value should have only one location op");
928
929 // See about constant-folding the expression.
930 // Copy the location operand in case we replace it.
931 SmallVector<SDDbgOperand, 1> LocationOps(1, SD->getLocationOps()[0]);
932 if (Expr && LocationOps[0].getKind() == SDDbgOperand::CONST) {
933 const Value *V = LocationOps[0].getConst();
934 if (auto *C = dyn_cast<ConstantInt>(Val: V)) {
935 std::tie(args&: Expr, args&: C) = Expr->constantFold(CI: C);
936 LocationOps[0] = SDDbgOperand::fromConst(Const: C);
937 }
938 }
939
940 // Emit non-variadic dbg_value nodes as DBG_VALUE.
941 // DBG_VALUE := "DBG_VALUE" loc, isIndirect, var, expr
942 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
943 AddDbgValueLocationOps(MIB, DbgValDesc: II, LocationOps, VRBaseMap);
944
945 if (SD->isIndirect())
946 MIB.addImm(Val: 0U);
947 else
948 MIB.addReg(RegNo: 0U);
949
950 return MIB.addMetadata(MD: Var).addMetadata(MD: Expr);
951}
952
953MachineInstr *
954InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
955 MDNode *Label = SD->getLabel();
956 DebugLoc DL = SD->getDebugLoc();
957 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
958 "Expected inlined-at fields to agree");
959
960 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_LABEL);
961 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
962 MIB.addMetadata(MD: Label);
963
964 return &*MIB;
965}
966
967/// EmitMachineNode - Generate machine code for a target-specific node and
968/// needed dependencies.
969///
970void InstrEmitter::
971EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
972 DenseMap<SDValue, Register> &VRBaseMap) {
973 unsigned Opc = Node->getMachineOpcode();
974
975 // Handle subreg insert/extract specially
976 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
977 Opc == TargetOpcode::INSERT_SUBREG ||
978 Opc == TargetOpcode::SUBREG_TO_REG) {
979 EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
980 return;
981 }
982
983 // Handle COPY_TO_REGCLASS specially.
984 if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
985 EmitCopyToRegClassNode(Node, VRBaseMap);
986 return;
987 }
988
989 // Handle REG_SEQUENCE specially.
990 if (Opc == TargetOpcode::REG_SEQUENCE) {
991 EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
992 return;
993 }
994
995 if (Opc == TargetOpcode::IMPLICIT_DEF)
996 // We want a unique VR for each IMPLICIT_DEF use.
997 return;
998
999 const MCInstrDesc &II = TII->get(Opcode: Opc);
1000 unsigned NumResults = CountResults(Node);
1001 unsigned NumDefs = II.getNumDefs();
1002 const MCPhysReg *ScratchRegs = nullptr;
1003
1004 // Handle STACKMAP and PATCHPOINT specially and then use the generic code.
1005 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
1006 // Stackmaps do not have arguments and do not preserve their calling
1007 // convention. However, to simplify runtime support, they clobber the same
1008 // scratch registers as AnyRegCC.
1009 unsigned CC = CallingConv::AnyReg;
1010 if (Opc == TargetOpcode::PATCHPOINT) {
1011 CC = Node->getConstantOperandVal(Num: PatchPointOpers::CCPos);
1012 NumDefs = NumResults;
1013 }
1014 ScratchRegs = TLI->getScratchRegisters(CC: (CallingConv::ID) CC);
1015 } else if (Opc == TargetOpcode::STATEPOINT) {
1016 NumDefs = NumResults;
1017 }
1018
1019 unsigned NumImpUses = 0;
1020 unsigned NodeOperands =
1021 countOperands(Node, NumExpUses: II.getNumOperands() - NumDefs, NumImpUses);
1022 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
1023 II.isVariadic() && II.variadicOpsAreDefs();
1024 bool HasPhysRegOuts = NumResults > NumDefs && !II.implicit_defs().empty() &&
1025 !HasVRegVariadicDefs;
1026#ifndef NDEBUG
1027 unsigned NumMIOperands = NodeOperands + NumResults;
1028 if (II.isVariadic())
1029 assert(NumMIOperands >= II.getNumOperands() &&
1030 "Too few operands for a variadic node!");
1031 else
1032 assert(NumMIOperands >= II.getNumOperands() &&
1033 NumMIOperands <=
1034 II.getNumOperands() + II.implicit_defs().size() + NumImpUses &&
1035 "#operands for dag node doesn't match .td file!");
1036#endif
1037
1038 // Create the new machine instruction.
1039 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II);
1040
1041 // Add result register values for things that are defined by this
1042 // instruction.
1043 if (NumResults) {
1044 CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
1045
1046 // Transfer any IR flags from the SDNode to the MachineInstr
1047 MachineInstr *MI = MIB.getInstr();
1048 const SDNodeFlags Flags = Node->getFlags();
1049 if (Flags.hasNoSignedZeros())
1050 MI->setFlag(MachineInstr::MIFlag::FmNsz);
1051
1052 if (Flags.hasAllowReciprocal())
1053 MI->setFlag(MachineInstr::MIFlag::FmArcp);
1054
1055 if (Flags.hasNoNaNs())
1056 MI->setFlag(MachineInstr::MIFlag::FmNoNans);
1057
1058 if (Flags.hasNoInfs())
1059 MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
1060
1061 if (Flags.hasAllowContract())
1062 MI->setFlag(MachineInstr::MIFlag::FmContract);
1063
1064 if (Flags.hasApproximateFuncs())
1065 MI->setFlag(MachineInstr::MIFlag::FmAfn);
1066
1067 if (Flags.hasAllowReassociation())
1068 MI->setFlag(MachineInstr::MIFlag::FmReassoc);
1069
1070 if (Flags.hasNoUnsignedWrap())
1071 MI->setFlag(MachineInstr::MIFlag::NoUWrap);
1072
1073 if (Flags.hasNoSignedWrap())
1074 MI->setFlag(MachineInstr::MIFlag::NoSWrap);
1075
1076 if (Flags.hasExact())
1077 MI->setFlag(MachineInstr::MIFlag::IsExact);
1078
1079 if (Flags.hasNoFPExcept())
1080 MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1081
1082 if (Flags.hasUnpredictable())
1083 MI->setFlag(MachineInstr::MIFlag::Unpredictable);
1084 }
1085
1086 // Emit all of the actual operands of this instruction, adding them to the
1087 // instruction as appropriate.
1088 bool HasOptPRefs = NumDefs > NumResults;
1089 assert((!HasOptPRefs || !HasPhysRegOuts) &&
1090 "Unable to cope with optional defs and phys regs defs!");
1091 unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
1092 for (unsigned i = NumSkip; i != NodeOperands; ++i)
1093 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: i-NumSkip+NumDefs, II: &II,
1094 VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
1095
1096 // Add scratch registers as implicit def and early clobber
1097 if (ScratchRegs)
1098 for (unsigned i = 0; ScratchRegs[i]; ++i)
1099 MIB.addReg(RegNo: ScratchRegs[i], flags: RegState::ImplicitDefine |
1100 RegState::EarlyClobber);
1101
1102 // Set the memory reference descriptions of this instruction now that it is
1103 // part of the function.
1104 MIB.setMemRefs(cast<MachineSDNode>(Val: Node)->memoperands());
1105
1106 // Set the CFI type.
1107 MIB->setCFIType(MF&: *MF, Type: Node->getCFIType());
1108
1109 // Insert the instruction into position in the block. This needs to
1110 // happen before any custom inserter hook is called so that the
1111 // hook knows where in the block to insert the replacement code.
1112 MBB->insert(I: InsertPos, MI: MIB);
1113
1114 // The MachineInstr may also define physregs instead of virtregs. These
1115 // physreg values can reach other instructions in different ways:
1116 //
1117 // 1. When there is a use of a Node value beyond the explicitly defined
1118 // virtual registers, we emit a CopyFromReg for one of the implicitly
1119 // defined physregs. This only happens when HasPhysRegOuts is true.
1120 //
1121 // 2. A CopyFromReg reading a physreg may be glued to this instruction.
1122 //
1123 // 3. A glued instruction may implicitly use a physreg.
1124 //
1125 // 4. A glued instruction may use a RegisterSDNode operand.
1126 //
1127 // Collect all the used physreg defs, and make sure that any unused physreg
1128 // defs are marked as dead.
1129 SmallVector<Register, 8> UsedRegs;
1130
1131 // Additional results must be physical register defs.
1132 if (HasPhysRegOuts) {
1133 for (unsigned i = NumDefs; i < NumResults; ++i) {
1134 Register Reg = II.implicit_defs()[i - NumDefs];
1135 if (!Node->hasAnyUseOfValue(Value: i))
1136 continue;
1137 // This implicitly defined physreg has a use.
1138 UsedRegs.push_back(Elt: Reg);
1139 EmitCopyFromReg(Node, ResNo: i, IsClone, SrcReg: Reg, VRBaseMap);
1140 }
1141 }
1142
1143 // Scan the glue chain for any used physregs.
1144 if (Node->getValueType(ResNo: Node->getNumValues()-1) == MVT::Glue) {
1145 for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
1146 if (F->getOpcode() == ISD::CopyFromReg) {
1147 UsedRegs.push_back(Elt: cast<RegisterSDNode>(Val: F->getOperand(Num: 1))->getReg());
1148 continue;
1149 } else if (F->getOpcode() == ISD::CopyToReg) {
1150 // Skip CopyToReg nodes that are internal to the glue chain.
1151 continue;
1152 }
1153 // Collect declared implicit uses.
1154 const MCInstrDesc &MCID = TII->get(Opcode: F->getMachineOpcode());
1155 append_range(C&: UsedRegs, R: MCID.implicit_uses());
1156 // In addition to declared implicit uses, we must also check for
1157 // direct RegisterSDNode operands.
1158 for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
1159 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: F->getOperand(Num: i))) {
1160 Register Reg = R->getReg();
1161 if (Reg.isPhysical())
1162 UsedRegs.push_back(Elt: Reg);
1163 }
1164 }
1165 }
1166
1167 // Add rounding control registers as implicit def for function call.
1168 if (II.isCall() && MF->getFunction().hasFnAttribute(Attribute::StrictFP)) {
1169 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
1170 for (MCPhysReg Reg : RCRegs)
1171 UsedRegs.push_back(Elt: Reg);
1172 }
1173
1174 // Finally mark unused registers as dead.
1175 if (!UsedRegs.empty() || !II.implicit_defs().empty() || II.hasOptionalDef())
1176 MIB->setPhysRegsDeadExcept(UsedRegs, TRI: *TRI);
1177
1178 // STATEPOINT is too 'dynamic' to have meaningful machine description.
1179 // We have to manually tie operands.
1180 if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
1181 assert(!HasPhysRegOuts && "STATEPOINT mishandled");
1182 MachineInstr *MI = MIB;
1183 unsigned Def = 0;
1184 int First = StatepointOpers(MI).getFirstGCPtrIdx();
1185 assert(First > 0 && "Statepoint has Defs but no GC ptr list");
1186 unsigned Use = (unsigned)First;
1187 while (Def < NumDefs) {
1188 if (MI->getOperand(i: Use).isReg())
1189 MI->tieOperands(DefIdx: Def++, UseIdx: Use);
1190 Use = StackMaps::getNextMetaArgIdx(MI, CurIdx: Use);
1191 }
1192 }
1193
1194 // Run post-isel target hook to adjust this instruction if needed.
1195 if (II.hasPostISelHook())
1196 TLI->AdjustInstrPostInstrSelection(MI&: *MIB, Node);
1197}
1198
1199/// EmitSpecialNode - Generate machine code for a target-independent node and
1200/// needed dependencies.
1201void InstrEmitter::
1202EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
1203 DenseMap<SDValue, Register> &VRBaseMap) {
1204 switch (Node->getOpcode()) {
1205 default:
1206#ifndef NDEBUG
1207 Node->dump();
1208#endif
1209 llvm_unreachable("This target-independent node should have been selected!");
1210 case ISD::EntryToken:
1211 case ISD::MERGE_VALUES:
1212 case ISD::TokenFactor: // fall thru
1213 break;
1214 case ISD::CopyToReg: {
1215 Register DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1216 SDValue SrcVal = Node->getOperand(Num: 2);
1217 if (DestReg.isVirtual() && SrcVal.isMachineOpcode() &&
1218 SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
1219 // Instead building a COPY to that vreg destination, build an
1220 // IMPLICIT_DEF instruction instead.
1221 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1222 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg);
1223 break;
1224 }
1225 Register SrcReg;
1226 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: SrcVal))
1227 SrcReg = R->getReg();
1228 else
1229 SrcReg = getVR(Op: SrcVal, VRBaseMap);
1230
1231 if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
1232 break;
1233
1234 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
1235 DestReg).addReg(RegNo: SrcReg);
1236 break;
1237 }
1238 case ISD::CopyFromReg: {
1239 unsigned SrcReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1240 EmitCopyFromReg(Node, ResNo: 0, IsClone, SrcReg, VRBaseMap);
1241 break;
1242 }
1243 case ISD::EH_LABEL:
1244 case ISD::ANNOTATION_LABEL: {
1245 unsigned Opc = (Node->getOpcode() == ISD::EH_LABEL)
1246 ? TargetOpcode::EH_LABEL
1247 : TargetOpcode::ANNOTATION_LABEL;
1248 MCSymbol *S = cast<LabelSDNode>(Val: Node)->getLabel();
1249 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1250 MCID: TII->get(Opcode: Opc)).addSym(Sym: S);
1251 break;
1252 }
1253
1254 case ISD::LIFETIME_START:
1255 case ISD::LIFETIME_END: {
1256 unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START)
1257 ? TargetOpcode::LIFETIME_START
1258 : TargetOpcode::LIFETIME_END;
1259 auto *FI = cast<FrameIndexSDNode>(Val: Node->getOperand(Num: 1));
1260 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1261 .addFrameIndex(Idx: FI->getIndex());
1262 break;
1263 }
1264
1265 case ISD::PSEUDO_PROBE: {
1266 unsigned TarOp = TargetOpcode::PSEUDO_PROBE;
1267 auto Guid = cast<PseudoProbeSDNode>(Val: Node)->getGuid();
1268 auto Index = cast<PseudoProbeSDNode>(Val: Node)->getIndex();
1269 auto Attr = cast<PseudoProbeSDNode>(Val: Node)->getAttributes();
1270
1271 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1272 .addImm(Val: Guid)
1273 .addImm(Val: Index)
1274 .addImm(Val: (uint8_t)PseudoProbeType::Block)
1275 .addImm(Val: Attr);
1276 break;
1277 }
1278
1279 case ISD::INLINEASM:
1280 case ISD::INLINEASM_BR: {
1281 unsigned NumOps = Node->getNumOperands();
1282 if (Node->getOperand(Num: NumOps-1).getValueType() == MVT::Glue)
1283 --NumOps; // Ignore the glue operand.
1284
1285 // Create the inline asm machine instruction.
1286 unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
1287 ? TargetOpcode::INLINEASM_BR
1288 : TargetOpcode::INLINEASM;
1289 MachineInstrBuilder MIB =
1290 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TgtOpc));
1291
1292 // Add the asm string as an external symbol operand.
1293 SDValue AsmStrV = Node->getOperand(Num: InlineAsm::Op_AsmString);
1294 const char *AsmStr = cast<ExternalSymbolSDNode>(Val&: AsmStrV)->getSymbol();
1295 MIB.addExternalSymbol(FnName: AsmStr);
1296
1297 // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
1298 // bits.
1299 int64_t ExtraInfo =
1300 cast<ConstantSDNode>(Val: Node->getOperand(Num: InlineAsm::Op_ExtraInfo))->
1301 getZExtValue();
1302 MIB.addImm(Val: ExtraInfo);
1303
1304 // Remember to operand index of the group flags.
1305 SmallVector<unsigned, 8> GroupIdx;
1306
1307 // Remember registers that are part of early-clobber defs.
1308 SmallVector<unsigned, 8> ECRegs;
1309
1310 // Add all of the operand registers to the instruction.
1311 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1312 unsigned Flags = Node->getConstantOperandVal(Num: i);
1313 const InlineAsm::Flag F(Flags);
1314 const unsigned NumVals = F.getNumOperandRegisters();
1315
1316 GroupIdx.push_back(Elt: MIB->getNumOperands());
1317 MIB.addImm(Val: Flags);
1318 ++i; // Skip the ID value.
1319
1320 switch (F.getKind()) {
1321 case InlineAsm::Kind::RegDef:
1322 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1323 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1324 // FIXME: Add dead flags for physical and virtual registers defined.
1325 // For now, mark physical register defs as implicit to help fast
1326 // regalloc. This makes inline asm look a lot like calls.
1327 MIB.addReg(RegNo: Reg, flags: RegState::Define | getImplRegState(B: Reg.isPhysical()));
1328 }
1329 break;
1330 case InlineAsm::Kind::RegDefEarlyClobber:
1331 case InlineAsm::Kind::Clobber:
1332 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1333 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1334 MIB.addReg(RegNo: Reg, flags: RegState::Define | RegState::EarlyClobber |
1335 getImplRegState(B: Reg.isPhysical()));
1336 ECRegs.push_back(Elt: Reg);
1337 }
1338 break;
1339 case InlineAsm::Kind::RegUse: // Use of register.
1340 case InlineAsm::Kind::Imm: // Immediate.
1341 case InlineAsm::Kind::Mem: // Non-function addressing mode.
1342 // The addressing mode has been selected, just add all of the
1343 // operands to the machine instruction.
1344 for (unsigned j = 0; j != NumVals; ++j, ++i)
1345 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: 0, II: nullptr, VRBaseMap,
1346 /*IsDebug=*/false, IsClone, IsCloned);
1347
1348 // Manually set isTied bits.
1349 if (F.isRegUseKind()) {
1350 unsigned DefGroup;
1351 if (F.isUseOperandTiedToDef(Idx&: DefGroup)) {
1352 unsigned DefIdx = GroupIdx[DefGroup] + 1;
1353 unsigned UseIdx = GroupIdx.back() + 1;
1354 for (unsigned j = 0; j != NumVals; ++j)
1355 MIB->tieOperands(DefIdx: DefIdx + j, UseIdx: UseIdx + j);
1356 }
1357 }
1358 break;
1359 case InlineAsm::Kind::Func: // Function addressing mode.
1360 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1361 SDValue Op = Node->getOperand(Num: i);
1362 AddOperand(MIB, Op, IIOpNum: 0, II: nullptr, VRBaseMap,
1363 /*IsDebug=*/false, IsClone, IsCloned);
1364
1365 // Adjust Target Flags for function reference.
1366 if (auto *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
1367 unsigned NewFlags =
1368 MF->getSubtarget().classifyGlobalFunctionReference(
1369 GV: TGA->getGlobal());
1370 unsigned LastIdx = MIB.getInstr()->getNumOperands() - 1;
1371 MIB.getInstr()->getOperand(i: LastIdx).setTargetFlags(NewFlags);
1372 }
1373 }
1374 }
1375 }
1376
1377 // GCC inline assembly allows input operands to also be early-clobber
1378 // output operands (so long as the operand is written only after it's
1379 // used), but this does not match the semantics of our early-clobber flag.
1380 // If an early-clobber operand register is also an input operand register,
1381 // then remove the early-clobber flag.
1382 for (unsigned Reg : ECRegs) {
1383 if (MIB->readsRegister(Reg, TRI)) {
1384 MachineOperand *MO =
1385 MIB->findRegisterDefOperand(Reg, isDead: false, Overlap: false, TRI);
1386 assert(MO && "No def operand for clobbered register?");
1387 MO->setIsEarlyClobber(false);
1388 }
1389 }
1390
1391 // Get the mdnode from the asm if it exists and add it to the instruction.
1392 SDValue MDV = Node->getOperand(Num: InlineAsm::Op_MDNode);
1393 const MDNode *MD = cast<MDNodeSDNode>(Val&: MDV)->getMD();
1394 if (MD)
1395 MIB.addMetadata(MD);
1396
1397 MBB->insert(I: InsertPos, MI: MIB);
1398 break;
1399 }
1400 }
1401}
1402
1403/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
1404/// at the given position in the given block.
1405InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
1406 MachineBasicBlock::iterator insertpos)
1407 : MF(mbb->getParent()), MRI(&MF->getRegInfo()),
1408 TII(MF->getSubtarget().getInstrInfo()),
1409 TRI(MF->getSubtarget().getRegisterInfo()),
1410 TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
1411 InsertPos(insertpos) {
1412 EmitDebugInstrRefs = mbb->getParent()->useDebugInstrRef();
1413}
1414

source code of llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp