1//===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the RISC-V assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVInstPrinter.h"
16#include "MCTargetDesc/RISCVMCExpr.h"
17#include "MCTargetDesc/RISCVTargetStreamer.h"
18#include "RISCV.h"
19#include "RISCVMachineFunctionInfo.h"
20#include "RISCVTargetMachine.h"
21#include "TargetInfo/RISCVTargetInfo.h"
22#include "llvm/ADT/APInt.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/BinaryFormat/ELF.h"
25#include "llvm/CodeGen/AsmPrinter.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/MC/MCAsmInfo.h"
31#include "llvm/MC/MCContext.h"
32#include "llvm/MC/MCInst.h"
33#include "llvm/MC/MCInstBuilder.h"
34#include "llvm/MC/MCObjectFileInfo.h"
35#include "llvm/MC/MCSectionELF.h"
36#include "llvm/MC/MCStreamer.h"
37#include "llvm/MC/MCSymbol.h"
38#include "llvm/MC/TargetRegistry.h"
39#include "llvm/Support/RISCVISAInfo.h"
40#include "llvm/Support/raw_ostream.h"
41#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
42
43using namespace llvm;
44
45#define DEBUG_TYPE "asm-printer"
46
47STATISTIC(RISCVNumInstrsCompressed,
48 "Number of RISC-V Compressed instructions emitted");
49
50namespace llvm {
51extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures];
52} // namespace llvm
53
54namespace {
55class RISCVAsmPrinter : public AsmPrinter {
56 const RISCVSubtarget *STI;
57
58public:
59 explicit RISCVAsmPrinter(TargetMachine &TM,
60 std::unique_ptr<MCStreamer> Streamer)
61 : AsmPrinter(TM, std::move(Streamer)) {}
62
63 StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
64
65 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
66 const MachineInstr &MI);
67
68 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
69 const MachineInstr &MI);
70
71 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
72 const MachineInstr &MI);
73
74 bool runOnMachineFunction(MachineFunction &MF) override;
75
76 void emitInstruction(const MachineInstr *MI) override;
77
78 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
79 const char *ExtraCode, raw_ostream &OS) override;
80 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
81 const char *ExtraCode, raw_ostream &OS) override;
82
83 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
84 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
85 const MachineInstr *MI);
86
87 typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
88 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
89 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
90 void LowerKCFI_CHECK(const MachineInstr &MI);
91 void EmitHwasanMemaccessSymbols(Module &M);
92
93 // Wrapper needed for tblgenned pseudo lowering.
94 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
95
96 void emitStartOfAsmFile(Module &M) override;
97 void emitEndOfAsmFile(Module &M) override;
98
99 void emitFunctionEntryLabel() override;
100 bool emitDirectiveOptionArch();
101
102private:
103 void emitAttributes();
104
105 void emitNTLHint(const MachineInstr *MI);
106
107 bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
108};
109}
110
111void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
112 const MachineInstr &MI) {
113 unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
114 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
115
116 auto &Ctx = OutStreamer.getContext();
117 MCSymbol *MILabel = Ctx.createTempSymbol();
118 OutStreamer.emitLabel(Symbol: MILabel);
119
120 SM.recordStackMap(L: *MILabel, MI);
121 assert(NumNOPBytes % NOPBytes == 0 &&
122 "Invalid number of NOP bytes requested!");
123
124 // Scan ahead to trim the shadow.
125 const MachineBasicBlock &MBB = *MI.getParent();
126 MachineBasicBlock::const_iterator MII(MI);
127 ++MII;
128 while (NumNOPBytes > 0) {
129 if (MII == MBB.end() || MII->isCall() ||
130 MII->getOpcode() == RISCV::DBG_VALUE ||
131 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
132 MII->getOpcode() == TargetOpcode::STACKMAP)
133 break;
134 ++MII;
135 NumNOPBytes -= 4;
136 }
137
138 // Emit nops.
139 emitNops(N: NumNOPBytes / NOPBytes);
140}
141
142// Lower a patchpoint of the form:
143// [<def>], <id>, <numBytes>, <target>, <numArgs>
144void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
145 const MachineInstr &MI) {
146 unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
147
148 auto &Ctx = OutStreamer.getContext();
149 MCSymbol *MILabel = Ctx.createTempSymbol();
150 OutStreamer.emitLabel(Symbol: MILabel);
151 SM.recordPatchPoint(L: *MILabel, MI);
152
153 PatchPointOpers Opers(&MI);
154
155 unsigned EncodedBytes = 0;
156
157 // Emit padding.
158 unsigned NumBytes = Opers.getNumPatchBytes();
159 assert(NumBytes >= EncodedBytes &&
160 "Patchpoint can't request size less than the length of a call.");
161 assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
162 "Invalid number of NOP bytes requested!");
163 emitNops(N: (NumBytes - EncodedBytes) / NOPBytes);
164}
165
166void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
167 const MachineInstr &MI) {
168 unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
169
170 StatepointOpers SOpers(&MI);
171 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
172 assert(PatchBytes % NOPBytes == 0 &&
173 "Invalid number of NOP bytes requested!");
174 emitNops(N: PatchBytes / NOPBytes);
175 }
176
177 auto &Ctx = OutStreamer.getContext();
178 MCSymbol *MILabel = Ctx.createTempSymbol();
179 OutStreamer.emitLabel(Symbol: MILabel);
180 SM.recordStatepoint(L: *MILabel, MI);
181}
182
183void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
184 MCInst CInst;
185 bool Res = RISCVRVC::compress(CInst, Inst, *STI);
186 if (Res)
187 ++RISCVNumInstrsCompressed;
188 AsmPrinter::EmitToStreamer(S&: *OutStreamer, Inst: Res ? CInst : Inst);
189}
190
191// Simple pseudo-instructions have their lowering (with expansion to real
192// instructions) auto-generated.
193#include "RISCVGenMCPseudoLowering.inc"
194
195// If the target supports Zihintntl and the instruction has a nontemporal
196// MachineMemOperand, emit an NTLH hint instruction before it.
197void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
198 if (!STI->hasStdExtZihintntl())
199 return;
200
201 if (MI->memoperands_empty())
202 return;
203
204 MachineMemOperand *MMO = *(MI->memoperands_begin());
205 if (!MMO->isNonTemporal())
206 return;
207
208 unsigned NontemporalMode = 0;
209 if (MMO->getFlags() & MONontemporalBit0)
210 NontemporalMode += 0b1;
211 if (MMO->getFlags() & MONontemporalBit1)
212 NontemporalMode += 0b10;
213
214 MCInst Hint;
215 if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
216 Hint.setOpcode(RISCV::C_ADD_HINT);
217 else
218 Hint.setOpcode(RISCV::ADD);
219
220 Hint.addOperand(Op: MCOperand::createReg(RISCV::Reg: X0));
221 Hint.addOperand(Op: MCOperand::createReg(RISCV::Reg: X0));
222 Hint.addOperand(Op: MCOperand::createReg(RISCV::Reg: X2 + NontemporalMode));
223
224 EmitToStreamer(S&: *OutStreamer, Inst: Hint);
225}
226
227void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
228 RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
229 getSubtargetInfo().getFeatureBits());
230
231 emitNTLHint(MI);
232
233 // Do any auto-generated pseudo lowerings.
234 if (emitPseudoExpansionLowering(OutStreamer&: *OutStreamer, MI))
235 return;
236
237
238 switch (MI->getOpcode()) {
239 case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
240 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
241 return;
242 case RISCV::KCFI_CHECK:
243 LowerKCFI_CHECK(MI: *MI);
244 return;
245 case RISCV::PseudoRVVInitUndefM1:
246 case RISCV::PseudoRVVInitUndefM2:
247 case RISCV::PseudoRVVInitUndefM4:
248 case RISCV::PseudoRVVInitUndefM8:
249 return;
250 case TargetOpcode::STACKMAP:
251 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
252 case TargetOpcode::PATCHPOINT:
253 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
254 case TargetOpcode::STATEPOINT:
255 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
256 }
257
258 MCInst OutInst;
259 if (!lowerToMCInst(MI, OutMI&: OutInst))
260 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
261}
262
263bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
264 const char *ExtraCode, raw_ostream &OS) {
265 // First try the generic code, which knows about modifiers like 'c' and 'n'.
266 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS))
267 return false;
268
269 const MachineOperand &MO = MI->getOperand(i: OpNo);
270 if (ExtraCode && ExtraCode[0]) {
271 if (ExtraCode[1] != 0)
272 return true; // Unknown modifier.
273
274 switch (ExtraCode[0]) {
275 default:
276 return true; // Unknown modifier.
277 case 'z': // Print zero register if zero, regular printing otherwise.
278 if (MO.isImm() && MO.getImm() == 0) {
279 OS << RISCVInstPrinter::getRegisterName(RISCV::X0);
280 return false;
281 }
282 break;
283 case 'i': // Literal 'i' if operand is not a register.
284 if (!MO.isReg())
285 OS << 'i';
286 return false;
287 }
288 }
289
290 switch (MO.getType()) {
291 case MachineOperand::MO_Immediate:
292 OS << MO.getImm();
293 return false;
294 case MachineOperand::MO_Register:
295 OS << RISCVInstPrinter::getRegisterName(Reg: MO.getReg());
296 return false;
297 case MachineOperand::MO_GlobalAddress:
298 PrintSymbolOperand(MO, OS);
299 return false;
300 case MachineOperand::MO_BlockAddress: {
301 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
302 Sym->print(OS, MAI);
303 return false;
304 }
305 default:
306 break;
307 }
308
309 return true;
310}
311
312bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
313 unsigned OpNo,
314 const char *ExtraCode,
315 raw_ostream &OS) {
316 if (ExtraCode)
317 return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS);
318
319 const MachineOperand &AddrReg = MI->getOperand(i: OpNo);
320 assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand");
321 const MachineOperand &Offset = MI->getOperand(i: OpNo + 1);
322 // All memory operands should have a register and an immediate operand (see
323 // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand).
324 if (!AddrReg.isReg())
325 return true;
326 if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() &&
327 !Offset.isMCSymbol())
328 return true;
329
330 MCOperand MCO;
331 if (!lowerOperand(MO: Offset, MCOp&: MCO))
332 return true;
333
334 if (Offset.isImm())
335 OS << MCO.getImm();
336 else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
337 OS << *MCO.getExpr();
338 OS << "(" << RISCVInstPrinter::getRegisterName(Reg: AddrReg.getReg()) << ")";
339 return false;
340}
341
342bool RISCVAsmPrinter::emitDirectiveOptionArch() {
343 RISCVTargetStreamer &RTS =
344 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
345 SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs;
346 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
347 for (const auto &Feature : RISCVFeatureKV) {
348 if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value))
349 continue;
350
351 if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
352 continue;
353
354 auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus
355 : RISCVOptionArchArgType::Minus;
356 NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key);
357 }
358 if (!NeedEmitStdOptionArgs.empty()) {
359 RTS.emitDirectiveOptionPush();
360 RTS.emitDirectiveOptionArch(Args: NeedEmitStdOptionArgs);
361 return true;
362 }
363
364 return false;
365}
366
367bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
368 STI = &MF.getSubtarget<RISCVSubtarget>();
369 RISCVTargetStreamer &RTS =
370 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
371
372 bool EmittedOptionArch = emitDirectiveOptionArch();
373
374 SetupMachineFunction(MF);
375 emitFunctionBody();
376
377 if (EmittedOptionArch)
378 RTS.emitDirectiveOptionPop();
379 return false;
380}
381
382void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
383 RISCVTargetStreamer &RTS =
384 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
385 if (const MDString *ModuleTargetABI =
386 dyn_cast_or_null<MDString>(Val: M.getModuleFlag(Key: "target-abi")))
387 RTS.setTargetABI(RISCVABI::getTargetABI(ABIName: ModuleTargetABI->getString()));
388 if (TM.getTargetTriple().isOSBinFormatELF())
389 emitAttributes();
390}
391
392void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
393 RISCVTargetStreamer &RTS =
394 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
395
396 if (TM.getTargetTriple().isOSBinFormatELF())
397 RTS.finishAttributeSection();
398 EmitHwasanMemaccessSymbols(M);
399}
400
401void RISCVAsmPrinter::emitAttributes() {
402 RISCVTargetStreamer &RTS =
403 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
404 // Use MCSubtargetInfo from TargetMachine. Individual functions may have
405 // attributes that differ from other functions in the module and we have no
406 // way to know which function is correct.
407 RTS.emitTargetAttributes(STI: *TM.getMCSubtargetInfo(), /*EmitStackAlign*/ true);
408}
409
410void RISCVAsmPrinter::emitFunctionEntryLabel() {
411 const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>();
412 if (RMFI->isVectorCall()) {
413 auto &RTS =
414 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
415 RTS.emitDirectiveVariantCC(Symbol&: *CurrentFnSym);
416 }
417 return AsmPrinter::emitFunctionEntryLabel();
418}
419
420// Force static initialization.
421extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
422 RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
423 RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
424}
425
426void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
427 Register Reg = MI.getOperand(i: 0).getReg();
428 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
429 MCSymbol *&Sym =
430 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)];
431 if (!Sym) {
432 // FIXME: Make this work on non-ELF.
433 if (!TM.getTargetTriple().isOSBinFormatELF())
434 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
435
436 std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" +
437 utostr(X: AccessInfo) + "_short";
438 Sym = OutContext.getOrCreateSymbol(Name: SymName);
439 }
440 auto Res = MCSymbolRefExpr::create(Symbol: Sym, Kind: MCSymbolRefExpr::VK_None, Ctx&: OutContext);
441 auto Expr = RISCVMCExpr::create(Expr: Res, Kind: RISCVMCExpr::VK_RISCV_CALL, Ctx&: OutContext);
442
443 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
444}
445
446void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
447 Register AddrReg = MI.getOperand(i: 0).getReg();
448 assert(std::next(MI.getIterator())->isCall() &&
449 "KCFI_CHECK not followed by a call instruction");
450 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
451 "KCFI_CHECK call target doesn't match call operand");
452
453 // Temporary registers for comparing the hashes. If a register is used
454 // for the call target, or reserved by the user, we can clobber another
455 // temporary register as the check is immediately followed by the
456 // call. The check defaults to X6/X7, but can fall back to X28-X31 if
457 // needed.
458 unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
459 unsigned NextReg = RISCV::X28;
460 auto isRegAvailable = [&](unsigned Reg) {
461 return Reg != AddrReg && !STI->isRegisterReservedByUser(i: Reg);
462 };
463 for (auto &Reg : ScratchRegs) {
464 if (isRegAvailable(Reg))
465 continue;
466 while (!isRegAvailable(NextReg))
467 ++NextReg;
468 Reg = NextReg++;
469 if (Reg > RISCV::X31)
470 report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
471 }
472
473 if (AddrReg == RISCV::X0) {
474 // Checking X0 makes no sense. Instead of emitting a load, zero
475 // ScratchRegs[0].
476 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
477 .addReg(ScratchRegs[0])
478 .addReg(RISCV::X0)
479 .addImm(0));
480 } else {
481 // Adjust the offset for patchable-function-prefix. This assumes that
482 // patchable-function-prefix is the same for all functions.
483 int NopSize = STI->hasStdExtCOrZca() ? 2 : 4;
484 int64_t PrefixNops = 0;
485 (void)MI.getMF()
486 ->getFunction()
487 .getFnAttribute(Kind: "patchable-function-prefix")
488 .getValueAsString()
489 .getAsInteger(Radix: 10, Result&: PrefixNops);
490
491 // Load the target function type hash.
492 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
493 .addReg(ScratchRegs[0])
494 .addReg(AddrReg)
495 .addImm(-(PrefixNops * NopSize + 4)));
496 }
497
498 // Load the expected 32-bit type hash.
499 const int64_t Type = MI.getOperand(i: 1).getImm();
500 const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
501 const int64_t Lo12 = SignExtend64<12>(x: Type);
502 if (Hi20) {
503 EmitToStreamer(
504 *OutStreamer,
505 MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
506 }
507 if (Lo12 || Hi20 == 0) {
508 EmitToStreamer(*OutStreamer,
509 MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
510 ? RISCV::ADDIW
511 : RISCV::ADDI)
512 .addReg(ScratchRegs[1])
513 .addReg(ScratchRegs[1])
514 .addImm(Lo12));
515 }
516
517 // Compare the hashes and trap if there's a mismatch.
518 MCSymbol *Pass = OutContext.createTempSymbol();
519 EmitToStreamer(*OutStreamer,
520 MCInstBuilder(RISCV::BEQ)
521 .addReg(ScratchRegs[0])
522 .addReg(ScratchRegs[1])
523 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
524
525 MCSymbol *Trap = OutContext.createTempSymbol();
526 OutStreamer->emitLabel(Symbol: Trap);
527 EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
528 emitKCFITrapEntry(MF: *MI.getMF(), Symbol: Trap);
529 OutStreamer->emitLabel(Symbol: Pass);
530}
531
532void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
533 if (HwasanMemaccessSymbols.empty())
534 return;
535
536 assert(TM.getTargetTriple().isOSBinFormatELF());
537 // Use MCSubtargetInfo from TargetMachine. Individual functions may have
538 // attributes that differ from other functions in the module and we have no
539 // way to know which function is correct.
540 const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
541
542 MCSymbol *HwasanTagMismatchV2Sym =
543 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
544 // Annotate symbol as one having incompatible calling convention, so
545 // run-time linkers can instead eagerly bind this function.
546 auto &RTS =
547 static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
548 RTS.emitDirectiveVariantCC(Symbol&: *HwasanTagMismatchV2Sym);
549
550 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
551 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
552 auto Expr = RISCVMCExpr::create(Expr: HwasanTagMismatchV2Ref,
553 Kind: RISCVMCExpr::VK_RISCV_CALL, Ctx&: OutContext);
554
555 for (auto &P : HwasanMemaccessSymbols) {
556 unsigned Reg = std::get<0>(t: P.first);
557 uint32_t AccessInfo = std::get<1>(t: P.first);
558 MCSymbol *Sym = P.second;
559
560 unsigned Size =
561 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
562 OutStreamer->switchSection(Section: OutContext.getELFSection(
563 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
564 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
565 /*IsComdat=*/true));
566
567 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
568 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
569 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
570 OutStreamer->emitLabel(Symbol: Sym);
571
572 // Extract shadow offset from ptr
573 OutStreamer->emitInstruction(
574 MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8),
575 MCSTI);
576 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI)
577 .addReg(RISCV::X6)
578 .addReg(RISCV::X6)
579 .addImm(12),
580 MCSTI);
581 // load shadow tag in X6, X5 contains shadow base
582 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD)
583 .addReg(RISCV::X6)
584 .addReg(RISCV::X5)
585 .addReg(RISCV::X6),
586 MCSTI);
587 OutStreamer->emitInstruction(
588 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
589 MCSTI);
590 // Extract tag from X5 and compare it with loaded tag from shadow
591 OutStreamer->emitInstruction(
592 MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56),
593 MCSTI);
594 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
595 // X7 contains tag from memory, while X6 contains tag from the pointer
596 OutStreamer->emitInstruction(
597 MCInstBuilder(RISCV::BNE)
598 .addReg(RISCV::X7)
599 .addReg(RISCV::X6)
600 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
601 OutContext)),
602 MCSTI);
603 MCSymbol *ReturnSym = OutContext.createTempSymbol();
604 OutStreamer->emitLabel(Symbol: ReturnSym);
605 OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR)
606 .addReg(RISCV::X0)
607 .addReg(RISCV::X1)
608 .addImm(0),
609 MCSTI);
610 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
611
612 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
613 .addReg(RISCV::X28)
614 .addReg(RISCV::X0)
615 .addImm(16),
616 MCSTI);
617 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
618 OutStreamer->emitInstruction(
619 MCInstBuilder(RISCV::BGEU)
620 .addReg(RISCV::X6)
621 .addReg(RISCV::X28)
622 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
623 MCSTI);
624
625 OutStreamer->emitInstruction(
626 MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF),
627 MCSTI);
628
629 if (Size != 1)
630 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
631 .addReg(RISCV::X28)
632 .addReg(RISCV::X28)
633 .addImm(Size - 1),
634 MCSTI);
635 OutStreamer->emitInstruction(
636 MCInstBuilder(RISCV::BGE)
637 .addReg(RISCV::X28)
638 .addReg(RISCV::X6)
639 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
640 MCSTI);
641
642 OutStreamer->emitInstruction(
643 MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF),
644 MCSTI);
645 OutStreamer->emitInstruction(
646 MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
647 MCSTI);
648 OutStreamer->emitInstruction(
649 MCInstBuilder(RISCV::BEQ)
650 .addReg(RISCV::X6)
651 .addReg(RISCV::X7)
652 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
653 MCSTI);
654
655 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
656
657 // | Previous stack frames... |
658 // +=================================+ <-- [SP + 256]
659 // | ... |
660 // | |
661 // | Stack frame space for x12 - x31.|
662 // | |
663 // | ... |
664 // +---------------------------------+ <-- [SP + 96]
665 // | Saved x11(arg1), as |
666 // | __hwasan_check_* clobbers it. |
667 // +---------------------------------+ <-- [SP + 88]
668 // | Saved x10(arg0), as |
669 // | __hwasan_check_* clobbers it. |
670 // +---------------------------------+ <-- [SP + 80]
671 // | |
672 // | Stack frame space for x9. |
673 // +---------------------------------+ <-- [SP + 72]
674 // | |
675 // | Saved x8(fp), as |
676 // | __hwasan_check_* clobbers it. |
677 // +---------------------------------+ <-- [SP + 64]
678 // | ... |
679 // | |
680 // | Stack frame space for x2 - x7. |
681 // | |
682 // | ... |
683 // +---------------------------------+ <-- [SP + 16]
684 // | Return address (x1) for caller |
685 // | of __hwasan_check_*. |
686 // +---------------------------------+ <-- [SP + 8]
687 // | Reserved place for x0, possibly |
688 // | junk, since we don't save it. |
689 // +---------------------------------+ <-- [x2 / SP]
690
691 // Adjust sp
692 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
693 .addReg(RISCV::X2)
694 .addReg(RISCV::X2)
695 .addImm(-256),
696 MCSTI);
697
698 // store x10(arg0) by new sp
699 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
700 .addReg(RISCV::X10)
701 .addReg(RISCV::X2)
702 .addImm(8 * 10),
703 MCSTI);
704 // store x11(arg1) by new sp
705 OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
706 .addReg(RISCV::X11)
707 .addReg(RISCV::X2)
708 .addImm(8 * 11),
709 MCSTI);
710
711 // store x8(fp) by new sp
712 OutStreamer->emitInstruction(
713 MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 *
714 8),
715 MCSTI);
716 // store x1(ra) by new sp
717 OutStreamer->emitInstruction(
718 MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 *
719 8),
720 MCSTI);
721 if (Reg != RISCV::X10)
722 OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
723 .addReg(RISCV::X10)
724 .addReg(Reg)
725 .addImm(0),
726 MCSTI);
727 OutStreamer->emitInstruction(
728 MCInstBuilder(RISCV::ADDI)
729 .addReg(RISCV::X11)
730 .addReg(RISCV::X0)
731 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask),
732 MCSTI);
733
734 OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr),
735 MCSTI);
736 }
737}
738
739static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
740 const AsmPrinter &AP) {
741 MCContext &Ctx = AP.OutContext;
742 RISCVMCExpr::VariantKind Kind;
743
744 switch (MO.getTargetFlags()) {
745 default:
746 llvm_unreachable("Unknown target flag on GV operand");
747 case RISCVII::MO_None:
748 Kind = RISCVMCExpr::VK_RISCV_None;
749 break;
750 case RISCVII::MO_CALL:
751 Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
752 break;
753 case RISCVII::MO_LO:
754 Kind = RISCVMCExpr::VK_RISCV_LO;
755 break;
756 case RISCVII::MO_HI:
757 Kind = RISCVMCExpr::VK_RISCV_HI;
758 break;
759 case RISCVII::MO_PCREL_LO:
760 Kind = RISCVMCExpr::VK_RISCV_PCREL_LO;
761 break;
762 case RISCVII::MO_PCREL_HI:
763 Kind = RISCVMCExpr::VK_RISCV_PCREL_HI;
764 break;
765 case RISCVII::MO_GOT_HI:
766 Kind = RISCVMCExpr::VK_RISCV_GOT_HI;
767 break;
768 case RISCVII::MO_TPREL_LO:
769 Kind = RISCVMCExpr::VK_RISCV_TPREL_LO;
770 break;
771 case RISCVII::MO_TPREL_HI:
772 Kind = RISCVMCExpr::VK_RISCV_TPREL_HI;
773 break;
774 case RISCVII::MO_TPREL_ADD:
775 Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD;
776 break;
777 case RISCVII::MO_TLS_GOT_HI:
778 Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI;
779 break;
780 case RISCVII::MO_TLS_GD_HI:
781 Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI;
782 break;
783 case RISCVII::MO_TLSDESC_HI:
784 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI;
785 break;
786 case RISCVII::MO_TLSDESC_LOAD_LO:
787 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO;
788 break;
789 case RISCVII::MO_TLSDESC_ADD_LO:
790 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO;
791 break;
792 case RISCVII::MO_TLSDESC_CALL:
793 Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL;
794 break;
795 }
796
797 const MCExpr *ME =
798 MCSymbolRefExpr::create(Symbol: Sym, Kind: MCSymbolRefExpr::VK_None, Ctx);
799
800 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
801 ME = MCBinaryExpr::createAdd(
802 LHS: ME, RHS: MCConstantExpr::create(Value: MO.getOffset(), Ctx), Ctx);
803
804 if (Kind != RISCVMCExpr::VK_RISCV_None)
805 ME = RISCVMCExpr::create(Expr: ME, Kind, Ctx);
806 return MCOperand::createExpr(Val: ME);
807}
808
809bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
810 MCOperand &MCOp) const {
811 switch (MO.getType()) {
812 default:
813 report_fatal_error(reason: "lowerOperand: unknown operand type");
814 case MachineOperand::MO_Register:
815 // Ignore all implicit register operands.
816 if (MO.isImplicit())
817 return false;
818 MCOp = MCOperand::createReg(Reg: MO.getReg());
819 break;
820 case MachineOperand::MO_RegisterMask:
821 // Regmasks are like implicit defs.
822 return false;
823 case MachineOperand::MO_Immediate:
824 MCOp = MCOperand::createImm(Val: MO.getImm());
825 break;
826 case MachineOperand::MO_MachineBasicBlock:
827 MCOp = lowerSymbolOperand(MO, Sym: MO.getMBB()->getSymbol(), AP: *this);
828 break;
829 case MachineOperand::MO_GlobalAddress:
830 MCOp = lowerSymbolOperand(MO, Sym: getSymbolPreferLocal(GV: *MO.getGlobal()), AP: *this);
831 break;
832 case MachineOperand::MO_BlockAddress:
833 MCOp = lowerSymbolOperand(MO, Sym: GetBlockAddressSymbol(BA: MO.getBlockAddress()),
834 AP: *this);
835 break;
836 case MachineOperand::MO_ExternalSymbol:
837 MCOp = lowerSymbolOperand(MO, Sym: GetExternalSymbolSymbol(Sym: MO.getSymbolName()),
838 AP: *this);
839 break;
840 case MachineOperand::MO_ConstantPoolIndex:
841 MCOp = lowerSymbolOperand(MO, Sym: GetCPISymbol(CPID: MO.getIndex()), AP: *this);
842 break;
843 case MachineOperand::MO_JumpTableIndex:
844 MCOp = lowerSymbolOperand(MO, Sym: GetJTISymbol(JTID: MO.getIndex()), AP: *this);
845 break;
846 case MachineOperand::MO_MCSymbol:
847 MCOp = lowerSymbolOperand(MO, Sym: MO.getMCSymbol(), AP: *this);
848 break;
849 }
850 return true;
851}
852
853static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
854 MCInst &OutMI) {
855 const RISCVVPseudosTable::PseudoInfo *RVV =
856 RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
857 if (!RVV)
858 return false;
859
860 OutMI.setOpcode(RVV->BaseInstr);
861
862 const MachineBasicBlock *MBB = MI->getParent();
863 assert(MBB && "MI expected to be in a basic block");
864 const MachineFunction *MF = MBB->getParent();
865 assert(MF && "MBB expected to be in a machine function");
866
867 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
868 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
869 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
870 assert(TRI && "TargetRegisterInfo expected");
871
872 const MCInstrDesc &MCID = MI->getDesc();
873 uint64_t TSFlags = MCID.TSFlags;
874 unsigned NumOps = MI->getNumExplicitOperands();
875
876 // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if
877 // present.
878 if (RISCVII::hasVecPolicyOp(TSFlags))
879 --NumOps;
880 if (RISCVII::hasSEWOp(TSFlags))
881 --NumOps;
882 if (RISCVII::hasVLOp(TSFlags))
883 --NumOps;
884 if (RISCVII::hasRoundModeOp(TSFlags))
885 --NumOps;
886
887 bool hasVLOutput = RISCV::isFaultFirstLoad(MI: *MI);
888 for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
889 const MachineOperand &MO = MI->getOperand(i: OpNo);
890 // Skip vl ouput. It should be the second output.
891 if (hasVLOutput && OpNo == 1)
892 continue;
893
894 // Skip merge op. It should be the first operand after the defs.
895 if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) {
896 assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 &&
897 "Expected tied to first def.");
898 const MCInstrDesc &OutMCID = TII->get(Opcode: OutMI.getOpcode());
899 // Skip if the next operand in OutMI is not supposed to be tied. Unless it
900 // is a _TIED instruction.
901 if (OutMCID.getOperandConstraint(OpNum: OutMI.getNumOperands(), Constraint: MCOI::TIED_TO) <
902 0 &&
903 !RISCVII::isTiedPseudo(TSFlags))
904 continue;
905 }
906
907 MCOperand MCOp;
908 switch (MO.getType()) {
909 default:
910 llvm_unreachable("Unknown operand type");
911 case MachineOperand::MO_Register: {
912 Register Reg = MO.getReg();
913
914 if (RISCV::VRM2RegClass.contains(Reg) ||
915 RISCV::VRM4RegClass.contains(Reg) ||
916 RISCV::VRM8RegClass.contains(Reg)) {
917 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
918 assert(Reg && "Subregister does not exist");
919 } else if (RISCV::FPR16RegClass.contains(Reg)) {
920 Reg =
921 TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass);
922 assert(Reg && "Subregister does not exist");
923 } else if (RISCV::FPR64RegClass.contains(Reg)) {
924 Reg = TRI->getSubReg(Reg, RISCV::sub_32);
925 assert(Reg && "Superregister does not exist");
926 } else if (RISCV::VRN2M1RegClass.contains(Reg) ||
927 RISCV::VRN2M2RegClass.contains(Reg) ||
928 RISCV::VRN2M4RegClass.contains(Reg) ||
929 RISCV::VRN3M1RegClass.contains(Reg) ||
930 RISCV::VRN3M2RegClass.contains(Reg) ||
931 RISCV::VRN4M1RegClass.contains(Reg) ||
932 RISCV::VRN4M2RegClass.contains(Reg) ||
933 RISCV::VRN5M1RegClass.contains(Reg) ||
934 RISCV::VRN6M1RegClass.contains(Reg) ||
935 RISCV::VRN7M1RegClass.contains(Reg) ||
936 RISCV::VRN8M1RegClass.contains(Reg)) {
937 Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
938 assert(Reg && "Subregister does not exist");
939 }
940
941 MCOp = MCOperand::createReg(Reg);
942 break;
943 }
944 case MachineOperand::MO_Immediate:
945 MCOp = MCOperand::createImm(Val: MO.getImm());
946 break;
947 }
948 OutMI.addOperand(Op: MCOp);
949 }
950
951 // Unmasked pseudo instructions need to append dummy mask operand to
952 // V instructions. All V instructions are modeled as the masked version.
953 const MCInstrDesc &OutMCID = TII->get(Opcode: OutMI.getOpcode());
954 if (OutMI.getNumOperands() < OutMCID.getNumOperands()) {
955 assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass ==
956 RISCV::VMV0RegClassID &&
957 "Expected only mask operand to be missing");
958 OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
959 }
960
961 assert(OutMI.getNumOperands() == OutMCID.getNumOperands());
962 return true;
963}
964
965bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
966 if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
967 return false;
968
969 OutMI.setOpcode(MI->getOpcode());
970
971 for (const MachineOperand &MO : MI->operands()) {
972 MCOperand MCOp;
973 if (lowerOperand(MO, MCOp))
974 OutMI.addOperand(Op: MCOp);
975 }
976
977 switch (OutMI.getOpcode()) {
978 case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
979 const Function &F = MI->getParent()->getParent()->getFunction();
980 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
981 unsigned Num;
982 if (F.getFnAttribute(Kind: "patchable-function-entry")
983 .getValueAsString()
984 .getAsInteger(Radix: 10, Result&: Num))
985 return false;
986 emitNops(N: Num);
987 return true;
988 }
989 break;
990 }
991 }
992 return false;
993}
994

source code of llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp