1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13#include "llvm/ADT/PostOrderIterator.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/Analysis/AliasAnalysis.h"
19#include "llvm/Analysis/AssumptionCache.h"
20#include "llvm/Analysis/BranchProbabilityInfo.h"
21#include "llvm/Analysis/Loads.h"
22#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
26#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
27#include "llvm/CodeGen/GlobalISel/CallLowering.h"
28#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
29#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
31#include "llvm/CodeGen/LowLevelTypeUtils.h"
32#include "llvm/CodeGen/MachineBasicBlock.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RuntimeLibcalls.h"
41#include "llvm/CodeGen/StackProtector.h"
42#include "llvm/CodeGen/SwitchLoweringUtils.h"
43#include "llvm/CodeGen/TargetFrameLowering.h"
44#include "llvm/CodeGen/TargetInstrInfo.h"
45#include "llvm/CodeGen/TargetLowering.h"
46#include "llvm/CodeGen/TargetOpcodes.h"
47#include "llvm/CodeGen/TargetPassConfig.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGenTypes/LowLevelType.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/CFG.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/DiagnosticInfo.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GetElementPtrTypeIterator.h"
60#include "llvm/IR/InlineAsm.h"
61#include "llvm/IR/InstrTypes.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/IntrinsicsAMDGPU.h"
66#include "llvm/IR/LLVMContext.h"
67#include "llvm/IR/Metadata.h"
68#include "llvm/IR/PatternMatch.h"
69#include "llvm/IR/Statepoint.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/User.h"
72#include "llvm/IR/Value.h"
73#include "llvm/InitializePasses.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/Pass.h"
76#include "llvm/Support/Casting.h"
77#include "llvm/Support/CodeGen.h"
78#include "llvm/Support/Debug.h"
79#include "llvm/Support/ErrorHandling.h"
80#include "llvm/Support/MathExtras.h"
81#include "llvm/Support/raw_ostream.h"
82#include "llvm/Target/TargetIntrinsicInfo.h"
83#include "llvm/Target/TargetMachine.h"
84#include "llvm/Transforms/Utils/Local.h"
85#include "llvm/Transforms/Utils/MemoryOpRemark.h"
86#include <algorithm>
87#include <cassert>
88#include <cstdint>
89#include <iterator>
90#include <optional>
91#include <string>
92#include <utility>
93#include <vector>
94
95#define DEBUG_TYPE "irtranslator"
96
97using namespace llvm;
98
99static cl::opt<bool>
100 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
101 cl::desc("Should enable CSE in irtranslator"),
102 cl::Optional, cl::init(Val: false));
103char IRTranslator::ID = 0;
104
105INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
106 false, false)
107INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
108INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
109INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
110INITIALIZE_PASS_DEPENDENCY(StackProtector)
111INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
112INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
113 false, false)
114
115static void reportTranslationError(MachineFunction &MF,
116 const TargetPassConfig &TPC,
117 OptimizationRemarkEmitter &ORE,
118 OptimizationRemarkMissed &R) {
119 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
120
121 // Print the function name explicitly if we don't have a debug location (which
122 // makes the diagnostic less useful) or if we're going to emit a raw error.
123 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
124 R << (" (in function: " + MF.getName() + ")").str();
125
126 if (TPC.isGlobalISelAbortEnabled())
127 report_fatal_error(reason: Twine(R.getMsg()));
128 else
129 ORE.emit(OptDiag&: R);
130}
131
132IRTranslator::IRTranslator(CodeGenOptLevel optlevel)
133 : MachineFunctionPass(ID), OptLevel(optlevel) {}
134
135#ifndef NDEBUG
136namespace {
137/// Verify that every instruction created has the same DILocation as the
138/// instruction being translated.
139class DILocationVerifier : public GISelChangeObserver {
140 const Instruction *CurrInst = nullptr;
141
142public:
143 DILocationVerifier() = default;
144 ~DILocationVerifier() = default;
145
146 const Instruction *getCurrentInst() const { return CurrInst; }
147 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
148
149 void erasingInstr(MachineInstr &MI) override {}
150 void changingInstr(MachineInstr &MI) override {}
151 void changedInstr(MachineInstr &MI) override {}
152
153 void createdInstr(MachineInstr &MI) override {
154 assert(getCurrentInst() && "Inserted instruction without a current MI");
155
156 // Only print the check message if we're actually checking it.
157#ifndef NDEBUG
158 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
159 << " was copied to " << MI);
160#endif
161 // We allow insts in the entry block to have no debug loc because
162 // they could have originated from constants, and we don't want a jumpy
163 // debug experience.
164 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
165 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
166 (MI.isDebugInstr())) &&
167 "Line info was not transferred to all instructions");
168 }
169};
170} // namespace
171#endif // ifndef NDEBUG
172
173
174void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
175 AU.addRequired<StackProtector>();
176 AU.addRequired<TargetPassConfig>();
177 AU.addRequired<GISelCSEAnalysisWrapperPass>();
178 AU.addRequired<AssumptionCacheTracker>();
179 if (OptLevel != CodeGenOptLevel::None) {
180 AU.addRequired<BranchProbabilityInfoWrapperPass>();
181 AU.addRequired<AAResultsWrapperPass>();
182 }
183 AU.addRequired<TargetLibraryInfoWrapperPass>();
184 AU.addPreserved<TargetLibraryInfoWrapperPass>();
185 getSelectionDAGFallbackAnalysisUsage(AU);
186 MachineFunctionPass::getAnalysisUsage(AU);
187}
188
189IRTranslator::ValueToVRegInfo::VRegListT &
190IRTranslator::allocateVRegs(const Value &Val) {
191 auto VRegsIt = VMap.findVRegs(V: Val);
192 if (VRegsIt != VMap.vregs_end())
193 return *VRegsIt->second;
194 auto *Regs = VMap.getVRegs(V: Val);
195 auto *Offsets = VMap.getOffsets(V: Val);
196 SmallVector<LLT, 4> SplitTys;
197 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueTys&: SplitTys,
198 Offsets: Offsets->empty() ? Offsets : nullptr);
199 for (unsigned i = 0; i < SplitTys.size(); ++i)
200 Regs->push_back(Elt: 0);
201 return *Regs;
202}
203
204ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
205 auto VRegsIt = VMap.findVRegs(V: Val);
206 if (VRegsIt != VMap.vregs_end())
207 return *VRegsIt->second;
208
209 if (Val.getType()->isVoidTy())
210 return *VMap.getVRegs(V: Val);
211
212 // Create entry for this type.
213 auto *VRegs = VMap.getVRegs(V: Val);
214 auto *Offsets = VMap.getOffsets(V: Val);
215
216 assert(Val.getType()->isSized() &&
217 "Don't know how to create an empty vreg");
218
219 SmallVector<LLT, 4> SplitTys;
220 computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueTys&: SplitTys,
221 Offsets: Offsets->empty() ? Offsets : nullptr);
222
223 if (!isa<Constant>(Val)) {
224 for (auto Ty : SplitTys)
225 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
226 return *VRegs;
227 }
228
229 if (Val.getType()->isAggregateType()) {
230 // UndefValue, ConstantAggregateZero
231 auto &C = cast<Constant>(Val);
232 unsigned Idx = 0;
233 while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
234 auto EltRegs = getOrCreateVRegs(Val: *Elt);
235 llvm::copy(Range&: EltRegs, Out: std::back_inserter(x&: *VRegs));
236 }
237 } else {
238 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
239 VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty: SplitTys[0]));
240 bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
241 if (!Success) {
242 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
243 MF->getFunction().getSubprogram(),
244 &MF->getFunction().getEntryBlock());
245 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
246 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
247 return *VRegs;
248 }
249 }
250
251 return *VRegs;
252}
253
254int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
255 auto MapEntry = FrameIndices.find(Val: &AI);
256 if (MapEntry != FrameIndices.end())
257 return MapEntry->second;
258
259 uint64_t ElementSize = DL->getTypeAllocSize(Ty: AI.getAllocatedType());
260 uint64_t Size =
261 ElementSize * cast<ConstantInt>(Val: AI.getArraySize())->getZExtValue();
262
263 // Always allocate at least one byte.
264 Size = std::max<uint64_t>(a: Size, b: 1u);
265
266 int &FI = FrameIndices[&AI];
267 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
268 return FI;
269}
270
271Align IRTranslator::getMemOpAlign(const Instruction &I) {
272 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
273 return SI->getAlign();
274 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
275 return LI->getAlign();
276 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
277 return AI->getAlign();
278 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
279 return AI->getAlign();
280
281 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
282 R << "unable to translate memop: " << ore::NV("Opcode", &I);
283 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
284 return Align(1);
285}
286
287MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
288 MachineBasicBlock *&MBB = BBToMBB[&BB];
289 assert(MBB && "BasicBlock was not encountered before");
290 return *MBB;
291}
292
293void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
294 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
295 MachinePreds[Edge].push_back(Elt: NewPred);
296}
297
298bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
299 MachineIRBuilder &MIRBuilder) {
300 // Get or create a virtual register for each value.
301 // Unless the value is a Constant => loadimm cst?
302 // or inline constant each time?
303 // Creation of a virtual register needs to have a size.
304 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
305 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
306 Register Res = getOrCreateVReg(Val: U);
307 uint32_t Flags = 0;
308 if (isa<Instruction>(Val: U)) {
309 const Instruction &I = cast<Instruction>(Val: U);
310 Flags = MachineInstr::copyFlagsFromInstruction(I);
311 }
312
313 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
314 return true;
315}
316
317bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
318 MachineIRBuilder &MIRBuilder) {
319 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
320 Register Res = getOrCreateVReg(Val: U);
321 uint32_t Flags = 0;
322 if (isa<Instruction>(Val: U)) {
323 const Instruction &I = cast<Instruction>(Val: U);
324 Flags = MachineInstr::copyFlagsFromInstruction(I);
325 }
326 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
327 return true;
328}
329
330bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
331 return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
332}
333
334bool IRTranslator::translateCompare(const User &U,
335 MachineIRBuilder &MIRBuilder) {
336 auto *CI = dyn_cast<CmpInst>(Val: &U);
337 Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
338 Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
339 Register Res = getOrCreateVReg(Val: U);
340 CmpInst::Predicate Pred =
341 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
342 cast<ConstantExpr>(Val: U).getPredicate());
343 if (CmpInst::isIntPredicate(P: Pred))
344 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
345 else if (Pred == CmpInst::FCMP_FALSE)
346 MIRBuilder.buildCopy(
347 Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
348 else if (Pred == CmpInst::FCMP_TRUE)
349 MIRBuilder.buildCopy(
350 Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
351 else {
352 uint32_t Flags = 0;
353 if (CI)
354 Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
355 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
356 }
357
358 return true;
359}
360
361bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
362 const ReturnInst &RI = cast<ReturnInst>(Val: U);
363 const Value *Ret = RI.getReturnValue();
364 if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
365 Ret = nullptr;
366
367 ArrayRef<Register> VRegs;
368 if (Ret)
369 VRegs = getOrCreateVRegs(Val: *Ret);
370
371 Register SwiftErrorVReg = 0;
372 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
373 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
374 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
375 }
376
377 // The target may mess up with the insertion point, but
378 // this is not important as a return is the last instruction
379 // of the block anyway.
380 return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
381}
382
383void IRTranslator::emitBranchForMergedCondition(
384 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
385 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
386 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
387 // If the leaf of the tree is a comparison, merge the condition into
388 // the caseblock.
389 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
390 CmpInst::Predicate Condition;
391 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
392 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
393 } else {
394 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
395 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
396 }
397
398 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
399 BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
400 CurBuilder->getDebugLoc(), TProb, FProb);
401 SL->SwitchCases.push_back(x: CB);
402 return;
403 }
404
405 // Create a CaseBlock record representing this branch.
406 CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
407 SwitchCG::CaseBlock CB(
408 Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
409 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
410 SL->SwitchCases.push_back(x: CB);
411}
412
413static bool isValInBlock(const Value *V, const BasicBlock *BB) {
414 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
415 return I->getParent() == BB;
416 return true;
417}
418
419void IRTranslator::findMergedConditions(
420 const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
421 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
422 Instruction::BinaryOps Opc, BranchProbability TProb,
423 BranchProbability FProb, bool InvertCond) {
424 using namespace PatternMatch;
425 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
426 "Expected Opc to be AND/OR");
427 // Skip over not part of the tree and remember to invert op and operands at
428 // next level.
429 Value *NotCond;
430 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
431 isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
432 findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
433 InvertCond: !InvertCond);
434 return;
435 }
436
437 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
438 const Value *BOpOp0, *BOpOp1;
439 // Compute the effective opcode for Cond, taking into account whether it needs
440 // to be inverted, e.g.
441 // and (not (or A, B)), C
442 // gets lowered as
443 // and (and (not A, not B), C)
444 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
445 if (BOp) {
446 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
447 ? Instruction::And
448 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
449 ? Instruction::Or
450 : (Instruction::BinaryOps)0);
451 if (InvertCond) {
452 if (BOpc == Instruction::And)
453 BOpc = Instruction::Or;
454 else if (BOpc == Instruction::Or)
455 BOpc = Instruction::And;
456 }
457 }
458
459 // If this node is not part of the or/and tree, emit it as a branch.
460 // Note that all nodes in the tree should have same opcode.
461 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
462 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
463 !isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
464 !isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
465 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
466 InvertCond);
467 return;
468 }
469
470 // Create TmpBB after CurBB.
471 MachineFunction::iterator BBI(CurBB);
472 MachineBasicBlock *TmpBB =
473 MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
474 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
475
476 if (Opc == Instruction::Or) {
477 // Codegen X | Y as:
478 // BB1:
479 // jmp_if_X TBB
480 // jmp TmpBB
481 // TmpBB:
482 // jmp_if_Y TBB
483 // jmp FBB
484 //
485
486 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
487 // The requirement is that
488 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
489 // = TrueProb for original BB.
490 // Assuming the original probabilities are A and B, one choice is to set
491 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
492 // A/(1+B) and 2B/(1+B). This choice assumes that
493 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
494 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
495 // TmpBB, but the math is more complicated.
496
497 auto NewTrueProb = TProb / 2;
498 auto NewFalseProb = TProb / 2 + FProb;
499 // Emit the LHS condition.
500 findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
501 FProb: NewFalseProb, InvertCond);
502
503 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
504 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
505 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
506 // Emit the RHS condition into TmpBB.
507 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
508 FProb: Probs[1], InvertCond);
509 } else {
510 assert(Opc == Instruction::And && "Unknown merge op!");
511 // Codegen X & Y as:
512 // BB1:
513 // jmp_if_X TmpBB
514 // jmp FBB
515 // TmpBB:
516 // jmp_if_Y TBB
517 // jmp FBB
518 //
519 // This requires creation of TmpBB after CurBB.
520
521 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
522 // The requirement is that
523 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
524 // = FalseProb for original BB.
525 // Assuming the original probabilities are A and B, one choice is to set
526 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
527 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
528 // TrueProb for BB1 * FalseProb for TmpBB.
529
530 auto NewTrueProb = TProb + FProb / 2;
531 auto NewFalseProb = FProb / 2;
532 // Emit the LHS condition.
533 findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
534 FProb: NewFalseProb, InvertCond);
535
536 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
537 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
538 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
539 // Emit the RHS condition into TmpBB.
540 findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
541 FProb: Probs[1], InvertCond);
542 }
543}
544
545bool IRTranslator::shouldEmitAsBranches(
546 const std::vector<SwitchCG::CaseBlock> &Cases) {
547 // For multiple cases, it's better to emit as branches.
548 if (Cases.size() != 2)
549 return true;
550
551 // If this is two comparisons of the same values or'd or and'd together, they
552 // will get folded into a single comparison, so don't emit two blocks.
553 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
554 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
555 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
556 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
557 return false;
558 }
559
560 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
561 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
562 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
563 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
564 isa<Constant>(Val: Cases[0].CmpRHS) &&
565 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
566 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
567 Cases[0].TrueBB == Cases[1].ThisBB)
568 return false;
569 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
570 Cases[0].FalseBB == Cases[1].ThisBB)
571 return false;
572 }
573
574 return true;
575}
576
577bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
578 const BranchInst &BrInst = cast<BranchInst>(Val: U);
579 auto &CurMBB = MIRBuilder.getMBB();
580 auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
581
582 if (BrInst.isUnconditional()) {
583 // If the unconditional target is the layout successor, fallthrough.
584 if (OptLevel == CodeGenOptLevel::None ||
585 !CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
586 MIRBuilder.buildBr(Dest&: *Succ0MBB);
587
588 // Link successors.
589 for (const BasicBlock *Succ : successors(I: &BrInst))
590 CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
591 return true;
592 }
593
594 // If this condition is one of the special cases we handle, do special stuff
595 // now.
596 const Value *CondVal = BrInst.getCondition();
597 MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
598
599 const auto &TLI = *MF->getSubtarget().getTargetLowering();
600
601 // If this is a series of conditions that are or'd or and'd together, emit
602 // this as a sequence of branches instead of setcc's with and/or operations.
603 // As long as jumps are not expensive (exceptions for multi-use logic ops,
604 // unpredictable branches, and vector extracts because those jumps are likely
605 // expensive for any target), this should improve performance.
606 // For example, instead of something like:
607 // cmp A, B
608 // C = seteq
609 // cmp D, E
610 // F = setle
611 // or C, F
612 // jnz foo
613 // Emit:
614 // cmp A, B
615 // je foo
616 // cmp D, E
617 // jle foo
618 using namespace PatternMatch;
619 const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
620 if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
621 !BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
622 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
623 Value *Vec;
624 const Value *BOp0, *BOp1;
625 if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
626 Opcode = Instruction::And;
627 else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
628 Opcode = Instruction::Or;
629
630 if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
631 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
632 findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
633 TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
634 FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
635 /*InvertCond=*/false);
636 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
637
638 // Allow some cases to be rejected.
639 if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
640 // Emit the branch for this block.
641 emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
642 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
643 return true;
644 }
645
646 // Okay, we decided not to do this, remove any inserted MBB's and clear
647 // SwitchCases.
648 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
649 MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
650
651 SL->SwitchCases.clear();
652 }
653 }
654
655 // Create a CaseBlock record representing this branch.
656 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
657 ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
658 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
659 CurBuilder->getDebugLoc());
660
661 // Use emitSwitchCase to actually insert the fast branch sequence for this
662 // cond branch.
663 emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
664 return true;
665}
666
667void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
668 MachineBasicBlock *Dst,
669 BranchProbability Prob) {
670 if (!FuncInfo.BPI) {
671 Src->addSuccessorWithoutProb(Succ: Dst);
672 return;
673 }
674 if (Prob.isUnknown())
675 Prob = getEdgeProbability(Src, Dst);
676 Src->addSuccessor(Succ: Dst, Prob);
677}
678
679BranchProbability
680IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
681 const MachineBasicBlock *Dst) const {
682 const BasicBlock *SrcBB = Src->getBasicBlock();
683 const BasicBlock *DstBB = Dst->getBasicBlock();
684 if (!FuncInfo.BPI) {
685 // If BPI is not available, set the default probability as 1 / N, where N is
686 // the number of successors.
687 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
688 return BranchProbability(1, SuccSize);
689 }
690 return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
691}
692
693bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
694 using namespace SwitchCG;
695 // Extract cases from the switch.
696 const SwitchInst &SI = cast<SwitchInst>(Val: U);
697 BranchProbabilityInfo *BPI = FuncInfo.BPI;
698 CaseClusterVector Clusters;
699 Clusters.reserve(n: SI.getNumCases());
700 for (const auto &I : SI.cases()) {
701 MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
702 assert(Succ && "Could not find successor mbb in mapping");
703 const ConstantInt *CaseVal = I.getCaseValue();
704 BranchProbability Prob =
705 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
706 : BranchProbability(1, SI.getNumCases() + 1);
707 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
708 }
709
710 MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
711
712 // Cluster adjacent cases with the same destination. We do this at all
713 // optimization levels because it's cheap to do and will make codegen faster
714 // if there are many clusters.
715 sortAndRangeify(Clusters);
716
717 MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
718
719 // If there is only the default destination, jump there directly.
720 if (Clusters.empty()) {
721 SwitchMBB->addSuccessor(Succ: DefaultMBB);
722 if (DefaultMBB != SwitchMBB->getNextNode())
723 MIB.buildBr(Dest&: *DefaultMBB);
724 return true;
725 }
726
727 SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
728 SL->findBitTestClusters(Clusters, SI: &SI);
729
730 LLVM_DEBUG({
731 dbgs() << "Case clusters: ";
732 for (const CaseCluster &C : Clusters) {
733 if (C.Kind == CC_JumpTable)
734 dbgs() << "JT:";
735 if (C.Kind == CC_BitTests)
736 dbgs() << "BT:";
737
738 C.Low->getValue().print(dbgs(), true);
739 if (C.Low != C.High) {
740 dbgs() << '-';
741 C.High->getValue().print(dbgs(), true);
742 }
743 dbgs() << ' ';
744 }
745 dbgs() << '\n';
746 });
747
748 assert(!Clusters.empty());
749 SwitchWorkList WorkList;
750 CaseClusterIt First = Clusters.begin();
751 CaseClusterIt Last = Clusters.end() - 1;
752 auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
753 WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
754
755 while (!WorkList.empty()) {
756 SwitchWorkListItem W = WorkList.pop_back_val();
757
758 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
759 // For optimized builds, lower large range as a balanced binary tree.
760 if (NumClusters > 3 &&
761 MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
762 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
763 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
764 continue;
765 }
766
767 if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
768 return false;
769 }
770 return true;
771}
772
773void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
774 const SwitchCG::SwitchWorkListItem &W,
775 Value *Cond, MachineBasicBlock *SwitchMBB,
776 MachineIRBuilder &MIB) {
777 using namespace SwitchCG;
778 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
779 "Clusters not sorted?");
780 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
781
782 auto [LastLeft, FirstRight, LeftProb, RightProb] =
783 SL->computeSplitWorkItemInfo(W);
784
785 // Use the first element on the right as pivot since we will make less-than
786 // comparisons against it.
787 CaseClusterIt PivotCluster = FirstRight;
788 assert(PivotCluster > W.FirstCluster);
789 assert(PivotCluster <= W.LastCluster);
790
791 CaseClusterIt FirstLeft = W.FirstCluster;
792 CaseClusterIt LastRight = W.LastCluster;
793
794 const ConstantInt *Pivot = PivotCluster->Low;
795
796 // New blocks will be inserted immediately after the current one.
797 MachineFunction::iterator BBI(W.MBB);
798 ++BBI;
799
800 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
801 // we can branch to its destination directly if it's squeezed exactly in
802 // between the known lower bound and Pivot - 1.
803 MachineBasicBlock *LeftMBB;
804 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
805 FirstLeft->Low == W.GE &&
806 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
807 LeftMBB = FirstLeft->MBB;
808 } else {
809 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
810 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
811 WorkList.push_back(
812 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
813 }
814
815 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
816 // single cluster, RHS.Low == Pivot, and we can branch to its destination
817 // directly if RHS.High equals the current upper bound.
818 MachineBasicBlock *RightMBB;
819 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
820 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
821 RightMBB = FirstRight->MBB;
822 } else {
823 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
824 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
825 WorkList.push_back(
826 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
827 }
828
829 // Create the CaseBlock record that will be used to lower the branch.
830 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
831 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
832 RightProb);
833
834 if (W.MBB == SwitchMBB)
835 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
836 else
837 SL->SwitchCases.push_back(x: CB);
838}
839
840void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
841 MachineBasicBlock *MBB) {
842 // Emit the code for the jump table
843 assert(JT.Reg != -1U && "Should lower JT Header first!");
844 MachineIRBuilder MIB(*MBB->getParent());
845 MIB.setMBB(*MBB);
846 MIB.setDebugLoc(CurBuilder->getDebugLoc());
847
848 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
849 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
850
851 auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
852 MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
853}
854
855bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
856 SwitchCG::JumpTableHeader &JTH,
857 MachineBasicBlock *HeaderBB) {
858 MachineIRBuilder MIB(*HeaderBB->getParent());
859 MIB.setMBB(*HeaderBB);
860 MIB.setDebugLoc(CurBuilder->getDebugLoc());
861
862 const Value &SValue = *JTH.SValue;
863 // Subtract the lowest switch case value from the value being switched on.
864 const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
865 Register SwitchOpReg = getOrCreateVReg(Val: SValue);
866 auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
867 auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
868
869 // This value may be smaller or larger than the target's pointer type, and
870 // therefore require extension or truncating.
871 auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
872 const LLT PtrScalarTy = LLT::scalar(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
873 Sub = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
874
875 JT.Reg = Sub.getReg(Idx: 0);
876
877 if (JTH.FallthroughUnreachable) {
878 if (JT.MBB != HeaderBB->getNextNode())
879 MIB.buildBr(Dest&: *JT.MBB);
880 return true;
881 }
882
883 // Emit the range check for the jump table, and branch to the default block
884 // for the switch statement if the value being switched on exceeds the
885 // largest case in the switch.
886 auto Cst = getOrCreateVReg(
887 Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
888 Cst = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Cst).getReg(Idx: 0);
889 auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1), Op0: Sub, Op1: Cst);
890
891 auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
892
893 // Avoid emitting unnecessary branches to the next block.
894 if (JT.MBB != HeaderBB->getNextNode())
895 BrCond = MIB.buildBr(Dest&: *JT.MBB);
896 return true;
897}
898
899void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
900 MachineBasicBlock *SwitchBB,
901 MachineIRBuilder &MIB) {
902 Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
903 Register Cond;
904 DebugLoc OldDbgLoc = MIB.getDebugLoc();
905 MIB.setDebugLoc(CB.DbgLoc);
906 MIB.setMBB(*CB.ThisBB);
907
908 if (CB.PredInfo.NoCmp) {
909 // Branch or fall through to TrueBB.
910 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
911 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
912 NewPred: CB.ThisBB);
913 CB.ThisBB->normalizeSuccProbs();
914 if (CB.TrueBB != CB.ThisBB->getNextNode())
915 MIB.buildBr(Dest&: *CB.TrueBB);
916 MIB.setDebugLoc(OldDbgLoc);
917 return;
918 }
919
920 const LLT i1Ty = LLT::scalar(SizeInBits: 1);
921 // Build the compare.
922 if (!CB.CmpMHS) {
923 const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
924 // For conditional branch lowering, we might try to do something silly like
925 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
926 // just re-use the existing condition vreg.
927 if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
928 CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
929 Cond = CondLHS;
930 } else {
931 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
932 if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
933 Cond =
934 MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
935 else
936 Cond =
937 MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
938 }
939 } else {
940 assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
941 "Can only handle SLE ranges");
942
943 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
944 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
945
946 Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
947 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
948 Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
949 Cond =
950 MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
951 } else {
952 const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
953 auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
954 auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
955 Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
956 }
957 }
958
959 // Update successor info
960 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
961
962 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
963 NewPred: CB.ThisBB);
964
965 // TrueBB and FalseBB are always different unless the incoming IR is
966 // degenerate. This only happens when running llc on weird IR.
967 if (CB.TrueBB != CB.FalseBB)
968 addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
969 CB.ThisBB->normalizeSuccProbs();
970
971 addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
972 NewPred: CB.ThisBB);
973
974 MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
975 MIB.buildBr(Dest&: *CB.FalseBB);
976 MIB.setDebugLoc(OldDbgLoc);
977}
978
979bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
980 MachineBasicBlock *SwitchMBB,
981 MachineBasicBlock *CurMBB,
982 MachineBasicBlock *DefaultMBB,
983 MachineIRBuilder &MIB,
984 MachineFunction::iterator BBI,
985 BranchProbability UnhandledProbs,
986 SwitchCG::CaseClusterIt I,
987 MachineBasicBlock *Fallthrough,
988 bool FallthroughUnreachable) {
989 using namespace SwitchCG;
990 MachineFunction *CurMF = SwitchMBB->getParent();
991 // FIXME: Optimize away range check based on pivot comparisons.
992 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
993 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
994 BranchProbability DefaultProb = W.DefaultProb;
995
996 // The jump block hasn't been inserted yet; insert it here.
997 MachineBasicBlock *JumpMBB = JT->MBB;
998 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
999
1000 // Since the jump table block is separate from the switch block, we need
1001 // to keep track of it as a machine predecessor to the default block,
1002 // otherwise we lose the phi edges.
1003 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1004 NewPred: CurMBB);
1005 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1006 NewPred: JumpMBB);
1007
1008 auto JumpProb = I->Prob;
1009 auto FallthroughProb = UnhandledProbs;
1010
1011 // If the default statement is a target of the jump table, we evenly
1012 // distribute the default probability to successors of CurMBB. Also
1013 // update the probability on the edge from JumpMBB to Fallthrough.
1014 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1015 SE = JumpMBB->succ_end();
1016 SI != SE; ++SI) {
1017 if (*SI == DefaultMBB) {
1018 JumpProb += DefaultProb / 2;
1019 FallthroughProb -= DefaultProb / 2;
1020 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1021 JumpMBB->normalizeSuccProbs();
1022 } else {
1023 // Also record edges from the jump table block to it's successors.
1024 addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1025 NewPred: JumpMBB);
1026 }
1027 }
1028
1029 if (FallthroughUnreachable)
1030 JTH->FallthroughUnreachable = true;
1031
1032 if (!JTH->FallthroughUnreachable)
1033 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1034 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1035 CurMBB->normalizeSuccProbs();
1036
1037 // The jump table header will be inserted in our current block, do the
1038 // range check, and fall through to our fallthrough block.
1039 JTH->HeaderBB = CurMBB;
1040 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1041
1042 // If we're in the right place, emit the jump table header right now.
1043 if (CurMBB == SwitchMBB) {
1044 if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1045 return false;
1046 JTH->Emitted = true;
1047 }
1048 return true;
1049}
1050bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1051 Value *Cond,
1052 MachineBasicBlock *Fallthrough,
1053 bool FallthroughUnreachable,
1054 BranchProbability UnhandledProbs,
1055 MachineBasicBlock *CurMBB,
1056 MachineIRBuilder &MIB,
1057 MachineBasicBlock *SwitchMBB) {
1058 using namespace SwitchCG;
1059 const Value *RHS, *LHS, *MHS;
1060 CmpInst::Predicate Pred;
1061 if (I->Low == I->High) {
1062 // Check Cond == I->Low.
1063 Pred = CmpInst::ICMP_EQ;
1064 LHS = Cond;
1065 RHS = I->Low;
1066 MHS = nullptr;
1067 } else {
1068 // Check I->Low <= Cond <= I->High.
1069 Pred = CmpInst::ICMP_SLE;
1070 LHS = I->Low;
1071 MHS = Cond;
1072 RHS = I->High;
1073 }
1074
1075 // If Fallthrough is unreachable, fold away the comparison.
1076 // The false probability is the sum of all unhandled cases.
1077 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1078 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1079
1080 emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1081 return true;
1082}
1083
1084void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1085 MachineBasicBlock *SwitchBB) {
1086 MachineIRBuilder &MIB = *CurBuilder;
1087 MIB.setMBB(*SwitchBB);
1088
1089 // Subtract the minimum value.
1090 Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1091
1092 LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1093 Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1094 auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1095
1096 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1097 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1098
1099 LLT MaskTy = SwitchOpTy;
1100 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1101 !llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1102 MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1103 else {
1104 // Ensure that the type will fit the mask value.
1105 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1106 if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: B.Cases[I].Mask)) {
1107 // Switch table case range are encoded into series of masks.
1108 // Just use pointer type, it's guaranteed to fit.
1109 MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1110 break;
1111 }
1112 }
1113 }
1114 Register SubReg = RangeSub.getReg(Idx: 0);
1115 if (SwitchOpTy != MaskTy)
1116 SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1117
1118 B.RegVT = getMVTForLLT(Ty: MaskTy);
1119 B.Reg = SubReg;
1120
1121 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1122
1123 if (!B.FallthroughUnreachable)
1124 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1125 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1126
1127 SwitchBB->normalizeSuccProbs();
1128
1129 if (!B.FallthroughUnreachable) {
1130 // Conditional branch to the default block.
1131 auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1132 auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1),
1133 Op0: RangeSub, Op1: RangeCst);
1134 MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1135 }
1136
1137 // Avoid emitting unnecessary branches to the next block.
1138 if (MBB != SwitchBB->getNextNode())
1139 MIB.buildBr(Dest&: *MBB);
1140}
1141
1142void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1143 MachineBasicBlock *NextMBB,
1144 BranchProbability BranchProbToNext,
1145 Register Reg, SwitchCG::BitTestCase &B,
1146 MachineBasicBlock *SwitchBB) {
1147 MachineIRBuilder &MIB = *CurBuilder;
1148 MIB.setMBB(*SwitchBB);
1149
1150 LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1151 Register Cmp;
1152 unsigned PopCount = llvm::popcount(Value: B.Mask);
1153 if (PopCount == 1) {
1154 // Testing for a single bit; just compare the shift count with what it
1155 // would need to be to shift a 1 bit in that position.
1156 auto MaskTrailingZeros =
1157 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1158 Cmp =
1159 MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingZeros)
1160 .getReg(Idx: 0);
1161 } else if (PopCount == BB.Range) {
1162 // There is only one zero bit in the range, test for it directly.
1163 auto MaskTrailingOnes =
1164 MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1165 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1166 .getReg(Idx: 0);
1167 } else {
1168 // Make desired shift.
1169 auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1170 auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1171
1172 // Emit bit tests and jumps.
1173 auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1174 auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1175 auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1176 Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1177 .getReg(Idx: 0);
1178 }
1179
1180 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1181 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1182 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1183 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1184 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1185 // one as they are relative probabilities (and thus work more like weights),
1186 // and hence we need to normalize them to let the sum of them become one.
1187 SwitchBB->normalizeSuccProbs();
1188
1189 // Record the fact that the IR edge from the header to the bit test target
1190 // will go through our new block. Neeeded for PHIs to have nodes added.
1191 addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1192 NewPred: SwitchBB);
1193
1194 MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1195
1196 // Avoid emitting unnecessary branches to the next block.
1197 if (NextMBB != SwitchBB->getNextNode())
1198 MIB.buildBr(Dest&: *NextMBB);
1199}
1200
1201bool IRTranslator::lowerBitTestWorkItem(
1202 SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1203 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1204 MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1205 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1206 SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1207 bool FallthroughUnreachable) {
1208 using namespace SwitchCG;
1209 MachineFunction *CurMF = SwitchMBB->getParent();
1210 // FIXME: Optimize away range check based on pivot comparisons.
1211 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1212 // The bit test blocks haven't been inserted yet; insert them here.
1213 for (BitTestCase &BTC : BTB->Cases)
1214 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1215
1216 // Fill in fields of the BitTestBlock.
1217 BTB->Parent = CurMBB;
1218 BTB->Default = Fallthrough;
1219
1220 BTB->DefaultProb = UnhandledProbs;
1221 // If the cases in bit test don't form a contiguous range, we evenly
1222 // distribute the probability on the edge to Fallthrough to two
1223 // successors of CurMBB.
1224 if (!BTB->ContiguousRange) {
1225 BTB->Prob += DefaultProb / 2;
1226 BTB->DefaultProb -= DefaultProb / 2;
1227 }
1228
1229 if (FallthroughUnreachable)
1230 BTB->FallthroughUnreachable = true;
1231
1232 // If we're in the right place, emit the bit test header right now.
1233 if (CurMBB == SwitchMBB) {
1234 emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
1235 BTB->Emitted = true;
1236 }
1237 return true;
1238}
1239
1240bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1241 Value *Cond,
1242 MachineBasicBlock *SwitchMBB,
1243 MachineBasicBlock *DefaultMBB,
1244 MachineIRBuilder &MIB) {
1245 using namespace SwitchCG;
1246 MachineFunction *CurMF = FuncInfo.MF;
1247 MachineBasicBlock *NextMBB = nullptr;
1248 MachineFunction::iterator BBI(W.MBB);
1249 if (++BBI != FuncInfo.MF->end())
1250 NextMBB = &*BBI;
1251
1252 if (EnableOpts) {
1253 // Here, we order cases by probability so the most likely case will be
1254 // checked first. However, two clusters can have the same probability in
1255 // which case their relative ordering is non-deterministic. So we use Low
1256 // as a tie-breaker as clusters are guaranteed to never overlap.
1257 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
1258 Comp: [](const CaseCluster &a, const CaseCluster &b) {
1259 return a.Prob != b.Prob
1260 ? a.Prob > b.Prob
1261 : a.Low->getValue().slt(RHS: b.Low->getValue());
1262 });
1263
1264 // Rearrange the case blocks so that the last one falls through if possible
1265 // without changing the order of probabilities.
1266 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1267 --I;
1268 if (I->Prob > W.LastCluster->Prob)
1269 break;
1270 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1271 std::swap(a&: *I, b&: *W.LastCluster);
1272 break;
1273 }
1274 }
1275 }
1276
1277 // Compute total probability.
1278 BranchProbability DefaultProb = W.DefaultProb;
1279 BranchProbability UnhandledProbs = DefaultProb;
1280 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1281 UnhandledProbs += I->Prob;
1282
1283 MachineBasicBlock *CurMBB = W.MBB;
1284 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1285 bool FallthroughUnreachable = false;
1286 MachineBasicBlock *Fallthrough;
1287 if (I == W.LastCluster) {
1288 // For the last cluster, fall through to the default destination.
1289 Fallthrough = DefaultMBB;
1290 FallthroughUnreachable = isa<UnreachableInst>(
1291 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1292 } else {
1293 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
1294 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
1295 }
1296 UnhandledProbs -= I->Prob;
1297
1298 switch (I->Kind) {
1299 case CC_BitTests: {
1300 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1301 DefaultProb, UnhandledProbs, I, Fallthrough,
1302 FallthroughUnreachable)) {
1303 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1304 return false;
1305 }
1306 break;
1307 }
1308
1309 case CC_JumpTable: {
1310 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1311 UnhandledProbs, I, Fallthrough,
1312 FallthroughUnreachable)) {
1313 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1314 return false;
1315 }
1316 break;
1317 }
1318 case CC_Range: {
1319 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1320 FallthroughUnreachable, UnhandledProbs,
1321 CurMBB, MIB, SwitchMBB)) {
1322 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1323 return false;
1324 }
1325 break;
1326 }
1327 }
1328 CurMBB = Fallthrough;
1329 }
1330
1331 return true;
1332}
1333
1334bool IRTranslator::translateIndirectBr(const User &U,
1335 MachineIRBuilder &MIRBuilder) {
1336 const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
1337
1338 const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
1339 MIRBuilder.buildBrIndirect(Tgt);
1340
1341 // Link successors.
1342 SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1343 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1344 for (const BasicBlock *Succ : successors(I: &BrInst)) {
1345 // It's legal for indirectbr instructions to have duplicate blocks in the
1346 // destination list. We don't allow this in MIR. Skip anything that's
1347 // already a successor.
1348 if (!AddedSuccessors.insert(Ptr: Succ).second)
1349 continue;
1350 CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1351 }
1352
1353 return true;
1354}
1355
1356static bool isSwiftError(const Value *V) {
1357 if (auto Arg = dyn_cast<Argument>(Val: V))
1358 return Arg->hasSwiftErrorAttr();
1359 if (auto AI = dyn_cast<AllocaInst>(Val: V))
1360 return AI->isSwiftError();
1361 return false;
1362}
1363
1364bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1365 const LoadInst &LI = cast<LoadInst>(Val: U);
1366
1367 unsigned StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
1368 if (StoreSize == 0)
1369 return true;
1370
1371 ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
1372 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
1373 Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
1374 AAMDNodes AAInfo = LI.getAAMetadata();
1375
1376 const Value *Ptr = LI.getPointerOperand();
1377 Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
1378 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1379
1380 if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
1381 assert(Regs.size() == 1 && "swifterror should be single pointer");
1382 Register VReg =
1383 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1384 MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
1385 return true;
1386 }
1387
1388 auto &TLI = *MF->getSubtarget().getTargetLowering();
1389 MachineMemOperand::Flags Flags =
1390 TLI.getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo);
1391 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1392 if (AA->pointsToConstantMemory(
1393 Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
1394 Flags |= MachineMemOperand::MOInvariant;
1395 }
1396 }
1397
1398 const MDNode *Ranges =
1399 Regs.size() == 1 ? LI.getMetadata(KindID: LLVMContext::MD_range) : nullptr;
1400 for (unsigned i = 0; i < Regs.size(); ++i) {
1401 Register Addr;
1402 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i] / 8);
1403
1404 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1405 Align BaseAlign = getMemOpAlign(I: LI);
1406 auto MMO = MF->getMachineMemOperand(
1407 PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Regs[i]),
1408 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i] / 8), AAInfo, Ranges,
1409 SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
1410 MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
1411 }
1412
1413 return true;
1414}
1415
1416bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1417 const StoreInst &SI = cast<StoreInst>(Val: U);
1418 if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()) == 0)
1419 return true;
1420
1421 ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
1422 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
1423 Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
1424
1425 Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
1426 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1427
1428 if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
1429 assert(Vals.size() == 1 && "swifterror should be single pointer");
1430
1431 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1432 SI.getPointerOperand());
1433 MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
1434 return true;
1435 }
1436
1437 auto &TLI = *MF->getSubtarget().getTargetLowering();
1438 MachineMemOperand::Flags Flags = TLI.getStoreMemOperandFlags(SI, DL: *DL);
1439
1440 for (unsigned i = 0; i < Vals.size(); ++i) {
1441 Register Addr;
1442 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i] / 8);
1443
1444 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1445 Align BaseAlign = getMemOpAlign(I: SI);
1446 auto MMO = MF->getMachineMemOperand(
1447 PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Vals[i]),
1448 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i] / 8), AAInfo: SI.getAAMetadata(), Ranges: nullptr,
1449 SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
1450 MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
1451 }
1452 return true;
1453}
1454
1455static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1456 const Value *Src = U.getOperand(i: 0);
1457 Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
1458
1459 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1460 // usual array element rather than looking into the actual aggregate.
1461 SmallVector<Value *, 1> Indices;
1462 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
1463
1464 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
1465 for (auto Idx : EVI->indices())
1466 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1467 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
1468 for (auto Idx : IVI->indices())
1469 Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1470 } else {
1471 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1472 Indices.push_back(Elt: U.getOperand(i));
1473 }
1474
1475 return 8 * static_cast<uint64_t>(
1476 DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
1477}
1478
1479bool IRTranslator::translateExtractValue(const User &U,
1480 MachineIRBuilder &MIRBuilder) {
1481 const Value *Src = U.getOperand(i: 0);
1482 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1483 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1484 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
1485 unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
1486 auto &DstRegs = allocateVRegs(Val: U);
1487
1488 for (unsigned i = 0; i < DstRegs.size(); ++i)
1489 DstRegs[i] = SrcRegs[Idx++];
1490
1491 return true;
1492}
1493
1494bool IRTranslator::translateInsertValue(const User &U,
1495 MachineIRBuilder &MIRBuilder) {
1496 const Value *Src = U.getOperand(i: 0);
1497 uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1498 auto &DstRegs = allocateVRegs(Val: U);
1499 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
1500 ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1501 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1502 auto *InsertedIt = InsertedRegs.begin();
1503
1504 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1505 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1506 DstRegs[i] = *InsertedIt++;
1507 else
1508 DstRegs[i] = SrcRegs[i];
1509 }
1510
1511 return true;
1512}
1513
1514bool IRTranslator::translateSelect(const User &U,
1515 MachineIRBuilder &MIRBuilder) {
1516 Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
1517 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
1518 ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1519 ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
1520
1521 uint32_t Flags = 0;
1522 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
1523 Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
1524
1525 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1526 MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
1527 }
1528
1529 return true;
1530}
1531
1532bool IRTranslator::translateCopy(const User &U, const Value &V,
1533 MachineIRBuilder &MIRBuilder) {
1534 Register Src = getOrCreateVReg(Val: V);
1535 auto &Regs = *VMap.getVRegs(V: U);
1536 if (Regs.empty()) {
1537 Regs.push_back(Elt: Src);
1538 VMap.getOffsets(V: U)->push_back(Elt: 0);
1539 } else {
1540 // If we already assigned a vreg for this instruction, we can't change that.
1541 // Emit a copy to satisfy the users we already emitted.
1542 MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
1543 }
1544 return true;
1545}
1546
1547bool IRTranslator::translateBitCast(const User &U,
1548 MachineIRBuilder &MIRBuilder) {
1549 // If we're bitcasting to the source type, we can reuse the source vreg.
1550 if (getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL) ==
1551 getLLTForType(Ty&: *U.getType(), DL: *DL)) {
1552 // If the source is a ConstantInt then it was probably created by
1553 // ConstantHoisting and we should leave it alone.
1554 if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
1555 return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1556 MIRBuilder);
1557 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
1558 }
1559
1560 return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
1561}
1562
1563bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1564 MachineIRBuilder &MIRBuilder) {
1565 if (U.getType()->getScalarType()->isBFloatTy() ||
1566 U.getOperand(i: 0)->getType()->getScalarType()->isBFloatTy())
1567 return false;
1568 Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
1569 Register Res = getOrCreateVReg(Val: U);
1570 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op});
1571 return true;
1572}
1573
1574bool IRTranslator::translateGetElementPtr(const User &U,
1575 MachineIRBuilder &MIRBuilder) {
1576 Value &Op0 = *U.getOperand(i: 0);
1577 Register BaseReg = getOrCreateVReg(Val: Op0);
1578 Type *PtrIRTy = Op0.getType();
1579 LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1580 Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1581 LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1582
1583 uint32_t Flags = 0;
1584 if (isa<Instruction>(Val: U)) {
1585 const Instruction &I = cast<Instruction>(Val: U);
1586 Flags = MachineInstr::copyFlagsFromInstruction(I);
1587 }
1588
1589 // Normalize Vector GEP - all scalar operands should be converted to the
1590 // splat vector.
1591 unsigned VectorWidth = 0;
1592
1593 // True if we should use a splat vector; using VectorWidth alone is not
1594 // sufficient.
1595 bool WantSplatVector = false;
1596 if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
1597 VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
1598 // We don't produce 1 x N vectors; those are treated as scalars.
1599 WantSplatVector = VectorWidth > 1;
1600 }
1601
1602 // We might need to splat the base pointer into a vector if the offsets
1603 // are vectors.
1604 if (WantSplatVector && !PtrTy.isVector()) {
1605 BaseReg =
1606 MIRBuilder
1607 .buildSplatVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy), Src: BaseReg)
1608 .getReg(Idx: 0);
1609 PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
1610 PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1611 OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1612 OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1613 }
1614
1615 int64_t Offset = 0;
1616 for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
1617 GTI != E; ++GTI) {
1618 const Value *Idx = GTI.getOperand();
1619 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1620 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
1621 Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
1622 continue;
1623 } else {
1624 uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
1625
1626 // If this is a scalar constant or a splat vector of constants,
1627 // handle it quickly.
1628 if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
1629 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1630 Offset += ElementSize * *Val;
1631 continue;
1632 }
1633 }
1634
1635 if (Offset != 0) {
1636 auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
1637 BaseReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0))
1638 .getReg(Idx: 0);
1639 Offset = 0;
1640 }
1641
1642 Register IdxReg = getOrCreateVReg(Val: *Idx);
1643 LLT IdxTy = MRI->getType(Reg: IdxReg);
1644 if (IdxTy != OffsetTy) {
1645 if (!IdxTy.isVector() && WantSplatVector) {
1646 IdxReg = MIRBuilder.buildSplatVector(
1647 Res: OffsetTy.changeElementType(NewEltTy: IdxTy), Src: IdxReg).getReg(Idx: 0);
1648 }
1649
1650 IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
1651 }
1652
1653 // N = N + Idx * ElementSize;
1654 // Avoid doing it for ElementSize of 1.
1655 Register GepOffsetReg;
1656 if (ElementSize != 1) {
1657 auto ElementSizeMIB = MIRBuilder.buildConstant(
1658 Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
1659 GepOffsetReg =
1660 MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB).getReg(Idx: 0);
1661 } else
1662 GepOffsetReg = IdxReg;
1663
1664 BaseReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg).getReg(Idx: 0);
1665 }
1666 }
1667
1668 if (Offset != 0) {
1669 auto OffsetMIB =
1670 MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
1671
1672 if (int64_t(Offset) >= 0 && cast<GEPOperator>(Val: U).isInBounds())
1673 Flags |= MachineInstr::MIFlag::NoUWrap;
1674
1675 MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1676 Flags);
1677 return true;
1678 }
1679
1680 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: U), Op: BaseReg);
1681 return true;
1682}
1683
1684bool IRTranslator::translateMemFunc(const CallInst &CI,
1685 MachineIRBuilder &MIRBuilder,
1686 unsigned Opcode) {
1687 const Value *SrcPtr = CI.getArgOperand(i: 1);
1688 // If the source is undef, then just emit a nop.
1689 if (isa<UndefValue>(Val: SrcPtr))
1690 return true;
1691
1692 SmallVector<Register, 3> SrcRegs;
1693
1694 unsigned MinPtrSize = UINT_MAX;
1695 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
1696 Register SrcReg = getOrCreateVReg(Val: **AI);
1697 LLT SrcTy = MRI->getType(Reg: SrcReg);
1698 if (SrcTy.isPointer())
1699 MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
1700 SrcRegs.push_back(Elt: SrcReg);
1701 }
1702
1703 LLT SizeTy = LLT::scalar(SizeInBits: MinPtrSize);
1704
1705 // The size operand should be the minimum of the pointer sizes.
1706 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1707 if (MRI->getType(Reg: SizeOpReg) != SizeTy)
1708 SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
1709
1710 auto ICall = MIRBuilder.buildInstr(Opcode);
1711 for (Register SrcReg : SrcRegs)
1712 ICall.addUse(RegNo: SrcReg);
1713
1714 Align DstAlign;
1715 Align SrcAlign;
1716 unsigned IsVol =
1717 cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
1718
1719 ConstantInt *CopySize = nullptr;
1720
1721 if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
1722 DstAlign = MCI->getDestAlign().valueOrOne();
1723 SrcAlign = MCI->getSourceAlign().valueOrOne();
1724 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1725 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(Val: &CI)) {
1726 DstAlign = MCI->getDestAlign().valueOrOne();
1727 SrcAlign = MCI->getSourceAlign().valueOrOne();
1728 CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1729 } else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
1730 DstAlign = MMI->getDestAlign().valueOrOne();
1731 SrcAlign = MMI->getSourceAlign().valueOrOne();
1732 CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
1733 } else {
1734 auto *MSI = cast<MemSetInst>(Val: &CI);
1735 DstAlign = MSI->getDestAlign().valueOrOne();
1736 }
1737
1738 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1739 // We need to propagate the tail call flag from the IR inst as an argument.
1740 // Otherwise, we have to pessimize and assume later that we cannot tail call
1741 // any memory intrinsics.
1742 ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
1743 }
1744
1745 // Create mem operands to store the alignment and volatile info.
1746 MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
1747 MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
1748 if (IsVol) {
1749 LoadFlags |= MachineMemOperand::MOVolatile;
1750 StoreFlags |= MachineMemOperand::MOVolatile;
1751 }
1752
1753 AAMDNodes AAInfo = CI.getAAMetadata();
1754 if (AA && CopySize &&
1755 AA->pointsToConstantMemory(Loc: MemoryLocation(
1756 SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
1757 LoadFlags |= MachineMemOperand::MOInvariant;
1758
1759 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1760 // but the previous usage implied it did. Probably should check
1761 // isDereferenceableAndAlignedPointer.
1762 LoadFlags |= MachineMemOperand::MODereferenceable;
1763 }
1764
1765 ICall.addMemOperand(
1766 MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
1767 f: StoreFlags, s: 1, base_alignment: DstAlign, AAInfo));
1768 if (Opcode != TargetOpcode::G_MEMSET)
1769 ICall.addMemOperand(MMO: MF->getMachineMemOperand(
1770 PtrInfo: MachinePointerInfo(SrcPtr), f: LoadFlags, s: 1, base_alignment: SrcAlign, AAInfo));
1771
1772 return true;
1773}
1774
1775void IRTranslator::getStackGuard(Register DstReg,
1776 MachineIRBuilder &MIRBuilder) {
1777 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1778 MRI->setRegClass(Reg: DstReg, RC: TRI->getPointerRegClass(MF: *MF));
1779 auto MIB =
1780 MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
1781
1782 auto &TLI = *MF->getSubtarget().getTargetLowering();
1783 Value *Global = TLI.getSDagStackGuard(M: *MF->getFunction().getParent());
1784 if (!Global)
1785 return;
1786
1787 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1788 LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
1789
1790 MachinePointerInfo MPInfo(Global);
1791 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1792 MachineMemOperand::MODereferenceable;
1793 MachineMemOperand *MemRef = MF->getMachineMemOperand(
1794 PtrInfo: MPInfo, f: Flags, MemTy: PtrTy, base_alignment: DL->getPointerABIAlignment(AS: AddrSpace));
1795 MIB.setMemRefs({MemRef});
1796}
1797
1798bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1799 MachineIRBuilder &MIRBuilder) {
1800 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
1801 MIRBuilder.buildInstr(
1802 Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
1803 SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
1804
1805 return true;
1806}
1807
1808bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1809 MachineIRBuilder &MIRBuilder) {
1810 Register Dst = getOrCreateVReg(Val: CI);
1811 Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1812 Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1813 uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
1814 MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
1815 return true;
1816}
1817
1818unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1819 switch (ID) {
1820 default:
1821 break;
1822 case Intrinsic::bswap:
1823 return TargetOpcode::G_BSWAP;
1824 case Intrinsic::bitreverse:
1825 return TargetOpcode::G_BITREVERSE;
1826 case Intrinsic::fshl:
1827 return TargetOpcode::G_FSHL;
1828 case Intrinsic::fshr:
1829 return TargetOpcode::G_FSHR;
1830 case Intrinsic::ceil:
1831 return TargetOpcode::G_FCEIL;
1832 case Intrinsic::cos:
1833 return TargetOpcode::G_FCOS;
1834 case Intrinsic::ctpop:
1835 return TargetOpcode::G_CTPOP;
1836 case Intrinsic::exp:
1837 return TargetOpcode::G_FEXP;
1838 case Intrinsic::exp2:
1839 return TargetOpcode::G_FEXP2;
1840 case Intrinsic::exp10:
1841 return TargetOpcode::G_FEXP10;
1842 case Intrinsic::fabs:
1843 return TargetOpcode::G_FABS;
1844 case Intrinsic::copysign:
1845 return TargetOpcode::G_FCOPYSIGN;
1846 case Intrinsic::minnum:
1847 return TargetOpcode::G_FMINNUM;
1848 case Intrinsic::maxnum:
1849 return TargetOpcode::G_FMAXNUM;
1850 case Intrinsic::minimum:
1851 return TargetOpcode::G_FMINIMUM;
1852 case Intrinsic::maximum:
1853 return TargetOpcode::G_FMAXIMUM;
1854 case Intrinsic::canonicalize:
1855 return TargetOpcode::G_FCANONICALIZE;
1856 case Intrinsic::floor:
1857 return TargetOpcode::G_FFLOOR;
1858 case Intrinsic::fma:
1859 return TargetOpcode::G_FMA;
1860 case Intrinsic::log:
1861 return TargetOpcode::G_FLOG;
1862 case Intrinsic::log2:
1863 return TargetOpcode::G_FLOG2;
1864 case Intrinsic::log10:
1865 return TargetOpcode::G_FLOG10;
1866 case Intrinsic::ldexp:
1867 return TargetOpcode::G_FLDEXP;
1868 case Intrinsic::nearbyint:
1869 return TargetOpcode::G_FNEARBYINT;
1870 case Intrinsic::pow:
1871 return TargetOpcode::G_FPOW;
1872 case Intrinsic::powi:
1873 return TargetOpcode::G_FPOWI;
1874 case Intrinsic::rint:
1875 return TargetOpcode::G_FRINT;
1876 case Intrinsic::round:
1877 return TargetOpcode::G_INTRINSIC_ROUND;
1878 case Intrinsic::roundeven:
1879 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1880 case Intrinsic::sin:
1881 return TargetOpcode::G_FSIN;
1882 case Intrinsic::sqrt:
1883 return TargetOpcode::G_FSQRT;
1884 case Intrinsic::trunc:
1885 return TargetOpcode::G_INTRINSIC_TRUNC;
1886 case Intrinsic::readcyclecounter:
1887 return TargetOpcode::G_READCYCLECOUNTER;
1888 case Intrinsic::readsteadycounter:
1889 return TargetOpcode::G_READSTEADYCOUNTER;
1890 case Intrinsic::ptrmask:
1891 return TargetOpcode::G_PTRMASK;
1892 case Intrinsic::lrint:
1893 return TargetOpcode::G_INTRINSIC_LRINT;
1894 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1895 case Intrinsic::vector_reduce_fmin:
1896 return TargetOpcode::G_VECREDUCE_FMIN;
1897 case Intrinsic::vector_reduce_fmax:
1898 return TargetOpcode::G_VECREDUCE_FMAX;
1899 case Intrinsic::vector_reduce_fminimum:
1900 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1901 case Intrinsic::vector_reduce_fmaximum:
1902 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1903 case Intrinsic::vector_reduce_add:
1904 return TargetOpcode::G_VECREDUCE_ADD;
1905 case Intrinsic::vector_reduce_mul:
1906 return TargetOpcode::G_VECREDUCE_MUL;
1907 case Intrinsic::vector_reduce_and:
1908 return TargetOpcode::G_VECREDUCE_AND;
1909 case Intrinsic::vector_reduce_or:
1910 return TargetOpcode::G_VECREDUCE_OR;
1911 case Intrinsic::vector_reduce_xor:
1912 return TargetOpcode::G_VECREDUCE_XOR;
1913 case Intrinsic::vector_reduce_smax:
1914 return TargetOpcode::G_VECREDUCE_SMAX;
1915 case Intrinsic::vector_reduce_smin:
1916 return TargetOpcode::G_VECREDUCE_SMIN;
1917 case Intrinsic::vector_reduce_umax:
1918 return TargetOpcode::G_VECREDUCE_UMAX;
1919 case Intrinsic::vector_reduce_umin:
1920 return TargetOpcode::G_VECREDUCE_UMIN;
1921 case Intrinsic::lround:
1922 return TargetOpcode::G_LROUND;
1923 case Intrinsic::llround:
1924 return TargetOpcode::G_LLROUND;
1925 case Intrinsic::get_fpenv:
1926 return TargetOpcode::G_GET_FPENV;
1927 case Intrinsic::get_fpmode:
1928 return TargetOpcode::G_GET_FPMODE;
1929 }
1930 return Intrinsic::not_intrinsic;
1931}
1932
1933bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1934 Intrinsic::ID ID,
1935 MachineIRBuilder &MIRBuilder) {
1936
1937 unsigned Op = getSimpleIntrinsicOpcode(ID);
1938
1939 // Is this a simple intrinsic?
1940 if (Op == Intrinsic::not_intrinsic)
1941 return false;
1942
1943 // Yes. Let's translate it.
1944 SmallVector<llvm::SrcOp, 4> VRegs;
1945 for (const auto &Arg : CI.args())
1946 VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
1947
1948 MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
1949 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
1950 return true;
1951}
1952
1953// TODO: Include ConstainedOps.def when all strict instructions are defined.
1954static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
1955 switch (ID) {
1956 case Intrinsic::experimental_constrained_fadd:
1957 return TargetOpcode::G_STRICT_FADD;
1958 case Intrinsic::experimental_constrained_fsub:
1959 return TargetOpcode::G_STRICT_FSUB;
1960 case Intrinsic::experimental_constrained_fmul:
1961 return TargetOpcode::G_STRICT_FMUL;
1962 case Intrinsic::experimental_constrained_fdiv:
1963 return TargetOpcode::G_STRICT_FDIV;
1964 case Intrinsic::experimental_constrained_frem:
1965 return TargetOpcode::G_STRICT_FREM;
1966 case Intrinsic::experimental_constrained_fma:
1967 return TargetOpcode::G_STRICT_FMA;
1968 case Intrinsic::experimental_constrained_sqrt:
1969 return TargetOpcode::G_STRICT_FSQRT;
1970 case Intrinsic::experimental_constrained_ldexp:
1971 return TargetOpcode::G_STRICT_FLDEXP;
1972 default:
1973 return 0;
1974 }
1975}
1976
1977bool IRTranslator::translateConstrainedFPIntrinsic(
1978 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
1979 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
1980
1981 unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
1982 if (!Opcode)
1983 return false;
1984
1985 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
1986 if (EB == fp::ExceptionBehavior::ebIgnore)
1987 Flags |= MachineInstr::NoFPExcept;
1988
1989 SmallVector<llvm::SrcOp, 4> VRegs;
1990 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 0)));
1991 if (!FPI.isUnaryOp())
1992 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 1)));
1993 if (FPI.isTernaryOp())
1994 VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 2)));
1995
1996 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
1997 return true;
1998}
1999
2000std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2001 auto VRegs = getOrCreateVRegs(Val: Arg);
2002 if (VRegs.size() != 1)
2003 return std::nullopt;
2004
2005 // Arguments are lowered as a copy of a livein physical register.
2006 auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2007 if (!VRegDef || !VRegDef->isCopy())
2008 return std::nullopt;
2009 return VRegDef->getOperand(i: 1).getReg().asMCReg();
2010}
2011
2012bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2013 const DILocalVariable *Var,
2014 const DIExpression *Expr,
2015 const DebugLoc &DL,
2016 MachineIRBuilder &MIRBuilder) {
2017 auto *Arg = dyn_cast<Argument>(Val);
2018 if (!Arg)
2019 return false;
2020
2021 if (!Expr->isEntryValue())
2022 return false;
2023
2024 std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2025 if (!PhysReg) {
2026 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2027 << ": expression is entry_value but "
2028 << "couldn't find a physical register\n");
2029 LLVM_DEBUG(dbgs() << *Var << "\n");
2030 return true;
2031 }
2032
2033 if (isDeclare) {
2034 // Append an op deref to account for the fact that this is a dbg_declare.
2035 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2036 MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2037 } else {
2038 MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2039 }
2040
2041 return true;
2042}
2043
2044bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2045 MachineIRBuilder &MIRBuilder) {
2046 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
2047 if (ORE->enabled()) {
2048 if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
2049 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2050 R.visit(I: MI);
2051 }
2052 }
2053 }
2054
2055 // If this is a simple intrinsic (that is, we just need to add a def of
2056 // a vreg, and uses for each arg operand, then translate it.
2057 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2058 return true;
2059
2060 switch (ID) {
2061 default:
2062 break;
2063 case Intrinsic::lifetime_start:
2064 case Intrinsic::lifetime_end: {
2065 // No stack colouring in O0, discard region information.
2066 if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None)
2067 return true;
2068
2069 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2070 : TargetOpcode::LIFETIME_END;
2071
2072 // Get the underlying objects for the location passed on the lifetime
2073 // marker.
2074 SmallVector<const Value *, 4> Allocas;
2075 getUnderlyingObjects(V: CI.getArgOperand(i: 1), Objects&: Allocas);
2076
2077 // Iterate over each underlying object, creating lifetime markers for each
2078 // static alloca. Quit if we find a non-static alloca.
2079 for (const Value *V : Allocas) {
2080 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V);
2081 if (!AI)
2082 continue;
2083
2084 if (!AI->isStaticAlloca())
2085 return true;
2086
2087 MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
2088 }
2089 return true;
2090 }
2091 case Intrinsic::dbg_declare: {
2092 const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
2093 assert(DI.getVariable() && "Missing variable");
2094 translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2095 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2096 return true;
2097 }
2098 case Intrinsic::dbg_label: {
2099 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
2100 assert(DI.getLabel() && "Missing label");
2101
2102 assert(DI.getLabel()->isValidLocationForIntrinsic(
2103 MIRBuilder.getDebugLoc()) &&
2104 "Expected inlined-at fields to agree");
2105
2106 MIRBuilder.buildDbgLabel(Label: DI.getLabel());
2107 return true;
2108 }
2109 case Intrinsic::vaend:
2110 // No target I know of cares about va_end. Certainly no in-tree target
2111 // does. Simplest intrinsic ever!
2112 return true;
2113 case Intrinsic::vastart: {
2114 auto &TLI = *MF->getSubtarget().getTargetLowering();
2115 Value *Ptr = CI.getArgOperand(i: 0);
2116 unsigned ListSize = TLI.getVaListSizeInBits(DL: *DL) / 8;
2117 Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
2118
2119 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
2120 .addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
2121 f: MachineMemOperand::MOStore,
2122 s: ListSize, base_alignment: Alignment));
2123 return true;
2124 }
2125 case Intrinsic::dbg_assign:
2126 // A dbg.assign is a dbg.value with more information about stack locations,
2127 // typically produced during optimisation of variables with leaked
2128 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2129 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2130 // need to register for and use the AssignmentTrackingAnalysis pass.
2131 LLVM_FALLTHROUGH;
2132 case Intrinsic::dbg_value: {
2133 // This form of DBG_VALUE is target-independent.
2134 const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
2135 translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2136 Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2137 return true;
2138 }
2139 case Intrinsic::uadd_with_overflow:
2140 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
2141 case Intrinsic::sadd_with_overflow:
2142 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
2143 case Intrinsic::usub_with_overflow:
2144 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
2145 case Intrinsic::ssub_with_overflow:
2146 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
2147 case Intrinsic::umul_with_overflow:
2148 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
2149 case Intrinsic::smul_with_overflow:
2150 return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
2151 case Intrinsic::uadd_sat:
2152 return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
2153 case Intrinsic::sadd_sat:
2154 return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
2155 case Intrinsic::usub_sat:
2156 return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
2157 case Intrinsic::ssub_sat:
2158 return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
2159 case Intrinsic::ushl_sat:
2160 return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
2161 case Intrinsic::sshl_sat:
2162 return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
2163 case Intrinsic::umin:
2164 return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
2165 case Intrinsic::umax:
2166 return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
2167 case Intrinsic::smin:
2168 return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
2169 case Intrinsic::smax:
2170 return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
2171 case Intrinsic::abs:
2172 // TODO: Preserve "int min is poison" arg in GMIR?
2173 return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
2174 case Intrinsic::smul_fix:
2175 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2176 case Intrinsic::umul_fix:
2177 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2178 case Intrinsic::smul_fix_sat:
2179 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2180 case Intrinsic::umul_fix_sat:
2181 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2182 case Intrinsic::sdiv_fix:
2183 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2184 case Intrinsic::udiv_fix:
2185 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2186 case Intrinsic::sdiv_fix_sat:
2187 return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2188 case Intrinsic::udiv_fix_sat:
2189 return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2190 case Intrinsic::fmuladd: {
2191 const TargetMachine &TM = MF->getTarget();
2192 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2193 Register Dst = getOrCreateVReg(Val: CI);
2194 Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2195 Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2196 Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
2197 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2198 TLI.isFMAFasterThanFMulAndFAdd(MF: *MF,
2199 TLI.getValueType(DL: *DL, Ty: CI.getType()))) {
2200 // TODO: Revisit this to see if we should move this part of the
2201 // lowering to the combiner.
2202 MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
2203 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2204 } else {
2205 LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
2206 auto FMul = MIRBuilder.buildFMul(
2207 Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2208 MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
2209 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2210 }
2211 return true;
2212 }
2213 case Intrinsic::convert_from_fp16:
2214 // FIXME: This intrinsic should probably be removed from the IR.
2215 MIRBuilder.buildFPExt(Res: getOrCreateVReg(Val: CI),
2216 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2217 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2218 return true;
2219 case Intrinsic::convert_to_fp16:
2220 // FIXME: This intrinsic should probably be removed from the IR.
2221 MIRBuilder.buildFPTrunc(Res: getOrCreateVReg(Val: CI),
2222 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2223 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2224 return true;
2225 case Intrinsic::frexp: {
2226 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2227 MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
2228 Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2229 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2230 return true;
2231 }
2232 case Intrinsic::memcpy_inline:
2233 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
2234 case Intrinsic::memcpy:
2235 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
2236 case Intrinsic::memmove:
2237 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
2238 case Intrinsic::memset:
2239 return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
2240 case Intrinsic::eh_typeid_for: {
2241 GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
2242 Register Reg = getOrCreateVReg(Val: CI);
2243 unsigned TypeID = MF->getTypeIDFor(TI: GV);
2244 MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
2245 return true;
2246 }
2247 case Intrinsic::objectsize:
2248 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2249
2250 case Intrinsic::is_constant:
2251 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2252
2253 case Intrinsic::stackguard:
2254 getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
2255 return true;
2256 case Intrinsic::stackprotector: {
2257 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2258 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2259 Register GuardVal;
2260 if (TLI.useLoadStackGuardNode()) {
2261 GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
2262 getStackGuard(DstReg: GuardVal, MIRBuilder);
2263 } else
2264 GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
2265
2266 AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
2267 int FI = getOrCreateFrameIndex(AI: *Slot);
2268 MF->getFrameInfo().setStackProtectorIndex(FI);
2269
2270 MIRBuilder.buildStore(
2271 Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
2272 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
2273 f: MachineMemOperand::MOStore |
2274 MachineMemOperand::MOVolatile,
2275 MemTy: PtrTy, base_alignment: Align(8)));
2276 return true;
2277 }
2278 case Intrinsic::stacksave: {
2279 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
2280 return true;
2281 }
2282 case Intrinsic::stackrestore: {
2283 MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
2284 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2285 return true;
2286 }
2287 case Intrinsic::cttz:
2288 case Intrinsic::ctlz: {
2289 ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
2290 bool isTrailing = ID == Intrinsic::cttz;
2291 unsigned Opcode = isTrailing
2292 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2293 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2294 : Cst->isZero() ? TargetOpcode::G_CTLZ
2295 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2296 MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
2297 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2298 return true;
2299 }
2300 case Intrinsic::invariant_start: {
2301 LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2302 Register Undef = MRI->createGenericVirtualRegister(Ty: PtrTy);
2303 MIRBuilder.buildUndef(Res: Undef);
2304 return true;
2305 }
2306 case Intrinsic::invariant_end:
2307 return true;
2308 case Intrinsic::expect:
2309 case Intrinsic::annotation:
2310 case Intrinsic::ptr_annotation:
2311 case Intrinsic::launder_invariant_group:
2312 case Intrinsic::strip_invariant_group: {
2313 // Drop the intrinsic, but forward the value.
2314 MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2315 Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2316 return true;
2317 }
2318 case Intrinsic::assume:
2319 case Intrinsic::experimental_noalias_scope_decl:
2320 case Intrinsic::var_annotation:
2321 case Intrinsic::sideeffect:
2322 // Discard annotate attributes, assumptions, and artificial side-effects.
2323 return true;
2324 case Intrinsic::read_volatile_register:
2325 case Intrinsic::read_register: {
2326 Value *Arg = CI.getArgOperand(i: 0);
2327 MIRBuilder
2328 .buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
2329 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
2330 return true;
2331 }
2332 case Intrinsic::write_register: {
2333 Value *Arg = CI.getArgOperand(i: 0);
2334 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
2335 .addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
2336 .addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
2337 return true;
2338 }
2339 case Intrinsic::localescape: {
2340 MachineBasicBlock &EntryMBB = MF->front();
2341 StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
2342
2343 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2344 // is the same on all targets.
2345 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2346 Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
2347 if (isa<ConstantPointerNull>(Val: Arg))
2348 continue; // Skip null pointers. They represent a hole in index space.
2349
2350 int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
2351 MCSymbol *FrameAllocSym =
2352 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName,
2353 Idx);
2354
2355 // This should be inserted at the start of the entry block.
2356 auto LocalEscape =
2357 MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
2358 .addSym(Sym: FrameAllocSym)
2359 .addFrameIndex(Idx: FI);
2360
2361 EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
2362 }
2363
2364 return true;
2365 }
2366 case Intrinsic::vector_reduce_fadd:
2367 case Intrinsic::vector_reduce_fmul: {
2368 // Need to check for the reassoc flag to decide whether we want a
2369 // sequential reduction opcode or not.
2370 Register Dst = getOrCreateVReg(Val: CI);
2371 Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2372 Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2373 unsigned Opc = 0;
2374 if (!CI.hasAllowReassoc()) {
2375 // The sequential ordering case.
2376 Opc = ID == Intrinsic::vector_reduce_fadd
2377 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2378 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2379 MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
2380 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2381 return true;
2382 }
2383 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2384 // since the associativity doesn't matter.
2385 unsigned ScalarOpc;
2386 if (ID == Intrinsic::vector_reduce_fadd) {
2387 Opc = TargetOpcode::G_VECREDUCE_FADD;
2388 ScalarOpc = TargetOpcode::G_FADD;
2389 } else {
2390 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2391 ScalarOpc = TargetOpcode::G_FMUL;
2392 }
2393 LLT DstTy = MRI->getType(Reg: Dst);
2394 auto Rdx = MIRBuilder.buildInstr(
2395 Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2396 MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
2397 Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2398
2399 return true;
2400 }
2401 case Intrinsic::trap:
2402 case Intrinsic::debugtrap:
2403 case Intrinsic::ubsantrap: {
2404 StringRef TrapFuncName =
2405 CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
2406 if (TrapFuncName.empty())
2407 break; // Use the default handling.
2408 CallLowering::CallLoweringInfo Info;
2409 if (ID == Intrinsic::ubsantrap) {
2410 Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
2411 CI.getArgOperand(i: 0)->getType(), 0});
2412 }
2413 Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
2414 Info.CB = &CI;
2415 Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
2416 return CLI->lowerCall(MIRBuilder, Info);
2417 }
2418 case Intrinsic::amdgcn_cs_chain:
2419 return translateCallBase(CB: CI, MIRBuilder);
2420 case Intrinsic::fptrunc_round: {
2421 uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
2422
2423 // Convert the metadata argument to a constant integer
2424 Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
2425 std::optional<RoundingMode> RoundMode =
2426 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
2427
2428 // Add the Rounding mode as an integer
2429 MIRBuilder
2430 .buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2431 DstOps: {getOrCreateVReg(Val: CI)},
2432 SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
2433 .addImm(Val: (int)*RoundMode);
2434
2435 return true;
2436 }
2437 case Intrinsic::is_fpclass: {
2438 Value *FpValue = CI.getOperand(i_nocapture: 0);
2439 ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
2440
2441 MIRBuilder
2442 .buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
2443 SrcOps: {getOrCreateVReg(Val: *FpValue)})
2444 .addImm(Val: TestMaskValue->getZExtValue());
2445
2446 return true;
2447 }
2448 case Intrinsic::set_fpenv: {
2449 Value *FPEnv = CI.getOperand(i_nocapture: 0);
2450 MIRBuilder.buildInstr(Opc: TargetOpcode::G_SET_FPENV, DstOps: {},
2451 SrcOps: {getOrCreateVReg(Val: *FPEnv)});
2452 return true;
2453 }
2454 case Intrinsic::reset_fpenv: {
2455 MIRBuilder.buildInstr(Opc: TargetOpcode::G_RESET_FPENV, DstOps: {}, SrcOps: {});
2456 return true;
2457 }
2458 case Intrinsic::set_fpmode: {
2459 Value *FPState = CI.getOperand(i_nocapture: 0);
2460 MIRBuilder.buildInstr(Opc: TargetOpcode::G_SET_FPMODE, DstOps: {},
2461 SrcOps: { getOrCreateVReg(Val: *FPState) });
2462 return true;
2463 }
2464 case Intrinsic::reset_fpmode: {
2465 MIRBuilder.buildInstr(Opc: TargetOpcode::G_RESET_FPMODE, DstOps: {}, SrcOps: {});
2466 return true;
2467 }
2468 case Intrinsic::prefetch: {
2469 Value *Addr = CI.getOperand(i_nocapture: 0);
2470 unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
2471 unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2472 unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
2473
2474 auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2475 auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), f: Flags,
2476 MemTy: LLT(), base_alignment: Align());
2477
2478 MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
2479 MMO);
2480
2481 return true;
2482 }
2483#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2484 case Intrinsic::INTRINSIC:
2485#include "llvm/IR/ConstrainedOps.def"
2486 return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
2487 MIRBuilder);
2488
2489 }
2490 return false;
2491}
2492
2493bool IRTranslator::translateInlineAsm(const CallBase &CB,
2494 MachineIRBuilder &MIRBuilder) {
2495
2496 const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
2497
2498 if (!ALI) {
2499 LLVM_DEBUG(
2500 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2501 return false;
2502 }
2503
2504 return ALI->lowerInlineAsm(
2505 MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
2506}
2507
2508bool IRTranslator::translateCallBase(const CallBase &CB,
2509 MachineIRBuilder &MIRBuilder) {
2510 ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
2511
2512 SmallVector<ArrayRef<Register>, 8> Args;
2513 Register SwiftInVReg = 0;
2514 Register SwiftErrorVReg = 0;
2515 for (const auto &Arg : CB.args()) {
2516 if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
2517 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2518 LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
2519 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2520 MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
2521 &CB, &MIRBuilder.getMBB(), Arg));
2522 Args.emplace_back(Args: ArrayRef(SwiftInVReg));
2523 SwiftErrorVReg =
2524 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2525 continue;
2526 }
2527 Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
2528 }
2529
2530 if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
2531 if (ORE->enabled()) {
2532 if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
2533 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2534 R.visit(I: CI);
2535 }
2536 }
2537 }
2538
2539 // We don't set HasCalls on MFI here yet because call lowering may decide to
2540 // optimize into tail calls. Instead, we defer that to selection where a final
2541 // scan is done to check if any instructions are calls.
2542 bool Success =
2543 CLI->lowerCall(MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg,
2544 GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
2545
2546 // Check if we just inserted a tail call.
2547 if (Success) {
2548 assert(!HasTailCall && "Can't tail call return twice from block?");
2549 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2550 HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
2551 }
2552
2553 return Success;
2554}
2555
2556bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2557 const CallInst &CI = cast<CallInst>(Val: U);
2558 auto TII = MF->getTarget().getIntrinsicInfo();
2559 const Function *F = CI.getCalledFunction();
2560
2561 // FIXME: support Windows dllimport function calls and calls through
2562 // weak symbols.
2563 if (F && (F->hasDLLImportStorageClass() ||
2564 (MF->getTarget().getTargetTriple().isOSWindows() &&
2565 F->hasExternalWeakLinkage())))
2566 return false;
2567
2568 // FIXME: support control flow guard targets.
2569 if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2570 return false;
2571
2572 // FIXME: support statepoints and related.
2573 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
2574 return false;
2575
2576 if (CI.isInlineAsm())
2577 return translateInlineAsm(CB: CI, MIRBuilder);
2578
2579 diagnoseDontCall(CI);
2580
2581 Intrinsic::ID ID = Intrinsic::not_intrinsic;
2582 if (F && F->isIntrinsic()) {
2583 ID = F->getIntrinsicID();
2584 if (TII && ID == Intrinsic::not_intrinsic)
2585 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2586 }
2587
2588 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2589 return translateCallBase(CB: CI, MIRBuilder);
2590
2591 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2592
2593 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2594 return true;
2595
2596 ArrayRef<Register> ResultRegs;
2597 if (!CI.getType()->isVoidTy())
2598 ResultRegs = getOrCreateVRegs(Val: CI);
2599
2600 // Ignore the callsite attributes. Backend code is most likely not expecting
2601 // an intrinsic to sometimes have side effects and sometimes not.
2602 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
2603 if (isa<FPMathOperator>(Val: CI))
2604 MIB->copyIRFlags(I: CI);
2605
2606 for (const auto &Arg : enumerate(First: CI.args())) {
2607 // If this is required to be an immediate, don't materialize it in a
2608 // register.
2609 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2610 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
2611 // imm arguments are more convenient than cimm (and realistically
2612 // probably sufficient), so use them.
2613 assert(CI->getBitWidth() <= 64 &&
2614 "large intrinsic immediates not handled");
2615 MIB.addImm(Val: CI->getSExtValue());
2616 } else {
2617 MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
2618 }
2619 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
2620 auto *MD = MDVal->getMetadata();
2621 auto *MDN = dyn_cast<MDNode>(Val: MD);
2622 if (!MDN) {
2623 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
2624 MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
2625 else // This was probably an MDString.
2626 return false;
2627 }
2628 MIB.addMetadata(MD: MDN);
2629 } else {
2630 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
2631 if (VRegs.size() > 1)
2632 return false;
2633 MIB.addUse(RegNo: VRegs[0]);
2634 }
2635 }
2636
2637 // Add a MachineMemOperand if it is a target mem intrinsic.
2638 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2639 TargetLowering::IntrinsicInfo Info;
2640 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2641 if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2642 Align Alignment = Info.align.value_or(
2643 u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: F->getContext())));
2644 LLT MemTy = Info.memVT.isSimple()
2645 ? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
2646 : LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
2647
2648 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2649 // didn't yield anything useful.
2650 MachinePointerInfo MPI;
2651 if (Info.ptrVal)
2652 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2653 else if (Info.fallbackAddressSpace)
2654 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2655 MIB.addMemOperand(
2656 MMO: MF->getMachineMemOperand(PtrInfo: MPI, f: Info.flags, MemTy, base_alignment: Alignment, AAInfo: CI.getAAMetadata()));
2657 }
2658
2659 return true;
2660}
2661
2662bool IRTranslator::findUnwindDestinations(
2663 const BasicBlock *EHPadBB,
2664 BranchProbability Prob,
2665 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2666 &UnwindDests) {
2667 EHPersonality Personality = classifyEHPersonality(
2668 Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
2669 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2670 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2671 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2672 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2673
2674 if (IsWasmCXX) {
2675 // Ignore this for now.
2676 return false;
2677 }
2678
2679 while (EHPadBB) {
2680 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2681 BasicBlock *NewEHPadBB = nullptr;
2682 if (isa<LandingPadInst>(Val: Pad)) {
2683 // Stop on landingpads. They are not funclets.
2684 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2685 break;
2686 }
2687 if (isa<CleanupPadInst>(Val: Pad)) {
2688 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2689 // personalities.
2690 UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2691 UnwindDests.back().first->setIsEHScopeEntry();
2692 UnwindDests.back().first->setIsEHFuncletEntry();
2693 break;
2694 }
2695 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val: Pad)) {
2696 // Add the catchpad handlers to the possible destinations.
2697 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2698 UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
2699 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2700 if (IsMSVCCXX || IsCoreCLR)
2701 UnwindDests.back().first->setIsEHFuncletEntry();
2702 if (!IsSEH)
2703 UnwindDests.back().first->setIsEHScopeEntry();
2704 }
2705 NewEHPadBB = CatchSwitch->getUnwindDest();
2706 } else {
2707 continue;
2708 }
2709
2710 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2711 if (BPI && NewEHPadBB)
2712 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2713 EHPadBB = NewEHPadBB;
2714 }
2715 return true;
2716}
2717
2718bool IRTranslator::translateInvoke(const User &U,
2719 MachineIRBuilder &MIRBuilder) {
2720 const InvokeInst &I = cast<InvokeInst>(Val: U);
2721 MCContext &Context = MF->getContext();
2722
2723 const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
2724 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
2725
2726 const Function *Fn = I.getCalledFunction();
2727
2728 // FIXME: support invoking patchpoint and statepoint intrinsics.
2729 if (Fn && Fn->isIntrinsic())
2730 return false;
2731
2732 // FIXME: support whatever these are.
2733 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_deopt))
2734 return false;
2735
2736 // FIXME: support control flow guard targets.
2737 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2738 return false;
2739
2740 // FIXME: support Windows exception handling.
2741 if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHI()))
2742 return false;
2743
2744 // FIXME: support Windows dllimport function calls and calls through
2745 // weak symbols.
2746 if (Fn && (Fn->hasDLLImportStorageClass() ||
2747 (MF->getTarget().getTargetTriple().isOSWindows() &&
2748 Fn->hasExternalWeakLinkage())))
2749 return false;
2750
2751 bool LowerInlineAsm = I.isInlineAsm();
2752 bool NeedEHLabel = true;
2753
2754 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2755 // the region covered by the try.
2756 MCSymbol *BeginSymbol = nullptr;
2757 if (NeedEHLabel) {
2758 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
2759 BeginSymbol = Context.createTempSymbol();
2760 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
2761 }
2762
2763 if (LowerInlineAsm) {
2764 if (!translateInlineAsm(CB: I, MIRBuilder))
2765 return false;
2766 } else if (!translateCallBase(CB: I, MIRBuilder))
2767 return false;
2768
2769 MCSymbol *EndSymbol = nullptr;
2770 if (NeedEHLabel) {
2771 EndSymbol = Context.createTempSymbol();
2772 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
2773 }
2774
2775 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2776 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2777 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2778 BranchProbability EHPadBBProb =
2779 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
2780 : BranchProbability::getZero();
2781
2782 if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
2783 return false;
2784
2785 MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
2786 &ReturnMBB = getMBB(BB: *ReturnBB);
2787 // Update successor info.
2788 addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
2789 for (auto &UnwindDest : UnwindDests) {
2790 UnwindDest.first->setIsEHPad();
2791 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2792 }
2793 InvokeMBB->normalizeSuccProbs();
2794
2795 if (NeedEHLabel) {
2796 assert(BeginSymbol && "Expected a begin symbol!");
2797 assert(EndSymbol && "Expected an end symbol!");
2798 MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
2799 }
2800
2801 MIRBuilder.buildBr(Dest&: ReturnMBB);
2802 return true;
2803}
2804
2805bool IRTranslator::translateCallBr(const User &U,
2806 MachineIRBuilder &MIRBuilder) {
2807 // FIXME: Implement this.
2808 return false;
2809}
2810
2811bool IRTranslator::translateLandingPad(const User &U,
2812 MachineIRBuilder &MIRBuilder) {
2813 const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
2814
2815 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2816
2817 MBB.setIsEHPad();
2818
2819 // If there aren't registers to copy the values into (e.g., during SjLj
2820 // exceptions), then don't bother.
2821 auto &TLI = *MF->getSubtarget().getTargetLowering();
2822 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2823 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2824 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2825 return true;
2826
2827 // If landingpad's return type is token type, we don't create DAG nodes
2828 // for its exception pointer and selector value. The extraction of exception
2829 // pointer or selector value from token type landingpads is not currently
2830 // supported.
2831 if (LP.getType()->isTokenTy())
2832 return true;
2833
2834 // Add a label to mark the beginning of the landing pad. Deletion of the
2835 // landing pad can thus be detected via the MachineModuleInfo.
2836 MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
2837 .addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
2838
2839 // If the unwinder does not preserve all registers, ensure that the
2840 // function marks the clobbered registers as used.
2841 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
2842 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
2843 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
2844
2845 LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
2846 Register Undef = MRI->createGenericVirtualRegister(Ty);
2847 MIRBuilder.buildUndef(Res: Undef);
2848
2849 SmallVector<LLT, 2> Tys;
2850 for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
2851 Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
2852 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2853
2854 // Mark exception register as live in.
2855 Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
2856 if (!ExceptionReg)
2857 return false;
2858
2859 MBB.addLiveIn(PhysReg: ExceptionReg);
2860 ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
2861 MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
2862
2863 Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
2864 if (!SelectorReg)
2865 return false;
2866
2867 MBB.addLiveIn(PhysReg: SelectorReg);
2868 Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
2869 MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
2870 MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
2871
2872 return true;
2873}
2874
2875bool IRTranslator::translateAlloca(const User &U,
2876 MachineIRBuilder &MIRBuilder) {
2877 auto &AI = cast<AllocaInst>(Val: U);
2878
2879 if (AI.isSwiftError())
2880 return true;
2881
2882 if (AI.isStaticAlloca()) {
2883 Register Res = getOrCreateVReg(Val: AI);
2884 int FI = getOrCreateFrameIndex(AI);
2885 MIRBuilder.buildFrameIndex(Res, Idx: FI);
2886 return true;
2887 }
2888
2889 // FIXME: support stack probing for Windows.
2890 if (MF->getTarget().getTargetTriple().isOSWindows())
2891 return false;
2892
2893 // Now we're in the harder dynamic case.
2894 Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
2895 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2896 LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
2897 if (MRI->getType(Reg: NumElts) != IntPtrTy) {
2898 Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
2899 MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
2900 NumElts = ExtElts;
2901 }
2902
2903 Type *Ty = AI.getAllocatedType();
2904
2905 Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
2906 Register TySize =
2907 getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: DL->getTypeAllocSize(Ty)));
2908 MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySize);
2909
2910 // Round the size of the allocation up to the stack alignment size
2911 // by add SA-1 to the size. This doesn't overflow because we're computing
2912 // an address inside an alloca.
2913 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
2914 auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
2915 auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
2916 Flags: MachineInstr::NoUWrap);
2917 auto AlignCst =
2918 MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
2919 auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
2920
2921 Align Alignment = std::max(a: AI.getAlign(), b: DL->getPrefTypeAlign(Ty));
2922 if (Alignment <= StackAlign)
2923 Alignment = Align(1);
2924 MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
2925
2926 MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
2927 assert(MF->getFrameInfo().hasVarSizedObjects());
2928 return true;
2929}
2930
2931bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2932 // FIXME: We may need more info about the type. Because of how LLT works,
2933 // we're completely discarding the i64/double distinction here (amongst
2934 // others). Fortunately the ABIs I know of where that matters don't use va_arg
2935 // anyway but that's not guaranteed.
2936 MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
2937 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
2938 DL->getABITypeAlign(Ty: U.getType()).value()});
2939 return true;
2940}
2941
2942bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
2943 if (!MF->getTarget().Options.TrapUnreachable)
2944 return true;
2945
2946 auto &UI = cast<UnreachableInst>(Val: U);
2947 // We may be able to ignore unreachable behind a noreturn call.
2948 if (MF->getTarget().Options.NoTrapAfterNoreturn) {
2949 const BasicBlock &BB = *UI.getParent();
2950 if (&UI != &BB.front()) {
2951 BasicBlock::const_iterator PredI =
2952 std::prev(x: BasicBlock::const_iterator(UI));
2953 if (const CallInst *Call = dyn_cast<CallInst>(Val: &*PredI)) {
2954 if (Call->doesNotReturn())
2955 return true;
2956 }
2957 }
2958 }
2959
2960 MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
2961 return true;
2962}
2963
2964bool IRTranslator::translateInsertElement(const User &U,
2965 MachineIRBuilder &MIRBuilder) {
2966 // If it is a <1 x Ty> vector, use the scalar as it is
2967 // not a legal vector type in LLT.
2968 if (cast<FixedVectorType>(Val: U.getType())->getNumElements() == 1)
2969 return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
2970
2971 Register Res = getOrCreateVReg(Val: U);
2972 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
2973 Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
2974 Register Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
2975 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
2976 return true;
2977}
2978
2979bool IRTranslator::translateExtractElement(const User &U,
2980 MachineIRBuilder &MIRBuilder) {
2981 // If it is a <1 x Ty> vector, use the scalar as it is
2982 // not a legal vector type in LLT.
2983 if (cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements() == 1)
2984 return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
2985
2986 Register Res = getOrCreateVReg(Val: U);
2987 Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
2988 const auto &TLI = *MF->getSubtarget().getTargetLowering();
2989 unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(DL: *DL).getSizeInBits();
2990 Register Idx;
2991 if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
2992 if (CI->getBitWidth() != PreferredVecIdxWidth) {
2993 APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
2994 auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
2995 Idx = getOrCreateVReg(Val: *NewIdxCI);
2996 }
2997 }
2998 if (!Idx)
2999 Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
3000 if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3001 const LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3002 Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3003 }
3004 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3005 return true;
3006}
3007
3008bool IRTranslator::translateShuffleVector(const User &U,
3009 MachineIRBuilder &MIRBuilder) {
3010 ArrayRef<int> Mask;
3011 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
3012 Mask = SVI->getShuffleMask();
3013 else
3014 Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
3015 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3016 MIRBuilder
3017 .buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
3018 SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3019 getOrCreateVReg(Val: *U.getOperand(i: 1))})
3020 .addShuffleMask(Val: MaskAlloc);
3021 return true;
3022}
3023
3024bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3025 const PHINode &PI = cast<PHINode>(Val: U);
3026
3027 SmallVector<MachineInstr *, 4> Insts;
3028 for (auto Reg : getOrCreateVRegs(Val: PI)) {
3029 auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
3030 Insts.push_back(Elt: MIB.getInstr());
3031 }
3032
3033 PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
3034 return true;
3035}
3036
3037bool IRTranslator::translateAtomicCmpXchg(const User &U,
3038 MachineIRBuilder &MIRBuilder) {
3039 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
3040
3041 auto &TLI = *MF->getSubtarget().getTargetLowering();
3042 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: *DL);
3043
3044 auto Res = getOrCreateVRegs(Val: I);
3045 Register OldValRes = Res[0];
3046 Register SuccessRes = Res[1];
3047 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3048 Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
3049 Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
3050
3051 MIRBuilder.buildAtomicCmpXchgWithSuccess(
3052 OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
3053 MMO&: *MF->getMachineMemOperand(
3054 PtrInfo: MachinePointerInfo(I.getPointerOperand()), f: Flags, MemTy: MRI->getType(Reg: Cmp),
3055 base_alignment: getMemOpAlign(I), AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3056 Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
3057 return true;
3058}
3059
3060bool IRTranslator::translateAtomicRMW(const User &U,
3061 MachineIRBuilder &MIRBuilder) {
3062 const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
3063 auto &TLI = *MF->getSubtarget().getTargetLowering();
3064 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: *DL);
3065
3066 Register Res = getOrCreateVReg(Val: I);
3067 Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3068 Register Val = getOrCreateVReg(Val: *I.getValOperand());
3069
3070 unsigned Opcode = 0;
3071 switch (I.getOperation()) {
3072 default:
3073 return false;
3074 case AtomicRMWInst::Xchg:
3075 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3076 break;
3077 case AtomicRMWInst::Add:
3078 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3079 break;
3080 case AtomicRMWInst::Sub:
3081 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3082 break;
3083 case AtomicRMWInst::And:
3084 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3085 break;
3086 case AtomicRMWInst::Nand:
3087 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3088 break;
3089 case AtomicRMWInst::Or:
3090 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3091 break;
3092 case AtomicRMWInst::Xor:
3093 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3094 break;
3095 case AtomicRMWInst::Max:
3096 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3097 break;
3098 case AtomicRMWInst::Min:
3099 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3100 break;
3101 case AtomicRMWInst::UMax:
3102 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3103 break;
3104 case AtomicRMWInst::UMin:
3105 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3106 break;
3107 case AtomicRMWInst::FAdd:
3108 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3109 break;
3110 case AtomicRMWInst::FSub:
3111 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3112 break;
3113 case AtomicRMWInst::FMax:
3114 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3115 break;
3116 case AtomicRMWInst::FMin:
3117 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3118 break;
3119 case AtomicRMWInst::UIncWrap:
3120 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3121 break;
3122 case AtomicRMWInst::UDecWrap:
3123 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3124 break;
3125 }
3126
3127 MIRBuilder.buildAtomicRMW(
3128 Opcode, OldValRes: Res, Addr, Val,
3129 MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
3130 f: Flags, MemTy: MRI->getType(Reg: Val), base_alignment: getMemOpAlign(I),
3131 AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3132 Ordering: I.getOrdering()));
3133 return true;
3134}
3135
3136bool IRTranslator::translateFence(const User &U,
3137 MachineIRBuilder &MIRBuilder) {
3138 const FenceInst &Fence = cast<FenceInst>(Val: U);
3139 MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
3140 Scope: Fence.getSyncScopeID());
3141 return true;
3142}
3143
3144bool IRTranslator::translateFreeze(const User &U,
3145 MachineIRBuilder &MIRBuilder) {
3146 const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
3147 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
3148
3149 assert(DstRegs.size() == SrcRegs.size() &&
3150 "Freeze with different source and destination type?");
3151
3152 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3153 MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
3154 }
3155
3156 return true;
3157}
3158
3159void IRTranslator::finishPendingPhis() {
3160#ifndef NDEBUG
3161 DILocationVerifier Verifier;
3162 GISelObserverWrapper WrapperObserver(&Verifier);
3163 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3164#endif // ifndef NDEBUG
3165 for (auto &Phi : PendingPHIs) {
3166 const PHINode *PI = Phi.first;
3167 if (PI->getType()->isEmptyTy())
3168 continue;
3169 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3170 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3171 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3172#ifndef NDEBUG
3173 Verifier.setCurrentInst(PI);
3174#endif // ifndef NDEBUG
3175
3176 SmallSet<const MachineBasicBlock *, 16> SeenPreds;
3177 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3178 auto IRPred = PI->getIncomingBlock(i);
3179 ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
3180 for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
3181 if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
3182 continue;
3183 SeenPreds.insert(Ptr: Pred);
3184 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3185 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3186 MIB.addUse(RegNo: ValRegs[j]);
3187 MIB.addMBB(MBB: Pred);
3188 }
3189 }
3190 }
3191 }
3192}
3193
3194void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3195 const DILocalVariable *Variable,
3196 const DIExpression *Expression,
3197 const DebugLoc &DL,
3198 MachineIRBuilder &MIRBuilder) {
3199 assert(Variable->isValidLocationForIntrinsic(DL) &&
3200 "Expected inlined-at fields to agree");
3201 // Act as if we're handling a debug intrinsic.
3202 MIRBuilder.setDebugLoc(DL);
3203
3204 if (!V || HasArgList) {
3205 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3206 // terminate any prior location.
3207 MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
3208 return;
3209 }
3210
3211 if (const auto *CI = dyn_cast<Constant>(Val: V)) {
3212 MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
3213 return;
3214 }
3215
3216 if (auto *AI = dyn_cast<AllocaInst>(Val: V);
3217 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3218 // If the value is an alloca and the expression starts with a
3219 // dereference, track a stack slot instead of a register, as registers
3220 // may be clobbered.
3221 auto ExprOperands = Expression->getElements();
3222 auto *ExprDerefRemoved =
3223 DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
3224 MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
3225 Expr: ExprDerefRemoved);
3226 return;
3227 }
3228 if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
3229 MIRBuilder))
3230 return;
3231 for (Register Reg : getOrCreateVRegs(Val: *V)) {
3232 // FIXME: This does not handle register-indirect values at offset 0. The
3233 // direct/indirect thing shouldn't really be handled by something as
3234 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3235 // pretty baked in right now.
3236 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
3237 }
3238 return;
3239}
3240
3241void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3242 const DILocalVariable *Variable,
3243 const DIExpression *Expression,
3244 const DebugLoc &DL,
3245 MachineIRBuilder &MIRBuilder) {
3246 if (!Address || isa<UndefValue>(Val: Address)) {
3247 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3248 return;
3249 }
3250
3251 assert(Variable->isValidLocationForIntrinsic(DL) &&
3252 "Expected inlined-at fields to agree");
3253 auto AI = dyn_cast<AllocaInst>(Val: Address);
3254 if (AI && AI->isStaticAlloca()) {
3255 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3256 // instructions (in fact, they get ignored if they *do* exist).
3257 MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
3258 Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
3259 return;
3260 }
3261
3262 if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
3263 Expr: Expression, DL,
3264 MIRBuilder))
3265 return;
3266
3267 // A dbg.declare describes the address of a source variable, so lower it
3268 // into an indirect DBG_VALUE.
3269 MIRBuilder.setDebugLoc(DL);
3270 MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address),
3271 Variable, Expr: Expression);
3272 return;
3273}
3274
3275void IRTranslator::translateDbgInfo(const Instruction &Inst,
3276 MachineIRBuilder &MIRBuilder) {
3277 for (DPValue &DPV : Inst.getDbgValueRange()) {
3278 const DILocalVariable *Variable = DPV.getVariable();
3279 const DIExpression *Expression = DPV.getExpression();
3280 Value *V = DPV.getVariableLocationOp(OpIdx: 0);
3281 if (DPV.isDbgDeclare())
3282 translateDbgDeclareRecord(Address: V, HasArgList: DPV.hasArgList(), Variable,
3283 Expression, DL: DPV.getDebugLoc(), MIRBuilder);
3284 else
3285 translateDbgValueRecord(V, HasArgList: DPV.hasArgList(), Variable,
3286 Expression, DL: DPV.getDebugLoc(), MIRBuilder);
3287 }
3288}
3289
3290bool IRTranslator::translate(const Instruction &Inst) {
3291 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3292 CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
3293
3294 auto &TLI = *MF->getSubtarget().getTargetLowering();
3295 if (TLI.fallBackToDAGISel(Inst))
3296 return false;
3297
3298 switch (Inst.getOpcode()) {
3299#define HANDLE_INST(NUM, OPCODE, CLASS) \
3300 case Instruction::OPCODE: \
3301 return translate##OPCODE(Inst, *CurBuilder.get());
3302#include "llvm/IR/Instruction.def"
3303 default:
3304 return false;
3305 }
3306}
3307
3308bool IRTranslator::translate(const Constant &C, Register Reg) {
3309 // We only emit constants into the entry block from here. To prevent jumpy
3310 // debug behaviour remove debug line.
3311 if (auto CurrInstDL = CurBuilder->getDL())
3312 EntryBuilder->setDebugLoc(DebugLoc());
3313
3314 if (auto CI = dyn_cast<ConstantInt>(Val: &C))
3315 EntryBuilder->buildConstant(Res: Reg, Val: *CI);
3316 else if (auto CF = dyn_cast<ConstantFP>(Val: &C))
3317 EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
3318 else if (isa<UndefValue>(Val: C))
3319 EntryBuilder->buildUndef(Res: Reg);
3320 else if (isa<ConstantPointerNull>(Val: C))
3321 EntryBuilder->buildConstant(Res: Reg, Val: 0);
3322 else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
3323 EntryBuilder->buildGlobalValue(Res: Reg, GV);
3324 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
3325 if (!isa<FixedVectorType>(Val: CAZ->getType()))
3326 return false;
3327 // Return the scalar if it is a <1 x Ty> vector.
3328 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3329 if (NumElts == 1)
3330 return translateCopy(U: C, V: *CAZ->getElementValue(Idx: 0u), MIRBuilder&: *EntryBuilder);
3331 SmallVector<Register, 4> Ops;
3332 for (unsigned I = 0; I < NumElts; ++I) {
3333 Constant &Elt = *CAZ->getElementValue(Idx: I);
3334 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3335 }
3336 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3337 } else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
3338 // Return the scalar if it is a <1 x Ty> vector.
3339 if (CV->getNumElements() == 1)
3340 return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
3341 SmallVector<Register, 4> Ops;
3342 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3343 Constant &Elt = *CV->getElementAsConstant(i);
3344 Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3345 }
3346 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3347 } else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
3348 switch(CE->getOpcode()) {
3349#define HANDLE_INST(NUM, OPCODE, CLASS) \
3350 case Instruction::OPCODE: \
3351 return translate##OPCODE(*CE, *EntryBuilder.get());
3352#include "llvm/IR/Instruction.def"
3353 default:
3354 return false;
3355 }
3356 } else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
3357 if (CV->getNumOperands() == 1)
3358 return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
3359 SmallVector<Register, 4> Ops;
3360 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3361 Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
3362 }
3363 EntryBuilder->buildBuildVector(Res: Reg, Ops);
3364 } else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
3365 EntryBuilder->buildBlockAddress(Res: Reg, BA);
3366 } else
3367 return false;
3368
3369 return true;
3370}
3371
3372bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3373 MachineBasicBlock &MBB) {
3374 for (auto &BTB : SL->BitTestCases) {
3375 // Emit header first, if it wasn't already emitted.
3376 if (!BTB.Emitted)
3377 emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
3378
3379 BranchProbability UnhandledProb = BTB.Prob;
3380 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3381 UnhandledProb -= BTB.Cases[j].ExtraProb;
3382 // Set the current basic block to the mbb we wish to insert the code into
3383 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3384 // If all cases cover a contiguous range, it is not necessary to jump to
3385 // the default block after the last bit test fails. This is because the
3386 // range check during bit test header creation has guaranteed that every
3387 // case here doesn't go outside the range. In this case, there is no need
3388 // to perform the last bit test, as it will always be true. Instead, make
3389 // the second-to-last bit-test fall through to the target of the last bit
3390 // test, and delete the last bit test.
3391
3392 MachineBasicBlock *NextMBB;
3393 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3394 // Second-to-last bit-test with contiguous range: fall through to the
3395 // target of the final bit test.
3396 NextMBB = BTB.Cases[j + 1].TargetBB;
3397 } else if (j + 1 == ej) {
3398 // For the last bit test, fall through to Default.
3399 NextMBB = BTB.Default;
3400 } else {
3401 // Otherwise, fall through to the next bit test.
3402 NextMBB = BTB.Cases[j + 1].ThisBB;
3403 }
3404
3405 emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
3406
3407 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3408 // We need to record the replacement phi edge here that normally
3409 // happens in emitBitTestCase before we delete the case, otherwise the
3410 // phi edge will be lost.
3411 addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
3412 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3413 NewPred: MBB);
3414 // Since we're not going to use the final bit test, remove it.
3415 BTB.Cases.pop_back();
3416 break;
3417 }
3418 }
3419 // This is "default" BB. We have two jumps to it. From "header" BB and from
3420 // last "case" BB, unless the latter was skipped.
3421 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3422 BTB.Default->getBasicBlock()};
3423 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
3424 if (!BTB.ContiguousRange) {
3425 addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
3426 }
3427 }
3428 SL->BitTestCases.clear();
3429
3430 for (auto &JTCase : SL->JTCases) {
3431 // Emit header first, if it wasn't already emitted.
3432 if (!JTCase.first.Emitted)
3433 emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
3434
3435 emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
3436 }
3437 SL->JTCases.clear();
3438
3439 for (auto &SwCase : SL->SwitchCases)
3440 emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
3441 SL->SwitchCases.clear();
3442
3443 // Check if we need to generate stack-protector guard checks.
3444 StackProtector &SP = getAnalysis<StackProtector>();
3445 if (SP.shouldEmitSDCheck(BB)) {
3446 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3447 bool FunctionBasedInstrumentation =
3448 TLI.getSSPStackGuardCheck(M: *MF->getFunction().getParent());
3449 SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
3450 }
3451 // Handle stack protector.
3452 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3453 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3454 return false;
3455 } else if (SPDescriptor.shouldEmitStackProtector()) {
3456 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3457 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3458
3459 // Find the split point to split the parent mbb. At the same time copy all
3460 // physical registers used in the tail of parent mbb into virtual registers
3461 // before the split point and back into physical registers after the split
3462 // point. This prevents us needing to deal with Live-ins and many other
3463 // register allocation issues caused by us splitting the parent mbb. The
3464 // register allocator will clean up said virtual copies later on.
3465 MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
3466 BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
3467
3468 // Splice the terminator of ParentMBB into SuccessMBB.
3469 SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
3470 To: ParentMBB->end());
3471
3472 // Add compare/jump on neq/jump to the parent BB.
3473 if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
3474 return false;
3475
3476 // CodeGen Failure MBB if we have not codegened it yet.
3477 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3478 if (FailureMBB->empty()) {
3479 if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
3480 return false;
3481 }
3482
3483 // Clear the Per-BB State.
3484 SPDescriptor.resetPerBBState();
3485 }
3486 return true;
3487}
3488
3489bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3490 MachineBasicBlock *ParentBB) {
3491 CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
3492 // First create the loads to the guard/stack slot for the comparison.
3493 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3494 Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
3495 const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
3496 LLT PtrMemTy = getLLTForMVT(Ty: TLI.getPointerMemTy(DL: *DL));
3497
3498 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3499 int FI = MFI.getStackProtectorIndex();
3500
3501 Register Guard;
3502 Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
3503 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3504 Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
3505
3506 // Generate code to load the content of the guard slot.
3507 Register GuardVal =
3508 CurBuilder
3509 ->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
3510 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
3511 MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
3512 .getReg(Idx: 0);
3513
3514 if (TLI.useStackGuardXorFP()) {
3515 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3516 return false;
3517 }
3518
3519 // Retrieve guard check function, nullptr if instrumentation is inlined.
3520 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3521 // This path is currently untestable on GlobalISel, since the only platform
3522 // that needs this seems to be Windows, and we fall back on that currently.
3523 // The code still lives here in case that changes.
3524 // Silence warning about unused variable until the code below that uses
3525 // 'GuardCheckFn' is enabled.
3526 (void)GuardCheckFn;
3527 return false;
3528#if 0
3529 // The target provides a guard check function to validate the guard value.
3530 // Generate a call to that function with the content of the guard slot as
3531 // argument.
3532 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3533 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3534 ISD::ArgFlagsTy Flags;
3535 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3536 Flags.setInReg();
3537 CallLowering::ArgInfo GuardArgInfo(
3538 {GuardVal, FnTy->getParamType(0), {Flags}});
3539
3540 CallLowering::CallLoweringInfo Info;
3541 Info.OrigArgs.push_back(GuardArgInfo);
3542 Info.CallConv = GuardCheckFn->getCallingConv();
3543 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3544 Info.OrigRet = {Register(), FnTy->getReturnType()};
3545 if (!CLI->lowerCall(MIRBuilder, Info)) {
3546 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3547 return false;
3548 }
3549 return true;
3550#endif
3551 }
3552
3553 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3554 // Otherwise, emit a volatile load to retrieve the stack guard value.
3555 if (TLI.useLoadStackGuardNode()) {
3556 Guard =
3557 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PtrTy.getSizeInBits()));
3558 getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
3559 } else {
3560 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3561 const Value *IRGuard = TLI.getSDagStackGuard(M);
3562 Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
3563
3564 Guard = CurBuilder
3565 ->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
3566 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
3567 MMOFlags: MachineMemOperand::MOLoad |
3568 MachineMemOperand::MOVolatile)
3569 .getReg(Idx: 0);
3570 }
3571
3572 // Perform the comparison.
3573 auto Cmp =
3574 CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
3575 // If the guard/stackslot do not equal, branch to failure MBB.
3576 CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
3577 // Otherwise branch to success MBB.
3578 CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
3579 return true;
3580}
3581
3582bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3583 MachineBasicBlock *FailureBB) {
3584 CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
3585 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3586
3587 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3588 const char *Name = TLI.getLibcallName(Call: Libcall);
3589
3590 CallLowering::CallLoweringInfo Info;
3591 Info.CallConv = TLI.getLibcallCallingConv(Call: Libcall);
3592 Info.Callee = MachineOperand::CreateES(SymName: Name);
3593 Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
3594 0};
3595 if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
3596 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3597 return false;
3598 }
3599
3600 // On PS4/PS5, the "return address" must still be within the calling
3601 // function, even if it's at the very end, so emit an explicit TRAP here.
3602 // WebAssembly needs an unreachable instruction after a non-returning call,
3603 // because the function return type can be different from __stack_chk_fail's
3604 // return type (void).
3605 const TargetMachine &TM = MF->getTarget();
3606 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3607 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3608 return false;
3609 }
3610 return true;
3611}
3612
3613void IRTranslator::finalizeFunction() {
3614 // Release the memory used by the different maps we
3615 // needed during the translation.
3616 PendingPHIs.clear();
3617 VMap.reset();
3618 FrameIndices.clear();
3619 MachinePreds.clear();
3620 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3621 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3622 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3623 EntryBuilder.reset();
3624 CurBuilder.reset();
3625 FuncInfo.clear();
3626 SPDescriptor.resetPerFunctionState();
3627}
3628
3629/// Returns true if a BasicBlock \p BB within a variadic function contains a
3630/// variadic musttail call.
3631static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3632 if (!IsVarArg)
3633 return false;
3634
3635 // Walk the block backwards, because tail calls usually only appear at the end
3636 // of a block.
3637 return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
3638 const auto *CI = dyn_cast<CallInst>(Val: &I);
3639 return CI && CI->isMustTailCall();
3640 });
3641}
3642
3643bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
3644 MF = &CurMF;
3645 const Function &F = MF->getFunction();
3646 GISelCSEAnalysisWrapper &Wrapper =
3647 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3648 // Set the CSEConfig and run the analysis.
3649 GISelCSEInfo *CSEInfo = nullptr;
3650 TPC = &getAnalysis<TargetPassConfig>();
3651 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3652 ? EnableCSEInIRTranslator
3653 : TPC->isGISelCSEEnabled();
3654
3655 if (EnableCSE) {
3656 EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
3657 CSEInfo = &Wrapper.get(CSEOpt: TPC->getCSEConfig());
3658 EntryBuilder->setCSEInfo(CSEInfo);
3659 CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
3660 CurBuilder->setCSEInfo(CSEInfo);
3661 } else {
3662 EntryBuilder = std::make_unique<MachineIRBuilder>();
3663 CurBuilder = std::make_unique<MachineIRBuilder>();
3664 }
3665 CLI = MF->getSubtarget().getCallLowering();
3666 CurBuilder->setMF(*MF);
3667 EntryBuilder->setMF(*MF);
3668 MRI = &MF->getRegInfo();
3669 DL = &F.getParent()->getDataLayout();
3670 ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
3671 const TargetMachine &TM = MF->getTarget();
3672 TM.resetTargetOptions(F);
3673 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3674 FuncInfo.MF = MF;
3675 if (EnableOpts) {
3676 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3677 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3678 } else {
3679 AA = nullptr;
3680 FuncInfo.BPI = nullptr;
3681 }
3682
3683 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3684 F&: MF->getFunction());
3685 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3686 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
3687
3688 const auto &TLI = *MF->getSubtarget().getTargetLowering();
3689
3690 SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
3691 SL->init(tli: TLI, tm: TM, dl: *DL);
3692
3693
3694
3695 assert(PendingPHIs.empty() && "stale PHIs");
3696
3697 // Targets which want to use big endian can enable it using
3698 // enableBigEndian()
3699 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3700 // Currently we don't properly handle big endian code.
3701 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3702 F.getSubprogram(), &F.getEntryBlock());
3703 R << "unable to translate in big endian mode";
3704 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3705 }
3706
3707 // Release the per-function state when we return, whether we succeeded or not.
3708 auto FinalizeOnReturn = make_scope_exit(F: [this]() { finalizeFunction(); });
3709
3710 // Setup a separate basic-block for the arguments and constants
3711 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
3712 MF->push_back(MBB: EntryBB);
3713 EntryBuilder->setMBB(*EntryBB);
3714
3715 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3716 SwiftError.setFunction(CurMF);
3717 SwiftError.createEntriesInEntryBlock(DbgLoc);
3718
3719 bool IsVarArg = F.isVarArg();
3720 bool HasMustTailInVarArgFn = false;
3721
3722 // Create all blocks, in IR order, to preserve the layout.
3723 for (const BasicBlock &BB: F) {
3724 auto *&MBB = BBToMBB[&BB];
3725
3726 MBB = MF->CreateMachineBasicBlock(BB: &BB);
3727 MF->push_back(MBB);
3728
3729 if (BB.hasAddressTaken())
3730 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3731
3732 if (!HasMustTailInVarArgFn)
3733 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3734 }
3735
3736 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3737
3738 // Make our arguments/constants entry block fallthrough to the IR entry block.
3739 EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
3740
3741 if (CLI->fallBackToDAGISel(MF: *MF)) {
3742 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3743 F.getSubprogram(), &F.getEntryBlock());
3744 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3745 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3746 return false;
3747 }
3748
3749 // Lower the actual args into this basic block.
3750 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3751 for (const Argument &Arg: F.args()) {
3752 if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
3753 continue; // Don't handle zero sized types.
3754 ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
3755 VRegArgs.push_back(Elt: VRegs);
3756
3757 if (Arg.hasSwiftErrorAttr()) {
3758 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3759 SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3760 }
3761 }
3762
3763 if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
3764 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3765 F.getSubprogram(), &F.getEntryBlock());
3766 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3767 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3768 return false;
3769 }
3770
3771 // Need to visit defs before uses when translating instructions.
3772 GISelObserverWrapper WrapperObserver;
3773 if (EnableCSE && CSEInfo)
3774 WrapperObserver.addObserver(O: CSEInfo);
3775 {
3776 ReversePostOrderTraversal<const Function *> RPOT(&F);
3777#ifndef NDEBUG
3778 DILocationVerifier Verifier;
3779 WrapperObserver.addObserver(O: &Verifier);
3780#endif // ifndef NDEBUG
3781 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3782 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3783 for (const BasicBlock *BB : RPOT) {
3784 MachineBasicBlock &MBB = getMBB(BB: *BB);
3785 // Set the insertion point of all the following translations to
3786 // the end of this basic block.
3787 CurBuilder->setMBB(MBB);
3788 HasTailCall = false;
3789 for (const Instruction &Inst : *BB) {
3790 // If we translated a tail call in the last step, then we know
3791 // everything after the call is either a return, or something that is
3792 // handled by the call itself. (E.g. a lifetime marker or assume
3793 // intrinsic.) In this case, we should stop translating the block and
3794 // move on.
3795 if (HasTailCall)
3796 break;
3797#ifndef NDEBUG
3798 Verifier.setCurrentInst(&Inst);
3799#endif // ifndef NDEBUG
3800
3801 // Translate any debug-info attached to the instruction.
3802 translateDbgInfo(Inst, MIRBuilder&: *CurBuilder.get());
3803
3804 if (translate(Inst))
3805 continue;
3806
3807 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3808 Inst.getDebugLoc(), BB);
3809 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3810
3811 if (ORE->allowExtraAnalysis(PassName: "gisel-irtranslator")) {
3812 std::string InstStrStorage;
3813 raw_string_ostream InstStr(InstStrStorage);
3814 InstStr << Inst;
3815
3816 R << ": '" << InstStr.str() << "'";
3817 }
3818
3819 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3820 return false;
3821 }
3822
3823 if (!finalizeBasicBlock(BB: *BB, MBB)) {
3824 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3825 BB->getTerminator()->getDebugLoc(), BB);
3826 R << "unable to translate basic block";
3827 reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3828 return false;
3829 }
3830 }
3831#ifndef NDEBUG
3832 WrapperObserver.removeObserver(O: &Verifier);
3833#endif
3834 }
3835
3836 finishPendingPhis();
3837
3838 SwiftError.propagateVRegs();
3839
3840 // Merge the argument lowering and constants block with its single
3841 // successor, the LLVM-IR entry block. We want the basic block to
3842 // be maximal.
3843 assert(EntryBB->succ_size() == 1 &&
3844 "Custom BB used for lowering should have only one successor");
3845 // Get the successor of the current entry block.
3846 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3847 assert(NewEntryBB.pred_size() == 1 &&
3848 "LLVM-IR entry block has a predecessor!?");
3849 // Move all the instruction from the current entry block to the
3850 // new entry block.
3851 NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
3852 To: EntryBB->end());
3853
3854 // Update the live-in information for the new entry block.
3855 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3856 NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
3857 NewEntryBB.sortUniqueLiveIns();
3858
3859 // Get rid of the now empty basic block.
3860 EntryBB->removeSuccessor(Succ: &NewEntryBB);
3861 MF->remove(MBBI: EntryBB);
3862 MF->deleteMachineBasicBlock(MBB: EntryBB);
3863
3864 assert(&MF->front() == &NewEntryBB &&
3865 "New entry wasn't next in the list of basic block!");
3866
3867 // Initialize stack protector information.
3868 StackProtector &SP = getAnalysis<StackProtector>();
3869 SP.copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
3870
3871 return false;
3872}
3873

source code of llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp