1//===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements GIMatchTableExecutor's `executeMatchTable`
10/// function. This is implemented in a separate file because the function is
11/// quite large.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
16#define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
17
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
20#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineOperand.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/RegisterBankInfo.h"
27#include "llvm/CodeGen/TargetInstrInfo.h"
28#include "llvm/CodeGen/TargetOpcodes.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/Type.h"
33#include "llvm/Support/CodeGenCoverage.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/LEB128.h"
37#include "llvm/Support/raw_ostream.h"
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41
42namespace llvm {
43
44template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
45 class CustomRendererFn>
46bool GIMatchTableExecutor::executeMatchTable(
47 TgtExecutor &Exec, MatcherState &State,
48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
49 &ExecInfo,
50 MachineIRBuilder &Builder, const uint8_t *MatchTable,
51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
53 const PredicateBitset &AvailableFeatures,
54 CodeGenCoverage *CoverageInfo) const {
55
56 uint64_t CurrentIdx = 0;
57 SmallVector<uint64_t, 4> OnFailResumeAt;
58 NewMIVector OutMIs;
59
60 GISelChangeObserver *Observer = Builder.getObserver();
61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63
64 const uint16_t Flags = State.MIs[0]->getFlags();
65
66 enum RejectAction { RejectAndGiveUp, RejectAndResume };
67 auto handleReject = [&]() -> RejectAction {
68 DEBUG_WITH_TYPE(TgtExecutor::getName(),
69 dbgs() << CurrentIdx << ": Rejected\n");
70 if (OnFailResumeAt.empty())
71 return RejectAndGiveUp;
72 CurrentIdx = OnFailResumeAt.pop_back_val();
73 DEBUG_WITH_TYPE(TgtExecutor::getName(),
74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75 << OnFailResumeAt.size() << " try-blocks remain)\n");
76 return RejectAndResume;
77 };
78
79 const auto propagateFlags = [&]() {
80 for (auto MIB : OutMIs) {
81 // Set the NoFPExcept flag when no original matched instruction could
82 // raise an FP exception, but the new instruction potentially might.
83 uint16_t MIBFlags = Flags;
84 if (NoFPException && MIB->mayRaiseFPException())
85 MIBFlags |= MachineInstr::NoFPExcept;
86 if (Observer)
87 Observer->changingInstr(MI&: *MIB);
88 MIB.setMIFlags(MIBFlags);
89 if (Observer)
90 Observer->changedInstr(MI&: *MIB);
91 }
92 };
93
94 // If the index is >= 0, it's an index in the type objects generated by
95 // TableGen. If the index is <0, it's an index in the recorded types object.
96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
97 if (Idx >= 0)
98 return ExecInfo.TypeObjects[Idx];
99 return State.RecordedTypes[1 - Idx];
100 };
101
102 const auto readULEB = [&]() {
103 unsigned N = 0;
104 uint64_t Val = decodeULEB128(p: MatchTable + CurrentIdx, n: &N);
105 CurrentIdx += N;
106 return Val;
107 };
108
109 // Convenience function to return a signed value. This avoids
110 // us forgetting to first cast to int8_t before casting to a
111 // wider signed int type.
112 // if we casted uint8 directly to a wider type we'd lose
113 // negative values.
114 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
115
116 const auto readU16 = [&]() {
117 auto V = readBytesAs<uint16_t>(MatchTable: MatchTable + CurrentIdx);
118 CurrentIdx += 2;
119 return V;
120 };
121
122 const auto readU32 = [&]() {
123 auto V = readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx);
124 CurrentIdx += 4;
125 return V;
126 };
127
128 const auto readU64 = [&]() {
129 auto V = readBytesAs<uint64_t>(MatchTable: MatchTable + CurrentIdx);
130 CurrentIdx += 8;
131 return V;
132 };
133
134 while (true) {
135 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
136 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
137 switch (MatcherOpcode) {
138 case GIM_Try: {
139 DEBUG_WITH_TYPE(TgtExecutor::getName(),
140 dbgs() << CurrentIdx << ": Begin try-block\n");
141 OnFailResumeAt.push_back(Elt: readU32());
142 break;
143 }
144
145 case GIM_RecordInsn:
146 case GIM_RecordInsnIgnoreCopies: {
147 uint64_t NewInsnID = readULEB();
148 uint64_t InsnID = readULEB();
149 uint64_t OpIdx = readULEB();
150
151 // As an optimisation we require that MIs[0] is always the root. Refuse
152 // any attempt to modify it.
153 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
154
155 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
156 if (!MO.isReg()) {
157 DEBUG_WITH_TYPE(TgtExecutor::getName(),
158 dbgs() << CurrentIdx << ": Not a register\n");
159 if (handleReject() == RejectAndGiveUp)
160 return false;
161 break;
162 }
163 if (MO.getReg().isPhysical()) {
164 DEBUG_WITH_TYPE(TgtExecutor::getName(),
165 dbgs() << CurrentIdx << ": Is a physical register\n");
166 if (handleReject() == RejectAndGiveUp)
167 return false;
168 break;
169 }
170
171 MachineInstr *NewMI;
172 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies)
173 NewMI = getDefIgnoringCopies(Reg: MO.getReg(), MRI);
174 else
175 NewMI = MRI.getVRegDef(Reg: MO.getReg());
176
177 if ((size_t)NewInsnID < State.MIs.size())
178 State.MIs[NewInsnID] = NewMI;
179 else {
180 assert((size_t)NewInsnID == State.MIs.size() &&
181 "Expected to store MIs in order");
182 State.MIs.push_back(Elt: NewMI);
183 }
184 DEBUG_WITH_TYPE(TgtExecutor::getName(),
185 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
186 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
187 << ")\n");
188 break;
189 }
190
191 case GIM_CheckFeatures: {
192 uint16_t ExpectedBitsetID = readU16();
193 DEBUG_WITH_TYPE(TgtExecutor::getName(),
194 dbgs() << CurrentIdx
195 << ": GIM_CheckFeatures(ExpectedBitsetID="
196 << ExpectedBitsetID << ")\n");
197 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=
198 ExecInfo.FeatureBitsets[ExpectedBitsetID]) {
199 if (handleReject() == RejectAndGiveUp)
200 return false;
201 }
202 break;
203 }
204 case GIM_CheckOpcode:
205 case GIM_CheckOpcodeIsEither: {
206 uint64_t InsnID = readULEB();
207 uint16_t Expected0 = readU16();
208 uint16_t Expected1 = -1;
209 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
210 Expected1 = readU16();
211
212 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
213 unsigned Opcode = State.MIs[InsnID]->getOpcode();
214
215 DEBUG_WITH_TYPE(TgtExecutor::getName(),
216 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
217 << "], ExpectedOpcode=" << Expected0;
218 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs()
219 << " || " << Expected1;
220 dbgs() << ") // Got=" << Opcode << "\n";);
221
222 if (Opcode != Expected0 && Opcode != Expected1) {
223 if (handleReject() == RejectAndGiveUp)
224 return false;
225 }
226 break;
227 }
228 case GIM_SwitchOpcode: {
229 uint64_t InsnID = readULEB();
230 uint16_t LowerBound = readU16();
231 uint16_t UpperBound = readU16();
232 uint32_t Default = readU32();
233
234 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
236
237 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
238 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
239 << LowerBound << ", " << UpperBound << "), Default=" << Default
240 << ", JumpTable...) // Got=" << Opcode << "\n";
241 });
242 if (Opcode < LowerBound || UpperBound <= Opcode) {
243 CurrentIdx = Default;
244 break;
245 }
246 const auto EntryIdx = (Opcode - LowerBound);
247 // Each entry is 4 bytes
248 CurrentIdx =
249 readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (EntryIdx * 4));
250 if (!CurrentIdx) {
251 CurrentIdx = Default;
252 break;
253 }
254 OnFailResumeAt.push_back(Elt: Default);
255 break;
256 }
257
258 case GIM_SwitchType: {
259 uint64_t InsnID = readULEB();
260 uint64_t OpIdx = readULEB();
261 uint16_t LowerBound = readU16();
262 uint16_t UpperBound = readU16();
263 int64_t Default = readU32();
264
265 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
266 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
267
268 DEBUG_WITH_TYPE(TgtExecutor::getName(), {
269 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
270 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
271 << UpperBound << "), Default=" << Default
272 << ", JumpTable...) // Got=";
273 if (!MO.isReg())
274 dbgs() << "Not a VReg\n";
275 else
276 dbgs() << MRI.getType(MO.getReg()) << "\n";
277 });
278 if (!MO.isReg()) {
279 CurrentIdx = Default;
280 break;
281 }
282 const LLT Ty = MRI.getType(Reg: MO.getReg());
283 const auto TyI = ExecInfo.TypeIDMap.find(Ty);
284 if (TyI == ExecInfo.TypeIDMap.end()) {
285 CurrentIdx = Default;
286 break;
287 }
288 const int64_t TypeID = TyI->second;
289 if (TypeID < LowerBound || UpperBound <= TypeID) {
290 CurrentIdx = Default;
291 break;
292 }
293 const auto NumEntry = (TypeID - LowerBound);
294 // Each entry is 4 bytes
295 CurrentIdx =
296 readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (NumEntry * 4));
297 if (!CurrentIdx) {
298 CurrentIdx = Default;
299 break;
300 }
301 OnFailResumeAt.push_back(Elt: Default);
302 break;
303 }
304
305 case GIM_CheckNumOperands: {
306 uint64_t InsnID = readULEB();
307 uint64_t Expected = readULEB();
308 DEBUG_WITH_TYPE(TgtExecutor::getName(),
309 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
310 << InsnID << "], Expected=" << Expected << ")\n");
311 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
312 if (State.MIs[InsnID]->getNumOperands() != Expected) {
313 if (handleReject() == RejectAndGiveUp)
314 return false;
315 }
316 break;
317 }
318 case GIM_CheckI64ImmPredicate:
319 case GIM_CheckImmOperandPredicate: {
320 uint64_t InsnID = readULEB();
321 unsigned OpIdx =
322 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
323 uint16_t Predicate = readU16();
324 DEBUG_WITH_TYPE(TgtExecutor::getName(),
325 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
326 << InsnID << "]->getOperand(" << OpIdx
327 << "), Predicate=" << Predicate << ")\n");
328 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
329 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
330 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
331 "Expected immediate operand");
332 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
333 int64_t Value = 0;
334 if (State.MIs[InsnID]->getOperand(i: OpIdx).isCImm())
335 Value = State.MIs[InsnID]->getOperand(i: OpIdx).getCImm()->getSExtValue();
336 else if (State.MIs[InsnID]->getOperand(i: OpIdx).isImm())
337 Value = State.MIs[InsnID]->getOperand(i: OpIdx).getImm();
338 else
339 llvm_unreachable("Expected Imm or CImm operand");
340
341 if (!testImmPredicate_I64(Predicate, Value))
342 if (handleReject() == RejectAndGiveUp)
343 return false;
344 break;
345 }
346 case GIM_CheckAPIntImmPredicate: {
347 uint64_t InsnID = readULEB();
348 uint16_t Predicate = readU16();
349 DEBUG_WITH_TYPE(TgtExecutor::getName(),
350 dbgs()
351 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
352 << InsnID << "], Predicate=" << Predicate << ")\n");
353 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
354 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
355 "Expected G_CONSTANT");
356 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
357 if (!State.MIs[InsnID]->getOperand(i: 1).isCImm())
358 llvm_unreachable("Expected Imm or CImm operand");
359
360 const APInt &Value =
361 State.MIs[InsnID]->getOperand(i: 1).getCImm()->getValue();
362 if (!testImmPredicate_APInt(Predicate, Value))
363 if (handleReject() == RejectAndGiveUp)
364 return false;
365 break;
366 }
367 case GIM_CheckAPFloatImmPredicate: {
368 uint64_t InsnID = readULEB();
369 uint16_t Predicate = readU16();
370 DEBUG_WITH_TYPE(TgtExecutor::getName(),
371 dbgs()
372 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
373 << InsnID << "], Predicate=" << Predicate << ")\n");
374 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
375 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
376 "Expected G_FCONSTANT");
377 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&
378 "Expected FPImm operand");
379 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
380 const APFloat &Value =
381 State.MIs[InsnID]->getOperand(i: 1).getFPImm()->getValueAPF();
382
383 if (!testImmPredicate_APFloat(Predicate, Value))
384 if (handleReject() == RejectAndGiveUp)
385 return false;
386 break;
387 }
388 case GIM_CheckIsBuildVectorAllOnes:
389 case GIM_CheckIsBuildVectorAllZeros: {
390 uint64_t InsnID = readULEB();
391
392 DEBUG_WITH_TYPE(TgtExecutor::getName(),
393 dbgs() << CurrentIdx
394 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
395 << InsnID << "])\n");
396 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
397
398 const MachineInstr *MI = State.MIs[InsnID];
399 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
400 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
401 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
402
403 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
404 if (!isBuildVectorAllOnes(MI: *MI, MRI)) {
405 if (handleReject() == RejectAndGiveUp)
406 return false;
407 }
408 } else {
409 if (!isBuildVectorAllZeros(MI: *MI, MRI)) {
410 if (handleReject() == RejectAndGiveUp)
411 return false;
412 }
413 }
414
415 break;
416 }
417 case GIM_CheckSimplePredicate: {
418 // Note: we don't check for invalid here because this is purely a hook to
419 // allow some executors (such as the combiner) to check arbitrary,
420 // contextless predicates, such as whether a rule is enabled or not.
421 uint16_t Predicate = readU16();
422 DEBUG_WITH_TYPE(TgtExecutor::getName(),
423 dbgs() << CurrentIdx
424 << ": GIM_CheckSimplePredicate(Predicate="
425 << Predicate << ")\n");
426 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
427 if (!testSimplePredicate(Predicate)) {
428 if (handleReject() == RejectAndGiveUp)
429 return false;
430 }
431 break;
432 }
433 case GIM_CheckCxxInsnPredicate: {
434 uint64_t InsnID = readULEB();
435 uint16_t Predicate = readU16();
436 DEBUG_WITH_TYPE(TgtExecutor::getName(),
437 dbgs()
438 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
439 << InsnID << "], Predicate=" << Predicate << ")\n");
440 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
441 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate");
442
443 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State))
444 if (handleReject() == RejectAndGiveUp)
445 return false;
446 break;
447 }
448 case GIM_CheckHasNoUse: {
449 uint64_t InsnID = readULEB();
450
451 DEBUG_WITH_TYPE(TgtExecutor::getName(),
452 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
453 << InsnID << "]\n");
454
455 const MachineInstr *MI = State.MIs[InsnID];
456 assert(MI && "Used insn before defined");
457 assert(MI->getNumDefs() > 0 && "No defs");
458 const Register Res = MI->getOperand(i: 0).getReg();
459
460 if (!MRI.use_nodbg_empty(RegNo: Res)) {
461 if (handleReject() == RejectAndGiveUp)
462 return false;
463 }
464
465 break;
466 }
467 case GIM_CheckAtomicOrdering: {
468 uint64_t InsnID = readULEB();
469 auto Ordering = (AtomicOrdering)readULEB();
470 DEBUG_WITH_TYPE(TgtExecutor::getName(),
471 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
472 << InsnID << "], " << (uint64_t)Ordering << ")\n");
473 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
474 if (!State.MIs[InsnID]->hasOneMemOperand())
475 if (handleReject() == RejectAndGiveUp)
476 return false;
477
478 for (const auto &MMO : State.MIs[InsnID]->memoperands())
479 if (MMO->getMergedOrdering() != Ordering)
480 if (handleReject() == RejectAndGiveUp)
481 return false;
482 break;
483 }
484 case GIM_CheckAtomicOrderingOrStrongerThan: {
485 uint64_t InsnID = readULEB();
486 auto Ordering = (AtomicOrdering)readULEB();
487 DEBUG_WITH_TYPE(TgtExecutor::getName(),
488 dbgs() << CurrentIdx
489 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
490 << InsnID << "], " << (uint64_t)Ordering << ")\n");
491 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
492 if (!State.MIs[InsnID]->hasOneMemOperand())
493 if (handleReject() == RejectAndGiveUp)
494 return false;
495
496 for (const auto &MMO : State.MIs[InsnID]->memoperands())
497 if (!isAtLeastOrStrongerThan(AO: MMO->getMergedOrdering(), Other: Ordering))
498 if (handleReject() == RejectAndGiveUp)
499 return false;
500 break;
501 }
502 case GIM_CheckAtomicOrderingWeakerThan: {
503 uint64_t InsnID = readULEB();
504 auto Ordering = (AtomicOrdering)readULEB();
505 DEBUG_WITH_TYPE(TgtExecutor::getName(),
506 dbgs() << CurrentIdx
507 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
508 << InsnID << "], " << (uint64_t)Ordering << ")\n");
509 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510 if (!State.MIs[InsnID]->hasOneMemOperand())
511 if (handleReject() == RejectAndGiveUp)
512 return false;
513
514 for (const auto &MMO : State.MIs[InsnID]->memoperands())
515 if (!isStrongerThan(AO: Ordering, Other: MMO->getMergedOrdering()))
516 if (handleReject() == RejectAndGiveUp)
517 return false;
518 break;
519 }
520 case GIM_CheckMemoryAddressSpace: {
521 uint64_t InsnID = readULEB();
522 uint64_t MMOIdx = readULEB();
523 // This accepts a list of possible address spaces.
524 const uint64_t NumAddrSpace = readULEB();
525
526 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
527 if (handleReject() == RejectAndGiveUp)
528 return false;
529 break;
530 }
531
532 // Need to still jump to the end of the list of address spaces if we find
533 // a match earlier.
534 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
535
536 const MachineMemOperand *MMO =
537 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
538 const unsigned MMOAddrSpace = MMO->getAddrSpace();
539
540 bool Success = false;
541 for (unsigned I = 0; I != NumAddrSpace; ++I) {
542 uint64_t AddrSpace = readULEB();
543 DEBUG_WITH_TYPE(TgtExecutor::getName(),
544 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
545 << AddrSpace << '\n');
546
547 if (AddrSpace == MMOAddrSpace) {
548 Success = true;
549 break;
550 }
551 }
552
553 CurrentIdx = LastIdx;
554 if (!Success && handleReject() == RejectAndGiveUp)
555 return false;
556 break;
557 }
558 case GIM_CheckMemoryAlignment: {
559 uint64_t InsnID = readULEB();
560 uint64_t MMOIdx = readULEB();
561 uint64_t MinAlign = readULEB();
562
563 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
564
565 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
566 if (handleReject() == RejectAndGiveUp)
567 return false;
568 break;
569 }
570
571 MachineMemOperand *MMO =
572 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
573 DEBUG_WITH_TYPE(TgtExecutor::getName(),
574 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
575 << "(MIs[" << InsnID << "]->memoperands() + "
576 << MMOIdx << ")->getAlignment() >= " << MinAlign
577 << ")\n");
578 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
579 return false;
580
581 break;
582 }
583 case GIM_CheckMemorySizeEqualTo: {
584 uint64_t InsnID = readULEB();
585 uint64_t MMOIdx = readULEB();
586 uint32_t Size = readU32();
587
588 DEBUG_WITH_TYPE(TgtExecutor::getName(),
589 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
590 << InsnID << "]->memoperands() + " << MMOIdx
591 << ", Size=" << Size << ")\n");
592 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
594 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
595 if (handleReject() == RejectAndGiveUp)
596 return false;
597 break;
598 }
599
600 MachineMemOperand *MMO =
601 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
602
603 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize()
604 << " bytes vs " << Size
605 << " bytes\n");
606 if (MMO->getSize() != Size)
607 if (handleReject() == RejectAndGiveUp)
608 return false;
609
610 break;
611 }
612 case GIM_CheckMemorySizeEqualToLLT:
613 case GIM_CheckMemorySizeLessThanLLT:
614 case GIM_CheckMemorySizeGreaterThanLLT: {
615 uint64_t InsnID = readULEB();
616 uint64_t MMOIdx = readULEB();
617 uint64_t OpIdx = readULEB();
618
619 DEBUG_WITH_TYPE(
620 TgtExecutor::getName(),
621 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
622 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo"
623 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
624 ? "GreaterThan"
625 : "LessThan")
626 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
627 << ", OpIdx=" << OpIdx << ")\n");
628 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
629
630 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
631 if (!MO.isReg()) {
632 DEBUG_WITH_TYPE(TgtExecutor::getName(),
633 dbgs() << CurrentIdx << ": Not a register\n");
634 if (handleReject() == RejectAndGiveUp)
635 return false;
636 break;
637 }
638
639 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
640 if (handleReject() == RejectAndGiveUp)
641 return false;
642 break;
643 }
644
645 MachineMemOperand *MMO =
646 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
647
648 unsigned Size = MRI.getType(Reg: MO.getReg()).getSizeInBits();
649 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
650 MMO->getSizeInBits() != Size) {
651 if (handleReject() == RejectAndGiveUp)
652 return false;
653 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
654 MMO->getSizeInBits() >= Size) {
655 if (handleReject() == RejectAndGiveUp)
656 return false;
657 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
658 MMO->getSizeInBits() <= Size)
659 if (handleReject() == RejectAndGiveUp)
660 return false;
661
662 break;
663 }
664 case GIM_CheckType: {
665 uint64_t InsnID = readULEB();
666 uint64_t OpIdx = readULEB();
667 int TypeID = readS8();
668 DEBUG_WITH_TYPE(TgtExecutor::getName(),
669 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
670 << "]->getOperand(" << OpIdx
671 << "), TypeID=" << TypeID << ")\n");
672 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
673 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
674 if (!MO.isReg() || MRI.getType(Reg: MO.getReg()) != getTypeFromIdx(TypeID)) {
675 if (handleReject() == RejectAndGiveUp)
676 return false;
677 }
678 break;
679 }
680 case GIM_CheckPointerToAny: {
681 uint64_t InsnID = readULEB();
682 uint64_t OpIdx = readULEB();
683 uint64_t SizeInBits = readULEB();
684
685 DEBUG_WITH_TYPE(TgtExecutor::getName(),
686 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
687 << InsnID << "]->getOperand(" << OpIdx
688 << "), SizeInBits=" << SizeInBits << ")\n");
689 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
690 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
691 const LLT Ty = MRI.getType(Reg: MO.getReg());
692
693 // iPTR must be looked up in the target.
694 if (SizeInBits == 0) {
695 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
696 const unsigned AddrSpace = Ty.getAddressSpace();
697 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AS: AddrSpace);
698 }
699
700 assert(SizeInBits != 0 && "Pointer size must be known");
701
702 if (MO.isReg()) {
703 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
704 if (handleReject() == RejectAndGiveUp)
705 return false;
706 } else if (handleReject() == RejectAndGiveUp)
707 return false;
708
709 break;
710 }
711 case GIM_RecordNamedOperand: {
712 uint64_t InsnID = readULEB();
713 uint64_t OpIdx = readULEB();
714 uint64_t StoreIdx = readULEB();
715
716 DEBUG_WITH_TYPE(TgtExecutor::getName(),
717 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
718 << InsnID << "]->getOperand(" << OpIdx
719 << "), StoreIdx=" << StoreIdx << ")\n");
720 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
721 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
722 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(i: OpIdx);
723 break;
724 }
725 case GIM_RecordRegType: {
726 uint64_t InsnID = readULEB();
727 uint64_t OpIdx = readULEB();
728 int TypeIdx = readS8();
729
730 DEBUG_WITH_TYPE(TgtExecutor::getName(),
731 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
732 << InsnID << "]->getOperand(" << OpIdx
733 << "), TypeIdx=" << TypeIdx << ")\n");
734 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
735 assert(TypeIdx < 0 && "Temp types always have negative indexes!");
736 // Indexes start at -1.
737 TypeIdx = 1 - TypeIdx;
738 const auto &Op = State.MIs[InsnID]->getOperand(i: OpIdx);
739 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)
740 State.RecordedTypes.resize(N: TypeIdx + 1, NV: LLT());
741 State.RecordedTypes[TypeIdx] = MRI.getType(Reg: Op.getReg());
742 break;
743 }
744 case GIM_CheckRegBankForClass: {
745 uint64_t InsnID = readULEB();
746 uint64_t OpIdx = readULEB();
747 uint16_t RCEnum = readU16();
748 DEBUG_WITH_TYPE(TgtExecutor::getName(),
749 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
750 << InsnID << "]->getOperand(" << OpIdx
751 << "), RCEnum=" << RCEnum << ")\n");
752 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
753 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
754 if (!MO.isReg() ||
755 &RBI.getRegBankFromRegClass(RC: *TRI.getRegClass(i: RCEnum),
756 Ty: MRI.getType(Reg: MO.getReg())) !=
757 RBI.getRegBank(Reg: MO.getReg(), MRI, TRI)) {
758 if (handleReject() == RejectAndGiveUp)
759 return false;
760 }
761 break;
762 }
763
764 case GIM_CheckComplexPattern: {
765 uint64_t InsnID = readULEB();
766 uint64_t OpIdx = readULEB();
767 uint16_t RendererID = readU16();
768 uint16_t ComplexPredicateID = readU16();
769 DEBUG_WITH_TYPE(TgtExecutor::getName(),
770 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
771 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
772 << "]->getOperand(" << OpIdx
773 << "), ComplexPredicateID=" << ComplexPredicateID
774 << ")\n");
775 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
776 // FIXME: Use std::invoke() when it's available.
777 ComplexRendererFns Renderer =
778 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])(
779 State.MIs[InsnID]->getOperand(i: OpIdx));
780 if (Renderer)
781 State.Renderers[RendererID] = *Renderer;
782 else if (handleReject() == RejectAndGiveUp)
783 return false;
784 break;
785 }
786
787 case GIM_CheckConstantInt:
788 case GIM_CheckConstantInt8: {
789 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
790
791 uint64_t InsnID = readULEB();
792 uint64_t OpIdx = readULEB();
793 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
794 DEBUG_WITH_TYPE(TgtExecutor::getName(),
795 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
796 << InsnID << "]->getOperand(" << OpIdx
797 << "), Value=" << Value << ")\n");
798 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
799 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
800 if (MO.isReg()) {
801 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
802 LLT Ty = MRI.getType(Reg: MO.getReg());
803 // If the type is > 64 bits, it can't be a constant int, so we bail
804 // early because SignExtend64 will assert otherwise.
805 if (Ty.getScalarSizeInBits() > 64) {
806 if (handleReject() == RejectAndGiveUp)
807 return false;
808 break;
809 }
810
811 Value = SignExtend64(X: Value, B: Ty.getScalarSizeInBits());
812 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/Splat: true)) {
813 if (handleReject() == RejectAndGiveUp)
814 return false;
815 }
816 } else if (handleReject() == RejectAndGiveUp)
817 return false;
818
819 break;
820 }
821
822 case GIM_CheckLiteralInt: {
823 uint64_t InsnID = readULEB();
824 uint64_t OpIdx = readULEB();
825 int64_t Value = readU64();
826 DEBUG_WITH_TYPE(TgtExecutor::getName(),
827 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
828 << InsnID << "]->getOperand(" << OpIdx
829 << "), Value=" << Value << ")\n");
830 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
831 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
832 if (MO.isImm() && MO.getImm() == Value)
833 break;
834
835 if (MO.isCImm() && MO.getCImm()->equalsInt(V: Value))
836 break;
837
838 if (handleReject() == RejectAndGiveUp)
839 return false;
840
841 break;
842 }
843
844 case GIM_CheckIntrinsicID: {
845 uint64_t InsnID = readULEB();
846 uint64_t OpIdx = readULEB();
847 uint16_t Value = readU16();
848 DEBUG_WITH_TYPE(TgtExecutor::getName(),
849 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
850 << InsnID << "]->getOperand(" << OpIdx
851 << "), Value=" << Value << ")\n");
852 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
853 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
854 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
855 if (handleReject() == RejectAndGiveUp)
856 return false;
857 break;
858 }
859 case GIM_CheckCmpPredicate: {
860 uint64_t InsnID = readULEB();
861 uint64_t OpIdx = readULEB();
862 uint16_t Value = readU16();
863 DEBUG_WITH_TYPE(TgtExecutor::getName(),
864 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
865 << InsnID << "]->getOperand(" << OpIdx
866 << "), Value=" << Value << ")\n");
867 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
868 MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx);
869 if (!MO.isPredicate() || MO.getPredicate() != Value)
870 if (handleReject() == RejectAndGiveUp)
871 return false;
872 break;
873 }
874 case GIM_CheckIsMBB: {
875 uint64_t InsnID = readULEB();
876 uint64_t OpIdx = readULEB();
877 DEBUG_WITH_TYPE(TgtExecutor::getName(),
878 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
879 << "]->getOperand(" << OpIdx << "))\n");
880 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
881 if (!State.MIs[InsnID]->getOperand(i: OpIdx).isMBB()) {
882 if (handleReject() == RejectAndGiveUp)
883 return false;
884 }
885 break;
886 }
887 case GIM_CheckIsImm: {
888 uint64_t InsnID = readULEB();
889 uint64_t OpIdx = readULEB();
890 DEBUG_WITH_TYPE(TgtExecutor::getName(),
891 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
892 << "]->getOperand(" << OpIdx << "))\n");
893 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
894 if (!State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) {
895 if (handleReject() == RejectAndGiveUp)
896 return false;
897 }
898 break;
899 }
900 case GIM_CheckIsSafeToFold: {
901 uint64_t InsnID = readULEB();
902 DEBUG_WITH_TYPE(TgtExecutor::getName(),
903 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
904 << InsnID << "])\n");
905 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
906 if (!isObviouslySafeToFold(MI&: *State.MIs[InsnID], IntoMI&: *State.MIs[0])) {
907 if (handleReject() == RejectAndGiveUp)
908 return false;
909 }
910 break;
911 }
912 case GIM_CheckIsSameOperand:
913 case GIM_CheckIsSameOperandIgnoreCopies: {
914 uint64_t InsnID = readULEB();
915 uint64_t OpIdx = readULEB();
916 uint64_t OtherInsnID = readULEB();
917 uint64_t OtherOpIdx = readULEB();
918 DEBUG_WITH_TYPE(TgtExecutor::getName(),
919 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
920 << InsnID << "][" << OpIdx << "], MIs["
921 << OtherInsnID << "][" << OtherOpIdx << "])\n");
922 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
923 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
924
925 MachineOperand &Op = State.MIs[InsnID]->getOperand(i: OpIdx);
926 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(i: OtherOpIdx);
927
928 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) {
929 if (Op.isReg() && OtherOp.isReg()) {
930 if (getSrcRegIgnoringCopies(Reg: Op.getReg(), MRI) ==
931 getSrcRegIgnoringCopies(Reg: OtherOp.getReg(), MRI))
932 break;
933 }
934 }
935
936 if (!Op.isIdenticalTo(Other: OtherOp)) {
937 if (handleReject() == RejectAndGiveUp)
938 return false;
939 }
940 break;
941 }
942 case GIM_CheckCanReplaceReg: {
943 uint64_t OldInsnID = readULEB();
944 uint64_t OldOpIdx = readULEB();
945 uint64_t NewInsnID = readULEB();
946 uint64_t NewOpIdx = readULEB();
947
948 DEBUG_WITH_TYPE(TgtExecutor::getName(),
949 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
950 << OldInsnID << "][" << OldOpIdx << "] = MIs["
951 << NewInsnID << "][" << NewOpIdx << "])\n");
952
953 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
954 Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg();
955 if (!canReplaceReg(DstReg: Old, SrcReg: New, MRI)) {
956 if (handleReject() == RejectAndGiveUp)
957 return false;
958 }
959 break;
960 }
961 case GIM_MIFlags: {
962 uint64_t InsnID = readULEB();
963 uint32_t Flags = readU32();
964
965 DEBUG_WITH_TYPE(TgtExecutor::getName(),
966 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
967 << "], " << Flags << ")\n");
968 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {
969 if (handleReject() == RejectAndGiveUp)
970 return false;
971 }
972 break;
973 }
974 case GIM_MIFlagsNot: {
975 uint64_t InsnID = readULEB();
976 uint32_t Flags = readU32();
977
978 DEBUG_WITH_TYPE(TgtExecutor::getName(),
979 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
980 << "], " << Flags << ")\n");
981 if ((State.MIs[InsnID]->getFlags() & Flags)) {
982 if (handleReject() == RejectAndGiveUp)
983 return false;
984 }
985 break;
986 }
987 case GIM_Reject:
988 DEBUG_WITH_TYPE(TgtExecutor::getName(),
989 dbgs() << CurrentIdx << ": GIM_Reject\n");
990 if (handleReject() == RejectAndGiveUp)
991 return false;
992 break;
993 case GIR_MutateOpcode: {
994 uint64_t OldInsnID = readULEB();
995 uint64_t NewInsnID = readULEB();
996 uint16_t NewOpcode = readU16();
997 if (NewInsnID >= OutMIs.size())
998 OutMIs.resize(N: NewInsnID + 1);
999
1000 MachineInstr *OldMI = State.MIs[OldInsnID];
1001 if (Observer)
1002 Observer->changingInstr(MI&: *OldMI);
1003 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
1004 OutMIs[NewInsnID]->setDesc(TII.get(Opcode: NewOpcode));
1005 if (Observer)
1006 Observer->changedInstr(MI&: *OldMI);
1007 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1008 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
1009 << NewInsnID << "], MIs[" << OldInsnID << "], "
1010 << NewOpcode << ")\n");
1011 break;
1012 }
1013
1014 case GIR_BuildMI: {
1015 uint64_t NewInsnID = readULEB();
1016 uint16_t Opcode = readU16();
1017 if (NewInsnID >= OutMIs.size())
1018 OutMIs.resize(N: NewInsnID + 1);
1019
1020 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
1021 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1022 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
1023 << NewInsnID << "], " << Opcode << ")\n");
1024 break;
1025 }
1026
1027 case GIR_BuildConstant: {
1028 uint64_t TempRegID = readULEB();
1029 uint64_t Imm = readU64();
1030 Builder.buildConstant(Res: State.TempRegisters[TempRegID], Val: Imm);
1031 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1032 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
1033 << TempRegID << "], Imm=" << Imm << ")\n");
1034 break;
1035 }
1036
1037 case GIR_Copy: {
1038 uint64_t NewInsnID = readULEB();
1039 uint64_t OldInsnID = readULEB();
1040 uint64_t OpIdx = readULEB();
1041 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1042 OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: OpIdx));
1043 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1044 dbgs()
1045 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
1046 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
1047 break;
1048 }
1049
1050 case GIR_CopyOrAddZeroReg: {
1051 uint64_t NewInsnID = readULEB();
1052 uint64_t OldInsnID = readULEB();
1053 uint64_t OpIdx = readULEB();
1054 uint16_t ZeroReg = readU16();
1055 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1056 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(i: OpIdx);
1057 if (isOperandImmEqual(MO, Value: 0, MRI))
1058 OutMIs[NewInsnID].addReg(RegNo: ZeroReg);
1059 else
1060 OutMIs[NewInsnID].add(MO);
1061 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1062 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
1063 << NewInsnID << "], MIs[" << OldInsnID << "], "
1064 << OpIdx << ", " << ZeroReg << ")\n");
1065 break;
1066 }
1067
1068 case GIR_CopySubReg: {
1069 uint64_t NewInsnID = readULEB();
1070 uint64_t OldInsnID = readULEB();
1071 uint64_t OpIdx = readULEB();
1072 uint16_t SubRegIdx = readU16();
1073 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1074 OutMIs[NewInsnID].addReg(RegNo: State.MIs[OldInsnID]->getOperand(i: OpIdx).getReg(),
1075 flags: 0, SubReg: SubRegIdx);
1076 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1077 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
1078 << NewInsnID << "], MIs[" << OldInsnID << "], "
1079 << OpIdx << ", " << SubRegIdx << ")\n");
1080 break;
1081 }
1082
1083 case GIR_AddImplicitDef: {
1084 uint64_t InsnID = readULEB();
1085 uint16_t RegNum = readU16();
1086 uint16_t Flags = readU16();
1087 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1088 Flags |= RegState::Implicit;
1089 OutMIs[InsnID].addDef(RegNo: RegNum, Flags);
1090 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1091 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
1092 << InsnID << "], " << RegNum << ")\n");
1093 break;
1094 }
1095
1096 case GIR_AddImplicitUse: {
1097 uint64_t InsnID = readULEB();
1098 uint16_t RegNum = readU16();
1099 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1100 OutMIs[InsnID].addUse(RegNo: RegNum, Flags: RegState::Implicit);
1101 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1102 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
1103 << InsnID << "], " << RegNum << ")\n");
1104 break;
1105 }
1106
1107 case GIR_AddRegister: {
1108 uint64_t InsnID = readULEB();
1109 uint16_t RegNum = readU16();
1110 uint16_t RegFlags = readU16();
1111 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1112 OutMIs[InsnID].addReg(RegNo: RegNum, flags: RegFlags);
1113 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1114 dbgs()
1115 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID
1116 << "], " << RegNum << ", " << RegFlags << ")\n");
1117 break;
1118 }
1119 case GIR_AddIntrinsicID: {
1120 uint64_t InsnID = readULEB();
1121 uint16_t Value = readU16();
1122 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1123 OutMIs[InsnID].addIntrinsicID(ID: (Intrinsic::ID)Value);
1124 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1125 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["
1126 << InsnID << "], " << Value << ")\n");
1127 break;
1128 }
1129 case GIR_SetImplicitDefDead: {
1130 uint64_t InsnID = readULEB();
1131 uint64_t OpIdx = readULEB();
1132 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1133 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
1134 << InsnID << "], OpIdx=" << OpIdx << ")\n");
1135 MachineInstr *MI = OutMIs[InsnID];
1136 assert(MI && "Modifying undefined instruction");
1137 MI->getOperand(i: MI->getNumExplicitOperands() + OpIdx).setIsDead();
1138 break;
1139 }
1140 case GIR_SetMIFlags: {
1141 uint64_t InsnID = readULEB();
1142 uint32_t Flags = readU32();
1143
1144 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1145 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
1146 << InsnID << "], " << Flags << ")\n");
1147 MachineInstr *MI = OutMIs[InsnID];
1148 MI->setFlags(MI->getFlags() | Flags);
1149 break;
1150 }
1151 case GIR_UnsetMIFlags: {
1152 uint64_t InsnID = readULEB();
1153 uint32_t Flags = readU32();
1154
1155 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1156 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
1157 << InsnID << "], " << Flags << ")\n");
1158 MachineInstr *MI = OutMIs[InsnID];
1159 MI->setFlags(MI->getFlags() & ~Flags);
1160 break;
1161 }
1162 case GIR_CopyMIFlags: {
1163 uint64_t InsnID = readULEB();
1164 uint64_t OldInsnID = readULEB();
1165
1166 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1167 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
1168 << InsnID << "], MIs[" << OldInsnID << "])\n");
1169 MachineInstr *MI = OutMIs[InsnID];
1170 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
1171 break;
1172 }
1173 case GIR_AddSimpleTempRegister:
1174 case GIR_AddTempRegister:
1175 case GIR_AddTempSubRegister: {
1176 uint64_t InsnID = readULEB();
1177 uint64_t TempRegID = readULEB();
1178 uint16_t TempRegFlags = 0;
1179 if (MatcherOpcode != GIR_AddSimpleTempRegister)
1180 TempRegFlags = readU16();
1181 uint16_t SubReg = 0;
1182 if (MatcherOpcode == GIR_AddTempSubRegister)
1183 SubReg = readU16();
1184
1185 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1186
1187 OutMIs[InsnID].addReg(RegNo: State.TempRegisters[TempRegID], flags: TempRegFlags,
1188 SubReg);
1189 DEBUG_WITH_TYPE(
1190 TgtExecutor::getName(),
1191 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID
1192 << "], TempRegisters[" << TempRegID << "]";
1193 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
1194 dbgs() << ", " << TempRegFlags << ")\n");
1195 break;
1196 }
1197
1198 case GIR_AddImm8:
1199 case GIR_AddImm: {
1200 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
1201 uint64_t InsnID = readULEB();
1202 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
1203 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1204 OutMIs[InsnID].addImm(Val: Imm);
1205 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1206 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
1207 << "], " << Imm << ")\n");
1208 break;
1209 }
1210
1211 case GIR_AddCImm: {
1212 uint64_t InsnID = readULEB();
1213 int TypeID = readS8();
1214 uint64_t Imm = readU64();
1215 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1216
1217 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
1218 LLVMContext &Ctx = MF->getFunction().getContext();
1219 OutMIs[InsnID].addCImm(
1220 Val: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: Width), V: Imm, /*signed*/ IsSigned: true));
1221 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1222 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID
1223 << "], TypeID=" << TypeID << ", Imm=" << Imm
1224 << ")\n");
1225 break;
1226 }
1227
1228 case GIR_ComplexRenderer: {
1229 uint64_t InsnID = readULEB();
1230 uint16_t RendererID = readU16();
1231 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1232 for (const auto &RenderOpFn : State.Renderers[RendererID])
1233 RenderOpFn(OutMIs[InsnID]);
1234 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1235 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
1236 << InsnID << "], " << RendererID << ")\n");
1237 break;
1238 }
1239 case GIR_ComplexSubOperandRenderer: {
1240 uint64_t InsnID = readULEB();
1241 uint16_t RendererID = readU16();
1242 uint64_t RenderOpID = readULEB();
1243 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1244 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1245 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1246 dbgs() << CurrentIdx
1247 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1248 << InsnID << "], " << RendererID << ", "
1249 << RenderOpID << ")\n");
1250 break;
1251 }
1252 case GIR_ComplexSubOperandSubRegRenderer: {
1253 uint64_t InsnID = readULEB();
1254 uint16_t RendererID = readU16();
1255 uint64_t RenderOpID = readULEB();
1256 uint16_t SubRegIdx = readU16();
1257 MachineInstrBuilder &MI = OutMIs[InsnID];
1258 assert(MI && "Attempted to add to undefined instruction");
1259 State.Renderers[RendererID][RenderOpID](MI);
1260 MI->getOperand(i: MI->getNumOperands() - 1).setSubReg(SubRegIdx);
1261 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1262 dbgs() << CurrentIdx
1263 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["
1264 << InsnID << "], " << RendererID << ", "
1265 << RenderOpID << ", " << SubRegIdx << ")\n");
1266 break;
1267 }
1268
1269 case GIR_CopyConstantAsSImm: {
1270 uint64_t NewInsnID = readULEB();
1271 uint64_t OldInsnID = readULEB();
1272 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1273 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
1274 "Expected G_CONSTANT");
1275 if (State.MIs[OldInsnID]->getOperand(i: 1).isCImm()) {
1276 OutMIs[NewInsnID].addImm(
1277 Val: State.MIs[OldInsnID]->getOperand(i: 1).getCImm()->getSExtValue());
1278 } else if (State.MIs[OldInsnID]->getOperand(i: 1).isImm())
1279 OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: 1));
1280 else
1281 llvm_unreachable("Expected Imm or CImm operand");
1282 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1283 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1284 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1285 break;
1286 }
1287
1288 // TODO: Needs a test case once we have a pattern that uses this.
1289 case GIR_CopyFConstantAsFPImm: {
1290 uint64_t NewInsnID = readULEB();
1291 uint64_t OldInsnID = readULEB();
1292 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1293 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
1294 "Expected G_FCONSTANT");
1295 if (State.MIs[OldInsnID]->getOperand(i: 1).isFPImm())
1296 OutMIs[NewInsnID].addFPImm(
1297 Val: State.MIs[OldInsnID]->getOperand(i: 1).getFPImm());
1298 else
1299 llvm_unreachable("Expected FPImm operand");
1300 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1301 dbgs()
1302 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1303 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1304 break;
1305 }
1306
1307 case GIR_CustomRenderer: {
1308 uint64_t InsnID = readULEB();
1309 uint64_t OldInsnID = readULEB();
1310 uint16_t RendererFnID = readU16();
1311 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1312 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1313 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1314 << InsnID << "], MIs[" << OldInsnID << "], "
1315 << RendererFnID << ")\n");
1316 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1317 OutMIs[InsnID], *State.MIs[OldInsnID],
1318 -1); // Not a source operand of the old instruction.
1319 break;
1320 }
1321 case GIR_CustomAction: {
1322 uint16_t FnID = readU16();
1323 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1324 dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
1325 << ")\n");
1326 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID");
1327 runCustomAction(FnID, State, OutMIs);
1328 break;
1329 }
1330 case GIR_CustomOperandRenderer: {
1331 uint64_t InsnID = readULEB();
1332 uint64_t OldInsnID = readULEB();
1333 uint64_t OpIdx = readULEB();
1334 uint16_t RendererFnID = readU16();
1335 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1336
1337 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1338 dbgs() << CurrentIdx
1339 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID
1340 << "], MIs[" << OldInsnID << "]->getOperand("
1341 << OpIdx << "), " << RendererFnID << ")\n");
1342 (Exec.*ExecInfo.CustomRenderers[RendererFnID])(
1343 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);
1344 break;
1345 }
1346 case GIR_ConstrainOperandRC: {
1347 uint64_t InsnID = readULEB();
1348 uint64_t OpIdx = readULEB();
1349 uint16_t RCEnum = readU16();
1350 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1351 MachineInstr &I = *OutMIs[InsnID].getInstr();
1352 MachineFunction &MF = *I.getParent()->getParent();
1353 MachineRegisterInfo &MRI = MF.getRegInfo();
1354 const TargetRegisterClass &RC = *TRI.getRegClass(i: RCEnum);
1355 MachineOperand &MO = I.getOperand(i: OpIdx);
1356 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, RegClass: RC, RegMO&: MO);
1357 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1358 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1359 << InsnID << "], " << OpIdx << ", " << RCEnum
1360 << ")\n");
1361 break;
1362 }
1363
1364 case GIR_ConstrainSelectedInstOperands: {
1365 uint64_t InsnID = readULEB();
1366 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1367 constrainSelectedInstRegOperands(I&: *OutMIs[InsnID].getInstr(), TII, TRI,
1368 RBI);
1369 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1370 dbgs() << CurrentIdx
1371 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1372 << InsnID << "])\n");
1373 break;
1374 }
1375
1376 case GIR_MergeMemOperands: {
1377 uint64_t InsnID = readULEB();
1378 uint64_t NumInsn = MatchTable[CurrentIdx++];
1379 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1380
1381 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1382 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1383 << InsnID << "]");
1384 for (unsigned K = 0; K < NumInsn; ++K) {
1385 uint64_t NextID = readULEB();
1386 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1387 dbgs() << ", MIs[" << NextID << "]");
1388 for (const auto &MMO : State.MIs[NextID]->memoperands())
1389 OutMIs[InsnID].addMemOperand(MMO);
1390 }
1391 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
1392 break;
1393 }
1394
1395 case GIR_EraseFromParent: {
1396 uint64_t InsnID = readULEB();
1397 MachineInstr *MI = State.MIs[InsnID];
1398 assert(MI && "Attempted to erase an undefined instruction");
1399 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1400 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1401 << InsnID << "])\n");
1402 // If we're erasing the insertion point, ensure we don't leave a dangling
1403 // pointer in the builder.
1404 if (Builder.getInsertPt() == MI)
1405 Builder.setInsertPt(MBB&: *MI->getParent(), II: ++MI->getIterator());
1406 if (Observer)
1407 Observer->erasingInstr(MI&: *MI);
1408 MI->eraseFromParent();
1409 break;
1410 }
1411
1412 case GIR_MakeTempReg: {
1413 uint64_t TempRegID = readULEB();
1414 int TypeID = readS8();
1415
1416 State.TempRegisters[TempRegID] =
1417 MRI.createGenericVirtualRegister(Ty: getTypeFromIdx(TypeID));
1418 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1419 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1420 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1421 break;
1422 }
1423 case GIR_ReplaceReg: {
1424 uint64_t OldInsnID = readULEB();
1425 uint64_t OldOpIdx = readULEB();
1426 uint64_t NewInsnID = readULEB();
1427 uint64_t NewOpIdx = readULEB();
1428
1429 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1430 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
1431 << OldInsnID << "][" << OldOpIdx << "] = MIs["
1432 << NewInsnID << "][" << NewOpIdx << "])\n");
1433
1434 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
1435 Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg();
1436 if (Observer)
1437 Observer->changingAllUsesOfReg(MRI, Reg: Old);
1438 MRI.replaceRegWith(FromReg: Old, ToReg: New);
1439 if (Observer)
1440 Observer->finishedChangingAllUsesOfReg();
1441 break;
1442 }
1443 case GIR_ReplaceRegWithTempReg: {
1444 uint64_t OldInsnID = readULEB();
1445 uint64_t OldOpIdx = readULEB();
1446 uint64_t TempRegID = readULEB();
1447
1448 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1449 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
1450 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["
1451 << TempRegID << "])\n");
1452
1453 Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg();
1454 Register New = State.TempRegisters[TempRegID];
1455 if (Observer)
1456 Observer->changingAllUsesOfReg(MRI, Reg: Old);
1457 MRI.replaceRegWith(FromReg: Old, ToReg: New);
1458 if (Observer)
1459 Observer->finishedChangingAllUsesOfReg();
1460 break;
1461 }
1462 case GIR_Coverage: {
1463 uint32_t RuleID = readU32();
1464 assert(CoverageInfo);
1465 CoverageInfo->setCovered(RuleID);
1466
1467 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx
1468 << ": GIR_Coverage("
1469 << RuleID << ")");
1470 break;
1471 }
1472
1473 case GIR_Done:
1474 DEBUG_WITH_TYPE(TgtExecutor::getName(),
1475 dbgs() << CurrentIdx << ": GIR_Done\n");
1476 propagateFlags();
1477 return true;
1478 default:
1479 llvm_unreachable("Unexpected command");
1480 }
1481 }
1482}
1483
1484} // end namespace llvm
1485
1486#endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H
1487

source code of llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h