1 | //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | /// \file This file implements GIMatchTableExecutor's `executeMatchTable` |
10 | /// function. This is implemented in a separate file because the function is |
11 | /// quite large. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
16 | #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
17 | |
18 | #include "llvm/ADT/SmallVector.h" |
19 | #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" |
20 | #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" |
21 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
22 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
23 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
24 | #include "llvm/CodeGen/MachineOperand.h" |
25 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
26 | #include "llvm/CodeGen/RegisterBankInfo.h" |
27 | #include "llvm/CodeGen/TargetInstrInfo.h" |
28 | #include "llvm/CodeGen/TargetOpcodes.h" |
29 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
30 | #include "llvm/IR/Constants.h" |
31 | #include "llvm/IR/DataLayout.h" |
32 | #include "llvm/IR/Type.h" |
33 | #include "llvm/Support/CodeGenCoverage.h" |
34 | #include "llvm/Support/Debug.h" |
35 | #include "llvm/Support/ErrorHandling.h" |
36 | #include "llvm/Support/LEB128.h" |
37 | #include "llvm/Support/raw_ostream.h" |
38 | #include <cassert> |
39 | #include <cstddef> |
40 | #include <cstdint> |
41 | |
42 | namespace llvm { |
43 | |
44 | template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, |
45 | class CustomRendererFn> |
46 | bool GIMatchTableExecutor::executeMatchTable( |
47 | TgtExecutor &Exec, MatcherState &State, |
48 | const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> |
49 | &ExecInfo, |
50 | MachineIRBuilder &Builder, const uint8_t *MatchTable, |
51 | const TargetInstrInfo &TII, MachineRegisterInfo &MRI, |
52 | const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, |
53 | const PredicateBitset &AvailableFeatures, |
54 | CodeGenCoverage *CoverageInfo) const { |
55 | |
56 | uint64_t CurrentIdx = 0; |
57 | SmallVector<uint64_t, 4> OnFailResumeAt; |
58 | NewMIVector OutMIs; |
59 | |
60 | GISelChangeObserver *Observer = Builder.getObserver(); |
61 | // Bypass the flag check on the instruction, and only look at the MCInstrDesc. |
62 | bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); |
63 | |
64 | const uint16_t Flags = State.MIs[0]->getFlags(); |
65 | |
66 | enum RejectAction { RejectAndGiveUp, RejectAndResume }; |
67 | auto handleReject = [&]() -> RejectAction { |
68 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
69 | dbgs() << CurrentIdx << ": Rejected\n" ); |
70 | if (OnFailResumeAt.empty()) |
71 | return RejectAndGiveUp; |
72 | CurrentIdx = OnFailResumeAt.pop_back_val(); |
73 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
74 | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" |
75 | << OnFailResumeAt.size() << " try-blocks remain)\n" ); |
76 | return RejectAndResume; |
77 | }; |
78 | |
79 | const auto propagateFlags = [&]() { |
80 | for (auto MIB : OutMIs) { |
81 | // Set the NoFPExcept flag when no original matched instruction could |
82 | // raise an FP exception, but the new instruction potentially might. |
83 | uint16_t MIBFlags = Flags; |
84 | if (NoFPException && MIB->mayRaiseFPException()) |
85 | MIBFlags |= MachineInstr::NoFPExcept; |
86 | if (Observer) |
87 | Observer->changingInstr(MI&: *MIB); |
88 | MIB.setMIFlags(MIBFlags); |
89 | if (Observer) |
90 | Observer->changedInstr(MI&: *MIB); |
91 | } |
92 | }; |
93 | |
94 | // If the index is >= 0, it's an index in the type objects generated by |
95 | // TableGen. If the index is <0, it's an index in the recorded types object. |
96 | const auto getTypeFromIdx = [&](int64_t Idx) -> LLT { |
97 | if (Idx >= 0) |
98 | return ExecInfo.TypeObjects[Idx]; |
99 | return State.RecordedTypes[1 - Idx]; |
100 | }; |
101 | |
102 | const auto readULEB = [&]() { |
103 | unsigned N = 0; |
104 | uint64_t Val = decodeULEB128(p: MatchTable + CurrentIdx, n: &N); |
105 | CurrentIdx += N; |
106 | return Val; |
107 | }; |
108 | |
109 | // Convenience function to return a signed value. This avoids |
110 | // us forgetting to first cast to int8_t before casting to a |
111 | // wider signed int type. |
112 | // if we casted uint8 directly to a wider type we'd lose |
113 | // negative values. |
114 | const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; }; |
115 | |
116 | const auto readU16 = [&]() { |
117 | auto V = readBytesAs<uint16_t>(MatchTable: MatchTable + CurrentIdx); |
118 | CurrentIdx += 2; |
119 | return V; |
120 | }; |
121 | |
122 | const auto readU32 = [&]() { |
123 | auto V = readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx); |
124 | CurrentIdx += 4; |
125 | return V; |
126 | }; |
127 | |
128 | const auto readU64 = [&]() { |
129 | auto V = readBytesAs<uint64_t>(MatchTable: MatchTable + CurrentIdx); |
130 | CurrentIdx += 8; |
131 | return V; |
132 | }; |
133 | |
134 | while (true) { |
135 | assert(CurrentIdx != ~0u && "Invalid MatchTable index" ); |
136 | uint8_t MatcherOpcode = MatchTable[CurrentIdx++]; |
137 | switch (MatcherOpcode) { |
138 | case GIM_Try: { |
139 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
140 | dbgs() << CurrentIdx << ": Begin try-block\n" ); |
141 | OnFailResumeAt.push_back(Elt: readU32()); |
142 | break; |
143 | } |
144 | |
145 | case GIM_RecordInsn: |
146 | case GIM_RecordInsnIgnoreCopies: { |
147 | uint64_t NewInsnID = readULEB(); |
148 | uint64_t InsnID = readULEB(); |
149 | uint64_t OpIdx = readULEB(); |
150 | |
151 | // As an optimisation we require that MIs[0] is always the root. Refuse |
152 | // any attempt to modify it. |
153 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]" ); |
154 | |
155 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
156 | if (!MO.isReg()) { |
157 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
158 | dbgs() << CurrentIdx << ": Not a register\n" ); |
159 | if (handleReject() == RejectAndGiveUp) |
160 | return false; |
161 | break; |
162 | } |
163 | if (MO.getReg().isPhysical()) { |
164 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
165 | dbgs() << CurrentIdx << ": Is a physical register\n" ); |
166 | if (handleReject() == RejectAndGiveUp) |
167 | return false; |
168 | break; |
169 | } |
170 | |
171 | MachineInstr *NewMI; |
172 | if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) |
173 | NewMI = getDefIgnoringCopies(Reg: MO.getReg(), MRI); |
174 | else |
175 | NewMI = MRI.getVRegDef(Reg: MO.getReg()); |
176 | |
177 | if ((size_t)NewInsnID < State.MIs.size()) |
178 | State.MIs[NewInsnID] = NewMI; |
179 | else { |
180 | assert((size_t)NewInsnID == State.MIs.size() && |
181 | "Expected to store MIs in order" ); |
182 | State.MIs.push_back(Elt: NewMI); |
183 | } |
184 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
185 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID |
186 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx |
187 | << ")\n" ); |
188 | break; |
189 | } |
190 | |
191 | case GIM_CheckFeatures: { |
192 | uint16_t ExpectedBitsetID = readU16(); |
193 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
194 | dbgs() << CurrentIdx |
195 | << ": GIM_CheckFeatures(ExpectedBitsetID=" |
196 | << ExpectedBitsetID << ")\n" ); |
197 | if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != |
198 | ExecInfo.FeatureBitsets[ExpectedBitsetID]) { |
199 | if (handleReject() == RejectAndGiveUp) |
200 | return false; |
201 | } |
202 | break; |
203 | } |
204 | case GIM_CheckOpcode: |
205 | case GIM_CheckOpcodeIsEither: { |
206 | uint64_t InsnID = readULEB(); |
207 | uint16_t Expected0 = readU16(); |
208 | uint16_t Expected1 = -1; |
209 | if (MatcherOpcode == GIM_CheckOpcodeIsEither) |
210 | Expected1 = readU16(); |
211 | |
212 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
213 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); |
214 | |
215 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
216 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID |
217 | << "], ExpectedOpcode=" << Expected0; |
218 | if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() |
219 | << " || " << Expected1; |
220 | dbgs() << ") // Got=" << Opcode << "\n" ;); |
221 | |
222 | if (Opcode != Expected0 && Opcode != Expected1) { |
223 | if (handleReject() == RejectAndGiveUp) |
224 | return false; |
225 | } |
226 | break; |
227 | } |
228 | case GIM_SwitchOpcode: { |
229 | uint64_t InsnID = readULEB(); |
230 | uint16_t LowerBound = readU16(); |
231 | uint16_t UpperBound = readU16(); |
232 | uint32_t Default = readU32(); |
233 | |
234 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
235 | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); |
236 | |
237 | DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
238 | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" |
239 | << LowerBound << ", " << UpperBound << "), Default=" << Default |
240 | << ", JumpTable...) // Got=" << Opcode << "\n" ; |
241 | }); |
242 | if (Opcode < LowerBound || UpperBound <= Opcode) { |
243 | CurrentIdx = Default; |
244 | break; |
245 | } |
246 | const auto EntryIdx = (Opcode - LowerBound); |
247 | // Each entry is 4 bytes |
248 | CurrentIdx = |
249 | readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (EntryIdx * 4)); |
250 | if (!CurrentIdx) { |
251 | CurrentIdx = Default; |
252 | break; |
253 | } |
254 | OnFailResumeAt.push_back(Elt: Default); |
255 | break; |
256 | } |
257 | |
258 | case GIM_SwitchType: { |
259 | uint64_t InsnID = readULEB(); |
260 | uint64_t OpIdx = readULEB(); |
261 | uint16_t LowerBound = readU16(); |
262 | uint16_t UpperBound = readU16(); |
263 | int64_t Default = readU32(); |
264 | |
265 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
266 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
267 | |
268 | DEBUG_WITH_TYPE(TgtExecutor::getName(), { |
269 | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID |
270 | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " |
271 | << UpperBound << "), Default=" << Default |
272 | << ", JumpTable...) // Got=" ; |
273 | if (!MO.isReg()) |
274 | dbgs() << "Not a VReg\n" ; |
275 | else |
276 | dbgs() << MRI.getType(MO.getReg()) << "\n" ; |
277 | }); |
278 | if (!MO.isReg()) { |
279 | CurrentIdx = Default; |
280 | break; |
281 | } |
282 | const LLT Ty = MRI.getType(Reg: MO.getReg()); |
283 | const auto TyI = ExecInfo.TypeIDMap.find(Ty); |
284 | if (TyI == ExecInfo.TypeIDMap.end()) { |
285 | CurrentIdx = Default; |
286 | break; |
287 | } |
288 | const int64_t TypeID = TyI->second; |
289 | if (TypeID < LowerBound || UpperBound <= TypeID) { |
290 | CurrentIdx = Default; |
291 | break; |
292 | } |
293 | const auto NumEntry = (TypeID - LowerBound); |
294 | // Each entry is 4 bytes |
295 | CurrentIdx = |
296 | readBytesAs<uint32_t>(MatchTable: MatchTable + CurrentIdx + (NumEntry * 4)); |
297 | if (!CurrentIdx) { |
298 | CurrentIdx = Default; |
299 | break; |
300 | } |
301 | OnFailResumeAt.push_back(Elt: Default); |
302 | break; |
303 | } |
304 | |
305 | case GIM_CheckNumOperands: { |
306 | uint64_t InsnID = readULEB(); |
307 | uint64_t Expected = readULEB(); |
308 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
309 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" |
310 | << InsnID << "], Expected=" << Expected << ")\n" ); |
311 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
312 | if (State.MIs[InsnID]->getNumOperands() != Expected) { |
313 | if (handleReject() == RejectAndGiveUp) |
314 | return false; |
315 | } |
316 | break; |
317 | } |
318 | case GIM_CheckI64ImmPredicate: |
319 | case GIM_CheckImmOperandPredicate: { |
320 | uint64_t InsnID = readULEB(); |
321 | unsigned OpIdx = |
322 | MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1; |
323 | uint16_t Predicate = readU16(); |
324 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
325 | dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" |
326 | << InsnID << "]->getOperand(" << OpIdx |
327 | << "), Predicate=" << Predicate << ")\n" ); |
328 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
329 | assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || |
330 | State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && |
331 | "Expected immediate operand" ); |
332 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
333 | int64_t Value = 0; |
334 | if (State.MIs[InsnID]->getOperand(i: OpIdx).isCImm()) |
335 | Value = State.MIs[InsnID]->getOperand(i: OpIdx).getCImm()->getSExtValue(); |
336 | else if (State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) |
337 | Value = State.MIs[InsnID]->getOperand(i: OpIdx).getImm(); |
338 | else |
339 | llvm_unreachable("Expected Imm or CImm operand" ); |
340 | |
341 | if (!testImmPredicate_I64(Predicate, Value)) |
342 | if (handleReject() == RejectAndGiveUp) |
343 | return false; |
344 | break; |
345 | } |
346 | case GIM_CheckAPIntImmPredicate: { |
347 | uint64_t InsnID = readULEB(); |
348 | uint16_t Predicate = readU16(); |
349 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
350 | dbgs() |
351 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" |
352 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
353 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
354 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
355 | "Expected G_CONSTANT" ); |
356 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
357 | if (!State.MIs[InsnID]->getOperand(i: 1).isCImm()) |
358 | llvm_unreachable("Expected Imm or CImm operand" ); |
359 | |
360 | const APInt &Value = |
361 | State.MIs[InsnID]->getOperand(i: 1).getCImm()->getValue(); |
362 | if (!testImmPredicate_APInt(Predicate, Value)) |
363 | if (handleReject() == RejectAndGiveUp) |
364 | return false; |
365 | break; |
366 | } |
367 | case GIM_CheckAPFloatImmPredicate: { |
368 | uint64_t InsnID = readULEB(); |
369 | uint16_t Predicate = readU16(); |
370 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
371 | dbgs() |
372 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" |
373 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
374 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
375 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
376 | "Expected G_FCONSTANT" ); |
377 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && |
378 | "Expected FPImm operand" ); |
379 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
380 | const APFloat &Value = |
381 | State.MIs[InsnID]->getOperand(i: 1).getFPImm()->getValueAPF(); |
382 | |
383 | if (!testImmPredicate_APFloat(Predicate, Value)) |
384 | if (handleReject() == RejectAndGiveUp) |
385 | return false; |
386 | break; |
387 | } |
388 | case GIM_CheckIsBuildVectorAllOnes: |
389 | case GIM_CheckIsBuildVectorAllZeros: { |
390 | uint64_t InsnID = readULEB(); |
391 | |
392 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
393 | dbgs() << CurrentIdx |
394 | << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" |
395 | << InsnID << "])\n" ); |
396 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
397 | |
398 | const MachineInstr *MI = State.MIs[InsnID]; |
399 | assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || |
400 | MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && |
401 | "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC" ); |
402 | |
403 | if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { |
404 | if (!isBuildVectorAllOnes(MI: *MI, MRI)) { |
405 | if (handleReject() == RejectAndGiveUp) |
406 | return false; |
407 | } |
408 | } else { |
409 | if (!isBuildVectorAllZeros(MI: *MI, MRI)) { |
410 | if (handleReject() == RejectAndGiveUp) |
411 | return false; |
412 | } |
413 | } |
414 | |
415 | break; |
416 | } |
417 | case GIM_CheckSimplePredicate: { |
418 | // Note: we don't check for invalid here because this is purely a hook to |
419 | // allow some executors (such as the combiner) to check arbitrary, |
420 | // contextless predicates, such as whether a rule is enabled or not. |
421 | uint16_t Predicate = readU16(); |
422 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
423 | dbgs() << CurrentIdx |
424 | << ": GIM_CheckSimplePredicate(Predicate=" |
425 | << Predicate << ")\n" ); |
426 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
427 | if (!testSimplePredicate(Predicate)) { |
428 | if (handleReject() == RejectAndGiveUp) |
429 | return false; |
430 | } |
431 | break; |
432 | } |
433 | case GIM_CheckCxxInsnPredicate: { |
434 | uint64_t InsnID = readULEB(); |
435 | uint16_t Predicate = readU16(); |
436 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
437 | dbgs() |
438 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" |
439 | << InsnID << "], Predicate=" << Predicate << ")\n" ); |
440 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
441 | assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate" ); |
442 | |
443 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) |
444 | if (handleReject() == RejectAndGiveUp) |
445 | return false; |
446 | break; |
447 | } |
448 | case GIM_CheckHasNoUse: { |
449 | uint64_t InsnID = readULEB(); |
450 | |
451 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
452 | dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" |
453 | << InsnID << "]\n" ); |
454 | |
455 | const MachineInstr *MI = State.MIs[InsnID]; |
456 | assert(MI && "Used insn before defined" ); |
457 | assert(MI->getNumDefs() > 0 && "No defs" ); |
458 | const Register Res = MI->getOperand(i: 0).getReg(); |
459 | |
460 | if (!MRI.use_nodbg_empty(RegNo: Res)) { |
461 | if (handleReject() == RejectAndGiveUp) |
462 | return false; |
463 | } |
464 | |
465 | break; |
466 | } |
467 | case GIM_CheckAtomicOrdering: { |
468 | uint64_t InsnID = readULEB(); |
469 | auto Ordering = (AtomicOrdering)readULEB(); |
470 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
471 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" |
472 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
473 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
474 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
475 | if (handleReject() == RejectAndGiveUp) |
476 | return false; |
477 | |
478 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
479 | if (MMO->getMergedOrdering() != Ordering) |
480 | if (handleReject() == RejectAndGiveUp) |
481 | return false; |
482 | break; |
483 | } |
484 | case GIM_CheckAtomicOrderingOrStrongerThan: { |
485 | uint64_t InsnID = readULEB(); |
486 | auto Ordering = (AtomicOrdering)readULEB(); |
487 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
488 | dbgs() << CurrentIdx |
489 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" |
490 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
491 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
492 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
493 | if (handleReject() == RejectAndGiveUp) |
494 | return false; |
495 | |
496 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
497 | if (!isAtLeastOrStrongerThan(AO: MMO->getMergedOrdering(), Other: Ordering)) |
498 | if (handleReject() == RejectAndGiveUp) |
499 | return false; |
500 | break; |
501 | } |
502 | case GIM_CheckAtomicOrderingWeakerThan: { |
503 | uint64_t InsnID = readULEB(); |
504 | auto Ordering = (AtomicOrdering)readULEB(); |
505 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
506 | dbgs() << CurrentIdx |
507 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" |
508 | << InsnID << "], " << (uint64_t)Ordering << ")\n" ); |
509 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
510 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
511 | if (handleReject() == RejectAndGiveUp) |
512 | return false; |
513 | |
514 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
515 | if (!isStrongerThan(AO: Ordering, Other: MMO->getMergedOrdering())) |
516 | if (handleReject() == RejectAndGiveUp) |
517 | return false; |
518 | break; |
519 | } |
520 | case GIM_CheckMemoryAddressSpace: { |
521 | uint64_t InsnID = readULEB(); |
522 | uint64_t MMOIdx = readULEB(); |
523 | // This accepts a list of possible address spaces. |
524 | const uint64_t NumAddrSpace = readULEB(); |
525 | |
526 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
527 | if (handleReject() == RejectAndGiveUp) |
528 | return false; |
529 | break; |
530 | } |
531 | |
532 | // Need to still jump to the end of the list of address spaces if we find |
533 | // a match earlier. |
534 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; |
535 | |
536 | const MachineMemOperand *MMO = |
537 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
538 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); |
539 | |
540 | bool Success = false; |
541 | for (unsigned I = 0; I != NumAddrSpace; ++I) { |
542 | uint64_t AddrSpace = readULEB(); |
543 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
544 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " |
545 | << AddrSpace << '\n'); |
546 | |
547 | if (AddrSpace == MMOAddrSpace) { |
548 | Success = true; |
549 | break; |
550 | } |
551 | } |
552 | |
553 | CurrentIdx = LastIdx; |
554 | if (!Success && handleReject() == RejectAndGiveUp) |
555 | return false; |
556 | break; |
557 | } |
558 | case GIM_CheckMemoryAlignment: { |
559 | uint64_t InsnID = readULEB(); |
560 | uint64_t MMOIdx = readULEB(); |
561 | uint64_t MinAlign = readULEB(); |
562 | |
563 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
564 | |
565 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
566 | if (handleReject() == RejectAndGiveUp) |
567 | return false; |
568 | break; |
569 | } |
570 | |
571 | MachineMemOperand *MMO = |
572 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
573 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
574 | dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" |
575 | << "(MIs[" << InsnID << "]->memoperands() + " |
576 | << MMOIdx << ")->getAlignment() >= " << MinAlign |
577 | << ")\n" ); |
578 | if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) |
579 | return false; |
580 | |
581 | break; |
582 | } |
583 | case GIM_CheckMemorySizeEqualTo: { |
584 | uint64_t InsnID = readULEB(); |
585 | uint64_t MMOIdx = readULEB(); |
586 | uint32_t Size = readU32(); |
587 | |
588 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
589 | dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" |
590 | << InsnID << "]->memoperands() + " << MMOIdx |
591 | << ", Size=" << Size << ")\n" ); |
592 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
593 | |
594 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
595 | if (handleReject() == RejectAndGiveUp) |
596 | return false; |
597 | break; |
598 | } |
599 | |
600 | MachineMemOperand *MMO = |
601 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
602 | |
603 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() |
604 | << " bytes vs " << Size |
605 | << " bytes\n" ); |
606 | if (MMO->getSize() != Size) |
607 | if (handleReject() == RejectAndGiveUp) |
608 | return false; |
609 | |
610 | break; |
611 | } |
612 | case GIM_CheckMemorySizeEqualToLLT: |
613 | case GIM_CheckMemorySizeLessThanLLT: |
614 | case GIM_CheckMemorySizeGreaterThanLLT: { |
615 | uint64_t InsnID = readULEB(); |
616 | uint64_t MMOIdx = readULEB(); |
617 | uint64_t OpIdx = readULEB(); |
618 | |
619 | DEBUG_WITH_TYPE( |
620 | TgtExecutor::getName(), |
621 | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" |
622 | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" |
623 | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT |
624 | ? "GreaterThan" |
625 | : "LessThan" ) |
626 | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx |
627 | << ", OpIdx=" << OpIdx << ")\n" ); |
628 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
629 | |
630 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
631 | if (!MO.isReg()) { |
632 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
633 | dbgs() << CurrentIdx << ": Not a register\n" ); |
634 | if (handleReject() == RejectAndGiveUp) |
635 | return false; |
636 | break; |
637 | } |
638 | |
639 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
640 | if (handleReject() == RejectAndGiveUp) |
641 | return false; |
642 | break; |
643 | } |
644 | |
645 | MachineMemOperand *MMO = |
646 | *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
647 | |
648 | unsigned Size = MRI.getType(Reg: MO.getReg()).getSizeInBits(); |
649 | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && |
650 | MMO->getSizeInBits() != Size) { |
651 | if (handleReject() == RejectAndGiveUp) |
652 | return false; |
653 | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && |
654 | MMO->getSizeInBits() >= Size) { |
655 | if (handleReject() == RejectAndGiveUp) |
656 | return false; |
657 | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && |
658 | MMO->getSizeInBits() <= Size) |
659 | if (handleReject() == RejectAndGiveUp) |
660 | return false; |
661 | |
662 | break; |
663 | } |
664 | case GIM_CheckType: { |
665 | uint64_t InsnID = readULEB(); |
666 | uint64_t OpIdx = readULEB(); |
667 | int TypeID = readS8(); |
668 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
669 | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID |
670 | << "]->getOperand(" << OpIdx |
671 | << "), TypeID=" << TypeID << ")\n" ); |
672 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
673 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
674 | if (!MO.isReg() || MRI.getType(Reg: MO.getReg()) != getTypeFromIdx(TypeID)) { |
675 | if (handleReject() == RejectAndGiveUp) |
676 | return false; |
677 | } |
678 | break; |
679 | } |
680 | case GIM_CheckPointerToAny: { |
681 | uint64_t InsnID = readULEB(); |
682 | uint64_t OpIdx = readULEB(); |
683 | uint64_t SizeInBits = readULEB(); |
684 | |
685 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
686 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" |
687 | << InsnID << "]->getOperand(" << OpIdx |
688 | << "), SizeInBits=" << SizeInBits << ")\n" ); |
689 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
690 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
691 | const LLT Ty = MRI.getType(Reg: MO.getReg()); |
692 | |
693 | // iPTR must be looked up in the target. |
694 | if (SizeInBits == 0) { |
695 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); |
696 | const unsigned AddrSpace = Ty.getAddressSpace(); |
697 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AS: AddrSpace); |
698 | } |
699 | |
700 | assert(SizeInBits != 0 && "Pointer size must be known" ); |
701 | |
702 | if (MO.isReg()) { |
703 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) |
704 | if (handleReject() == RejectAndGiveUp) |
705 | return false; |
706 | } else if (handleReject() == RejectAndGiveUp) |
707 | return false; |
708 | |
709 | break; |
710 | } |
711 | case GIM_RecordNamedOperand: { |
712 | uint64_t InsnID = readULEB(); |
713 | uint64_t OpIdx = readULEB(); |
714 | uint64_t StoreIdx = readULEB(); |
715 | |
716 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
717 | dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" |
718 | << InsnID << "]->getOperand(" << OpIdx |
719 | << "), StoreIdx=" << StoreIdx << ")\n" ); |
720 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
721 | assert(StoreIdx < State.RecordedOperands.size() && "Index out of range" ); |
722 | State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(i: OpIdx); |
723 | break; |
724 | } |
725 | case GIM_RecordRegType: { |
726 | uint64_t InsnID = readULEB(); |
727 | uint64_t OpIdx = readULEB(); |
728 | int TypeIdx = readS8(); |
729 | |
730 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
731 | dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs[" |
732 | << InsnID << "]->getOperand(" << OpIdx |
733 | << "), TypeIdx=" << TypeIdx << ")\n" ); |
734 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
735 | assert(TypeIdx < 0 && "Temp types always have negative indexes!" ); |
736 | // Indexes start at -1. |
737 | TypeIdx = 1 - TypeIdx; |
738 | const auto &Op = State.MIs[InsnID]->getOperand(i: OpIdx); |
739 | if (State.RecordedTypes.size() <= (uint64_t)TypeIdx) |
740 | State.RecordedTypes.resize(N: TypeIdx + 1, NV: LLT()); |
741 | State.RecordedTypes[TypeIdx] = MRI.getType(Reg: Op.getReg()); |
742 | break; |
743 | } |
744 | case GIM_CheckRegBankForClass: { |
745 | uint64_t InsnID = readULEB(); |
746 | uint64_t OpIdx = readULEB(); |
747 | uint16_t RCEnum = readU16(); |
748 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
749 | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" |
750 | << InsnID << "]->getOperand(" << OpIdx |
751 | << "), RCEnum=" << RCEnum << ")\n" ); |
752 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
753 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
754 | if (!MO.isReg() || |
755 | &RBI.getRegBankFromRegClass(RC: *TRI.getRegClass(i: RCEnum), |
756 | Ty: MRI.getType(Reg: MO.getReg())) != |
757 | RBI.getRegBank(Reg: MO.getReg(), MRI, TRI)) { |
758 | if (handleReject() == RejectAndGiveUp) |
759 | return false; |
760 | } |
761 | break; |
762 | } |
763 | |
764 | case GIM_CheckComplexPattern: { |
765 | uint64_t InsnID = readULEB(); |
766 | uint64_t OpIdx = readULEB(); |
767 | uint16_t RendererID = readU16(); |
768 | uint16_t ComplexPredicateID = readU16(); |
769 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
770 | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID |
771 | << "] = GIM_CheckComplexPattern(MIs[" << InsnID |
772 | << "]->getOperand(" << OpIdx |
773 | << "), ComplexPredicateID=" << ComplexPredicateID |
774 | << ")\n" ); |
775 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
776 | // FIXME: Use std::invoke() when it's available. |
777 | ComplexRendererFns Renderer = |
778 | (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( |
779 | State.MIs[InsnID]->getOperand(i: OpIdx)); |
780 | if (Renderer) |
781 | State.Renderers[RendererID] = *Renderer; |
782 | else if (handleReject() == RejectAndGiveUp) |
783 | return false; |
784 | break; |
785 | } |
786 | |
787 | case GIM_CheckConstantInt: |
788 | case GIM_CheckConstantInt8: { |
789 | const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8); |
790 | |
791 | uint64_t InsnID = readULEB(); |
792 | uint64_t OpIdx = readULEB(); |
793 | uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64(); |
794 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
795 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" |
796 | << InsnID << "]->getOperand(" << OpIdx |
797 | << "), Value=" << Value << ")\n" ); |
798 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
799 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
800 | if (MO.isReg()) { |
801 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. |
802 | LLT Ty = MRI.getType(Reg: MO.getReg()); |
803 | // If the type is > 64 bits, it can't be a constant int, so we bail |
804 | // early because SignExtend64 will assert otherwise. |
805 | if (Ty.getScalarSizeInBits() > 64) { |
806 | if (handleReject() == RejectAndGiveUp) |
807 | return false; |
808 | break; |
809 | } |
810 | |
811 | Value = SignExtend64(X: Value, B: Ty.getScalarSizeInBits()); |
812 | if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/Splat: true)) { |
813 | if (handleReject() == RejectAndGiveUp) |
814 | return false; |
815 | } |
816 | } else if (handleReject() == RejectAndGiveUp) |
817 | return false; |
818 | |
819 | break; |
820 | } |
821 | |
822 | case GIM_CheckLiteralInt: { |
823 | uint64_t InsnID = readULEB(); |
824 | uint64_t OpIdx = readULEB(); |
825 | int64_t Value = readU64(); |
826 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
827 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" |
828 | << InsnID << "]->getOperand(" << OpIdx |
829 | << "), Value=" << Value << ")\n" ); |
830 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
831 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
832 | if (MO.isImm() && MO.getImm() == Value) |
833 | break; |
834 | |
835 | if (MO.isCImm() && MO.getCImm()->equalsInt(V: Value)) |
836 | break; |
837 | |
838 | if (handleReject() == RejectAndGiveUp) |
839 | return false; |
840 | |
841 | break; |
842 | } |
843 | |
844 | case GIM_CheckIntrinsicID: { |
845 | uint64_t InsnID = readULEB(); |
846 | uint64_t OpIdx = readULEB(); |
847 | uint16_t Value = readU16(); |
848 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
849 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" |
850 | << InsnID << "]->getOperand(" << OpIdx |
851 | << "), Value=" << Value << ")\n" ); |
852 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
853 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
854 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) |
855 | if (handleReject() == RejectAndGiveUp) |
856 | return false; |
857 | break; |
858 | } |
859 | case GIM_CheckCmpPredicate: { |
860 | uint64_t InsnID = readULEB(); |
861 | uint64_t OpIdx = readULEB(); |
862 | uint16_t Value = readU16(); |
863 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
864 | dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" |
865 | << InsnID << "]->getOperand(" << OpIdx |
866 | << "), Value=" << Value << ")\n" ); |
867 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
868 | MachineOperand &MO = State.MIs[InsnID]->getOperand(i: OpIdx); |
869 | if (!MO.isPredicate() || MO.getPredicate() != Value) |
870 | if (handleReject() == RejectAndGiveUp) |
871 | return false; |
872 | break; |
873 | } |
874 | case GIM_CheckIsMBB: { |
875 | uint64_t InsnID = readULEB(); |
876 | uint64_t OpIdx = readULEB(); |
877 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
878 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID |
879 | << "]->getOperand(" << OpIdx << "))\n" ); |
880 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
881 | if (!State.MIs[InsnID]->getOperand(i: OpIdx).isMBB()) { |
882 | if (handleReject() == RejectAndGiveUp) |
883 | return false; |
884 | } |
885 | break; |
886 | } |
887 | case GIM_CheckIsImm: { |
888 | uint64_t InsnID = readULEB(); |
889 | uint64_t OpIdx = readULEB(); |
890 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
891 | dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID |
892 | << "]->getOperand(" << OpIdx << "))\n" ); |
893 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
894 | if (!State.MIs[InsnID]->getOperand(i: OpIdx).isImm()) { |
895 | if (handleReject() == RejectAndGiveUp) |
896 | return false; |
897 | } |
898 | break; |
899 | } |
900 | case GIM_CheckIsSafeToFold: { |
901 | uint64_t InsnID = readULEB(); |
902 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
903 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" |
904 | << InsnID << "])\n" ); |
905 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
906 | if (!isObviouslySafeToFold(MI&: *State.MIs[InsnID], IntoMI&: *State.MIs[0])) { |
907 | if (handleReject() == RejectAndGiveUp) |
908 | return false; |
909 | } |
910 | break; |
911 | } |
912 | case GIM_CheckIsSameOperand: |
913 | case GIM_CheckIsSameOperandIgnoreCopies: { |
914 | uint64_t InsnID = readULEB(); |
915 | uint64_t OpIdx = readULEB(); |
916 | uint64_t OtherInsnID = readULEB(); |
917 | uint64_t OtherOpIdx = readULEB(); |
918 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
919 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" |
920 | << InsnID << "][" << OpIdx << "], MIs[" |
921 | << OtherInsnID << "][" << OtherOpIdx << "])\n" ); |
922 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined" ); |
923 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined" ); |
924 | |
925 | MachineOperand &Op = State.MIs[InsnID]->getOperand(i: OpIdx); |
926 | MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(i: OtherOpIdx); |
927 | |
928 | if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { |
929 | if (Op.isReg() && OtherOp.isReg()) { |
930 | if (getSrcRegIgnoringCopies(Reg: Op.getReg(), MRI) == |
931 | getSrcRegIgnoringCopies(Reg: OtherOp.getReg(), MRI)) |
932 | break; |
933 | } |
934 | } |
935 | |
936 | if (!Op.isIdenticalTo(Other: OtherOp)) { |
937 | if (handleReject() == RejectAndGiveUp) |
938 | return false; |
939 | } |
940 | break; |
941 | } |
942 | case GIM_CheckCanReplaceReg: { |
943 | uint64_t OldInsnID = readULEB(); |
944 | uint64_t OldOpIdx = readULEB(); |
945 | uint64_t NewInsnID = readULEB(); |
946 | uint64_t NewOpIdx = readULEB(); |
947 | |
948 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
949 | dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs[" |
950 | << OldInsnID << "][" << OldOpIdx << "] = MIs[" |
951 | << NewInsnID << "][" << NewOpIdx << "])\n" ); |
952 | |
953 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
954 | Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg(); |
955 | if (!canReplaceReg(DstReg: Old, SrcReg: New, MRI)) { |
956 | if (handleReject() == RejectAndGiveUp) |
957 | return false; |
958 | } |
959 | break; |
960 | } |
961 | case GIM_MIFlags: { |
962 | uint64_t InsnID = readULEB(); |
963 | uint32_t Flags = readU32(); |
964 | |
965 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
966 | dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID |
967 | << "], " << Flags << ")\n" ); |
968 | if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) { |
969 | if (handleReject() == RejectAndGiveUp) |
970 | return false; |
971 | } |
972 | break; |
973 | } |
974 | case GIM_MIFlagsNot: { |
975 | uint64_t InsnID = readULEB(); |
976 | uint32_t Flags = readU32(); |
977 | |
978 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
979 | dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID |
980 | << "], " << Flags << ")\n" ); |
981 | if ((State.MIs[InsnID]->getFlags() & Flags)) { |
982 | if (handleReject() == RejectAndGiveUp) |
983 | return false; |
984 | } |
985 | break; |
986 | } |
987 | case GIM_Reject: |
988 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
989 | dbgs() << CurrentIdx << ": GIM_Reject\n" ); |
990 | if (handleReject() == RejectAndGiveUp) |
991 | return false; |
992 | break; |
993 | case GIR_MutateOpcode: { |
994 | uint64_t OldInsnID = readULEB(); |
995 | uint64_t NewInsnID = readULEB(); |
996 | uint16_t NewOpcode = readU16(); |
997 | if (NewInsnID >= OutMIs.size()) |
998 | OutMIs.resize(N: NewInsnID + 1); |
999 | |
1000 | MachineInstr *OldMI = State.MIs[OldInsnID]; |
1001 | if (Observer) |
1002 | Observer->changingInstr(MI&: *OldMI); |
1003 | OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI); |
1004 | OutMIs[NewInsnID]->setDesc(TII.get(Opcode: NewOpcode)); |
1005 | if (Observer) |
1006 | Observer->changedInstr(MI&: *OldMI); |
1007 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1008 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" |
1009 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1010 | << NewOpcode << ")\n" ); |
1011 | break; |
1012 | } |
1013 | |
1014 | case GIR_BuildMI: { |
1015 | uint64_t NewInsnID = readULEB(); |
1016 | uint16_t Opcode = readU16(); |
1017 | if (NewInsnID >= OutMIs.size()) |
1018 | OutMIs.resize(N: NewInsnID + 1); |
1019 | |
1020 | OutMIs[NewInsnID] = Builder.buildInstr(Opcode); |
1021 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1022 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" |
1023 | << NewInsnID << "], " << Opcode << ")\n" ); |
1024 | break; |
1025 | } |
1026 | |
1027 | case GIR_BuildConstant: { |
1028 | uint64_t TempRegID = readULEB(); |
1029 | uint64_t Imm = readU64(); |
1030 | Builder.buildConstant(Res: State.TempRegisters[TempRegID], Val: Imm); |
1031 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1032 | dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg[" |
1033 | << TempRegID << "], Imm=" << Imm << ")\n" ); |
1034 | break; |
1035 | } |
1036 | |
1037 | case GIR_Copy: { |
1038 | uint64_t NewInsnID = readULEB(); |
1039 | uint64_t OldInsnID = readULEB(); |
1040 | uint64_t OpIdx = readULEB(); |
1041 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1042 | OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: OpIdx)); |
1043 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1044 | dbgs() |
1045 | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID |
1046 | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n" ); |
1047 | break; |
1048 | } |
1049 | |
1050 | case GIR_CopyOrAddZeroReg: { |
1051 | uint64_t NewInsnID = readULEB(); |
1052 | uint64_t OldInsnID = readULEB(); |
1053 | uint64_t OpIdx = readULEB(); |
1054 | uint16_t ZeroReg = readU16(); |
1055 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1056 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(i: OpIdx); |
1057 | if (isOperandImmEqual(MO, Value: 0, MRI)) |
1058 | OutMIs[NewInsnID].addReg(RegNo: ZeroReg); |
1059 | else |
1060 | OutMIs[NewInsnID].add(MO); |
1061 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1062 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" |
1063 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1064 | << OpIdx << ", " << ZeroReg << ")\n" ); |
1065 | break; |
1066 | } |
1067 | |
1068 | case GIR_CopySubReg: { |
1069 | uint64_t NewInsnID = readULEB(); |
1070 | uint64_t OldInsnID = readULEB(); |
1071 | uint64_t OpIdx = readULEB(); |
1072 | uint16_t SubRegIdx = readU16(); |
1073 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1074 | OutMIs[NewInsnID].addReg(RegNo: State.MIs[OldInsnID]->getOperand(i: OpIdx).getReg(), |
1075 | flags: 0, SubReg: SubRegIdx); |
1076 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1077 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" |
1078 | << NewInsnID << "], MIs[" << OldInsnID << "], " |
1079 | << OpIdx << ", " << SubRegIdx << ")\n" ); |
1080 | break; |
1081 | } |
1082 | |
1083 | case GIR_AddImplicitDef: { |
1084 | uint64_t InsnID = readULEB(); |
1085 | uint16_t RegNum = readU16(); |
1086 | uint16_t Flags = readU16(); |
1087 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1088 | Flags |= RegState::Implicit; |
1089 | OutMIs[InsnID].addDef(RegNo: RegNum, Flags); |
1090 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1091 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" |
1092 | << InsnID << "], " << RegNum << ")\n" ); |
1093 | break; |
1094 | } |
1095 | |
1096 | case GIR_AddImplicitUse: { |
1097 | uint64_t InsnID = readULEB(); |
1098 | uint16_t RegNum = readU16(); |
1099 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1100 | OutMIs[InsnID].addUse(RegNo: RegNum, Flags: RegState::Implicit); |
1101 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1102 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" |
1103 | << InsnID << "], " << RegNum << ")\n" ); |
1104 | break; |
1105 | } |
1106 | |
1107 | case GIR_AddRegister: { |
1108 | uint64_t InsnID = readULEB(); |
1109 | uint16_t RegNum = readU16(); |
1110 | uint16_t RegFlags = readU16(); |
1111 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1112 | OutMIs[InsnID].addReg(RegNo: RegNum, flags: RegFlags); |
1113 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1114 | dbgs() |
1115 | << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID |
1116 | << "], " << RegNum << ", " << RegFlags << ")\n" ); |
1117 | break; |
1118 | } |
1119 | case GIR_AddIntrinsicID: { |
1120 | uint64_t InsnID = readULEB(); |
1121 | uint16_t Value = readU16(); |
1122 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1123 | OutMIs[InsnID].addIntrinsicID(ID: (Intrinsic::ID)Value); |
1124 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1125 | dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs[" |
1126 | << InsnID << "], " << Value << ")\n" ); |
1127 | break; |
1128 | } |
1129 | case GIR_SetImplicitDefDead: { |
1130 | uint64_t InsnID = readULEB(); |
1131 | uint64_t OpIdx = readULEB(); |
1132 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1133 | dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs[" |
1134 | << InsnID << "], OpIdx=" << OpIdx << ")\n" ); |
1135 | MachineInstr *MI = OutMIs[InsnID]; |
1136 | assert(MI && "Modifying undefined instruction" ); |
1137 | MI->getOperand(i: MI->getNumExplicitOperands() + OpIdx).setIsDead(); |
1138 | break; |
1139 | } |
1140 | case GIR_SetMIFlags: { |
1141 | uint64_t InsnID = readULEB(); |
1142 | uint32_t Flags = readU32(); |
1143 | |
1144 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1145 | dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs[" |
1146 | << InsnID << "], " << Flags << ")\n" ); |
1147 | MachineInstr *MI = OutMIs[InsnID]; |
1148 | MI->setFlags(MI->getFlags() | Flags); |
1149 | break; |
1150 | } |
1151 | case GIR_UnsetMIFlags: { |
1152 | uint64_t InsnID = readULEB(); |
1153 | uint32_t Flags = readU32(); |
1154 | |
1155 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1156 | dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs[" |
1157 | << InsnID << "], " << Flags << ")\n" ); |
1158 | MachineInstr *MI = OutMIs[InsnID]; |
1159 | MI->setFlags(MI->getFlags() & ~Flags); |
1160 | break; |
1161 | } |
1162 | case GIR_CopyMIFlags: { |
1163 | uint64_t InsnID = readULEB(); |
1164 | uint64_t OldInsnID = readULEB(); |
1165 | |
1166 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1167 | dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs[" |
1168 | << InsnID << "], MIs[" << OldInsnID << "])\n" ); |
1169 | MachineInstr *MI = OutMIs[InsnID]; |
1170 | MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags()); |
1171 | break; |
1172 | } |
1173 | case GIR_AddSimpleTempRegister: |
1174 | case GIR_AddTempRegister: |
1175 | case GIR_AddTempSubRegister: { |
1176 | uint64_t InsnID = readULEB(); |
1177 | uint64_t TempRegID = readULEB(); |
1178 | uint16_t TempRegFlags = 0; |
1179 | if (MatcherOpcode != GIR_AddSimpleTempRegister) |
1180 | TempRegFlags = readU16(); |
1181 | uint16_t SubReg = 0; |
1182 | if (MatcherOpcode == GIR_AddTempSubRegister) |
1183 | SubReg = readU16(); |
1184 | |
1185 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1186 | |
1187 | OutMIs[InsnID].addReg(RegNo: State.TempRegisters[TempRegID], flags: TempRegFlags, |
1188 | SubReg); |
1189 | DEBUG_WITH_TYPE( |
1190 | TgtExecutor::getName(), |
1191 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID |
1192 | << "], TempRegisters[" << TempRegID << "]" ; |
1193 | if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); |
1194 | dbgs() << ", " << TempRegFlags << ")\n" ); |
1195 | break; |
1196 | } |
1197 | |
1198 | case GIR_AddImm8: |
1199 | case GIR_AddImm: { |
1200 | const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8); |
1201 | uint64_t InsnID = readULEB(); |
1202 | uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64(); |
1203 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1204 | OutMIs[InsnID].addImm(Val: Imm); |
1205 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1206 | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID |
1207 | << "], " << Imm << ")\n" ); |
1208 | break; |
1209 | } |
1210 | |
1211 | case GIR_AddCImm: { |
1212 | uint64_t InsnID = readULEB(); |
1213 | int TypeID = readS8(); |
1214 | uint64_t Imm = readU64(); |
1215 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1216 | |
1217 | unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits(); |
1218 | LLVMContext &Ctx = MF->getFunction().getContext(); |
1219 | OutMIs[InsnID].addCImm( |
1220 | Val: ConstantInt::get(Ty: IntegerType::get(C&: Ctx, NumBits: Width), V: Imm, /*signed*/ IsSigned: true)); |
1221 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1222 | dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID |
1223 | << "], TypeID=" << TypeID << ", Imm=" << Imm |
1224 | << ")\n" ); |
1225 | break; |
1226 | } |
1227 | |
1228 | case GIR_ComplexRenderer: { |
1229 | uint64_t InsnID = readULEB(); |
1230 | uint16_t RendererID = readU16(); |
1231 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1232 | for (const auto &RenderOpFn : State.Renderers[RendererID]) |
1233 | RenderOpFn(OutMIs[InsnID]); |
1234 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1235 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" |
1236 | << InsnID << "], " << RendererID << ")\n" ); |
1237 | break; |
1238 | } |
1239 | case GIR_ComplexSubOperandRenderer: { |
1240 | uint64_t InsnID = readULEB(); |
1241 | uint16_t RendererID = readU16(); |
1242 | uint64_t RenderOpID = readULEB(); |
1243 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1244 | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); |
1245 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1246 | dbgs() << CurrentIdx |
1247 | << ": GIR_ComplexSubOperandRenderer(OutMIs[" |
1248 | << InsnID << "], " << RendererID << ", " |
1249 | << RenderOpID << ")\n" ); |
1250 | break; |
1251 | } |
1252 | case GIR_ComplexSubOperandSubRegRenderer: { |
1253 | uint64_t InsnID = readULEB(); |
1254 | uint16_t RendererID = readU16(); |
1255 | uint64_t RenderOpID = readULEB(); |
1256 | uint16_t SubRegIdx = readU16(); |
1257 | MachineInstrBuilder &MI = OutMIs[InsnID]; |
1258 | assert(MI && "Attempted to add to undefined instruction" ); |
1259 | State.Renderers[RendererID][RenderOpID](MI); |
1260 | MI->getOperand(i: MI->getNumOperands() - 1).setSubReg(SubRegIdx); |
1261 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1262 | dbgs() << CurrentIdx |
1263 | << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" |
1264 | << InsnID << "], " << RendererID << ", " |
1265 | << RenderOpID << ", " << SubRegIdx << ")\n" ); |
1266 | break; |
1267 | } |
1268 | |
1269 | case GIR_CopyConstantAsSImm: { |
1270 | uint64_t NewInsnID = readULEB(); |
1271 | uint64_t OldInsnID = readULEB(); |
1272 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1273 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
1274 | "Expected G_CONSTANT" ); |
1275 | if (State.MIs[OldInsnID]->getOperand(i: 1).isCImm()) { |
1276 | OutMIs[NewInsnID].addImm( |
1277 | Val: State.MIs[OldInsnID]->getOperand(i: 1).getCImm()->getSExtValue()); |
1278 | } else if (State.MIs[OldInsnID]->getOperand(i: 1).isImm()) |
1279 | OutMIs[NewInsnID].add(MO: State.MIs[OldInsnID]->getOperand(i: 1)); |
1280 | else |
1281 | llvm_unreachable("Expected Imm or CImm operand" ); |
1282 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1283 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" |
1284 | << NewInsnID << "], MIs[" << OldInsnID << "])\n" ); |
1285 | break; |
1286 | } |
1287 | |
1288 | // TODO: Needs a test case once we have a pattern that uses this. |
1289 | case GIR_CopyFConstantAsFPImm: { |
1290 | uint64_t NewInsnID = readULEB(); |
1291 | uint64_t OldInsnID = readULEB(); |
1292 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction" ); |
1293 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
1294 | "Expected G_FCONSTANT" ); |
1295 | if (State.MIs[OldInsnID]->getOperand(i: 1).isFPImm()) |
1296 | OutMIs[NewInsnID].addFPImm( |
1297 | Val: State.MIs[OldInsnID]->getOperand(i: 1).getFPImm()); |
1298 | else |
1299 | llvm_unreachable("Expected FPImm operand" ); |
1300 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1301 | dbgs() |
1302 | << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" |
1303 | << NewInsnID << "], MIs[" << OldInsnID << "])\n" ); |
1304 | break; |
1305 | } |
1306 | |
1307 | case GIR_CustomRenderer: { |
1308 | uint64_t InsnID = readULEB(); |
1309 | uint64_t OldInsnID = readULEB(); |
1310 | uint16_t RendererFnID = readU16(); |
1311 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1312 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1313 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" |
1314 | << InsnID << "], MIs[" << OldInsnID << "], " |
1315 | << RendererFnID << ")\n" ); |
1316 | (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
1317 | OutMIs[InsnID], *State.MIs[OldInsnID], |
1318 | -1); // Not a source operand of the old instruction. |
1319 | break; |
1320 | } |
1321 | case GIR_CustomAction: { |
1322 | uint16_t FnID = readU16(); |
1323 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1324 | dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID |
1325 | << ")\n" ); |
1326 | assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID" ); |
1327 | runCustomAction(FnID, State, OutMIs); |
1328 | break; |
1329 | } |
1330 | case GIR_CustomOperandRenderer: { |
1331 | uint64_t InsnID = readULEB(); |
1332 | uint64_t OldInsnID = readULEB(); |
1333 | uint64_t OpIdx = readULEB(); |
1334 | uint16_t RendererFnID = readU16(); |
1335 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1336 | |
1337 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1338 | dbgs() << CurrentIdx |
1339 | << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID |
1340 | << "], MIs[" << OldInsnID << "]->getOperand(" |
1341 | << OpIdx << "), " << RendererFnID << ")\n" ); |
1342 | (Exec.*ExecInfo.CustomRenderers[RendererFnID])( |
1343 | OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); |
1344 | break; |
1345 | } |
1346 | case GIR_ConstrainOperandRC: { |
1347 | uint64_t InsnID = readULEB(); |
1348 | uint64_t OpIdx = readULEB(); |
1349 | uint16_t RCEnum = readU16(); |
1350 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1351 | MachineInstr &I = *OutMIs[InsnID].getInstr(); |
1352 | MachineFunction &MF = *I.getParent()->getParent(); |
1353 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
1354 | const TargetRegisterClass &RC = *TRI.getRegClass(i: RCEnum); |
1355 | MachineOperand &MO = I.getOperand(i: OpIdx); |
1356 | constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt&: I, RegClass: RC, RegMO&: MO); |
1357 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1358 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" |
1359 | << InsnID << "], " << OpIdx << ", " << RCEnum |
1360 | << ")\n" ); |
1361 | break; |
1362 | } |
1363 | |
1364 | case GIR_ConstrainSelectedInstOperands: { |
1365 | uint64_t InsnID = readULEB(); |
1366 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1367 | constrainSelectedInstRegOperands(I&: *OutMIs[InsnID].getInstr(), TII, TRI, |
1368 | RBI); |
1369 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1370 | dbgs() << CurrentIdx |
1371 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" |
1372 | << InsnID << "])\n" ); |
1373 | break; |
1374 | } |
1375 | |
1376 | case GIR_MergeMemOperands: { |
1377 | uint64_t InsnID = readULEB(); |
1378 | uint64_t NumInsn = MatchTable[CurrentIdx++]; |
1379 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction" ); |
1380 | |
1381 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1382 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" |
1383 | << InsnID << "]" ); |
1384 | for (unsigned K = 0; K < NumInsn; ++K) { |
1385 | uint64_t NextID = readULEB(); |
1386 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1387 | dbgs() << ", MIs[" << NextID << "]" ); |
1388 | for (const auto &MMO : State.MIs[NextID]->memoperands()) |
1389 | OutMIs[InsnID].addMemOperand(MMO); |
1390 | } |
1391 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n" ); |
1392 | break; |
1393 | } |
1394 | |
1395 | case GIR_EraseFromParent: { |
1396 | uint64_t InsnID = readULEB(); |
1397 | MachineInstr *MI = State.MIs[InsnID]; |
1398 | assert(MI && "Attempted to erase an undefined instruction" ); |
1399 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1400 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" |
1401 | << InsnID << "])\n" ); |
1402 | // If we're erasing the insertion point, ensure we don't leave a dangling |
1403 | // pointer in the builder. |
1404 | if (Builder.getInsertPt() == MI) |
1405 | Builder.setInsertPt(MBB&: *MI->getParent(), II: ++MI->getIterator()); |
1406 | if (Observer) |
1407 | Observer->erasingInstr(MI&: *MI); |
1408 | MI->eraseFromParent(); |
1409 | break; |
1410 | } |
1411 | |
1412 | case GIR_MakeTempReg: { |
1413 | uint64_t TempRegID = readULEB(); |
1414 | int TypeID = readS8(); |
1415 | |
1416 | State.TempRegisters[TempRegID] = |
1417 | MRI.createGenericVirtualRegister(Ty: getTypeFromIdx(TypeID)); |
1418 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1419 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID |
1420 | << "] = GIR_MakeTempReg(" << TypeID << ")\n" ); |
1421 | break; |
1422 | } |
1423 | case GIR_ReplaceReg: { |
1424 | uint64_t OldInsnID = readULEB(); |
1425 | uint64_t OldOpIdx = readULEB(); |
1426 | uint64_t NewInsnID = readULEB(); |
1427 | uint64_t NewOpIdx = readULEB(); |
1428 | |
1429 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1430 | dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs[" |
1431 | << OldInsnID << "][" << OldOpIdx << "] = MIs[" |
1432 | << NewInsnID << "][" << NewOpIdx << "])\n" ); |
1433 | |
1434 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
1435 | Register New = State.MIs[NewInsnID]->getOperand(i: NewOpIdx).getReg(); |
1436 | if (Observer) |
1437 | Observer->changingAllUsesOfReg(MRI, Reg: Old); |
1438 | MRI.replaceRegWith(FromReg: Old, ToReg: New); |
1439 | if (Observer) |
1440 | Observer->finishedChangingAllUsesOfReg(); |
1441 | break; |
1442 | } |
1443 | case GIR_ReplaceRegWithTempReg: { |
1444 | uint64_t OldInsnID = readULEB(); |
1445 | uint64_t OldOpIdx = readULEB(); |
1446 | uint64_t TempRegID = readULEB(); |
1447 | |
1448 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1449 | dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs[" |
1450 | << OldInsnID << "][" << OldOpIdx << "] = TempRegs[" |
1451 | << TempRegID << "])\n" ); |
1452 | |
1453 | Register Old = State.MIs[OldInsnID]->getOperand(i: OldOpIdx).getReg(); |
1454 | Register New = State.TempRegisters[TempRegID]; |
1455 | if (Observer) |
1456 | Observer->changingAllUsesOfReg(MRI, Reg: Old); |
1457 | MRI.replaceRegWith(FromReg: Old, ToReg: New); |
1458 | if (Observer) |
1459 | Observer->finishedChangingAllUsesOfReg(); |
1460 | break; |
1461 | } |
1462 | case GIR_Coverage: { |
1463 | uint32_t RuleID = readU32(); |
1464 | assert(CoverageInfo); |
1465 | CoverageInfo->setCovered(RuleID); |
1466 | |
1467 | DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx |
1468 | << ": GIR_Coverage(" |
1469 | << RuleID << ")" ); |
1470 | break; |
1471 | } |
1472 | |
1473 | case GIR_Done: |
1474 | DEBUG_WITH_TYPE(TgtExecutor::getName(), |
1475 | dbgs() << CurrentIdx << ": GIR_Done\n" ); |
1476 | propagateFlags(); |
1477 | return true; |
1478 | default: |
1479 | llvm_unreachable("Unexpected command" ); |
1480 | } |
1481 | } |
1482 | } |
1483 | |
1484 | } // end namespace llvm |
1485 | |
1486 | #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H |
1487 | |