1	//===- MIParser.cpp - Machine instructions parser implementation ----------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the parsing of machine instructions.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "llvm/CodeGen/MIRParser/MIParser.h"
14	#include "MILexer.h"
15	#include "llvm/ADT/APInt.h"
16	#include "llvm/ADT/APSInt.h"
17	#include "llvm/ADT/ArrayRef.h"
18	#include "llvm/ADT/DenseMap.h"
19	#include "llvm/ADT/SmallVector.h"
20	#include "llvm/ADT/StringMap.h"
21	#include "llvm/ADT/StringRef.h"
22	#include "llvm/ADT/StringSwitch.h"
23	#include "llvm/ADT/Twine.h"
24	#include "llvm/Analysis/MemoryLocation.h"
25	#include "llvm/AsmParser/Parser.h"
26	#include "llvm/AsmParser/SlotMapping.h"
27	#include "llvm/CodeGen/MIRFormatter.h"
28	#include "llvm/CodeGen/MIRPrinter.h"
29	#include "llvm/CodeGen/MachineBasicBlock.h"
30	#include "llvm/CodeGen/MachineFrameInfo.h"
31	#include "llvm/CodeGen/MachineFunction.h"
32	#include "llvm/CodeGen/MachineInstr.h"
33	#include "llvm/CodeGen/MachineInstrBuilder.h"
34	#include "llvm/CodeGen/MachineMemOperand.h"
35	#include "llvm/CodeGen/MachineOperand.h"
36	#include "llvm/CodeGen/MachineRegisterInfo.h"
37	#include "llvm/CodeGen/PseudoSourceValueManager.h"
38	#include "llvm/CodeGen/RegisterBank.h"
39	#include "llvm/CodeGen/RegisterBankInfo.h"
40	#include "llvm/CodeGen/TargetInstrInfo.h"
41	#include "llvm/CodeGen/TargetRegisterInfo.h"
42	#include "llvm/CodeGen/TargetSubtargetInfo.h"
43	#include "llvm/CodeGenTypes/LowLevelType.h"
44	#include "llvm/IR/BasicBlock.h"
45	#include "llvm/IR/Constants.h"
46	#include "llvm/IR/DataLayout.h"
47	#include "llvm/IR/DebugInfoMetadata.h"
48	#include "llvm/IR/DebugLoc.h"
49	#include "llvm/IR/Function.h"
50	#include "llvm/IR/InstrTypes.h"
51	#include "llvm/IR/Instructions.h"
52	#include "llvm/IR/Intrinsics.h"
53	#include "llvm/IR/Metadata.h"
54	#include "llvm/IR/Module.h"
55	#include "llvm/IR/ModuleSlotTracker.h"
56	#include "llvm/IR/Type.h"
57	#include "llvm/IR/Value.h"
58	#include "llvm/IR/ValueSymbolTable.h"
59	#include "llvm/MC/LaneBitmask.h"
60	#include "llvm/MC/MCContext.h"
61	#include "llvm/MC/MCDwarf.h"
62	#include "llvm/MC/MCInstrDesc.h"
63	#include "llvm/Support/AtomicOrdering.h"
64	#include "llvm/Support/BranchProbability.h"
65	#include "llvm/Support/Casting.h"
66	#include "llvm/Support/ErrorHandling.h"
67	#include "llvm/Support/MemoryBuffer.h"
68	#include "llvm/Support/SMLoc.h"
69	#include "llvm/Support/SourceMgr.h"
70	#include "llvm/Target/TargetIntrinsicInfo.h"
71	#include "llvm/Target/TargetMachine.h"
72	#include <cassert>
73	#include <cctype>
74	#include <cstddef>
75	#include <cstdint>
76	#include <limits>
77	#include <string>
78	#include <utility>
79
80	using namespace llvm;
81
82	void PerTargetMIParsingState::setTarget(
83	const TargetSubtargetInfo &NewSubtarget) {
84
85	// If the subtarget changed, over conservatively assume everything is invalid.
86	if (&Subtarget == &NewSubtarget)
87	return;
88
89	Names2InstrOpCodes.clear();
90	Names2Regs.clear();
91	Names2RegMasks.clear();
92	Names2SubRegIndices.clear();
93	Names2TargetIndices.clear();
94	Names2DirectTargetFlags.clear();
95	Names2BitmaskTargetFlags.clear();
96	Names2MMOTargetFlags.clear();
97
98	initNames2RegClasses();
99	initNames2RegBanks();
100	}
101
102	void PerTargetMIParsingState::initNames2Regs() {
103	if (!Names2Regs.empty())
104	return;
105
106	// The '%noreg' register is the register 0.
107	Names2Regs.insert(KV: std::make_pair(x: "noreg", y: 0));
108	const auto *TRI = Subtarget.getRegisterInfo();
109	assert(TRI && "Expected target register info");
110
111	for (unsigned I = 0, E = TRI->getNumRegs(); I < E; ++I) {
112	bool WasInserted =
113	Names2Regs.insert(KV: std::make_pair(x: StringRef(TRI->getName(RegNo: I)).lower(), y&: I))
114	.second;
115	(void)WasInserted;
116	assert(WasInserted && "Expected registers to be unique case-insensitively");
117	}
118	}
119
120	bool PerTargetMIParsingState::getRegisterByName(StringRef RegName,
121	Register &Reg) {
122	initNames2Regs();
123	auto RegInfo = Names2Regs.find(Key: RegName);
124	if (RegInfo == Names2Regs.end())
125	return true;
126	Reg = RegInfo->getValue();
127	return false;
128	}
129
130	void PerTargetMIParsingState::initNames2InstrOpCodes() {
131	if (!Names2InstrOpCodes.empty())
132	return;
133	const auto *TII = Subtarget.getInstrInfo();
134	assert(TII && "Expected target instruction info");
135	for (unsigned I = 0, E = TII->getNumOpcodes(); I < E; ++I)
136	Names2InstrOpCodes.insert(KV: std::make_pair(x: StringRef(TII->getName(Opcode: I)), y&: I));
137	}
138
139	bool PerTargetMIParsingState::parseInstrName(StringRef InstrName,
140	unsigned &OpCode) {
141	initNames2InstrOpCodes();
142	auto InstrInfo = Names2InstrOpCodes.find(Key: InstrName);
143	if (InstrInfo == Names2InstrOpCodes.end())
144	return true;
145	OpCode = InstrInfo->getValue();
146	return false;
147	}
148
149	void PerTargetMIParsingState::initNames2RegMasks() {
150	if (!Names2RegMasks.empty())
151	return;
152	const auto *TRI = Subtarget.getRegisterInfo();
153	assert(TRI && "Expected target register info");
154	ArrayRef<const uint32_t *> RegMasks = TRI->getRegMasks();
155	ArrayRef<const char *> RegMaskNames = TRI->getRegMaskNames();
156	assert(RegMasks.size() == RegMaskNames.size());
157	for (size_t I = 0, E = RegMasks.size(); I < E; ++I)
158	Names2RegMasks.insert(
159	KV: std::make_pair(x: StringRef(RegMaskNames[I]).lower(), y: RegMasks[I]));
160	}
161
162	const uint32_t *PerTargetMIParsingState::getRegMask(StringRef Identifier) {
163	initNames2RegMasks();
164	auto RegMaskInfo = Names2RegMasks.find(Key: Identifier);
165	if (RegMaskInfo == Names2RegMasks.end())
166	return nullptr;
167	return RegMaskInfo->getValue();
168	}
169
170	void PerTargetMIParsingState::initNames2SubRegIndices() {
171	if (!Names2SubRegIndices.empty())
172	return;
173	const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
174	for (unsigned I = 1, E = TRI->getNumSubRegIndices(); I < E; ++I)
175	Names2SubRegIndices.insert(
176	KV: std::make_pair(x: TRI->getSubRegIndexName(SubIdx: I), y&: I));
177	}
178
179	unsigned PerTargetMIParsingState::getSubRegIndex(StringRef Name) {
180	initNames2SubRegIndices();
181	auto SubRegInfo = Names2SubRegIndices.find(Key: Name);
182	if (SubRegInfo == Names2SubRegIndices.end())
183	return 0;
184	return SubRegInfo->getValue();
185	}
186
187	void PerTargetMIParsingState::initNames2TargetIndices() {
188	if (!Names2TargetIndices.empty())
189	return;
190	const auto *TII = Subtarget.getInstrInfo();
191	assert(TII && "Expected target instruction info");
192	auto Indices = TII->getSerializableTargetIndices();
193	for (const auto &I : Indices)
194	Names2TargetIndices.insert(KV: std::make_pair(x: StringRef(I.second), y: I.first));
195	}
196
197	bool PerTargetMIParsingState::getTargetIndex(StringRef Name, int &Index) {
198	initNames2TargetIndices();
199	auto IndexInfo = Names2TargetIndices.find(Key: Name);
200	if (IndexInfo == Names2TargetIndices.end())
201	return true;
202	Index = IndexInfo->second;
203	return false;
204	}
205
206	void PerTargetMIParsingState::initNames2DirectTargetFlags() {
207	if (!Names2DirectTargetFlags.empty())
208	return;
209
210	const auto *TII = Subtarget.getInstrInfo();
211	assert(TII && "Expected target instruction info");
212	auto Flags = TII->getSerializableDirectMachineOperandTargetFlags();
213	for (const auto &I : Flags)
214	Names2DirectTargetFlags.insert(
215	KV: std::make_pair(x: StringRef(I.second), y: I.first));
216	}
217
218	bool PerTargetMIParsingState::getDirectTargetFlag(StringRef Name,
219	unsigned &Flag) {
220	initNames2DirectTargetFlags();
221	auto FlagInfo = Names2DirectTargetFlags.find(Key: Name);
222	if (FlagInfo == Names2DirectTargetFlags.end())
223	return true;
224	Flag = FlagInfo->second;
225	return false;
226	}
227
228	void PerTargetMIParsingState::initNames2BitmaskTargetFlags() {
229	if (!Names2BitmaskTargetFlags.empty())
230	return;
231
232	const auto *TII = Subtarget.getInstrInfo();
233	assert(TII && "Expected target instruction info");
234	auto Flags = TII->getSerializableBitmaskMachineOperandTargetFlags();
235	for (const auto &I : Flags)
236	Names2BitmaskTargetFlags.insert(
237	KV: std::make_pair(x: StringRef(I.second), y: I.first));
238	}
239
240	bool PerTargetMIParsingState::getBitmaskTargetFlag(StringRef Name,
241	unsigned &Flag) {
242	initNames2BitmaskTargetFlags();
243	auto FlagInfo = Names2BitmaskTargetFlags.find(Key: Name);
244	if (FlagInfo == Names2BitmaskTargetFlags.end())
245	return true;
246	Flag = FlagInfo->second;
247	return false;
248	}
249
250	void PerTargetMIParsingState::initNames2MMOTargetFlags() {
251	if (!Names2MMOTargetFlags.empty())
252	return;
253
254	const auto *TII = Subtarget.getInstrInfo();
255	assert(TII && "Expected target instruction info");
256	auto Flags = TII->getSerializableMachineMemOperandTargetFlags();
257	for (const auto &I : Flags)
258	Names2MMOTargetFlags.insert(KV: std::make_pair(x: StringRef(I.second), y: I.first));
259	}
260
261	bool PerTargetMIParsingState::getMMOTargetFlag(StringRef Name,
262	MachineMemOperand::Flags &Flag) {
263	initNames2MMOTargetFlags();
264	auto FlagInfo = Names2MMOTargetFlags.find(Key: Name);
265	if (FlagInfo == Names2MMOTargetFlags.end())
266	return true;
267	Flag = FlagInfo->second;
268	return false;
269	}
270
271	void PerTargetMIParsingState::initNames2RegClasses() {
272	if (!Names2RegClasses.empty())
273	return;
274
275	const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
276	for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; ++I) {
277	const auto *RC = TRI->getRegClass(i: I);
278	Names2RegClasses.insert(
279	KV: std::make_pair(x: StringRef(TRI->getRegClassName(Class: RC)).lower(), y&: RC));
280	}
281	}
282
283	void PerTargetMIParsingState::initNames2RegBanks() {
284	if (!Names2RegBanks.empty())
285	return;
286
287	const RegisterBankInfo *RBI = Subtarget.getRegBankInfo();
288	// If the target does not support GlobalISel, we may not have a
289	// register bank info.
290	if (!RBI)
291	return;
292
293	for (unsigned I = 0, E = RBI->getNumRegBanks(); I < E; ++I) {
294	const auto &RegBank = RBI->getRegBank(ID: I);
295	Names2RegBanks.insert(
296	KV: std::make_pair(x: StringRef(RegBank.getName()).lower(), y: &RegBank));
297	}
298	}
299
300	const TargetRegisterClass *
301	PerTargetMIParsingState::getRegClass(StringRef Name) {
302	auto RegClassInfo = Names2RegClasses.find(Key: Name);
303	if (RegClassInfo == Names2RegClasses.end())
304	return nullptr;
305	return RegClassInfo->getValue();
306	}
307
308	const RegisterBank *PerTargetMIParsingState::getRegBank(StringRef Name) {
309	auto RegBankInfo = Names2RegBanks.find(Key: Name);
310	if (RegBankInfo == Names2RegBanks.end())
311	return nullptr;
312	return RegBankInfo->getValue();
313	}
314
315	PerFunctionMIParsingState::PerFunctionMIParsingState(MachineFunction &MF,
316	SourceMgr &SM, const SlotMapping &IRSlots, PerTargetMIParsingState &T)
317	: MF(MF), SM(&SM), IRSlots(IRSlots), Target(T) {
318	}
319
320	VRegInfo &PerFunctionMIParsingState::getVRegInfo(Register Num) {
321	auto I = VRegInfos.insert(KV: std::make_pair(x&: Num, y: nullptr));
322	if (I.second) {
323	MachineRegisterInfo &MRI = MF.getRegInfo();
324	VRegInfo *Info = new (Allocator) VRegInfo;
325	Info->VReg = MRI.createIncompleteVirtualRegister();
326	I.first->second = Info;
327	}
328	return *I.first->second;
329	}
330
331	VRegInfo &PerFunctionMIParsingState::getVRegInfoNamed(StringRef RegName) {
332	assert(RegName != "" && "Expected named reg.");
333
334	auto I = VRegInfosNamed.insert(KV: std::make_pair(x: RegName.str(), y: nullptr));
335	if (I.second) {
336	VRegInfo *Info = new (Allocator) VRegInfo;
337	Info->VReg = MF.getRegInfo().createIncompleteVirtualRegister(Name: RegName);
338	I.first->second = Info;
339	}
340	return *I.first->second;
341	}
342
343	static void mapValueToSlot(const Value *V, ModuleSlotTracker &MST,
344	DenseMap<unsigned, const Value *> &Slots2Values) {
345	int Slot = MST.getLocalSlot(V);
346	if (Slot == -1)
347	return;
348	Slots2Values.insert(KV: std::make_pair(x: unsigned(Slot), y&: V));
349	}
350
351	/// Creates the mapping from slot numbers to function's unnamed IR values.
352	static void initSlots2Values(const Function &F,
353	DenseMap<unsigned, const Value *> &Slots2Values) {
354	ModuleSlotTracker MST(F.getParent(), /*ShouldInitializeAllMetadata=*/false);
355	MST.incorporateFunction(F);
356	for (const auto &Arg : F.args())
357	mapValueToSlot(V: &Arg, MST, Slots2Values);
358	for (const auto &BB : F) {
359	mapValueToSlot(V: &BB, MST, Slots2Values);
360	for (const auto &I : BB)
361	mapValueToSlot(V: &I, MST, Slots2Values);
362	}
363	}
364
365	const Value* PerFunctionMIParsingState::getIRValue(unsigned Slot) {
366	if (Slots2Values.empty())
367	initSlots2Values(F: MF.getFunction(), Slots2Values);
368	return Slots2Values.lookup(Val: Slot);
369	}
370
371	namespace {
372
373	/// A wrapper struct around the 'MachineOperand' struct that includes a source
374	/// range and other attributes.
375	struct ParsedMachineOperand {
376	MachineOperand Operand;
377	StringRef::iterator Begin;
378	StringRef::iterator End;
379	std::optional<unsigned> TiedDefIdx;
380
381	ParsedMachineOperand(const MachineOperand &Operand, StringRef::iterator Begin,
382	StringRef::iterator End,
383	std::optional<unsigned> &TiedDefIdx)
384	: Operand(Operand), Begin(Begin), End(End), TiedDefIdx(TiedDefIdx) {
385	if (TiedDefIdx)
386	assert(Operand.isReg() && Operand.isUse() &&
387	"Only used register operands can be tied");
388	}
389	};
390
391	class MIParser {
392	MachineFunction &MF;
393	SMDiagnostic &Error;
394	StringRef Source, CurrentSource;
395	SMRange SourceRange;
396	MIToken Token;
397	PerFunctionMIParsingState &PFS;
398	/// Maps from slot numbers to function's unnamed basic blocks.
399	DenseMap<unsigned, const BasicBlock *> Slots2BasicBlocks;
400
401	public:
402	MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
403	StringRef Source);
404	MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
405	StringRef Source, SMRange SourceRange);
406
407	/// \p SkipChar gives the number of characters to skip before looking
408	/// for the next token.
409	void lex(unsigned SkipChar = 0);
410
411	/// Report an error at the current location with the given message.
412	///
413	/// This function always return true.
414	bool error(const Twine &Msg);
415
416	/// Report an error at the given location with the given message.
417	///
418	/// This function always return true.
419	bool error(StringRef::iterator Loc, const Twine &Msg);
420
421	bool
422	parseBasicBlockDefinitions(DenseMap<unsigned, MachineBasicBlock *> &MBBSlots);
423	bool parseBasicBlocks();
424	bool parse(MachineInstr *&MI);
425	bool parseStandaloneMBB(MachineBasicBlock *&MBB);
426	bool parseStandaloneNamedRegister(Register &Reg);
427	bool parseStandaloneVirtualRegister(VRegInfo *&Info);
428	bool parseStandaloneRegister(Register &Reg);
429	bool parseStandaloneStackObject(int &FI);
430	bool parseStandaloneMDNode(MDNode *&Node);
431	bool parseMachineMetadata();
432	bool parseMDTuple(MDNode *&MD, bool IsDistinct);
433	bool parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts);
434	bool parseMetadata(Metadata *&MD);
435
436	bool
437	parseBasicBlockDefinition(DenseMap<unsigned, MachineBasicBlock *> &MBBSlots);
438	bool parseBasicBlock(MachineBasicBlock &MBB,
439	MachineBasicBlock *&AddFalthroughFrom);
440	bool parseBasicBlockLiveins(MachineBasicBlock &MBB);
441	bool parseBasicBlockSuccessors(MachineBasicBlock &MBB);
442
443	bool parseNamedRegister(Register &Reg);
444	bool parseVirtualRegister(VRegInfo *&Info);
445	bool parseNamedVirtualRegister(VRegInfo *&Info);
446	bool parseRegister(Register &Reg, VRegInfo *&VRegInfo);
447	bool parseRegisterFlag(unsigned &Flags);
448	bool parseRegisterClassOrBank(VRegInfo &RegInfo);
449	bool parseSubRegisterIndex(unsigned &SubReg);
450	bool parseRegisterTiedDefIndex(unsigned &TiedDefIdx);
451	bool parseRegisterOperand(MachineOperand &Dest,
452	std::optional<unsigned> &TiedDefIdx,
453	bool IsDef = false);
454	bool parseImmediateOperand(MachineOperand &Dest);
455	bool parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
456	const Constant *&C);
457	bool parseIRConstant(StringRef::iterator Loc, const Constant *&C);
458	bool parseLowLevelType(StringRef::iterator Loc, LLT &Ty);
459	bool parseTypedImmediateOperand(MachineOperand &Dest);
460	bool parseFPImmediateOperand(MachineOperand &Dest);
461	bool parseMBBReference(MachineBasicBlock *&MBB);
462	bool parseMBBOperand(MachineOperand &Dest);
463	bool parseStackFrameIndex(int &FI);
464	bool parseStackObjectOperand(MachineOperand &Dest);
465	bool parseFixedStackFrameIndex(int &FI);
466	bool parseFixedStackObjectOperand(MachineOperand &Dest);
467	bool parseGlobalValue(GlobalValue *&GV);
468	bool parseGlobalAddressOperand(MachineOperand &Dest);
469	bool parseConstantPoolIndexOperand(MachineOperand &Dest);
470	bool parseSubRegisterIndexOperand(MachineOperand &Dest);
471	bool parseJumpTableIndexOperand(MachineOperand &Dest);
472	bool parseExternalSymbolOperand(MachineOperand &Dest);
473	bool parseMCSymbolOperand(MachineOperand &Dest);
474	[[nodiscard]] bool parseMDNode(MDNode *&Node);
475	bool parseDIExpression(MDNode *&Expr);
476	bool parseDILocation(MDNode *&Expr);
477	bool parseMetadataOperand(MachineOperand &Dest);
478	bool parseCFIOffset(int &Offset);
479	bool parseCFIRegister(Register &Reg);
480	bool parseCFIAddressSpace(unsigned &AddressSpace);
481	bool parseCFIEscapeValues(std::string& Values);
482	bool parseCFIOperand(MachineOperand &Dest);
483	bool parseIRBlock(BasicBlock *&BB, const Function &F);
484	bool parseBlockAddressOperand(MachineOperand &Dest);
485	bool parseIntrinsicOperand(MachineOperand &Dest);
486	bool parsePredicateOperand(MachineOperand &Dest);
487	bool parseShuffleMaskOperand(MachineOperand &Dest);
488	bool parseTargetIndexOperand(MachineOperand &Dest);
489	bool parseDbgInstrRefOperand(MachineOperand &Dest);
490	bool parseCustomRegisterMaskOperand(MachineOperand &Dest);
491	bool parseLiveoutRegisterMaskOperand(MachineOperand &Dest);
492	bool parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
493	MachineOperand &Dest,
494	std::optional<unsigned> &TiedDefIdx);
495	bool parseMachineOperandAndTargetFlags(const unsigned OpCode,
496	const unsigned OpIdx,
497	MachineOperand &Dest,
498	std::optional<unsigned> &TiedDefIdx);
499	bool parseOffset(int64_t &Offset);
500	bool parseIRBlockAddressTaken(BasicBlock *&BB);
501	bool parseAlignment(uint64_t &Alignment);
502	bool parseAddrspace(unsigned &Addrspace);
503	bool parseSectionID(std::optional<MBBSectionID> &SID);
504	bool parseBBID(std::optional<UniqueBBID> &BBID);
505	bool parseCallFrameSize(unsigned &CallFrameSize);
506	bool parseOperandsOffset(MachineOperand &Op);
507	bool parseIRValue(const Value *&V);
508	bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags);
509	bool parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV);
510	bool parseMachinePointerInfo(MachinePointerInfo &Dest);
511	bool parseOptionalScope(LLVMContext &Context, SyncScope::ID &SSID);
512	bool parseOptionalAtomicOrdering(AtomicOrdering &Order);
513	bool parseMachineMemoryOperand(MachineMemOperand *&Dest);
514	bool parsePreOrPostInstrSymbol(MCSymbol *&Symbol);
515	bool parseHeapAllocMarker(MDNode *&Node);
516	bool parsePCSections(MDNode *&Node);
517
518	bool parseTargetImmMnemonic(const unsigned OpCode, const unsigned OpIdx,
519	MachineOperand &Dest, const MIRFormatter &MF);
520
521	private:
522	/// Convert the integer literal in the current token into an unsigned integer.
523	///
524	/// Return true if an error occurred.
525	bool getUnsigned(unsigned &Result);
526
527	/// Convert the integer literal in the current token into an uint64.
528	///
529	/// Return true if an error occurred.
530	bool getUint64(uint64_t &Result);
531
532	/// Convert the hexadecimal literal in the current token into an unsigned
533	/// APInt with a minimum bitwidth required to represent the value.
534	///
535	/// Return true if the literal does not represent an integer value.
536	bool getHexUint(APInt &Result);
537
538	/// If the current token is of the given kind, consume it and return false.
539	/// Otherwise report an error and return true.
540	bool expectAndConsume(MIToken::TokenKind TokenKind);
541
542	/// If the current token is of the given kind, consume it and return true.
543	/// Otherwise return false.
544	bool consumeIfPresent(MIToken::TokenKind TokenKind);
545
546	bool parseInstruction(unsigned &OpCode, unsigned &Flags);
547
548	bool assignRegisterTies(MachineInstr &MI,
549	ArrayRef<ParsedMachineOperand> Operands);
550
551	bool verifyImplicitOperands(ArrayRef<ParsedMachineOperand> Operands,
552	const MCInstrDesc &MCID);
553
554	const BasicBlock *getIRBlock(unsigned Slot);
555	const BasicBlock *getIRBlock(unsigned Slot, const Function &F);
556
557	/// Get or create an MCSymbol for a given name.
558	MCSymbol *getOrCreateMCSymbol(StringRef Name);
559
560	/// parseStringConstant
561	/// ::= StringConstant
562	bool parseStringConstant(std::string &Result);
563
564	/// Map the location in the MI string to the corresponding location specified
565	/// in `SourceRange`.
566	SMLoc mapSMLoc(StringRef::iterator Loc);
567	};
568
569	} // end anonymous namespace
570
571	MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
572	StringRef Source)
573	: MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source), PFS(PFS)
574	{}
575
576	MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
577	StringRef Source, SMRange SourceRange)
578	: MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source),
579	SourceRange(SourceRange), PFS(PFS) {}
580
581	void MIParser::lex(unsigned SkipChar) {
582	CurrentSource = lexMIToken(
583	Source: CurrentSource.slice(Start: SkipChar, End: StringRef::npos), Token,
584	ErrorCallback: [this](StringRef::iterator Loc, const Twine &Msg) { error(Loc, Msg); });
585	}
586
587	bool MIParser::error(const Twine &Msg) { return error(Loc: Token.location(), Msg); }
588
589	bool MIParser::error(StringRef::iterator Loc, const Twine &Msg) {
590	const SourceMgr &SM = *PFS.SM;
591	assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
592	const MemoryBuffer &Buffer = *SM.getMemoryBuffer(i: SM.getMainFileID());
593	if (Loc >= Buffer.getBufferStart() && Loc <= Buffer.getBufferEnd()) {
594	// Create an ordinary diagnostic when the source manager's buffer is the
595	// source string.
596	Error = SM.GetMessage(Loc: SMLoc::getFromPointer(Ptr: Loc), Kind: SourceMgr::DK_Error, Msg);
597	return true;
598	}
599	// Create a diagnostic for a YAML string literal.
600	Error = SMDiagnostic(SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
601	Loc - Source.data(), SourceMgr::DK_Error, Msg.str(),
602	Source, std::nullopt, std::nullopt);
603	return true;
604	}
605
606	SMLoc MIParser::mapSMLoc(StringRef::iterator Loc) {
607	assert(SourceRange.isValid() && "Invalid source range");
608	assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
609	return SMLoc::getFromPointer(Ptr: SourceRange.Start.getPointer() +
610	(Loc - Source.data()));
611	}
612
613	typedef function_ref<bool(StringRef::iterator Loc, const Twine &)>
614	ErrorCallbackType;
615
616	static const char *toString(MIToken::TokenKind TokenKind) {
617	switch (TokenKind) {
618	case MIToken::comma:
619	return "','";
620	case MIToken::equal:
621	return "'='";
622	case MIToken::colon:
623	return "':'";
624	case MIToken::lparen:
625	return "'('";
626	case MIToken::rparen:
627	return "')'";
628	default:
629	return "<unknown token>";
630	}
631	}
632
633	bool MIParser::expectAndConsume(MIToken::TokenKind TokenKind) {
634	if (Token.isNot(K: TokenKind))
635	return error(Msg: Twine("expected ") + toString(TokenKind));
636	lex();
637	return false;
638	}
639
640	bool MIParser::consumeIfPresent(MIToken::TokenKind TokenKind) {
641	if (Token.isNot(K: TokenKind))
642	return false;
643	lex();
644	return true;
645	}
646
647	// Parse Machine Basic Block Section ID.
648	bool MIParser::parseSectionID(std::optional<MBBSectionID> &SID) {
649	assert(Token.is(MIToken::kw_bbsections));
650	lex();
651	if (Token.is(K: MIToken::IntegerLiteral)) {
652	unsigned Value = 0;
653	if (getUnsigned(Result&: Value))
654	return error(Msg: "Unknown Section ID");
655	SID = MBBSectionID{Value};
656	} else {
657	const StringRef &S = Token.stringValue();
658	if (S == "Exception")
659	SID = MBBSectionID::ExceptionSectionID;
660	else if (S == "Cold")
661	SID = MBBSectionID::ColdSectionID;
662	else
663	return error(Msg: "Unknown Section ID");
664	}
665	lex();
666	return false;
667	}
668
669	// Parse Machine Basic Block ID.
670	bool MIParser::parseBBID(std::optional<UniqueBBID> &BBID) {
671	assert(Token.is(MIToken::kw_bb_id));
672	lex();
673	unsigned BaseID = 0;
674	unsigned CloneID = 0;
675	if (getUnsigned(Result&: BaseID))
676	return error(Msg: "Unknown BB ID");
677	lex();
678	if (Token.is(K: MIToken::IntegerLiteral)) {
679	if (getUnsigned(Result&: CloneID))
680	return error(Msg: "Unknown Clone ID");
681	lex();
682	}
683	BBID = {.BaseID: BaseID, .CloneID: CloneID};
684	return false;
685	}
686
687	// Parse basic block call frame size.
688	bool MIParser::parseCallFrameSize(unsigned &CallFrameSize) {
689	assert(Token.is(MIToken::kw_call_frame_size));
690	lex();
691	unsigned Value = 0;
692	if (getUnsigned(Result&: Value))
693	return error(Msg: "Unknown call frame size");
694	CallFrameSize = Value;
695	lex();
696	return false;
697	}
698
699	bool MIParser::parseBasicBlockDefinition(
700	DenseMap<unsigned, MachineBasicBlock *> &MBBSlots) {
701	assert(Token.is(MIToken::MachineBasicBlockLabel));
702	unsigned ID = 0;
703	if (getUnsigned(Result&: ID))
704	return true;
705	auto Loc = Token.location();
706	auto Name = Token.stringValue();
707	lex();
708	bool MachineBlockAddressTaken = false;
709	BasicBlock *AddressTakenIRBlock = nullptr;
710	bool IsLandingPad = false;
711	bool IsInlineAsmBrIndirectTarget = false;
712	bool IsEHFuncletEntry = false;
713	std::optional<MBBSectionID> SectionID;
714	uint64_t Alignment = 0;
715	std::optional<UniqueBBID> BBID;
716	unsigned CallFrameSize = 0;
717	BasicBlock *BB = nullptr;
718	if (consumeIfPresent(TokenKind: MIToken::lparen)) {
719	do {
720	// TODO: Report an error when multiple same attributes are specified.
721	switch (Token.kind()) {
722	case MIToken::kw_machine_block_address_taken:
723	MachineBlockAddressTaken = true;
724	lex();
725	break;
726	case MIToken::kw_ir_block_address_taken:
727	if (parseIRBlockAddressTaken(BB&: AddressTakenIRBlock))
728	return true;
729	break;
730	case MIToken::kw_landing_pad:
731	IsLandingPad = true;
732	lex();
733	break;
734	case MIToken::kw_inlineasm_br_indirect_target:
735	IsInlineAsmBrIndirectTarget = true;
736	lex();
737	break;
738	case MIToken::kw_ehfunclet_entry:
739	IsEHFuncletEntry = true;
740	lex();
741	break;
742	case MIToken::kw_align:
743	if (parseAlignment(Alignment))
744	return true;
745	break;
746	case MIToken::IRBlock:
747	case MIToken::NamedIRBlock:
748	// TODO: Report an error when both name and ir block are specified.
749	if (parseIRBlock(BB, F: MF.getFunction()))
750	return true;
751	lex();
752	break;
753	case MIToken::kw_bbsections:
754	if (parseSectionID(SID&: SectionID))
755	return true;
756	break;
757	case MIToken::kw_bb_id:
758	if (parseBBID(BBID))
759	return true;
760	break;
761	case MIToken::kw_call_frame_size:
762	if (parseCallFrameSize(CallFrameSize))
763	return true;
764	break;
765	default:
766	break;
767	}
768	} while (consumeIfPresent(TokenKind: MIToken::comma));
769	if (expectAndConsume(TokenKind: MIToken::rparen))
770	return true;
771	}
772	if (expectAndConsume(TokenKind: MIToken::colon))
773	return true;
774
775	if (!Name.empty()) {
776	BB = dyn_cast_or_null<BasicBlock>(
777	Val: MF.getFunction().getValueSymbolTable()->lookup(Name));
778	if (!BB)
779	return error(Loc, Msg: Twine("basic block '") + Name +
780	"' is not defined in the function '" +
781	MF.getName() + "'");
782	}
783	auto *MBB = MF.CreateMachineBasicBlock(BB);
784	MF.insert(MBBI: MF.end(), MBB);
785	bool WasInserted = MBBSlots.insert(KV: std::make_pair(x&: ID, y&: MBB)).second;
786	if (!WasInserted)
787	return error(Loc, Msg: Twine("redefinition of machine basic block with id #") +
788	Twine(ID));
789	if (Alignment)
790	MBB->setAlignment(Align(Alignment));
791	if (MachineBlockAddressTaken)
792	MBB->setMachineBlockAddressTaken();
793	if (AddressTakenIRBlock)
794	MBB->setAddressTakenIRBlock(AddressTakenIRBlock);
795	MBB->setIsEHPad(IsLandingPad);
796	MBB->setIsInlineAsmBrIndirectTarget(IsInlineAsmBrIndirectTarget);
797	MBB->setIsEHFuncletEntry(IsEHFuncletEntry);
798	if (SectionID) {
799	MBB->setSectionID(*SectionID);
800	MF.setBBSectionsType(BasicBlockSection::List);
801	}
802	if (BBID.has_value()) {
803	// BBSectionsType is set to `List` if any basic blocks has `SectionID`.
804	// Here, we set it to `Labels` if it hasn't been set above.
805	if (!MF.hasBBSections())
806	MF.setBBSectionsType(BasicBlockSection::Labels);
807	MBB->setBBID(BBID.value());
808	}
809	MBB->setCallFrameSize(CallFrameSize);
810	return false;
811	}
812
813	bool MIParser::parseBasicBlockDefinitions(
814	DenseMap<unsigned, MachineBasicBlock *> &MBBSlots) {
815	lex();
816	// Skip until the first machine basic block.
817	while (Token.is(K: MIToken::Newline))
818	lex();
819	if (Token.isErrorOrEOF())
820	return Token.isError();
821	if (Token.isNot(K: MIToken::MachineBasicBlockLabel))
822	return error(Msg: "expected a basic block definition before instructions");
823	unsigned BraceDepth = 0;
824	do {
825	if (parseBasicBlockDefinition(MBBSlots))
826	return true;
827	bool IsAfterNewline = false;
828	// Skip until the next machine basic block.
829	while (true) {
830	if ((Token.is(K: MIToken::MachineBasicBlockLabel) && IsAfterNewline) ||
831	Token.isErrorOrEOF())
832	break;
833	else if (Token.is(K: MIToken::MachineBasicBlockLabel))
834	return error(Msg: "basic block definition should be located at the start of "
835	"the line");
836	else if (consumeIfPresent(TokenKind: MIToken::Newline)) {
837	IsAfterNewline = true;
838	continue;
839	}
840	IsAfterNewline = false;
841	if (Token.is(K: MIToken::lbrace))
842	++BraceDepth;
843	if (Token.is(K: MIToken::rbrace)) {
844	if (!BraceDepth)
845	return error(Msg: "extraneous closing brace ('}')");
846	--BraceDepth;
847	}
848	lex();
849	}
850	// Verify that we closed all of the '{' at the end of a file or a block.
851	if (!Token.isError() && BraceDepth)
852	return error(Msg: "expected '}'"); // FIXME: Report a note that shows '{'.
853	} while (!Token.isErrorOrEOF());
854	return Token.isError();
855	}
856
857	bool MIParser::parseBasicBlockLiveins(MachineBasicBlock &MBB) {
858	assert(Token.is(MIToken::kw_liveins));
859	lex();
860	if (expectAndConsume(TokenKind: MIToken::colon))
861	return true;
862	if (Token.isNewlineOrEOF()) // Allow an empty list of liveins.
863	return false;
864	do {
865	if (Token.isNot(K: MIToken::NamedRegister))
866	return error(Msg: "expected a named register");
867	Register Reg;
868	if (parseNamedRegister(Reg))
869	return true;
870	lex();
871	LaneBitmask Mask = LaneBitmask::getAll();
872	if (consumeIfPresent(TokenKind: MIToken::colon)) {
873	// Parse lane mask.
874	if (Token.isNot(K: MIToken::IntegerLiteral) &&
875	Token.isNot(K: MIToken::HexLiteral))
876	return error(Msg: "expected a lane mask");
877	static_assert(sizeof(LaneBitmask::Type) == sizeof(uint64_t),
878	"Use correct get-function for lane mask");
879	LaneBitmask::Type V;
880	if (getUint64(Result&: V))
881	return error(Msg: "invalid lane mask value");
882	Mask = LaneBitmask(V);
883	lex();
884	}
885	MBB.addLiveIn(PhysReg: Reg, LaneMask: Mask);
886	} while (consumeIfPresent(TokenKind: MIToken::comma));
887	return false;
888	}
889
890	bool MIParser::parseBasicBlockSuccessors(MachineBasicBlock &MBB) {
891	assert(Token.is(MIToken::kw_successors));
892	lex();
893	if (expectAndConsume(TokenKind: MIToken::colon))
894	return true;
895	if (Token.isNewlineOrEOF()) // Allow an empty list of successors.
896	return false;
897	do {
898	if (Token.isNot(K: MIToken::MachineBasicBlock))
899	return error(Msg: "expected a machine basic block reference");
900	MachineBasicBlock *SuccMBB = nullptr;
901	if (parseMBBReference(MBB&: SuccMBB))
902	return true;
903	lex();
904	unsigned Weight = 0;
905	if (consumeIfPresent(TokenKind: MIToken::lparen)) {
906	if (Token.isNot(K: MIToken::IntegerLiteral) &&
907	Token.isNot(K: MIToken::HexLiteral))
908	return error(Msg: "expected an integer literal after '('");
909	if (getUnsigned(Result&: Weight))
910	return true;
911	lex();
912	if (expectAndConsume(TokenKind: MIToken::rparen))
913	return true;
914	}
915	MBB.addSuccessor(Succ: SuccMBB, Prob: BranchProbability::getRaw(N: Weight));
916	} while (consumeIfPresent(TokenKind: MIToken::comma));
917	MBB.normalizeSuccProbs();
918	return false;
919	}
920
921	bool MIParser::parseBasicBlock(MachineBasicBlock &MBB,
922	MachineBasicBlock *&AddFalthroughFrom) {
923	// Skip the definition.
924	assert(Token.is(MIToken::MachineBasicBlockLabel));
925	lex();
926	if (consumeIfPresent(TokenKind: MIToken::lparen)) {
927	while (Token.isNot(K: MIToken::rparen) && !Token.isErrorOrEOF())
928	lex();
929	consumeIfPresent(TokenKind: MIToken::rparen);
930	}
931	consumeIfPresent(TokenKind: MIToken::colon);
932
933	// Parse the liveins and successors.
934	// N.B: Multiple lists of successors and liveins are allowed and they're
935	// merged into one.
936	// Example:
937	// liveins: $edi
938	// liveins: $esi
939	//
940	// is equivalent to
941	// liveins: $edi, $esi
942	bool ExplicitSuccessors = false;
943	while (true) {
944	if (Token.is(K: MIToken::kw_successors)) {
945	if (parseBasicBlockSuccessors(MBB))
946	return true;
947	ExplicitSuccessors = true;
948	} else if (Token.is(K: MIToken::kw_liveins)) {
949	if (parseBasicBlockLiveins(MBB))
950	return true;
951	} else if (consumeIfPresent(TokenKind: MIToken::Newline)) {
952	continue;
953	} else
954	break;
955	if (!Token.isNewlineOrEOF())
956	return error(Msg: "expected line break at the end of a list");
957	lex();
958	}
959
960	// Parse the instructions.
961	bool IsInBundle = false;
962	MachineInstr *PrevMI = nullptr;
963	while (!Token.is(K: MIToken::MachineBasicBlockLabel) &&
964	!Token.is(K: MIToken::Eof)) {
965	if (consumeIfPresent(TokenKind: MIToken::Newline))
966	continue;
967	if (consumeIfPresent(TokenKind: MIToken::rbrace)) {
968	// The first parsing pass should verify that all closing '}' have an
969	// opening '{'.
970	assert(IsInBundle);
971	IsInBundle = false;
972	continue;
973	}
974	MachineInstr *MI = nullptr;
975	if (parse(MI))
976	return true;
977	MBB.insert(I: MBB.end(), MI);
978	if (IsInBundle) {
979	PrevMI->setFlag(MachineInstr::BundledSucc);
980	MI->setFlag(MachineInstr::BundledPred);
981	}
982	PrevMI = MI;
983	if (Token.is(K: MIToken::lbrace)) {
984	if (IsInBundle)
985	return error(Msg: "nested instruction bundles are not allowed");
986	lex();
987	// This instruction is the start of the bundle.
988	MI->setFlag(MachineInstr::BundledSucc);
989	IsInBundle = true;
990	if (!Token.is(K: MIToken::Newline))
991	// The next instruction can be on the same line.
992	continue;
993	}
994	assert(Token.isNewlineOrEOF() && "MI is not fully parsed");
995	lex();
996	}
997
998	// Construct successor list by searching for basic block machine operands.
999	if (!ExplicitSuccessors) {
1000	SmallVector<MachineBasicBlock*,4> Successors;
1001	bool IsFallthrough;
1002	guessSuccessors(MBB, Result&: Successors, IsFallthrough);
1003	for (MachineBasicBlock *Succ : Successors)
1004	MBB.addSuccessor(Succ);
1005
1006	if (IsFallthrough) {
1007	AddFalthroughFrom = &MBB;
1008	} else {
1009	MBB.normalizeSuccProbs();
1010	}
1011	}
1012
1013	return false;
1014	}
1015
1016	bool MIParser::parseBasicBlocks() {
1017	lex();
1018	// Skip until the first machine basic block.
1019	while (Token.is(K: MIToken::Newline))
1020	lex();
1021	if (Token.isErrorOrEOF())
1022	return Token.isError();
1023	// The first parsing pass should have verified that this token is a MBB label
1024	// in the 'parseBasicBlockDefinitions' method.
1025	assert(Token.is(MIToken::MachineBasicBlockLabel));
1026	MachineBasicBlock *AddFalthroughFrom = nullptr;
1027	do {
1028	MachineBasicBlock *MBB = nullptr;
1029	if (parseMBBReference(MBB))
1030	return true;
1031	if (AddFalthroughFrom) {
1032	if (!AddFalthroughFrom->isSuccessor(MBB))
1033	AddFalthroughFrom->addSuccessor(Succ: MBB);
1034	AddFalthroughFrom->normalizeSuccProbs();
1035	AddFalthroughFrom = nullptr;
1036	}
1037	if (parseBasicBlock(MBB&: *MBB, AddFalthroughFrom))
1038	return true;
1039	// The method 'parseBasicBlock' should parse the whole block until the next
1040	// block or the end of file.
1041	assert(Token.is(MIToken::MachineBasicBlockLabel) || Token.is(MIToken::Eof));
1042	} while (Token.isNot(K: MIToken::Eof));
1043	return false;
1044	}
1045
1046	bool MIParser::parse(MachineInstr *&MI) {
1047	// Parse any register operands before '='
1048	MachineOperand MO = MachineOperand::CreateImm(Val: 0);
1049	SmallVector<ParsedMachineOperand, 8> Operands;
1050	while (Token.isRegister() || Token.isRegisterFlag()) {
1051	auto Loc = Token.location();
1052	std::optional<unsigned> TiedDefIdx;
1053	if (parseRegisterOperand(Dest&: MO, TiedDefIdx, /*IsDef=*/true))
1054	return true;
1055	Operands.push_back(
1056	Elt: ParsedMachineOperand(MO, Loc, Token.location(), TiedDefIdx));
1057	if (Token.isNot(K: MIToken::comma))
1058	break;
1059	lex();
1060	}
1061	if (!Operands.empty() && expectAndConsume(TokenKind: MIToken::equal))
1062	return true;
1063
1064	unsigned OpCode, Flags = 0;
1065	if (Token.isError() || parseInstruction(OpCode, Flags))
1066	return true;
1067
1068	// Parse the remaining machine operands.
1069	while (!Token.isNewlineOrEOF() && Token.isNot(K: MIToken::kw_pre_instr_symbol) &&
1070	Token.isNot(K: MIToken::kw_post_instr_symbol) &&
1071	Token.isNot(K: MIToken::kw_heap_alloc_marker) &&
1072	Token.isNot(K: MIToken::kw_pcsections) &&
1073	Token.isNot(K: MIToken::kw_cfi_type) &&
1074	Token.isNot(K: MIToken::kw_debug_location) &&
1075	Token.isNot(K: MIToken::kw_debug_instr_number) &&
1076	Token.isNot(K: MIToken::coloncolon) && Token.isNot(K: MIToken::lbrace)) {
1077	auto Loc = Token.location();
1078	std::optional<unsigned> TiedDefIdx;
1079	if (parseMachineOperandAndTargetFlags(OpCode, OpIdx: Operands.size(), Dest&: MO, TiedDefIdx))
1080	return true;
1081	Operands.push_back(
1082	Elt: ParsedMachineOperand(MO, Loc, Token.location(), TiedDefIdx));
1083	if (Token.isNewlineOrEOF() || Token.is(K: MIToken::coloncolon) ||
1084	Token.is(K: MIToken::lbrace))
1085	break;
1086	if (Token.isNot(K: MIToken::comma))
1087	return error(Msg: "expected ',' before the next machine operand");
1088	lex();
1089	}
1090
1091	MCSymbol *PreInstrSymbol = nullptr;
1092	if (Token.is(K: MIToken::kw_pre_instr_symbol))
1093	if (parsePreOrPostInstrSymbol(Symbol&: PreInstrSymbol))
1094	return true;
1095	MCSymbol *PostInstrSymbol = nullptr;
1096	if (Token.is(K: MIToken::kw_post_instr_symbol))
1097	if (parsePreOrPostInstrSymbol(Symbol&: PostInstrSymbol))
1098	return true;
1099	MDNode *HeapAllocMarker = nullptr;
1100	if (Token.is(K: MIToken::kw_heap_alloc_marker))
1101	if (parseHeapAllocMarker(Node&: HeapAllocMarker))
1102	return true;
1103	MDNode *PCSections = nullptr;
1104	if (Token.is(K: MIToken::kw_pcsections))
1105	if (parsePCSections(Node&: PCSections))
1106	return true;
1107
1108	unsigned CFIType = 0;
1109	if (Token.is(K: MIToken::kw_cfi_type)) {
1110	lex();
1111	if (Token.isNot(K: MIToken::IntegerLiteral))
1112	return error(Msg: "expected an integer literal after 'cfi-type'");
1113	// getUnsigned is sufficient for 32-bit integers.
1114	if (getUnsigned(Result&: CFIType))
1115	return true;
1116	lex();
1117	// Lex past trailing comma if present.
1118	if (Token.is(K: MIToken::comma))
1119	lex();
1120	}
1121
1122	unsigned InstrNum = 0;
1123	if (Token.is(K: MIToken::kw_debug_instr_number)) {
1124	lex();
1125	if (Token.isNot(K: MIToken::IntegerLiteral))
1126	return error(Msg: "expected an integer literal after 'debug-instr-number'");
1127	if (getUnsigned(Result&: InstrNum))
1128	return true;
1129	lex();
1130	// Lex past trailing comma if present.
1131	if (Token.is(K: MIToken::comma))
1132	lex();
1133	}
1134
1135	DebugLoc DebugLocation;
1136	if (Token.is(K: MIToken::kw_debug_location)) {
1137	lex();
1138	MDNode *Node = nullptr;
1139	if (Token.is(K: MIToken::exclaim)) {
1140	if (parseMDNode(Node))
1141	return true;
1142	} else if (Token.is(K: MIToken::md_dilocation)) {
1143	if (parseDILocation(Expr&: Node))
1144	return true;
1145	} else
1146	return error(Msg: "expected a metadata node after 'debug-location'");
1147	if (!isa<DILocation>(Val: Node))
1148	return error(Msg: "referenced metadata is not a DILocation");
1149	DebugLocation = DebugLoc(Node);
1150	}
1151
1152	// Parse the machine memory operands.
1153	SmallVector<MachineMemOperand *, 2> MemOperands;
1154	if (Token.is(K: MIToken::coloncolon)) {
1155	lex();
1156	while (!Token.isNewlineOrEOF()) {
1157	MachineMemOperand *MemOp = nullptr;
1158	if (parseMachineMemoryOperand(Dest&: MemOp))
1159	return true;
1160	MemOperands.push_back(Elt: MemOp);
1161	if (Token.isNewlineOrEOF())
1162	break;
1163	if (Token.isNot(K: MIToken::comma))
1164	return error(Msg: "expected ',' before the next machine memory operand");
1165	lex();
1166	}
1167	}
1168
1169	const auto &MCID = MF.getSubtarget().getInstrInfo()->get(Opcode: OpCode);
1170	if (!MCID.isVariadic()) {
1171	// FIXME: Move the implicit operand verification to the machine verifier.
1172	if (verifyImplicitOperands(Operands, MCID))
1173	return true;
1174	}
1175
1176	MI = MF.CreateMachineInstr(MCID, DL: DebugLocation, /*NoImplicit=*/true);
1177	MI->setFlags(Flags);
1178
1179	// Don't check the operands make sense, let the verifier catch any
1180	// improprieties.
1181	for (const auto &Operand : Operands)
1182	MI->addOperand(MF, Op: Operand.Operand);
1183
1184	if (assignRegisterTies(MI&: *MI, Operands))
1185	return true;
1186	if (PreInstrSymbol)
1187	MI->setPreInstrSymbol(MF, Symbol: PreInstrSymbol);
1188	if (PostInstrSymbol)
1189	MI->setPostInstrSymbol(MF, Symbol: PostInstrSymbol);
1190	if (HeapAllocMarker)
1191	MI->setHeapAllocMarker(MF, MD: HeapAllocMarker);
1192	if (PCSections)
1193	MI->setPCSections(MF, MD: PCSections);
1194	if (CFIType)
1195	MI->setCFIType(MF, Type: CFIType);
1196	if (!MemOperands.empty())
1197	MI->setMemRefs(MF, MemRefs: MemOperands);
1198	if (InstrNum)
1199	MI->setDebugInstrNum(InstrNum);
1200	return false;
1201	}
1202
1203	bool MIParser::parseStandaloneMBB(MachineBasicBlock *&MBB) {
1204	lex();
1205	if (Token.isNot(K: MIToken::MachineBasicBlock))
1206	return error(Msg: "expected a machine basic block reference");
1207	if (parseMBBReference(MBB))
1208	return true;
1209	lex();
1210	if (Token.isNot(K: MIToken::Eof))
1211	return error(
1212	Msg: "expected end of string after the machine basic block reference");
1213	return false;
1214	}
1215
1216	bool MIParser::parseStandaloneNamedRegister(Register &Reg) {
1217	lex();
1218	if (Token.isNot(K: MIToken::NamedRegister))
1219	return error(Msg: "expected a named register");
1220	if (parseNamedRegister(Reg))
1221	return true;
1222	lex();
1223	if (Token.isNot(K: MIToken::Eof))
1224	return error(Msg: "expected end of string after the register reference");
1225	return false;
1226	}
1227
1228	bool MIParser::parseStandaloneVirtualRegister(VRegInfo *&Info) {
1229	lex();
1230	if (Token.isNot(K: MIToken::VirtualRegister))
1231	return error(Msg: "expected a virtual register");
1232	if (parseVirtualRegister(Info))
1233	return true;
1234	lex();
1235	if (Token.isNot(K: MIToken::Eof))
1236	return error(Msg: "expected end of string after the register reference");
1237	return false;
1238	}
1239
1240	bool MIParser::parseStandaloneRegister(Register &Reg) {
1241	lex();
1242	if (Token.isNot(K: MIToken::NamedRegister) &&
1243	Token.isNot(K: MIToken::VirtualRegister))
1244	return error(Msg: "expected either a named or virtual register");
1245
1246	VRegInfo *Info;
1247	if (parseRegister(Reg, VRegInfo&: Info))
1248	return true;
1249
1250	lex();
1251	if (Token.isNot(K: MIToken::Eof))
1252	return error(Msg: "expected end of string after the register reference");
1253	return false;
1254	}
1255
1256	bool MIParser::parseStandaloneStackObject(int &FI) {
1257	lex();
1258	if (Token.isNot(K: MIToken::StackObject))
1259	return error(Msg: "expected a stack object");
1260	if (parseStackFrameIndex(FI))
1261	return true;
1262	if (Token.isNot(K: MIToken::Eof))
1263	return error(Msg: "expected end of string after the stack object reference");
1264	return false;
1265	}
1266
1267	bool MIParser::parseStandaloneMDNode(MDNode *&Node) {
1268	lex();
1269	if (Token.is(K: MIToken::exclaim)) {
1270	if (parseMDNode(Node))
1271	return true;
1272	} else if (Token.is(K: MIToken::md_diexpr)) {
1273	if (parseDIExpression(Expr&: Node))
1274	return true;
1275	} else if (Token.is(K: MIToken::md_dilocation)) {
1276	if (parseDILocation(Expr&: Node))
1277	return true;
1278	} else
1279	return error(Msg: "expected a metadata node");
1280	if (Token.isNot(K: MIToken::Eof))
1281	return error(Msg: "expected end of string after the metadata node");
1282	return false;
1283	}
1284
1285	bool MIParser::parseMachineMetadata() {
1286	lex();
1287	if (Token.isNot(K: MIToken::exclaim))
1288	return error(Msg: "expected a metadata node");
1289
1290	lex();
1291	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isSigned())
1292	return error(Msg: "expected metadata id after '!'");
1293	unsigned ID = 0;
1294	if (getUnsigned(Result&: ID))
1295	return true;
1296	lex();
1297	if (expectAndConsume(TokenKind: MIToken::equal))
1298	return true;
1299	bool IsDistinct = Token.is(K: MIToken::kw_distinct);
1300	if (IsDistinct)
1301	lex();
1302	if (Token.isNot(K: MIToken::exclaim))
1303	return error(Msg: "expected a metadata node");
1304	lex();
1305
1306	MDNode *MD;
1307	if (parseMDTuple(MD, IsDistinct))
1308	return true;
1309
1310	auto FI = PFS.MachineForwardRefMDNodes.find(x: ID);
1311	if (FI != PFS.MachineForwardRefMDNodes.end()) {
1312	FI->second.first->replaceAllUsesWith(MD);
1313	PFS.MachineForwardRefMDNodes.erase(position: FI);
1314
1315	assert(PFS.MachineMetadataNodes[ID] == MD && "Tracking VH didn't work");
1316	} else {
1317	if (PFS.MachineMetadataNodes.count(x: ID))
1318	return error(Msg: "Metadata id is already used");
1319	PFS.MachineMetadataNodes[ID].reset(MD);
1320	}
1321
1322	return false;
1323	}
1324
1325	bool MIParser::parseMDTuple(MDNode *&MD, bool IsDistinct) {
1326	SmallVector<Metadata *, 16> Elts;
1327	if (parseMDNodeVector(Elts))
1328	return true;
1329	MD = (IsDistinct ? MDTuple::getDistinct
1330	: MDTuple::get)(MF.getFunction().getContext(), Elts);
1331	return false;
1332	}
1333
1334	bool MIParser::parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts) {
1335	if (Token.isNot(K: MIToken::lbrace))
1336	return error(Msg: "expected '{' here");
1337	lex();
1338
1339	if (Token.is(K: MIToken::rbrace)) {
1340	lex();
1341	return false;
1342	}
1343
1344	do {
1345	Metadata *MD;
1346	if (parseMetadata(MD))
1347	return true;
1348
1349	Elts.push_back(Elt: MD);
1350
1351	if (Token.isNot(K: MIToken::comma))
1352	break;
1353	lex();
1354	} while (true);
1355
1356	if (Token.isNot(K: MIToken::rbrace))
1357	return error(Msg: "expected end of metadata node");
1358	lex();
1359
1360	return false;
1361	}
1362
1363	// ::= !42
1364	// ::= !"string"
1365	bool MIParser::parseMetadata(Metadata *&MD) {
1366	if (Token.isNot(K: MIToken::exclaim))
1367	return error(Msg: "expected '!' here");
1368	lex();
1369
1370	if (Token.is(K: MIToken::StringConstant)) {
1371	std::string Str;
1372	if (parseStringConstant(Result&: Str))
1373	return true;
1374	MD = MDString::get(Context&: MF.getFunction().getContext(), Str);
1375	return false;
1376	}
1377
1378	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isSigned())
1379	return error(Msg: "expected metadata id after '!'");
1380
1381	SMLoc Loc = mapSMLoc(Loc: Token.location());
1382
1383	unsigned ID = 0;
1384	if (getUnsigned(Result&: ID))
1385	return true;
1386	lex();
1387
1388	auto NodeInfo = PFS.IRSlots.MetadataNodes.find(x: ID);
1389	if (NodeInfo != PFS.IRSlots.MetadataNodes.end()) {
1390	MD = NodeInfo->second.get();
1391	return false;
1392	}
1393	// Check machine metadata.
1394	NodeInfo = PFS.MachineMetadataNodes.find(x: ID);
1395	if (NodeInfo != PFS.MachineMetadataNodes.end()) {
1396	MD = NodeInfo->second.get();
1397	return false;
1398	}
1399	// Forward reference.
1400	auto &FwdRef = PFS.MachineForwardRefMDNodes[ID];
1401	FwdRef = std::make_pair(
1402	x: MDTuple::getTemporary(Context&: MF.getFunction().getContext(), MDs: std::nullopt), y&: Loc);
1403	PFS.MachineMetadataNodes[ID].reset(MD: FwdRef.first.get());
1404	MD = FwdRef.first.get();
1405
1406	return false;
1407	}
1408
1409	static const char *printImplicitRegisterFlag(const MachineOperand &MO) {
1410	assert(MO.isImplicit());
1411	return MO.isDef() ? "implicit-def" : "implicit";
1412	}
1413
1414	static std::string getRegisterName(const TargetRegisterInfo *TRI,
1415	Register Reg) {
1416	assert(Reg.isPhysical() && "expected phys reg");
1417	return StringRef(TRI->getName(RegNo: Reg)).lower();
1418	}
1419
1420	/// Return true if the parsed machine operands contain a given machine operand.
1421	static bool isImplicitOperandIn(const MachineOperand &ImplicitOperand,
1422	ArrayRef<ParsedMachineOperand> Operands) {
1423	for (const auto &I : Operands) {
1424	if (ImplicitOperand.isIdenticalTo(Other: I.Operand))
1425	return true;
1426	}
1427	return false;
1428	}
1429
1430	bool MIParser::verifyImplicitOperands(ArrayRef<ParsedMachineOperand> Operands,
1431	const MCInstrDesc &MCID) {
1432	if (MCID.isCall())
1433	// We can't verify call instructions as they can contain arbitrary implicit
1434	// register and register mask operands.
1435	return false;
1436
1437	// Gather all the expected implicit operands.
1438	SmallVector<MachineOperand, 4> ImplicitOperands;
1439	for (MCPhysReg ImpDef : MCID.implicit_defs())
1440	ImplicitOperands.push_back(Elt: MachineOperand::CreateReg(Reg: ImpDef, isDef: true, isImp: true));
1441	for (MCPhysReg ImpUse : MCID.implicit_uses())
1442	ImplicitOperands.push_back(Elt: MachineOperand::CreateReg(Reg: ImpUse, isDef: false, isImp: true));
1443
1444	const auto *TRI = MF.getSubtarget().getRegisterInfo();
1445	assert(TRI && "Expected target register info");
1446	for (const auto &I : ImplicitOperands) {
1447	if (isImplicitOperandIn(ImplicitOperand: I, Operands))
1448	continue;
1449	return error(Loc: Operands.empty() ? Token.location() : Operands.back().End,
1450	Msg: Twine("missing implicit register operand '") +
1451	printImplicitRegisterFlag(MO: I) + " $" +
1452	getRegisterName(TRI, Reg: I.getReg()) + "'");
1453	}
1454	return false;
1455	}
1456
1457	bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
1458	// Allow frame and fast math flags for OPCODE
1459	// clang-format off
1460	while (Token.is(K: MIToken::kw_frame_setup) ||
1461	Token.is(K: MIToken::kw_frame_destroy) ||
1462	Token.is(K: MIToken::kw_nnan) ||
1463	Token.is(K: MIToken::kw_ninf) ||
1464	Token.is(K: MIToken::kw_nsz) ||
1465	Token.is(K: MIToken::kw_arcp) ||
1466	Token.is(K: MIToken::kw_contract) ||
1467	Token.is(K: MIToken::kw_afn) ||
1468	Token.is(K: MIToken::kw_reassoc) ||
1469	Token.is(K: MIToken::kw_nuw) ||
1470	Token.is(K: MIToken::kw_nsw) ||
1471	Token.is(K: MIToken::kw_exact) ||
1472	Token.is(K: MIToken::kw_nofpexcept) ||
1473	Token.is(K: MIToken::kw_noconvergent) ||
1474	Token.is(K: MIToken::kw_unpredictable)) {
1475	// clang-format on
1476	// Mine frame and fast math flags
1477	if (Token.is(K: MIToken::kw_frame_setup))
1478	Flags |= MachineInstr::FrameSetup;
1479	if (Token.is(K: MIToken::kw_frame_destroy))
1480	Flags |= MachineInstr::FrameDestroy;
1481	if (Token.is(K: MIToken::kw_nnan))
1482	Flags |= MachineInstr::FmNoNans;
1483	if (Token.is(K: MIToken::kw_ninf))
1484	Flags |= MachineInstr::FmNoInfs;
1485	if (Token.is(K: MIToken::kw_nsz))
1486	Flags |= MachineInstr::FmNsz;
1487	if (Token.is(K: MIToken::kw_arcp))
1488	Flags |= MachineInstr::FmArcp;
1489	if (Token.is(K: MIToken::kw_contract))
1490	Flags |= MachineInstr::FmContract;
1491	if (Token.is(K: MIToken::kw_afn))
1492	Flags |= MachineInstr::FmAfn;
1493	if (Token.is(K: MIToken::kw_reassoc))
1494	Flags |= MachineInstr::FmReassoc;
1495	if (Token.is(K: MIToken::kw_nuw))
1496	Flags |= MachineInstr::NoUWrap;
1497	if (Token.is(K: MIToken::kw_nsw))
1498	Flags |= MachineInstr::NoSWrap;
1499	if (Token.is(K: MIToken::kw_exact))
1500	Flags |= MachineInstr::IsExact;
1501	if (Token.is(K: MIToken::kw_nofpexcept))
1502	Flags |= MachineInstr::NoFPExcept;
1503	if (Token.is(K: MIToken::kw_unpredictable))
1504	Flags |= MachineInstr::Unpredictable;
1505	if (Token.is(K: MIToken::kw_noconvergent))
1506	Flags |= MachineInstr::NoConvergent;
1507
1508	lex();
1509	}
1510	if (Token.isNot(K: MIToken::Identifier))
1511	return error(Msg: "expected a machine instruction");
1512	StringRef InstrName = Token.stringValue();
1513	if (PFS.Target.parseInstrName(InstrName, OpCode))
1514	return error(Msg: Twine("unknown machine instruction name '") + InstrName + "'");
1515	lex();
1516	return false;
1517	}
1518
1519	bool MIParser::parseNamedRegister(Register &Reg) {
1520	assert(Token.is(MIToken::NamedRegister) && "Needs NamedRegister token");
1521	StringRef Name = Token.stringValue();
1522	if (PFS.Target.getRegisterByName(RegName: Name, Reg))
1523	return error(Msg: Twine("unknown register name '") + Name + "'");
1524	return false;
1525	}
1526
1527	bool MIParser::parseNamedVirtualRegister(VRegInfo *&Info) {
1528	assert(Token.is(MIToken::NamedVirtualRegister) && "Expected NamedVReg token");
1529	StringRef Name = Token.stringValue();
1530	// TODO: Check that the VReg name is not the same as a physical register name.
1531	// If it is, then print a warning (when warnings are implemented).
1532	Info = &PFS.getVRegInfoNamed(RegName: Name);
1533	return false;
1534	}
1535
1536	bool MIParser::parseVirtualRegister(VRegInfo *&Info) {
1537	if (Token.is(K: MIToken::NamedVirtualRegister))
1538	return parseNamedVirtualRegister(Info);
1539	assert(Token.is(MIToken::VirtualRegister) && "Needs VirtualRegister token");
1540	unsigned ID;
1541	if (getUnsigned(Result&: ID))
1542	return true;
1543	Info = &PFS.getVRegInfo(Num: ID);
1544	return false;
1545	}
1546
1547	bool MIParser::parseRegister(Register &Reg, VRegInfo *&Info) {
1548	switch (Token.kind()) {
1549	case MIToken::underscore:
1550	Reg = 0;
1551	return false;
1552	case MIToken::NamedRegister:
1553	return parseNamedRegister(Reg);
1554	case MIToken::NamedVirtualRegister:
1555	case MIToken::VirtualRegister:
1556	if (parseVirtualRegister(Info))
1557	return true;
1558	Reg = Info->VReg;
1559	return false;
1560	// TODO: Parse other register kinds.
1561	default:
1562	llvm_unreachable("The current token should be a register");
1563	}
1564	}
1565
1566	bool MIParser::parseRegisterClassOrBank(VRegInfo &RegInfo) {
1567	if (Token.isNot(K: MIToken::Identifier) && Token.isNot(K: MIToken::underscore))
1568	return error(Msg: "expected '_', register class, or register bank name");
1569	StringRef::iterator Loc = Token.location();
1570	StringRef Name = Token.stringValue();
1571
1572	// Was it a register class?
1573	const TargetRegisterClass *RC = PFS.Target.getRegClass(Name);
1574	if (RC) {
1575	lex();
1576
1577	switch (RegInfo.Kind) {
1578	case VRegInfo::UNKNOWN:
1579	case VRegInfo::NORMAL:
1580	RegInfo.Kind = VRegInfo::NORMAL;
1581	if (RegInfo.Explicit && RegInfo.D.RC != RC) {
1582	const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1583	return error(Loc, Msg: Twine("conflicting register classes, previously: ") +
1584	Twine(TRI.getRegClassName(Class: RegInfo.D.RC)));
1585	}
1586	RegInfo.D.RC = RC;
1587	RegInfo.Explicit = true;
1588	return false;
1589
1590	case VRegInfo::GENERIC:
1591	case VRegInfo::REGBANK:
1592	return error(Loc, Msg: "register class specification on generic register");
1593	}
1594	llvm_unreachable("Unexpected register kind");
1595	}
1596
1597	// Should be a register bank or a generic register.
1598	const RegisterBank *RegBank = nullptr;
1599	if (Name != "_") {
1600	RegBank = PFS.Target.getRegBank(Name);
1601	if (!RegBank)
1602	return error(Loc, Msg: "expected '_', register class, or register bank name");
1603	}
1604
1605	lex();
1606
1607	switch (RegInfo.Kind) {
1608	case VRegInfo::UNKNOWN:
1609	case VRegInfo::GENERIC:
1610	case VRegInfo::REGBANK:
1611	RegInfo.Kind = RegBank ? VRegInfo::REGBANK : VRegInfo::GENERIC;
1612	if (RegInfo.Explicit && RegInfo.D.RegBank != RegBank)
1613	return error(Loc, Msg: "conflicting generic register banks");
1614	RegInfo.D.RegBank = RegBank;
1615	RegInfo.Explicit = true;
1616	return false;
1617
1618	case VRegInfo::NORMAL:
1619	return error(Loc, Msg: "register bank specification on normal register");
1620	}
1621	llvm_unreachable("Unexpected register kind");
1622	}
1623
1624	bool MIParser::parseRegisterFlag(unsigned &Flags) {
1625	const unsigned OldFlags = Flags;
1626	switch (Token.kind()) {
1627	case MIToken::kw_implicit:
1628	Flags |= RegState::Implicit;
1629	break;
1630	case MIToken::kw_implicit_define:
1631	Flags |= RegState::ImplicitDefine;
1632	break;
1633	case MIToken::kw_def:
1634	Flags |= RegState::Define;
1635	break;
1636	case MIToken::kw_dead:
1637	Flags |= RegState::Dead;
1638	break;
1639	case MIToken::kw_killed:
1640	Flags |= RegState::Kill;
1641	break;
1642	case MIToken::kw_undef:
1643	Flags |= RegState::Undef;
1644	break;
1645	case MIToken::kw_internal:
1646	Flags |= RegState::InternalRead;
1647	break;
1648	case MIToken::kw_early_clobber:
1649	Flags |= RegState::EarlyClobber;
1650	break;
1651	case MIToken::kw_debug_use:
1652	Flags |= RegState::Debug;
1653	break;
1654	case MIToken::kw_renamable:
1655	Flags |= RegState::Renamable;
1656	break;
1657	default:
1658	llvm_unreachable("The current token should be a register flag");
1659	}
1660	if (OldFlags == Flags)
1661	// We know that the same flag is specified more than once when the flags
1662	// weren't modified.
1663	return error(Msg: "duplicate '" + Token.stringValue() + "' register flag");
1664	lex();
1665	return false;
1666	}
1667
1668	bool MIParser::parseSubRegisterIndex(unsigned &SubReg) {
1669	assert(Token.is(MIToken::dot));
1670	lex();
1671	if (Token.isNot(K: MIToken::Identifier))
1672	return error(Msg: "expected a subregister index after '.'");
1673	auto Name = Token.stringValue();
1674	SubReg = PFS.Target.getSubRegIndex(Name);
1675	if (!SubReg)
1676	return error(Msg: Twine("use of unknown subregister index '") + Name + "'");
1677	lex();
1678	return false;
1679	}
1680
1681	bool MIParser::parseRegisterTiedDefIndex(unsigned &TiedDefIdx) {
1682	if (!consumeIfPresent(TokenKind: MIToken::kw_tied_def))
1683	return true;
1684	if (Token.isNot(K: MIToken::IntegerLiteral))
1685	return error(Msg: "expected an integer literal after 'tied-def'");
1686	if (getUnsigned(Result&: TiedDefIdx))
1687	return true;
1688	lex();
1689	if (expectAndConsume(TokenKind: MIToken::rparen))
1690	return true;
1691	return false;
1692	}
1693
1694	bool MIParser::assignRegisterTies(MachineInstr &MI,
1695	ArrayRef<ParsedMachineOperand> Operands) {
1696	SmallVector<std::pair<unsigned, unsigned>, 4> TiedRegisterPairs;
1697	for (unsigned I = 0, E = Operands.size(); I != E; ++I) {
1698	if (!Operands[I].TiedDefIdx)
1699	continue;
1700	// The parser ensures that this operand is a register use, so we just have
1701	// to check the tied-def operand.
1702	unsigned DefIdx = *Operands[I].TiedDefIdx;
1703	if (DefIdx >= E)
1704	return error(Loc: Operands[I].Begin,
1705	Msg: Twine("use of invalid tied-def operand index '" +
1706	Twine(DefIdx) + "'; instruction has only ") +
1707	Twine(E) + " operands");
1708	const auto &DefOperand = Operands[DefIdx].Operand;
1709	if (!DefOperand.isReg() || !DefOperand.isDef())
1710	// FIXME: add note with the def operand.
1711	return error(Loc: Operands[I].Begin,
1712	Msg: Twine("use of invalid tied-def operand index '") +
1713	Twine(DefIdx) + "'; the operand #" + Twine(DefIdx) +
1714	" isn't a defined register");
1715	// Check that the tied-def operand wasn't tied elsewhere.
1716	for (const auto &TiedPair : TiedRegisterPairs) {
1717	if (TiedPair.first == DefIdx)
1718	return error(Loc: Operands[I].Begin,
1719	Msg: Twine("the tied-def operand #") + Twine(DefIdx) +
1720	" is already tied with another register operand");
1721	}
1722	TiedRegisterPairs.push_back(Elt: std::make_pair(x&: DefIdx, y&: I));
1723	}
1724	// FIXME: Verify that for non INLINEASM instructions, the def and use tied
1725	// indices must be less than tied max.
1726	for (const auto &TiedPair : TiedRegisterPairs)
1727	MI.tieOperands(DefIdx: TiedPair.first, UseIdx: TiedPair.second);
1728	return false;
1729	}
1730
1731	bool MIParser::parseRegisterOperand(MachineOperand &Dest,
1732	std::optional<unsigned> &TiedDefIdx,
1733	bool IsDef) {
1734	unsigned Flags = IsDef ? RegState::Define : 0;
1735	while (Token.isRegisterFlag()) {
1736	if (parseRegisterFlag(Flags))
1737	return true;
1738	}
1739	if (!Token.isRegister())
1740	return error(Msg: "expected a register after register flags");
1741	Register Reg;
1742	VRegInfo *RegInfo;
1743	if (parseRegister(Reg, Info&: RegInfo))
1744	return true;
1745	lex();
1746	unsigned SubReg = 0;
1747	if (Token.is(K: MIToken::dot)) {
1748	if (parseSubRegisterIndex(SubReg))
1749	return true;
1750	if (!Reg.isVirtual())
1751	return error(Msg: "subregister index expects a virtual register");
1752	}
1753	if (Token.is(K: MIToken::colon)) {
1754	if (!Reg.isVirtual())
1755	return error(Msg: "register class specification expects a virtual register");
1756	lex();
1757	if (parseRegisterClassOrBank(RegInfo&: *RegInfo))
1758	return true;
1759	}
1760	MachineRegisterInfo &MRI = MF.getRegInfo();
1761	if ((Flags & RegState::Define) == 0) {
1762	if (consumeIfPresent(TokenKind: MIToken::lparen)) {
1763	unsigned Idx;
1764	if (!parseRegisterTiedDefIndex(TiedDefIdx&: Idx))
1765	TiedDefIdx = Idx;
1766	else {
1767	// Try a redundant low-level type.
1768	LLT Ty;
1769	if (parseLowLevelType(Loc: Token.location(), Ty))
1770	return error(Msg: "expected tied-def or low-level type after '('");
1771
1772	if (expectAndConsume(TokenKind: MIToken::rparen))
1773	return true;
1774
1775	if (MRI.getType(Reg).isValid() && MRI.getType(Reg) != Ty)
1776	return error(Msg: "inconsistent type for generic virtual register");
1777
1778	MRI.setRegClassOrRegBank(Reg, RCOrRB: static_cast<RegisterBank *>(nullptr));
1779	MRI.setType(VReg: Reg, Ty);
1780	}
1781	}
1782	} else if (consumeIfPresent(TokenKind: MIToken::lparen)) {
1783	// Virtual registers may have a tpe with GlobalISel.
1784	if (!Reg.isVirtual())
1785	return error(Msg: "unexpected type on physical register");
1786
1787	LLT Ty;
1788	if (parseLowLevelType(Loc: Token.location(), Ty))
1789	return true;
1790
1791	if (expectAndConsume(TokenKind: MIToken::rparen))
1792	return true;
1793
1794	if (MRI.getType(Reg).isValid() && MRI.getType(Reg) != Ty)
1795	return error(Msg: "inconsistent type for generic virtual register");
1796
1797	MRI.setRegClassOrRegBank(Reg, RCOrRB: static_cast<RegisterBank *>(nullptr));
1798	MRI.setType(VReg: Reg, Ty);
1799	} else if (Reg.isVirtual()) {
1800	// Generic virtual registers must have a type.
1801	// If we end up here this means the type hasn't been specified and
1802	// this is bad!
1803	if (RegInfo->Kind == VRegInfo::GENERIC ||
1804	RegInfo->Kind == VRegInfo::REGBANK)
1805	return error(Msg: "generic virtual registers must have a type");
1806	}
1807
1808	if (Flags & RegState::Define) {
1809	if (Flags & RegState::Kill)
1810	return error(Msg: "cannot have a killed def operand");
1811	} else {
1812	if (Flags & RegState::Dead)
1813	return error(Msg: "cannot have a dead use operand");
1814	}
1815
1816	Dest = MachineOperand::CreateReg(
1817	Reg, isDef: Flags & RegState::Define, isImp: Flags & RegState::Implicit,
1818	isKill: Flags & RegState::Kill, isDead: Flags & RegState::Dead, isUndef: Flags & RegState::Undef,
1819	isEarlyClobber: Flags & RegState::EarlyClobber, SubReg, isDebug: Flags & RegState::Debug,
1820	isInternalRead: Flags & RegState::InternalRead, isRenamable: Flags & RegState::Renamable);
1821
1822	return false;
1823	}
1824
1825	bool MIParser::parseImmediateOperand(MachineOperand &Dest) {
1826	assert(Token.is(MIToken::IntegerLiteral));
1827	const APSInt &Int = Token.integerValue();
1828	if (auto SImm = Int.trySExtValue(); Int.isSigned() && SImm.has_value())
1829	Dest = MachineOperand::CreateImm(Val: *SImm);
1830	else if (auto UImm = Int.tryZExtValue(); !Int.isSigned() && UImm.has_value())
1831	Dest = MachineOperand::CreateImm(Val: *UImm);
1832	else
1833	return error(Msg: "integer literal is too large to be an immediate operand");
1834	lex();
1835	return false;
1836	}
1837
1838	bool MIParser::parseTargetImmMnemonic(const unsigned OpCode,
1839	const unsigned OpIdx,
1840	MachineOperand &Dest,
1841	const MIRFormatter &MF) {
1842	assert(Token.is(MIToken::dot));
1843	auto Loc = Token.location(); // record start position
1844	size_t Len = 1; // for "."
1845	lex();
1846
1847	// Handle the case that mnemonic starts with number.
1848	if (Token.is(K: MIToken::IntegerLiteral)) {
1849	Len += Token.range().size();
1850	lex();
1851	}
1852
1853	StringRef Src;
1854	if (Token.is(K: MIToken::comma))
1855	Src = StringRef(Loc, Len);
1856	else {
1857	assert(Token.is(MIToken::Identifier));
1858	Src = StringRef(Loc, Len + Token.stringValue().size());
1859	}
1860	int64_t Val;
1861	if (MF.parseImmMnemonic(OpCode, OpIdx, Src, Imm&: Val,
1862	ErrorCallback: [this](StringRef::iterator Loc, const Twine &Msg)
1863	-> bool { return error(Loc, Msg); }))
1864	return true;
1865
1866	Dest = MachineOperand::CreateImm(Val);
1867	if (!Token.is(K: MIToken::comma))
1868	lex();
1869	return false;
1870	}
1871
1872	static bool parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
1873	PerFunctionMIParsingState &PFS, const Constant *&C,
1874	ErrorCallbackType ErrCB) {
1875	auto Source = StringValue.str(); // The source has to be null terminated.
1876	SMDiagnostic Err;
1877	C = parseConstantValue(Asm: Source, Err, M: *PFS.MF.getFunction().getParent(),
1878	Slots: &PFS.IRSlots);
1879	if (!C)
1880	return ErrCB(Loc + Err.getColumnNo(), Err.getMessage());
1881	return false;
1882	}
1883
1884	bool MIParser::parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
1885	const Constant *&C) {
1886	return ::parseIRConstant(
1887	Loc, StringValue, PFS, C,
1888	ErrCB: [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
1889	return error(Loc, Msg);
1890	});
1891	}
1892
1893	bool MIParser::parseIRConstant(StringRef::iterator Loc, const Constant *&C) {
1894	if (parseIRConstant(Loc, StringValue: StringRef(Loc, Token.range().end() - Loc), C))
1895	return true;
1896	lex();
1897	return false;
1898	}
1899
1900	// See LLT implementation for bit size limits.
1901	static bool verifyScalarSize(uint64_t Size) {
1902	return Size != 0 && isUInt<16>(x: Size);
1903	}
1904
1905	static bool verifyVectorElementCount(uint64_t NumElts) {
1906	return NumElts != 0 && isUInt<16>(x: NumElts);
1907	}
1908
1909	static bool verifyAddrSpace(uint64_t AddrSpace) {
1910	return isUInt<24>(x: AddrSpace);
1911	}
1912
1913	bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
1914	if (Token.range().front() == 's' || Token.range().front() == 'p') {
1915	StringRef SizeStr = Token.range().drop_front();
1916	if (SizeStr.size() == 0 || !llvm::all_of(Range&: SizeStr, P: isdigit))
1917	return error(Msg: "expected integers after 's'/'p' type character");
1918	}
1919
1920	if (Token.range().front() == 's') {
1921	auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
1922	if (!verifyScalarSize(Size: ScalarSize))
1923	return error(Msg: "invalid size for scalar type");
1924
1925	Ty = LLT::scalar(SizeInBits: ScalarSize);
1926	lex();
1927	return false;
1928	} else if (Token.range().front() == 'p') {
1929	const DataLayout &DL = MF.getDataLayout();
1930	uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
1931	if (!verifyAddrSpace(AddrSpace: AS))
1932	return error(Msg: "invalid address space number");
1933
1934	Ty = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
1935	lex();
1936	return false;
1937	}
1938
1939	// Now we're looking for a vector.
1940	if (Token.isNot(K: MIToken::less))
1941	return error(Loc, Msg: "expected sN, pA, <M x sN>, <M x pA>, <vscale x M x sN>, "
1942	"or <vscale x M x pA> for GlobalISel type");
1943	lex();
1944
1945	bool HasVScale =
1946	Token.is(K: MIToken::Identifier) && Token.stringValue() == "vscale";
1947	if (HasVScale) {
1948	lex();
1949	if (Token.isNot(K: MIToken::Identifier) || Token.stringValue() != "x")
1950	return error(Msg: "expected <vscale x M x sN> or <vscale x M x pA>");
1951	lex();
1952	}
1953
1954	auto GetError = [this, &HasVScale, Loc]() {
1955	if (HasVScale)
1956	return error(
1957	Loc, Msg: "expected <vscale x M x sN> or <vscale M x pA> for vector type");
1958	return error(Loc, Msg: "expected <M x sN> or <M x pA> for vector type");
1959	};
1960
1961	if (Token.isNot(K: MIToken::IntegerLiteral))
1962	return GetError();
1963	uint64_t NumElements = Token.integerValue().getZExtValue();
1964	if (!verifyVectorElementCount(NumElts: NumElements))
1965	return error(Msg: "invalid number of vector elements");
1966
1967	lex();
1968
1969	if (Token.isNot(K: MIToken::Identifier) || Token.stringValue() != "x")
1970	return GetError();
1971	lex();
1972
1973	if (Token.range().front() != 's' && Token.range().front() != 'p')
1974	return GetError();
1975
1976	StringRef SizeStr = Token.range().drop_front();
1977	if (SizeStr.size() == 0 || !llvm::all_of(Range&: SizeStr, P: isdigit))
1978	return error(Msg: "expected integers after 's'/'p' type character");
1979
1980	if (Token.range().front() == 's') {
1981	auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
1982	if (!verifyScalarSize(Size: ScalarSize))
1983	return error(Msg: "invalid size for scalar type");
1984	Ty = LLT::scalar(SizeInBits: ScalarSize);
1985	} else if (Token.range().front() == 'p') {
1986	const DataLayout &DL = MF.getDataLayout();
1987	uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
1988	if (!verifyAddrSpace(AddrSpace: AS))
1989	return error(Msg: "invalid address space number");
1990
1991	Ty = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
1992	} else
1993	return GetError();
1994	lex();
1995
1996	if (Token.isNot(K: MIToken::greater))
1997	return GetError();
1998
1999	lex();
2000
2001	Ty = LLT::vector(EC: ElementCount::get(MinVal: NumElements, Scalable: HasVScale), ScalarTy: Ty);
2002	return false;
2003	}
2004
2005	bool MIParser::parseTypedImmediateOperand(MachineOperand &Dest) {
2006	assert(Token.is(MIToken::Identifier));
2007	StringRef TypeStr = Token.range();
2008	if (TypeStr.front() != 'i' && TypeStr.front() != 's' &&
2009	TypeStr.front() != 'p')
2010	return error(
2011	Msg: "a typed immediate operand should start with one of 'i', 's', or 'p'");
2012	StringRef SizeStr = Token.range().drop_front();
2013	if (SizeStr.size() == 0 || !llvm::all_of(Range&: SizeStr, P: isdigit))
2014	return error(Msg: "expected integers after 'i'/'s'/'p' type character");
2015
2016	auto Loc = Token.location();
2017	lex();
2018	if (Token.isNot(K: MIToken::IntegerLiteral)) {
2019	if (Token.isNot(K: MIToken::Identifier) ||
2020	!(Token.range() == "true" || Token.range() == "false"))
2021	return error(Msg: "expected an integer literal");
2022	}
2023	const Constant *C = nullptr;
2024	if (parseIRConstant(Loc, C))
2025	return true;
2026	Dest = MachineOperand::CreateCImm(CI: cast<ConstantInt>(Val: C));
2027	return false;
2028	}
2029
2030	bool MIParser::parseFPImmediateOperand(MachineOperand &Dest) {
2031	auto Loc = Token.location();
2032	lex();
2033	if (Token.isNot(K: MIToken::FloatingPointLiteral) &&
2034	Token.isNot(K: MIToken::HexLiteral))
2035	return error(Msg: "expected a floating point literal");
2036	const Constant *C = nullptr;
2037	if (parseIRConstant(Loc, C))
2038	return true;
2039	Dest = MachineOperand::CreateFPImm(CFP: cast<ConstantFP>(Val: C));
2040	return false;
2041	}
2042
2043	static bool getHexUint(const MIToken &Token, APInt &Result) {
2044	assert(Token.is(MIToken::HexLiteral));
2045	StringRef S = Token.range();
2046	assert(S[0] == '0' && tolower(S[1]) == 'x');
2047	// This could be a floating point literal with a special prefix.
2048	if (!isxdigit(S[2]))
2049	return true;
2050	StringRef V = S.substr(Start: 2);
2051	APInt A(V.size()*4, V, 16);
2052
2053	// If A is 0, then A.getActiveBits() is 0. This isn't a valid bitwidth. Make
2054	// sure it isn't the case before constructing result.
2055	unsigned NumBits = (A == 0) ? 32 : A.getActiveBits();
2056	Result = APInt(NumBits, ArrayRef<uint64_t>(A.getRawData(), A.getNumWords()));
2057	return false;
2058	}
2059
2060	static bool getUnsigned(const MIToken &Token, unsigned &Result,
2061	ErrorCallbackType ErrCB) {
2062	if (Token.hasIntegerValue()) {
2063	const uint64_t Limit = uint64_t(std::numeric_limits<unsigned>::max()) + 1;
2064	uint64_t Val64 = Token.integerValue().getLimitedValue(Limit);
2065	if (Val64 == Limit)
2066	return ErrCB(Token.location(), "expected 32-bit integer (too large)");
2067	Result = Val64;
2068	return false;
2069	}
2070	if (Token.is(K: MIToken::HexLiteral)) {
2071	APInt A;
2072	if (getHexUint(Token, Result&: A))
2073	return true;
2074	if (A.getBitWidth() > 32)
2075	return ErrCB(Token.location(), "expected 32-bit integer (too large)");
2076	Result = A.getZExtValue();
2077	return false;
2078	}
2079	return true;
2080	}
2081
2082	bool MIParser::getUnsigned(unsigned &Result) {
2083	return ::getUnsigned(
2084	Token, Result, ErrCB: [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
2085	return error(Loc, Msg);
2086	});
2087	}
2088
2089	bool MIParser::parseMBBReference(MachineBasicBlock *&MBB) {
2090	assert(Token.is(MIToken::MachineBasicBlock) ||
2091	Token.is(MIToken::MachineBasicBlockLabel));
2092	unsigned Number;
2093	if (getUnsigned(Result&: Number))
2094	return true;
2095	auto MBBInfo = PFS.MBBSlots.find(Val: Number);
2096	if (MBBInfo == PFS.MBBSlots.end())
2097	return error(Msg: Twine("use of undefined machine basic block #") +
2098	Twine(Number));
2099	MBB = MBBInfo->second;
2100	// TODO: Only parse the name if it's a MachineBasicBlockLabel. Deprecate once
2101	// we drop the <irname> from the bb.<id>.<irname> format.
2102	if (!Token.stringValue().empty() && Token.stringValue() != MBB->getName())
2103	return error(Msg: Twine("the name of machine basic block #") + Twine(Number) +
2104	" isn't '" + Token.stringValue() + "'");
2105	return false;
2106	}
2107
2108	bool MIParser::parseMBBOperand(MachineOperand &Dest) {
2109	MachineBasicBlock *MBB;
2110	if (parseMBBReference(MBB))
2111	return true;
2112	Dest = MachineOperand::CreateMBB(MBB);
2113	lex();
2114	return false;
2115	}
2116
2117	bool MIParser::parseStackFrameIndex(int &FI) {
2118	assert(Token.is(MIToken::StackObject));
2119	unsigned ID;
2120	if (getUnsigned(Result&: ID))
2121	return true;
2122	auto ObjectInfo = PFS.StackObjectSlots.find(Val: ID);
2123	if (ObjectInfo == PFS.StackObjectSlots.end())
2124	return error(Msg: Twine("use of undefined stack object '%stack.") + Twine(ID) +
2125	"'");
2126	StringRef Name;
2127	if (const auto *Alloca =
2128	MF.getFrameInfo().getObjectAllocation(ObjectIdx: ObjectInfo->second))
2129	Name = Alloca->getName();
2130	if (!Token.stringValue().empty() && Token.stringValue() != Name)
2131	return error(Msg: Twine("the name of the stack object '%stack.") + Twine(ID) +
2132	"' isn't '" + Token.stringValue() + "'");
2133	lex();
2134	FI = ObjectInfo->second;
2135	return false;
2136	}
2137
2138	bool MIParser::parseStackObjectOperand(MachineOperand &Dest) {
2139	int FI;
2140	if (parseStackFrameIndex(FI))
2141	return true;
2142	Dest = MachineOperand::CreateFI(Idx: FI);
2143	return false;
2144	}
2145
2146	bool MIParser::parseFixedStackFrameIndex(int &FI) {
2147	assert(Token.is(MIToken::FixedStackObject));
2148	unsigned ID;
2149	if (getUnsigned(Result&: ID))
2150	return true;
2151	auto ObjectInfo = PFS.FixedStackObjectSlots.find(Val: ID);
2152	if (ObjectInfo == PFS.FixedStackObjectSlots.end())
2153	return error(Msg: Twine("use of undefined fixed stack object '%fixed-stack.") +
2154	Twine(ID) + "'");
2155	lex();
2156	FI = ObjectInfo->second;
2157	return false;
2158	}
2159
2160	bool MIParser::parseFixedStackObjectOperand(MachineOperand &Dest) {
2161	int FI;
2162	if (parseFixedStackFrameIndex(FI))
2163	return true;
2164	Dest = MachineOperand::CreateFI(Idx: FI);
2165	return false;
2166	}
2167
2168	static bool parseGlobalValue(const MIToken &Token,
2169	PerFunctionMIParsingState &PFS, GlobalValue *&GV,
2170	ErrorCallbackType ErrCB) {
2171	switch (Token.kind()) {
2172	case MIToken::NamedGlobalValue: {
2173	const Module *M = PFS.MF.getFunction().getParent();
2174	GV = M->getNamedValue(Name: Token.stringValue());
2175	if (!GV)
2176	return ErrCB(Token.location(), Twine("use of undefined global value '") +
2177	Token.range() + "'");
2178	break;
2179	}
2180	case MIToken::GlobalValue: {
2181	unsigned GVIdx;
2182	if (getUnsigned(Token, Result&: GVIdx, ErrCB))
2183	return true;
2184	GV = PFS.IRSlots.GlobalValues.get(ID: GVIdx);
2185	if (!GV)
2186	return ErrCB(Token.location(), Twine("use of undefined global value '@") +
2187	Twine(GVIdx) + "'");
2188	break;
2189	}
2190	default:
2191	llvm_unreachable("The current token should be a global value");
2192	}
2193	return false;
2194	}
2195
2196	bool MIParser::parseGlobalValue(GlobalValue *&GV) {
2197	return ::parseGlobalValue(
2198	Token, PFS, GV,
2199	ErrCB: [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
2200	return error(Loc, Msg);
2201	});
2202	}
2203
2204	bool MIParser::parseGlobalAddressOperand(MachineOperand &Dest) {
2205	GlobalValue *GV = nullptr;
2206	if (parseGlobalValue(GV))
2207	return true;
2208	lex();
2209	Dest = MachineOperand::CreateGA(GV, /*Offset=*/0);
2210	if (parseOperandsOffset(Op&: Dest))
2211	return true;
2212	return false;
2213	}
2214
2215	bool MIParser::parseConstantPoolIndexOperand(MachineOperand &Dest) {
2216	assert(Token.is(MIToken::ConstantPoolItem));
2217	unsigned ID;
2218	if (getUnsigned(Result&: ID))
2219	return true;
2220	auto ConstantInfo = PFS.ConstantPoolSlots.find(Val: ID);
2221	if (ConstantInfo == PFS.ConstantPoolSlots.end())
2222	return error(Msg: "use of undefined constant '%const." + Twine(ID) + "'");
2223	lex();
2224	Dest = MachineOperand::CreateCPI(Idx: ID, /*Offset=*/0);
2225	if (parseOperandsOffset(Op&: Dest))
2226	return true;
2227	return false;
2228	}
2229
2230	bool MIParser::parseJumpTableIndexOperand(MachineOperand &Dest) {
2231	assert(Token.is(MIToken::JumpTableIndex));
2232	unsigned ID;
2233	if (getUnsigned(Result&: ID))
2234	return true;
2235	auto JumpTableEntryInfo = PFS.JumpTableSlots.find(Val: ID);
2236	if (JumpTableEntryInfo == PFS.JumpTableSlots.end())
2237	return error(Msg: "use of undefined jump table '%jump-table." + Twine(ID) + "'");
2238	lex();
2239	Dest = MachineOperand::CreateJTI(Idx: JumpTableEntryInfo->second);
2240	return false;
2241	}
2242
2243	bool MIParser::parseExternalSymbolOperand(MachineOperand &Dest) {
2244	assert(Token.is(MIToken::ExternalSymbol));
2245	const char *Symbol = MF.createExternalSymbolName(Name: Token.stringValue());
2246	lex();
2247	Dest = MachineOperand::CreateES(SymName: Symbol);
2248	if (parseOperandsOffset(Op&: Dest))
2249	return true;
2250	return false;
2251	}
2252
2253	bool MIParser::parseMCSymbolOperand(MachineOperand &Dest) {
2254	assert(Token.is(MIToken::MCSymbol));
2255	MCSymbol *Symbol = getOrCreateMCSymbol(Name: Token.stringValue());
2256	lex();
2257	Dest = MachineOperand::CreateMCSymbol(Sym: Symbol);
2258	if (parseOperandsOffset(Op&: Dest))
2259	return true;
2260	return false;
2261	}
2262
2263	bool MIParser::parseSubRegisterIndexOperand(MachineOperand &Dest) {
2264	assert(Token.is(MIToken::SubRegisterIndex));
2265	StringRef Name = Token.stringValue();
2266	unsigned SubRegIndex = PFS.Target.getSubRegIndex(Name: Token.stringValue());
2267	if (SubRegIndex == 0)
2268	return error(Msg: Twine("unknown subregister index '") + Name + "'");
2269	lex();
2270	Dest = MachineOperand::CreateImm(Val: SubRegIndex);
2271	return false;
2272	}
2273
2274	bool MIParser::parseMDNode(MDNode *&Node) {
2275	assert(Token.is(MIToken::exclaim));
2276
2277	auto Loc = Token.location();
2278	lex();
2279	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isSigned())
2280	return error(Msg: "expected metadata id after '!'");
2281	unsigned ID;
2282	if (getUnsigned(Result&: ID))
2283	return true;
2284	auto NodeInfo = PFS.IRSlots.MetadataNodes.find(x: ID);
2285	if (NodeInfo == PFS.IRSlots.MetadataNodes.end()) {
2286	NodeInfo = PFS.MachineMetadataNodes.find(x: ID);
2287	if (NodeInfo == PFS.MachineMetadataNodes.end())
2288	return error(Loc, Msg: "use of undefined metadata '!" + Twine(ID) + "'");
2289	}
2290	lex();
2291	Node = NodeInfo->second.get();
2292	return false;
2293	}
2294
2295	bool MIParser::parseDIExpression(MDNode *&Expr) {
2296	assert(Token.is(MIToken::md_diexpr));
2297	lex();
2298
2299	// FIXME: Share this parsing with the IL parser.
2300	SmallVector<uint64_t, 8> Elements;
2301
2302	if (expectAndConsume(TokenKind: MIToken::lparen))
2303	return true;
2304
2305	if (Token.isNot(K: MIToken::rparen)) {
2306	do {
2307	if (Token.is(K: MIToken::Identifier)) {
2308	if (unsigned Op = dwarf::getOperationEncoding(OperationEncodingString: Token.stringValue())) {
2309	lex();
2310	Elements.push_back(Elt: Op);
2311	continue;
2312	}
2313	if (unsigned Enc = dwarf::getAttributeEncoding(EncodingString: Token.stringValue())) {
2314	lex();
2315	Elements.push_back(Elt: Enc);
2316	continue;
2317	}
2318	return error(Msg: Twine("invalid DWARF op '") + Token.stringValue() + "'");
2319	}
2320
2321	if (Token.isNot(K: MIToken::IntegerLiteral) ||
2322	Token.integerValue().isSigned())
2323	return error(Msg: "expected unsigned integer");
2324
2325	auto &U = Token.integerValue();
2326	if (U.ugt(UINT64_MAX))
2327	return error(Msg: "element too large, limit is " + Twine(UINT64_MAX));
2328	Elements.push_back(Elt: U.getZExtValue());
2329	lex();
2330
2331	} while (consumeIfPresent(TokenKind: MIToken::comma));
2332	}
2333
2334	if (expectAndConsume(TokenKind: MIToken::rparen))
2335	return true;
2336
2337	Expr = DIExpression::get(Context&: MF.getFunction().getContext(), Elements);
2338	return false;
2339	}
2340
2341	bool MIParser::parseDILocation(MDNode *&Loc) {
2342	assert(Token.is(MIToken::md_dilocation));
2343	lex();
2344
2345	bool HaveLine = false;
2346	unsigned Line = 0;
2347	unsigned Column = 0;
2348	MDNode *Scope = nullptr;
2349	MDNode *InlinedAt = nullptr;
2350	bool ImplicitCode = false;
2351
2352	if (expectAndConsume(TokenKind: MIToken::lparen))
2353	return true;
2354
2355	if (Token.isNot(K: MIToken::rparen)) {
2356	do {
2357	if (Token.is(K: MIToken::Identifier)) {
2358	if (Token.stringValue() == "line") {
2359	lex();
2360	if (expectAndConsume(TokenKind: MIToken::colon))
2361	return true;
2362	if (Token.isNot(K: MIToken::IntegerLiteral) ||
2363	Token.integerValue().isSigned())
2364	return error(Msg: "expected unsigned integer");
2365	Line = Token.integerValue().getZExtValue();
2366	HaveLine = true;
2367	lex();
2368	continue;
2369	}
2370	if (Token.stringValue() == "column") {
2371	lex();
2372	if (expectAndConsume(TokenKind: MIToken::colon))
2373	return true;
2374	if (Token.isNot(K: MIToken::IntegerLiteral) ||
2375	Token.integerValue().isSigned())
2376	return error(Msg: "expected unsigned integer");
2377	Column = Token.integerValue().getZExtValue();
2378	lex();
2379	continue;
2380	}
2381	if (Token.stringValue() == "scope") {
2382	lex();
2383	if (expectAndConsume(TokenKind: MIToken::colon))
2384	return true;
2385	if (parseMDNode(Node&: Scope))
2386	return error(Msg: "expected metadata node");
2387	if (!isa<DIScope>(Val: Scope))
2388	return error(Msg: "expected DIScope node");
2389	continue;
2390	}
2391	if (Token.stringValue() == "inlinedAt") {
2392	lex();
2393	if (expectAndConsume(TokenKind: MIToken::colon))
2394	return true;
2395	if (Token.is(K: MIToken::exclaim)) {
2396	if (parseMDNode(Node&: InlinedAt))
2397	return true;
2398	} else if (Token.is(K: MIToken::md_dilocation)) {
2399	if (parseDILocation(Loc&: InlinedAt))
2400	return true;
2401	} else
2402	return error(Msg: "expected metadata node");
2403	if (!isa<DILocation>(Val: InlinedAt))
2404	return error(Msg: "expected DILocation node");
2405	continue;
2406	}
2407	if (Token.stringValue() == "isImplicitCode") {
2408	lex();
2409	if (expectAndConsume(TokenKind: MIToken::colon))
2410	return true;
2411	if (!Token.is(K: MIToken::Identifier))
2412	return error(Msg: "expected true/false");
2413	// As far as I can see, we don't have any existing need for parsing
2414	// true/false in MIR yet. Do it ad-hoc until there's something else
2415	// that needs it.
2416	if (Token.stringValue() == "true")
2417	ImplicitCode = true;
2418	else if (Token.stringValue() == "false")
2419	ImplicitCode = false;
2420	else
2421	return error(Msg: "expected true/false");
2422	lex();
2423	continue;
2424	}
2425	}
2426	return error(Msg: Twine("invalid DILocation argument '") +
2427	Token.stringValue() + "'");
2428	} while (consumeIfPresent(TokenKind: MIToken::comma));
2429	}
2430
2431	if (expectAndConsume(TokenKind: MIToken::rparen))
2432	return true;
2433
2434	if (!HaveLine)
2435	return error(Msg: "DILocation requires line number");
2436	if (!Scope)
2437	return error(Msg: "DILocation requires a scope");
2438
2439	Loc = DILocation::get(Context&: MF.getFunction().getContext(), Line, Column, Scope,
2440	InlinedAt, ImplicitCode);
2441	return false;
2442	}
2443
2444	bool MIParser::parseMetadataOperand(MachineOperand &Dest) {
2445	MDNode *Node = nullptr;
2446	if (Token.is(K: MIToken::exclaim)) {
2447	if (parseMDNode(Node))
2448	return true;
2449	} else if (Token.is(K: MIToken::md_diexpr)) {
2450	if (parseDIExpression(Expr&: Node))
2451	return true;
2452	}
2453	Dest = MachineOperand::CreateMetadata(Meta: Node);
2454	return false;
2455	}
2456
2457	bool MIParser::parseCFIOffset(int &Offset) {
2458	if (Token.isNot(K: MIToken::IntegerLiteral))
2459	return error(Msg: "expected a cfi offset");
2460	if (Token.integerValue().getSignificantBits() > 32)
2461	return error(Msg: "expected a 32 bit integer (the cfi offset is too large)");
2462	Offset = (int)Token.integerValue().getExtValue();
2463	lex();
2464	return false;
2465	}
2466
2467	bool MIParser::parseCFIRegister(Register &Reg) {
2468	if (Token.isNot(K: MIToken::NamedRegister))
2469	return error(Msg: "expected a cfi register");
2470	Register LLVMReg;
2471	if (parseNamedRegister(Reg&: LLVMReg))
2472	return true;
2473	const auto *TRI = MF.getSubtarget().getRegisterInfo();
2474	assert(TRI && "Expected target register info");
2475	int DwarfReg = TRI->getDwarfRegNum(RegNum: LLVMReg, isEH: true);
2476	if (DwarfReg < 0)
2477	return error(Msg: "invalid DWARF register");
2478	Reg = (unsigned)DwarfReg;
2479	lex();
2480	return false;
2481	}
2482
2483	bool MIParser::parseCFIAddressSpace(unsigned &AddressSpace) {
2484	if (Token.isNot(K: MIToken::IntegerLiteral))
2485	return error(Msg: "expected a cfi address space literal");
2486	if (Token.integerValue().isSigned())
2487	return error(Msg: "expected an unsigned integer (cfi address space)");
2488	AddressSpace = Token.integerValue().getZExtValue();
2489	lex();
2490	return false;
2491	}
2492
2493	bool MIParser::parseCFIEscapeValues(std::string &Values) {
2494	do {
2495	if (Token.isNot(K: MIToken::HexLiteral))
2496	return error(Msg: "expected a hexadecimal literal");
2497	unsigned Value;
2498	if (getUnsigned(Result&: Value))
2499	return true;
2500	if (Value > UINT8_MAX)
2501	return error(Msg: "expected a 8-bit integer (too large)");
2502	Values.push_back(c: static_cast<uint8_t>(Value));
2503	lex();
2504	} while (consumeIfPresent(TokenKind: MIToken::comma));
2505	return false;
2506	}
2507
2508	bool MIParser::parseCFIOperand(MachineOperand &Dest) {
2509	auto Kind = Token.kind();
2510	lex();
2511	int Offset;
2512	Register Reg;
2513	unsigned AddressSpace;
2514	unsigned CFIIndex;
2515	switch (Kind) {
2516	case MIToken::kw_cfi_same_value:
2517	if (parseCFIRegister(Reg))
2518	return true;
2519	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createSameValue(L: nullptr, Register: Reg));
2520	break;
2521	case MIToken::kw_cfi_offset:
2522	if (parseCFIRegister(Reg) || expectAndConsume(TokenKind: MIToken::comma) ||
2523	parseCFIOffset(Offset))
2524	return true;
2525	CFIIndex =
2526	MF.addFrameInst(Inst: MCCFIInstruction::createOffset(L: nullptr, Register: Reg, Offset));
2527	break;
2528	case MIToken::kw_cfi_rel_offset:
2529	if (parseCFIRegister(Reg) || expectAndConsume(TokenKind: MIToken::comma) ||
2530	parseCFIOffset(Offset))
2531	return true;
2532	CFIIndex = MF.addFrameInst(
2533	Inst: MCCFIInstruction::createRelOffset(L: nullptr, Register: Reg, Offset));
2534	break;
2535	case MIToken::kw_cfi_def_cfa_register:
2536	if (parseCFIRegister(Reg))
2537	return true;
2538	CFIIndex =
2539	MF.addFrameInst(Inst: MCCFIInstruction::createDefCfaRegister(L: nullptr, Register: Reg));
2540	break;
2541	case MIToken::kw_cfi_def_cfa_offset:
2542	if (parseCFIOffset(Offset))
2543	return true;
2544	CFIIndex =
2545	MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset));
2546	break;
2547	case MIToken::kw_cfi_adjust_cfa_offset:
2548	if (parseCFIOffset(Offset))
2549	return true;
2550	CFIIndex = MF.addFrameInst(
2551	Inst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: Offset));
2552	break;
2553	case MIToken::kw_cfi_def_cfa:
2554	if (parseCFIRegister(Reg) || expectAndConsume(TokenKind: MIToken::comma) ||
2555	parseCFIOffset(Offset))
2556	return true;
2557	CFIIndex =
2558	MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(L: nullptr, Register: Reg, Offset));
2559	break;
2560	case MIToken::kw_cfi_llvm_def_aspace_cfa:
2561	if (parseCFIRegister(Reg) || expectAndConsume(TokenKind: MIToken::comma) ||
2562	parseCFIOffset(Offset) || expectAndConsume(TokenKind: MIToken::comma) ||
2563	parseCFIAddressSpace(AddressSpace))
2564	return true;
2565	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createLLVMDefAspaceCfa(
2566	L: nullptr, Register: Reg, Offset, AddressSpace, Loc: SMLoc()));
2567	break;
2568	case MIToken::kw_cfi_remember_state:
2569	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRememberState(L: nullptr));
2570	break;
2571	case MIToken::kw_cfi_restore:
2572	if (parseCFIRegister(Reg))
2573	return true;
2574	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestore(L: nullptr, Register: Reg));
2575	break;
2576	case MIToken::kw_cfi_restore_state:
2577	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestoreState(L: nullptr));
2578	break;
2579	case MIToken::kw_cfi_undefined:
2580	if (parseCFIRegister(Reg))
2581	return true;
2582	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createUndefined(L: nullptr, Register: Reg));
2583	break;
2584	case MIToken::kw_cfi_register: {
2585	Register Reg2;
2586	if (parseCFIRegister(Reg) || expectAndConsume(TokenKind: MIToken::comma) ||
2587	parseCFIRegister(Reg&: Reg2))
2588	return true;
2589
2590	CFIIndex =
2591	MF.addFrameInst(Inst: MCCFIInstruction::createRegister(L: nullptr, Register1: Reg, Register2: Reg2));
2592	break;
2593	}
2594	case MIToken::kw_cfi_window_save:
2595	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createWindowSave(L: nullptr));
2596	break;
2597	case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
2598	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr));
2599	break;
2600	case MIToken::kw_cfi_escape: {
2601	std::string Values;
2602	if (parseCFIEscapeValues(Values))
2603	return true;
2604	CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createEscape(L: nullptr, Vals: Values));
2605	break;
2606	}
2607	default:
2608	// TODO: Parse the other CFI operands.
2609	llvm_unreachable("The current token should be a cfi operand");
2610	}
2611	Dest = MachineOperand::CreateCFIIndex(CFIIndex);
2612	return false;
2613	}
2614
2615	bool MIParser::parseIRBlock(BasicBlock *&BB, const Function &F) {
2616	switch (Token.kind()) {
2617	case MIToken::NamedIRBlock: {
2618	BB = dyn_cast_or_null<BasicBlock>(
2619	Val: F.getValueSymbolTable()->lookup(Name: Token.stringValue()));
2620	if (!BB)
2621	return error(Msg: Twine("use of undefined IR block '") + Token.range() + "'");
2622	break;
2623	}
2624	case MIToken::IRBlock: {
2625	unsigned SlotNumber = 0;
2626	if (getUnsigned(Result&: SlotNumber))
2627	return true;
2628	BB = const_cast<BasicBlock *>(getIRBlock(Slot: SlotNumber, F));
2629	if (!BB)
2630	return error(Msg: Twine("use of undefined IR block '%ir-block.") +
2631	Twine(SlotNumber) + "'");
2632	break;
2633	}
2634	default:
2635	llvm_unreachable("The current token should be an IR block reference");
2636	}
2637	return false;
2638	}
2639
2640	bool MIParser::parseBlockAddressOperand(MachineOperand &Dest) {
2641	assert(Token.is(MIToken::kw_blockaddress));
2642	lex();
2643	if (expectAndConsume(TokenKind: MIToken::lparen))
2644	return true;
2645	if (Token.isNot(K: MIToken::GlobalValue) &&
2646	Token.isNot(K: MIToken::NamedGlobalValue))
2647	return error(Msg: "expected a global value");
2648	GlobalValue *GV = nullptr;
2649	if (parseGlobalValue(GV))
2650	return true;
2651	auto *F = dyn_cast<Function>(Val: GV);
2652	if (!F)
2653	return error(Msg: "expected an IR function reference");
2654	lex();
2655	if (expectAndConsume(TokenKind: MIToken::comma))
2656	return true;
2657	BasicBlock *BB = nullptr;
2658	if (Token.isNot(K: MIToken::IRBlock) && Token.isNot(K: MIToken::NamedIRBlock))
2659	return error(Msg: "expected an IR block reference");
2660	if (parseIRBlock(BB, F: *F))
2661	return true;
2662	lex();
2663	if (expectAndConsume(TokenKind: MIToken::rparen))
2664	return true;
2665	Dest = MachineOperand::CreateBA(BA: BlockAddress::get(F, BB), /*Offset=*/0);
2666	if (parseOperandsOffset(Op&: Dest))
2667	return true;
2668	return false;
2669	}
2670
2671	bool MIParser::parseIntrinsicOperand(MachineOperand &Dest) {
2672	assert(Token.is(MIToken::kw_intrinsic));
2673	lex();
2674	if (expectAndConsume(TokenKind: MIToken::lparen))
2675	return error(Msg: "expected syntax intrinsic(@llvm.whatever)");
2676
2677	if (Token.isNot(K: MIToken::NamedGlobalValue))
2678	return error(Msg: "expected syntax intrinsic(@llvm.whatever)");
2679
2680	std::string Name = std::string(Token.stringValue());
2681	lex();
2682
2683	if (expectAndConsume(TokenKind: MIToken::rparen))
2684	return error(Msg: "expected ')' to terminate intrinsic name");
2685
2686	// Find out what intrinsic we're dealing with, first try the global namespace
2687	// and then the target's private intrinsics if that fails.
2688	const TargetIntrinsicInfo *TII = MF.getTarget().getIntrinsicInfo();
2689	Intrinsic::ID ID = Function::lookupIntrinsicID(Name);
2690	if (ID == Intrinsic::not_intrinsic && TII)
2691	ID = static_cast<Intrinsic::ID>(TII->lookupName(Name));
2692
2693	if (ID == Intrinsic::not_intrinsic)
2694	return error(Msg: "unknown intrinsic name");
2695	Dest = MachineOperand::CreateIntrinsicID(ID);
2696
2697	return false;
2698	}
2699
2700	bool MIParser::parsePredicateOperand(MachineOperand &Dest) {
2701	assert(Token.is(MIToken::kw_intpred) || Token.is(MIToken::kw_floatpred));
2702	bool IsFloat = Token.is(K: MIToken::kw_floatpred);
2703	lex();
2704
2705	if (expectAndConsume(TokenKind: MIToken::lparen))
2706	return error(Msg: "expected syntax intpred(whatever) or floatpred(whatever");
2707
2708	if (Token.isNot(K: MIToken::Identifier))
2709	return error(Msg: "whatever");
2710
2711	CmpInst::Predicate Pred;
2712	if (IsFloat) {
2713	Pred = StringSwitch<CmpInst::Predicate>(Token.stringValue())
2714	.Case(S: "false", Value: CmpInst::FCMP_FALSE)
2715	.Case(S: "oeq", Value: CmpInst::FCMP_OEQ)
2716	.Case(S: "ogt", Value: CmpInst::FCMP_OGT)
2717	.Case(S: "oge", Value: CmpInst::FCMP_OGE)
2718	.Case(S: "olt", Value: CmpInst::FCMP_OLT)
2719	.Case(S: "ole", Value: CmpInst::FCMP_OLE)
2720	.Case(S: "one", Value: CmpInst::FCMP_ONE)
2721	.Case(S: "ord", Value: CmpInst::FCMP_ORD)
2722	.Case(S: "uno", Value: CmpInst::FCMP_UNO)
2723	.Case(S: "ueq", Value: CmpInst::FCMP_UEQ)
2724	.Case(S: "ugt", Value: CmpInst::FCMP_UGT)
2725	.Case(S: "uge", Value: CmpInst::FCMP_UGE)
2726	.Case(S: "ult", Value: CmpInst::FCMP_ULT)
2727	.Case(S: "ule", Value: CmpInst::FCMP_ULE)
2728	.Case(S: "une", Value: CmpInst::FCMP_UNE)
2729	.Case(S: "true", Value: CmpInst::FCMP_TRUE)
2730	.Default(Value: CmpInst::BAD_FCMP_PREDICATE);
2731	if (!CmpInst::isFPPredicate(P: Pred))
2732	return error(Msg: "invalid floating-point predicate");
2733	} else {
2734	Pred = StringSwitch<CmpInst::Predicate>(Token.stringValue())
2735	.Case(S: "eq", Value: CmpInst::ICMP_EQ)
2736	.Case(S: "ne", Value: CmpInst::ICMP_NE)
2737	.Case(S: "sgt", Value: CmpInst::ICMP_SGT)
2738	.Case(S: "sge", Value: CmpInst::ICMP_SGE)
2739	.Case(S: "slt", Value: CmpInst::ICMP_SLT)
2740	.Case(S: "sle", Value: CmpInst::ICMP_SLE)
2741	.Case(S: "ugt", Value: CmpInst::ICMP_UGT)
2742	.Case(S: "uge", Value: CmpInst::ICMP_UGE)
2743	.Case(S: "ult", Value: CmpInst::ICMP_ULT)
2744	.Case(S: "ule", Value: CmpInst::ICMP_ULE)
2745	.Default(Value: CmpInst::BAD_ICMP_PREDICATE);
2746	if (!CmpInst::isIntPredicate(P: Pred))
2747	return error(Msg: "invalid integer predicate");
2748	}
2749
2750	lex();
2751	Dest = MachineOperand::CreatePredicate(Pred);
2752	if (expectAndConsume(TokenKind: MIToken::rparen))
2753	return error(Msg: "predicate should be terminated by ')'.");
2754
2755	return false;
2756	}
2757
2758	bool MIParser::parseShuffleMaskOperand(MachineOperand &Dest) {
2759	assert(Token.is(MIToken::kw_shufflemask));
2760
2761	lex();
2762	if (expectAndConsume(TokenKind: MIToken::lparen))
2763	return error(Msg: "expected syntax shufflemask(<integer or undef>, ...)");
2764
2765	SmallVector<int, 32> ShufMask;
2766	do {
2767	if (Token.is(K: MIToken::kw_undef)) {
2768	ShufMask.push_back(Elt: -1);
2769	} else if (Token.is(K: MIToken::IntegerLiteral)) {
2770	const APSInt &Int = Token.integerValue();
2771	ShufMask.push_back(Elt: Int.getExtValue());
2772	} else
2773	return error(Msg: "expected integer constant");
2774
2775	lex();
2776	} while (consumeIfPresent(TokenKind: MIToken::comma));
2777
2778	if (expectAndConsume(TokenKind: MIToken::rparen))
2779	return error(Msg: "shufflemask should be terminated by ')'.");
2780
2781	ArrayRef<int> MaskAlloc = MF.allocateShuffleMask(Mask: ShufMask);
2782	Dest = MachineOperand::CreateShuffleMask(Mask: MaskAlloc);
2783	return false;
2784	}
2785
2786	bool MIParser::parseDbgInstrRefOperand(MachineOperand &Dest) {
2787	assert(Token.is(MIToken::kw_dbg_instr_ref));
2788
2789	lex();
2790	if (expectAndConsume(TokenKind: MIToken::lparen))
2791	return error(Msg: "expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2792
2793	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isNegative())
2794	return error(Msg: "expected unsigned integer for instruction index");
2795	uint64_t InstrIdx = Token.integerValue().getZExtValue();
2796	assert(InstrIdx <= std::numeric_limits<unsigned>::max() &&
2797	"Instruction reference's instruction index is too large");
2798	lex();
2799
2800	if (expectAndConsume(TokenKind: MIToken::comma))
2801	return error(Msg: "expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2802
2803	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isNegative())
2804	return error(Msg: "expected unsigned integer for operand index");
2805	uint64_t OpIdx = Token.integerValue().getZExtValue();
2806	assert(OpIdx <= std::numeric_limits<unsigned>::max() &&
2807	"Instruction reference's operand index is too large");
2808	lex();
2809
2810	if (expectAndConsume(TokenKind: MIToken::rparen))
2811	return error(Msg: "expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2812
2813	Dest = MachineOperand::CreateDbgInstrRef(InstrIdx, OpIdx);
2814	return false;
2815	}
2816
2817	bool MIParser::parseTargetIndexOperand(MachineOperand &Dest) {
2818	assert(Token.is(MIToken::kw_target_index));
2819	lex();
2820	if (expectAndConsume(TokenKind: MIToken::lparen))
2821	return true;
2822	if (Token.isNot(K: MIToken::Identifier))
2823	return error(Msg: "expected the name of the target index");
2824	int Index = 0;
2825	if (PFS.Target.getTargetIndex(Name: Token.stringValue(), Index))
2826	return error(Msg: "use of undefined target index '" + Token.stringValue() + "'");
2827	lex();
2828	if (expectAndConsume(TokenKind: MIToken::rparen))
2829	return true;
2830	Dest = MachineOperand::CreateTargetIndex(Idx: unsigned(Index), /*Offset=*/0);
2831	if (parseOperandsOffset(Op&: Dest))
2832	return true;
2833	return false;
2834	}
2835
2836	bool MIParser::parseCustomRegisterMaskOperand(MachineOperand &Dest) {
2837	assert(Token.stringValue() == "CustomRegMask" && "Expected a custom RegMask");
2838	lex();
2839	if (expectAndConsume(TokenKind: MIToken::lparen))
2840	return true;
2841
2842	uint32_t *Mask = MF.allocateRegMask();
2843	do {
2844	if (Token.isNot(K: MIToken::rparen)) {
2845	if (Token.isNot(K: MIToken::NamedRegister))
2846	return error(Msg: "expected a named register");
2847	Register Reg;
2848	if (parseNamedRegister(Reg))
2849	return true;
2850	lex();
2851	Mask[Reg / 32] |= 1U << (Reg % 32);
2852	}
2853
2854	// TODO: Report an error if the same register is used more than once.
2855	} while (consumeIfPresent(TokenKind: MIToken::comma));
2856
2857	if (expectAndConsume(TokenKind: MIToken::rparen))
2858	return true;
2859	Dest = MachineOperand::CreateRegMask(Mask);
2860	return false;
2861	}
2862
2863	bool MIParser::parseLiveoutRegisterMaskOperand(MachineOperand &Dest) {
2864	assert(Token.is(MIToken::kw_liveout));
2865	uint32_t *Mask = MF.allocateRegMask();
2866	lex();
2867	if (expectAndConsume(TokenKind: MIToken::lparen))
2868	return true;
2869	while (true) {
2870	if (Token.isNot(K: MIToken::NamedRegister))
2871	return error(Msg: "expected a named register");
2872	Register Reg;
2873	if (parseNamedRegister(Reg))
2874	return true;
2875	lex();
2876	Mask[Reg / 32] |= 1U << (Reg % 32);
2877	// TODO: Report an error if the same register is used more than once.
2878	if (Token.isNot(K: MIToken::comma))
2879	break;
2880	lex();
2881	}
2882	if (expectAndConsume(TokenKind: MIToken::rparen))
2883	return true;
2884	Dest = MachineOperand::CreateRegLiveOut(Mask);
2885	return false;
2886	}
2887
2888	bool MIParser::parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
2889	MachineOperand &Dest,
2890	std::optional<unsigned> &TiedDefIdx) {
2891	switch (Token.kind()) {
2892	case MIToken::kw_implicit:
2893	case MIToken::kw_implicit_define:
2894	case MIToken::kw_def:
2895	case MIToken::kw_dead:
2896	case MIToken::kw_killed:
2897	case MIToken::kw_undef:
2898	case MIToken::kw_internal:
2899	case MIToken::kw_early_clobber:
2900	case MIToken::kw_debug_use:
2901	case MIToken::kw_renamable:
2902	case MIToken::underscore:
2903	case MIToken::NamedRegister:
2904	case MIToken::VirtualRegister:
2905	case MIToken::NamedVirtualRegister:
2906	return parseRegisterOperand(Dest, TiedDefIdx);
2907	case MIToken::IntegerLiteral:
2908	return parseImmediateOperand(Dest);
2909	case MIToken::kw_half:
2910	case MIToken::kw_float:
2911	case MIToken::kw_double:
2912	case MIToken::kw_x86_fp80:
2913	case MIToken::kw_fp128:
2914	case MIToken::kw_ppc_fp128:
2915	return parseFPImmediateOperand(Dest);
2916	case MIToken::MachineBasicBlock:
2917	return parseMBBOperand(Dest);
2918	case MIToken::StackObject:
2919	return parseStackObjectOperand(Dest);
2920	case MIToken::FixedStackObject:
2921	return parseFixedStackObjectOperand(Dest);
2922	case MIToken::GlobalValue:
2923	case MIToken::NamedGlobalValue:
2924	return parseGlobalAddressOperand(Dest);
2925	case MIToken::ConstantPoolItem:
2926	return parseConstantPoolIndexOperand(Dest);
2927	case MIToken::JumpTableIndex:
2928	return parseJumpTableIndexOperand(Dest);
2929	case MIToken::ExternalSymbol:
2930	return parseExternalSymbolOperand(Dest);
2931	case MIToken::MCSymbol:
2932	return parseMCSymbolOperand(Dest);
2933	case MIToken::SubRegisterIndex:
2934	return parseSubRegisterIndexOperand(Dest);
2935	case MIToken::md_diexpr:
2936	case MIToken::exclaim:
2937	return parseMetadataOperand(Dest);
2938	case MIToken::kw_cfi_same_value:
2939	case MIToken::kw_cfi_offset:
2940	case MIToken::kw_cfi_rel_offset:
2941	case MIToken::kw_cfi_def_cfa_register:
2942	case MIToken::kw_cfi_def_cfa_offset:
2943	case MIToken::kw_cfi_adjust_cfa_offset:
2944	case MIToken::kw_cfi_escape:
2945	case MIToken::kw_cfi_def_cfa:
2946	case MIToken::kw_cfi_llvm_def_aspace_cfa:
2947	case MIToken::kw_cfi_register:
2948	case MIToken::kw_cfi_remember_state:
2949	case MIToken::kw_cfi_restore:
2950	case MIToken::kw_cfi_restore_state:
2951	case MIToken::kw_cfi_undefined:
2952	case MIToken::kw_cfi_window_save:
2953	case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
2954	return parseCFIOperand(Dest);
2955	case MIToken::kw_blockaddress:
2956	return parseBlockAddressOperand(Dest);
2957	case MIToken::kw_intrinsic:
2958	return parseIntrinsicOperand(Dest);
2959	case MIToken::kw_target_index:
2960	return parseTargetIndexOperand(Dest);
2961	case MIToken::kw_liveout:
2962	return parseLiveoutRegisterMaskOperand(Dest);
2963	case MIToken::kw_floatpred:
2964	case MIToken::kw_intpred:
2965	return parsePredicateOperand(Dest);
2966	case MIToken::kw_shufflemask:
2967	return parseShuffleMaskOperand(Dest);
2968	case MIToken::kw_dbg_instr_ref:
2969	return parseDbgInstrRefOperand(Dest);
2970	case MIToken::Error:
2971	return true;
2972	case MIToken::Identifier:
2973	if (const auto *RegMask = PFS.Target.getRegMask(Identifier: Token.stringValue())) {
2974	Dest = MachineOperand::CreateRegMask(Mask: RegMask);
2975	lex();
2976	break;
2977	} else if (Token.stringValue() == "CustomRegMask") {
2978	return parseCustomRegisterMaskOperand(Dest);
2979	} else
2980	return parseTypedImmediateOperand(Dest);
2981	case MIToken::dot: {
2982	const auto *TII = MF.getSubtarget().getInstrInfo();
2983	if (const auto *Formatter = TII->getMIRFormatter()) {
2984	return parseTargetImmMnemonic(OpCode, OpIdx, Dest, MF: *Formatter);
2985	}
2986	[[fallthrough]];
2987	}
2988	default:
2989	// FIXME: Parse the MCSymbol machine operand.
2990	return error(Msg: "expected a machine operand");
2991	}
2992	return false;
2993	}
2994
2995	bool MIParser::parseMachineOperandAndTargetFlags(
2996	const unsigned OpCode, const unsigned OpIdx, MachineOperand &Dest,
2997	std::optional<unsigned> &TiedDefIdx) {
2998	unsigned TF = 0;
2999	bool HasTargetFlags = false;
3000	if (Token.is(K: MIToken::kw_target_flags)) {
3001	HasTargetFlags = true;
3002	lex();
3003	if (expectAndConsume(TokenKind: MIToken::lparen))
3004	return true;
3005	if (Token.isNot(K: MIToken::Identifier))
3006	return error(Msg: "expected the name of the target flag");
3007	if (PFS.Target.getDirectTargetFlag(Name: Token.stringValue(), Flag&: TF)) {
3008	if (PFS.Target.getBitmaskTargetFlag(Name: Token.stringValue(), Flag&: TF))
3009	return error(Msg: "use of undefined target flag '" + Token.stringValue() +
3010	"'");
3011	}
3012	lex();
3013	while (Token.is(K: MIToken::comma)) {
3014	lex();
3015	if (Token.isNot(K: MIToken::Identifier))
3016	return error(Msg: "expected the name of the target flag");
3017	unsigned BitFlag = 0;
3018	if (PFS.Target.getBitmaskTargetFlag(Name: Token.stringValue(), Flag&: BitFlag))
3019	return error(Msg: "use of undefined target flag '" + Token.stringValue() +
3020	"'");
3021	// TODO: Report an error when using a duplicate bit target flag.
3022	TF |= BitFlag;
3023	lex();
3024	}
3025	if (expectAndConsume(TokenKind: MIToken::rparen))
3026	return true;
3027	}
3028	auto Loc = Token.location();
3029	if (parseMachineOperand(OpCode, OpIdx, Dest, TiedDefIdx))
3030	return true;
3031	if (!HasTargetFlags)
3032	return false;
3033	if (Dest.isReg())
3034	return error(Loc, Msg: "register operands can't have target flags");
3035	Dest.setTargetFlags(TF);
3036	return false;
3037	}
3038
3039	bool MIParser::parseOffset(int64_t &Offset) {
3040	if (Token.isNot(K: MIToken::plus) && Token.isNot(K: MIToken::minus))
3041	return false;
3042	StringRef Sign = Token.range();
3043	bool IsNegative = Token.is(K: MIToken::minus);
3044	lex();
3045	if (Token.isNot(K: MIToken::IntegerLiteral))
3046	return error(Msg: "expected an integer literal after '" + Sign + "'");
3047	if (Token.integerValue().getSignificantBits() > 64)
3048	return error(Msg: "expected 64-bit integer (too large)");
3049	Offset = Token.integerValue().getExtValue();
3050	if (IsNegative)
3051	Offset = -Offset;
3052	lex();
3053	return false;
3054	}
3055
3056	bool MIParser::parseIRBlockAddressTaken(BasicBlock *&BB) {
3057	assert(Token.is(MIToken::kw_ir_block_address_taken));
3058	lex();
3059	if (Token.isNot(K: MIToken::IRBlock) && Token.isNot(K: MIToken::NamedIRBlock))
3060	return error(Msg: "expected basic block after 'ir_block_address_taken'");
3061
3062	if (parseIRBlock(BB, F: MF.getFunction()))
3063	return true;
3064
3065	lex();
3066	return false;
3067	}
3068
3069	bool MIParser::parseAlignment(uint64_t &Alignment) {
3070	assert(Token.is(MIToken::kw_align) || Token.is(MIToken::kw_basealign));
3071	lex();
3072	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isSigned())
3073	return error(Msg: "expected an integer literal after 'align'");
3074	if (getUint64(Result&: Alignment))
3075	return true;
3076	lex();
3077
3078	if (!isPowerOf2_64(Value: Alignment))
3079	return error(Msg: "expected a power-of-2 literal after 'align'");
3080
3081	return false;
3082	}
3083
3084	bool MIParser::parseAddrspace(unsigned &Addrspace) {
3085	assert(Token.is(MIToken::kw_addrspace));
3086	lex();
3087	if (Token.isNot(K: MIToken::IntegerLiteral) || Token.integerValue().isSigned())
3088	return error(Msg: "expected an integer literal after 'addrspace'");
3089	if (getUnsigned(Result&: Addrspace))
3090	return true;
3091	lex();
3092	return false;
3093	}
3094
3095	bool MIParser::parseOperandsOffset(MachineOperand &Op) {
3096	int64_t Offset = 0;
3097	if (parseOffset(Offset))
3098	return true;
3099	Op.setOffset(Offset);
3100	return false;
3101	}
3102
3103	static bool parseIRValue(const MIToken &Token, PerFunctionMIParsingState &PFS,
3104	const Value *&V, ErrorCallbackType ErrCB) {
3105	switch (Token.kind()) {
3106	case MIToken::NamedIRValue: {
3107	V = PFS.MF.getFunction().getValueSymbolTable()->lookup(Name: Token.stringValue());
3108	break;
3109	}
3110	case MIToken::IRValue: {
3111	unsigned SlotNumber = 0;
3112	if (getUnsigned(Token, Result&: SlotNumber, ErrCB))
3113	return true;
3114	V = PFS.getIRValue(Slot: SlotNumber);
3115	break;
3116	}
3117	case MIToken::NamedGlobalValue:
3118	case MIToken::GlobalValue: {
3119	GlobalValue *GV = nullptr;
3120	if (parseGlobalValue(Token, PFS, GV, ErrCB))
3121	return true;
3122	V = GV;
3123	break;
3124	}
3125	case MIToken::QuotedIRValue: {
3126	const Constant *C = nullptr;
3127	if (parseIRConstant(Loc: Token.location(), StringValue: Token.stringValue(), PFS, C, ErrCB))
3128	return true;
3129	V = C;
3130	break;
3131	}
3132	case MIToken::kw_unknown_address:
3133	V = nullptr;
3134	return false;
3135	default:
3136	llvm_unreachable("The current token should be an IR block reference");
3137	}
3138	if (!V)
3139	return ErrCB(Token.location(), Twine("use of undefined IR value '") + Token.range() + "'");
3140	return false;
3141	}
3142
3143	bool MIParser::parseIRValue(const Value *&V) {
3144	return ::parseIRValue(
3145	Token, PFS, V, ErrCB: [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
3146	return error(Loc, Msg);
3147	});
3148	}
3149
3150	bool MIParser::getUint64(uint64_t &Result) {
3151	if (Token.hasIntegerValue()) {
3152	if (Token.integerValue().getActiveBits() > 64)
3153	return error(Msg: "expected 64-bit integer (too large)");
3154	Result = Token.integerValue().getZExtValue();
3155	return false;
3156	}
3157	if (Token.is(K: MIToken::HexLiteral)) {
3158	APInt A;
3159	if (getHexUint(Result&: A))
3160	return true;
3161	if (A.getBitWidth() > 64)
3162	return error(Msg: "expected 64-bit integer (too large)");
3163	Result = A.getZExtValue();
3164	return false;
3165	}
3166	return true;
3167	}
3168
3169	bool MIParser::getHexUint(APInt &Result) {
3170	return ::getHexUint(Token, Result);
3171	}
3172
3173	bool MIParser::parseMemoryOperandFlag(MachineMemOperand::Flags &Flags) {
3174	const auto OldFlags = Flags;
3175	switch (Token.kind()) {
3176	case MIToken::kw_volatile:
3177	Flags |= MachineMemOperand::MOVolatile;
3178	break;
3179	case MIToken::kw_non_temporal:
3180	Flags |= MachineMemOperand::MONonTemporal;
3181	break;
3182	case MIToken::kw_dereferenceable:
3183	Flags |= MachineMemOperand::MODereferenceable;
3184	break;
3185	case MIToken::kw_invariant:
3186	Flags |= MachineMemOperand::MOInvariant;
3187	break;
3188	case MIToken::StringConstant: {
3189	MachineMemOperand::Flags TF;
3190	if (PFS.Target.getMMOTargetFlag(Name: Token.stringValue(), Flag&: TF))
3191	return error(Msg: "use of undefined target MMO flag '" + Token.stringValue() +
3192	"'");
3193	Flags |= TF;
3194	break;
3195	}
3196	default:
3197	llvm_unreachable("The current token should be a memory operand flag");
3198	}
3199	if (OldFlags == Flags)
3200	// We know that the same flag is specified more than once when the flags
3201	// weren't modified.
3202	return error(Msg: "duplicate '" + Token.stringValue() + "' memory operand flag");
3203	lex();
3204	return false;
3205	}
3206
3207	bool MIParser::parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV) {
3208	switch (Token.kind()) {
3209	case MIToken::kw_stack:
3210	PSV = MF.getPSVManager().getStack();
3211	break;
3212	case MIToken::kw_got:
3213	PSV = MF.getPSVManager().getGOT();
3214	break;
3215	case MIToken::kw_jump_table:
3216	PSV = MF.getPSVManager().getJumpTable();
3217	break;
3218	case MIToken::kw_constant_pool:
3219	PSV = MF.getPSVManager().getConstantPool();
3220	break;
3221	case MIToken::FixedStackObject: {
3222	int FI;
3223	if (parseFixedStackFrameIndex(FI))
3224	return true;
3225	PSV = MF.getPSVManager().getFixedStack(FI);
3226	// The token was already consumed, so use return here instead of break.
3227	return false;
3228	}
3229	case MIToken::StackObject: {
3230	int FI;
3231	if (parseStackFrameIndex(FI))
3232	return true;
3233	PSV = MF.getPSVManager().getFixedStack(FI);
3234	// The token was already consumed, so use return here instead of break.
3235	return false;
3236	}
3237	case MIToken::kw_call_entry:
3238	lex();
3239	switch (Token.kind()) {
3240	case MIToken::GlobalValue:
3241	case MIToken::NamedGlobalValue: {
3242	GlobalValue *GV = nullptr;
3243	if (parseGlobalValue(GV))
3244	return true;
3245	PSV = MF.getPSVManager().getGlobalValueCallEntry(GV);
3246	break;
3247	}
3248	case MIToken::ExternalSymbol:
3249	PSV = MF.getPSVManager().getExternalSymbolCallEntry(
3250	ES: MF.createExternalSymbolName(Name: Token.stringValue()));
3251	break;
3252	default:
3253	return error(
3254	Msg: "expected a global value or an external symbol after 'call-entry'");
3255	}
3256	break;
3257	case MIToken::kw_custom: {
3258	lex();
3259	const auto *TII = MF.getSubtarget().getInstrInfo();
3260	if (const auto *Formatter = TII->getMIRFormatter()) {
3261	if (Formatter->parseCustomPseudoSourceValue(
3262	Src: Token.stringValue(), MF, PFS, PSV,
3263	ErrorCallback: [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
3264	return error(Loc, Msg);
3265	}))
3266	return true;
3267	} else
3268	return error(Msg: "unable to parse target custom pseudo source value");
3269	break;
3270	}
3271	default:
3272	llvm_unreachable("The current token should be pseudo source value");
3273	}
3274	lex();
3275	return false;
3276	}
3277
3278	bool MIParser::parseMachinePointerInfo(MachinePointerInfo &Dest) {
3279	if (Token.is(K: MIToken::kw_constant_pool) || Token.is(K: MIToken::kw_stack) ||
3280	Token.is(K: MIToken::kw_got) || Token.is(K: MIToken::kw_jump_table) ||
3281	Token.is(K: MIToken::FixedStackObject) || Token.is(K: MIToken::StackObject) ||
3282	Token.is(K: MIToken::kw_call_entry) || Token.is(K: MIToken::kw_custom)) {
3283	const PseudoSourceValue *PSV = nullptr;
3284	if (parseMemoryPseudoSourceValue(PSV))
3285	return true;
3286	int64_t Offset = 0;
3287	if (parseOffset(Offset))
3288	return true;
3289	Dest = MachinePointerInfo(PSV, Offset);
3290	return false;
3291	}
3292	if (Token.isNot(K: MIToken::NamedIRValue) && Token.isNot(K: MIToken::IRValue) &&
3293	Token.isNot(K: MIToken::GlobalValue) &&
3294	Token.isNot(K: MIToken::NamedGlobalValue) &&
3295	Token.isNot(K: MIToken::QuotedIRValue) &&
3296	Token.isNot(K: MIToken::kw_unknown_address))
3297	return error(Msg: "expected an IR value reference");
3298	const Value *V = nullptr;
3299	if (parseIRValue(V))
3300	return true;
3301	if (V && !V->getType()->isPointerTy())
3302	return error(Msg: "expected a pointer IR value");
3303	lex();
3304	int64_t Offset = 0;
3305	if (parseOffset(Offset))
3306	return true;
3307	Dest = MachinePointerInfo(V, Offset);
3308	return false;
3309	}
3310
3311	bool MIParser::parseOptionalScope(LLVMContext &Context,
3312	SyncScope::ID &SSID) {
3313	SSID = SyncScope::System;
3314	if (Token.is(K: MIToken::Identifier) && Token.stringValue() == "syncscope") {
3315	lex();
3316	if (expectAndConsume(TokenKind: MIToken::lparen))
3317	return error(Msg: "expected '(' in syncscope");
3318
3319	std::string SSN;
3320	if (parseStringConstant(Result&: SSN))
3321	return true;
3322
3323	SSID = Context.getOrInsertSyncScopeID(SSN);
3324	if (expectAndConsume(TokenKind: MIToken::rparen))
3325	return error(Msg: "expected ')' in syncscope");
3326	}
3327
3328	return false;
3329	}
3330
3331	bool MIParser::parseOptionalAtomicOrdering(AtomicOrdering &Order) {
3332	Order = AtomicOrdering::NotAtomic;
3333	if (Token.isNot(K: MIToken::Identifier))
3334	return false;
3335
3336	Order = StringSwitch<AtomicOrdering>(Token.stringValue())
3337	.Case(S: "unordered", Value: AtomicOrdering::Unordered)
3338	.Case(S: "monotonic", Value: AtomicOrdering::Monotonic)
3339	.Case(S: "acquire", Value: AtomicOrdering::Acquire)
3340	.Case(S: "release", Value: AtomicOrdering::Release)
3341	.Case(S: "acq_rel", Value: AtomicOrdering::AcquireRelease)
3342	.Case(S: "seq_cst", Value: AtomicOrdering::SequentiallyConsistent)
3343	.Default(Value: AtomicOrdering::NotAtomic);
3344
3345	if (Order != AtomicOrdering::NotAtomic) {
3346	lex();
3347	return false;
3348	}
3349
3350	return error(Msg: "expected an atomic scope, ordering or a size specification");
3351	}
3352
3353	bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
3354	if (expectAndConsume(TokenKind: MIToken::lparen))
3355	return true;
3356	MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
3357	while (Token.isMemoryOperandFlag()) {
3358	if (parseMemoryOperandFlag(Flags))
3359	return true;
3360	}
3361	if (Token.isNot(K: MIToken::Identifier) ||
3362	(Token.stringValue() != "load" && Token.stringValue() != "store"))
3363	return error(Msg: "expected 'load' or 'store' memory operation");
3364	if (Token.stringValue() == "load")
3365	Flags |= MachineMemOperand::MOLoad;
3366	else
3367	Flags |= MachineMemOperand::MOStore;
3368	lex();
3369
3370	// Optional 'store' for operands that both load and store.
3371	if (Token.is(K: MIToken::Identifier) && Token.stringValue() == "store") {
3372	Flags |= MachineMemOperand::MOStore;
3373	lex();
3374	}
3375
3376	// Optional synchronization scope.
3377	SyncScope::ID SSID;
3378	if (parseOptionalScope(Context&: MF.getFunction().getContext(), SSID))
3379	return true;
3380
3381	// Up to two atomic orderings (cmpxchg provides guarantees on failure).
3382	AtomicOrdering Order, FailureOrder;
3383	if (parseOptionalAtomicOrdering(Order))
3384	return true;
3385
3386	if (parseOptionalAtomicOrdering(Order&: FailureOrder))
3387	return true;
3388
3389	LLT MemoryType;
3390	if (Token.isNot(K: MIToken::IntegerLiteral) &&
3391	Token.isNot(K: MIToken::kw_unknown_size) &&
3392	Token.isNot(K: MIToken::lparen))
3393	return error(Msg: "expected memory LLT, the size integer literal or 'unknown-size' after "
3394	"memory operation");
3395
3396	uint64_t Size = MemoryLocation::UnknownSize;
3397	if (Token.is(K: MIToken::IntegerLiteral)) {
3398	if (getUint64(Result&: Size))
3399	return true;
3400
3401	// Convert from bytes to bits for storage.
3402	MemoryType = LLT::scalar(SizeInBits: 8 * Size);
3403	lex();
3404	} else if (Token.is(K: MIToken::kw_unknown_size)) {
3405	Size = MemoryLocation::UnknownSize;
3406	lex();
3407	} else {
3408	if (expectAndConsume(TokenKind: MIToken::lparen))
3409	return true;
3410	if (parseLowLevelType(Loc: Token.location(), Ty&: MemoryType))
3411	return true;
3412	if (expectAndConsume(TokenKind: MIToken::rparen))
3413	return true;
3414
3415	Size = MemoryType.getSizeInBytes();
3416	}
3417
3418	MachinePointerInfo Ptr = MachinePointerInfo();
3419	if (Token.is(K: MIToken::Identifier)) {
3420	const char *Word =
3421	((Flags & MachineMemOperand::MOLoad) &&
3422	(Flags & MachineMemOperand::MOStore))
3423	? "on"
3424	: Flags & MachineMemOperand::MOLoad ? "from" : "into";
3425	if (Token.stringValue() != Word)
3426	return error(Msg: Twine("expected '") + Word + "'");
3427	lex();
3428
3429	if (parseMachinePointerInfo(Dest&: Ptr))
3430	return true;
3431	}
3432	uint64_t BaseAlignment =
3433	(Size != MemoryLocation::UnknownSize ? PowerOf2Ceil(A: Size) : 1);
3434	AAMDNodes AAInfo;
3435	MDNode *Range = nullptr;
3436	while (consumeIfPresent(TokenKind: MIToken::comma)) {
3437	switch (Token.kind()) {
3438	case MIToken::kw_align: {
3439	// align is printed if it is different than size.
3440	uint64_t Alignment;
3441	if (parseAlignment(Alignment))
3442	return true;
3443	if (Ptr.Offset & (Alignment - 1)) {
3444	// MachineMemOperand::getAlign never returns a value greater than the
3445	// alignment of offset, so this just guards against hand-written MIR
3446	// that specifies a large "align" value when it should probably use
3447	// "basealign" instead.
3448	return error(Msg: "specified alignment is more aligned than offset");
3449	}
3450	BaseAlignment = Alignment;
3451	break;
3452	}
3453	case MIToken::kw_basealign:
3454	// basealign is printed if it is different than align.
3455	if (parseAlignment(Alignment&: BaseAlignment))
3456	return true;
3457	break;
3458	case MIToken::kw_addrspace:
3459	if (parseAddrspace(Addrspace&: Ptr.AddrSpace))
3460	return true;
3461	break;
3462	case MIToken::md_tbaa:
3463	lex();
3464	if (parseMDNode(Node&: AAInfo.TBAA))
3465	return true;
3466	break;
3467	case MIToken::md_alias_scope:
3468	lex();
3469	if (parseMDNode(Node&: AAInfo.Scope))
3470	return true;
3471	break;
3472	case MIToken::md_noalias:
3473	lex();
3474	if (parseMDNode(Node&: AAInfo.NoAlias))
3475	return true;
3476	break;
3477	case MIToken::md_range:
3478	lex();
3479	if (parseMDNode(Node&: Range))
3480	return true;
3481	break;
3482	// TODO: Report an error on duplicate metadata nodes.
3483	default:
3484	return error(Msg: "expected 'align' or '!tbaa' or '!alias.scope' or "
3485	"'!noalias' or '!range'");
3486	}
3487	}
3488	if (expectAndConsume(TokenKind: MIToken::rparen))
3489	return true;
3490	Dest = MF.getMachineMemOperand(PtrInfo: Ptr, f: Flags, MemTy: MemoryType, base_alignment: Align(BaseAlignment),
3491	AAInfo, Ranges: Range, SSID, Ordering: Order, FailureOrdering: FailureOrder);
3492	return false;
3493	}
3494
3495	bool MIParser::parsePreOrPostInstrSymbol(MCSymbol *&Symbol) {
3496	assert((Token.is(MIToken::kw_pre_instr_symbol) ||
3497	Token.is(MIToken::kw_post_instr_symbol)) &&
3498	"Invalid token for a pre- post-instruction symbol!");
3499	lex();
3500	if (Token.isNot(K: MIToken::MCSymbol))
3501	return error(Msg: "expected a symbol after 'pre-instr-symbol'");
3502	Symbol = getOrCreateMCSymbol(Name: Token.stringValue());
3503	lex();
3504	if (Token.isNewlineOrEOF() || Token.is(K: MIToken::coloncolon) ||
3505	Token.is(K: MIToken::lbrace))
3506	return false;
3507	if (Token.isNot(K: MIToken::comma))
3508	return error(Msg: "expected ',' before the next machine operand");
3509	lex();
3510	return false;
3511	}
3512
3513	bool MIParser::parseHeapAllocMarker(MDNode *&Node) {
3514	assert(Token.is(MIToken::kw_heap_alloc_marker) &&
3515	"Invalid token for a heap alloc marker!");
3516	lex();
3517	if (parseMDNode(Node))
3518	return true;
3519	if (!Node)
3520	return error(Msg: "expected a MDNode after 'heap-alloc-marker'");
3521	if (Token.isNewlineOrEOF() || Token.is(K: MIToken::coloncolon) ||
3522	Token.is(K: MIToken::lbrace))
3523	return false;
3524	if (Token.isNot(K: MIToken::comma))
3525	return error(Msg: "expected ',' before the next machine operand");
3526	lex();
3527	return false;
3528	}
3529
3530	bool MIParser::parsePCSections(MDNode *&Node) {
3531	assert(Token.is(MIToken::kw_pcsections) &&
3532	"Invalid token for a PC sections!");
3533	lex();
3534	if (parseMDNode(Node))
3535	return true;
3536	if (!Node)
3537	return error(Msg: "expected a MDNode after 'pcsections'");
3538	if (Token.isNewlineOrEOF() || Token.is(K: MIToken::coloncolon) ||
3539	Token.is(K: MIToken::lbrace))
3540	return false;
3541	if (Token.isNot(K: MIToken::comma))
3542	return error(Msg: "expected ',' before the next machine operand");
3543	lex();
3544	return false;
3545	}
3546
3547	static void initSlots2BasicBlocks(
3548	const Function &F,
3549	DenseMap<unsigned, const BasicBlock *> &Slots2BasicBlocks) {
3550	ModuleSlotTracker MST(F.getParent(), /*ShouldInitializeAllMetadata=*/false);
3551	MST.incorporateFunction(F);
3552	for (const auto &BB : F) {
3553	if (BB.hasName())
3554	continue;
3555	int Slot = MST.getLocalSlot(V: &BB);
3556	if (Slot == -1)
3557	continue;
3558	Slots2BasicBlocks.insert(KV: std::make_pair(x: unsigned(Slot), y: &BB));
3559	}
3560	}
3561
3562	static const BasicBlock *getIRBlockFromSlot(
3563	unsigned Slot,
3564	const DenseMap<unsigned, const BasicBlock *> &Slots2BasicBlocks) {
3565	return Slots2BasicBlocks.lookup(Val: Slot);
3566	}
3567
3568	const BasicBlock *MIParser::getIRBlock(unsigned Slot) {
3569	if (Slots2BasicBlocks.empty())
3570	initSlots2BasicBlocks(F: MF.getFunction(), Slots2BasicBlocks);
3571	return getIRBlockFromSlot(Slot, Slots2BasicBlocks);
3572	}
3573
3574	const BasicBlock *MIParser::getIRBlock(unsigned Slot, const Function &F) {
3575	if (&F == &MF.getFunction())
3576	return getIRBlock(Slot);
3577	DenseMap<unsigned, const BasicBlock *> CustomSlots2BasicBlocks;
3578	initSlots2BasicBlocks(F, Slots2BasicBlocks&: CustomSlots2BasicBlocks);
3579	return getIRBlockFromSlot(Slot, Slots2BasicBlocks: CustomSlots2BasicBlocks);
3580	}
3581
3582	MCSymbol *MIParser::getOrCreateMCSymbol(StringRef Name) {
3583	// FIXME: Currently we can't recognize temporary or local symbols and call all
3584	// of the appropriate forms to create them. However, this handles basic cases
3585	// well as most of the special aspects are recognized by a prefix on their
3586	// name, and the input names should already be unique. For test cases, keeping
3587	// the symbol name out of the symbol table isn't terribly important.
3588	return MF.getContext().getOrCreateSymbol(Name);
3589	}
3590
3591	bool MIParser::parseStringConstant(std::string &Result) {
3592	if (Token.isNot(K: MIToken::StringConstant))
3593	return error(Msg: "expected string constant");
3594	Result = std::string(Token.stringValue());
3595	lex();
3596	return false;
3597	}
3598
3599	bool llvm::parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS,
3600	StringRef Src,
3601	SMDiagnostic &Error) {
3602	return MIParser(PFS, Error, Src).parseBasicBlockDefinitions(MBBSlots&: PFS.MBBSlots);
3603	}
3604
3605	bool llvm::parseMachineInstructions(PerFunctionMIParsingState &PFS,
3606	StringRef Src, SMDiagnostic &Error) {
3607	return MIParser(PFS, Error, Src).parseBasicBlocks();
3608	}
3609
3610	bool llvm::parseMBBReference(PerFunctionMIParsingState &PFS,
3611	MachineBasicBlock *&MBB, StringRef Src,
3612	SMDiagnostic &Error) {
3613	return MIParser(PFS, Error, Src).parseStandaloneMBB(MBB);
3614	}
3615
3616	bool llvm::parseRegisterReference(PerFunctionMIParsingState &PFS,
3617	Register &Reg, StringRef Src,
3618	SMDiagnostic &Error) {
3619	return MIParser(PFS, Error, Src).parseStandaloneRegister(Reg);
3620	}
3621
3622	bool llvm::parseNamedRegisterReference(PerFunctionMIParsingState &PFS,
3623	Register &Reg, StringRef Src,
3624	SMDiagnostic &Error) {
3625	return MIParser(PFS, Error, Src).parseStandaloneNamedRegister(Reg);
3626	}
3627
3628	bool llvm::parseVirtualRegisterReference(PerFunctionMIParsingState &PFS,
3629	VRegInfo *&Info, StringRef Src,
3630	SMDiagnostic &Error) {
3631	return MIParser(PFS, Error, Src).parseStandaloneVirtualRegister(Info);
3632	}
3633
3634	bool llvm::parseStackObjectReference(PerFunctionMIParsingState &PFS,
3635	int &FI, StringRef Src,
3636	SMDiagnostic &Error) {
3637	return MIParser(PFS, Error, Src).parseStandaloneStackObject(FI);
3638	}
3639
3640	bool llvm::parseMDNode(PerFunctionMIParsingState &PFS,
3641	MDNode *&Node, StringRef Src, SMDiagnostic &Error) {
3642	return MIParser(PFS, Error, Src).parseStandaloneMDNode(Node);
3643	}
3644
3645	bool llvm::parseMachineMetadata(PerFunctionMIParsingState &PFS, StringRef Src,
3646	SMRange SrcRange, SMDiagnostic &Error) {
3647	return MIParser(PFS, Error, Src, SrcRange).parseMachineMetadata();
3648	}
3649
3650	bool MIRFormatter::parseIRValue(StringRef Src, MachineFunction &MF,
3651	PerFunctionMIParsingState &PFS, const Value *&V,
3652	ErrorCallbackType ErrorCallback) {
3653	MIToken Token;
3654	Src = lexMIToken(Source: Src, Token, ErrorCallback: [&](StringRef::iterator Loc, const Twine &Msg) {
3655	ErrorCallback(Loc, Msg);
3656	});
3657	V = nullptr;
3658
3659	return ::parseIRValue(Token, PFS, V, ErrCB: ErrorCallback);
3660	}
3661