Preprocessor.h source code [include/clang/Lex/Preprocessor.h]

Warning: That file was not part of the compilation database. It may have many parsing errors.

1	//===- Preprocessor.h - C Language Family Preprocessor ----------- C++ --===//
2	//
3	// The LLVM Compiler Infrastructure
4	//
5	// This file is distributed under the University of Illinois Open Source
6	// License. See LICENSE.TXT for details.
7	//
8	//===----------------------------------------------------------------------===//
9	//
10	/// \file
11	/// Defines the clang::Preprocessor interface.
12	//
13	//===----------------------------------------------------------------------===//
14
15	#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16	#define LLVM_CLANG_LEX_PREPROCESSOR_H
17
18	#include "clang/Basic/Builtins.h"
19	#include "clang/Basic/Diagnostic.h"
20	#include "clang/Basic/IdentifierTable.h"
21	#include "clang/Basic/LLVM.h"
22	#include "clang/Basic/LangOptions.h"
23	#include "clang/Basic/Module.h"
24	#include "clang/Basic/SourceLocation.h"
25	#include "clang/Basic/SourceManager.h"
26	#include "clang/Basic/TokenKinds.h"
27	#include "clang/Lex/Lexer.h"
28	#include "clang/Lex/MacroInfo.h"
29	#include "clang/Lex/ModuleLoader.h"
30	#include "clang/Lex/ModuleMap.h"
31	#include "clang/Lex/PPCallbacks.h"
32	#include "clang/Lex/PTHLexer.h"
33	#include "clang/Lex/Token.h"
34	#include "clang/Lex/TokenLexer.h"
35	#include "llvm/ADT/ArrayRef.h"
36	#include "llvm/ADT/DenseMap.h"
37	#include "llvm/ADT/FoldingSet.h"
38	#include "llvm/ADT/None.h"
39	#include "llvm/ADT/Optional.h"
40	#include "llvm/ADT/PointerUnion.h"
41	#include "llvm/ADT/STLExtras.h"
42	#include "llvm/ADT/SmallPtrSet.h"
43	#include "llvm/ADT/SmallVector.h"
44	#include "llvm/ADT/StringRef.h"
45	#include "llvm/ADT/TinyPtrVector.h"
46	#include "llvm/ADT/iterator_range.h"
47	#include "llvm/Support/Allocator.h"
48	#include "llvm/Support/Casting.h"
49	#include "llvm/Support/Registry.h"
50	#include <cassert>
51	#include <cstddef>
52	#include <cstdint>
53	#include <memory>
54	#include <map>
55	#include <string>
56	#include <utility>
57	#include <vector>
58
59	namespace llvm {
60
61	template<unsigned InternalLen> class SmallString;
62
63	} // namespace llvm
64
65	namespace clang {
66
67	class CodeCompletionHandler;
68	class CommentHandler;
69	class DirectoryEntry;
70	class DirectoryLookup;
71	class ExternalPreprocessorSource;
72	class FileEntry;
73	class FileManager;
74	class HeaderSearch;
75	class MacroArgs;
76	class MemoryBufferCache;
77	class PragmaHandler;
78	class PragmaNamespace;
79	class PreprocessingRecord;
80	class PreprocessorLexer;
81	class PreprocessorOptions;
82	class PTHManager;
83	class ScratchBuffer;
84	class TargetInfo;
85
86	/// Stores token information for comparing actual tokens with
87	/// predefined values. Only handles simple tokens and identifiers.
88	class TokenValue {
89	tok::TokenKind Kind;
90	IdentifierInfo *II;
91
92	public:
93	TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
94	assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
95	assert(Kind != tok::identifier &&
96	"Identifiers should be created by TokenValue(IdentifierInfo *)");
97	assert(!tok::isLiteral(Kind) && "Literals are not supported.");
98	assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
99	}
100
101	TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
102
103	bool operator==(const Token &Tok) const {
104	return Tok.getKind() == Kind &&
105	(!II \|\| II == Tok.getIdentifierInfo());
106	}
107	};
108
109	/// Context in which macro name is used.
110	enum MacroUse {
111	// other than #define or #undef
112	MU_Other = `0`,
113
114	// macro name specified in #define
115	MU_Define = `1`,
116
117	// macro name specified in #undef
118	MU_Undef = `2`
119	};
120
121	/// Engages in a tight little dance with the lexer to efficiently
122	/// preprocess tokens.
123	///
124	/// Lexers know only about tokens within a single source file, and don't
125	/// know anything about preprocessor-level issues like the \#include stack,
126	/// token expansion, etc.
127	class Preprocessor {
128	friend class VAOptDefinitionContext;
129	friend class VariadicMacroScopeGuard;
130
131	std::shared_ptr<PreprocessorOptions> PPOpts;
132	DiagnosticsEngine *Diags;
133	LangOptions &LangOpts;
134	const TargetInfo *Target = nullptr;
135	const TargetInfo *AuxTarget = nullptr;
136	FileManager &FileMgr;
137	SourceManager &SourceMgr;
138	MemoryBufferCache &PCMCache;
139	std::unique_ptr<ScratchBuffer> ScratchBuf;
140	HeaderSearch &HeaderInfo;
141	ModuleLoader &TheModuleLoader;
142
143	/// External source of macros.
144	ExternalPreprocessorSource *ExternalSource;
145
146	/// An optional PTHManager object used for getting tokens from
147	/// a token cache rather than lexing the original source file.
148	std::unique_ptr<PTHManager> PTH;
149
150	/// A BumpPtrAllocator object used to quickly allocate and release
151	/// objects internal to the Preprocessor.
152	llvm::BumpPtrAllocator BP;
153
154	/// Identifiers for builtin macros and other builtins.
155	IdentifierInfo Ident__LINE__, Ident__FILE__; // __LINE__, __FILE__
156	IdentifierInfo Ident__DATE__, Ident__TIME__; // __DATE__, __TIME__
157	IdentifierInfo Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__*
158	IdentifierInfo Ident__BASE_FILE__; // __BASE_FILE__*
159	IdentifierInfo Ident__TIMESTAMP__; // __TIMESTAMP__*
160	IdentifierInfo Ident__COUNTER__; // __COUNTER__*
161	IdentifierInfo Ident_Pragma, Ident__pragma; // _Pragma, __pragma
162	IdentifierInfo Ident__identifier; // __identifier*
163	IdentifierInfo Ident__VA_ARGS__; // __VA_ARGS__*
164	IdentifierInfo Ident__VA_OPT__; // __VA_OPT__*
165	IdentifierInfo Ident__has_feature; // __has_feature*
166	IdentifierInfo Ident__has_extension; // __has_extension*
167	IdentifierInfo Ident__has_builtin; // __has_builtin*
168	IdentifierInfo Ident__has_attribute; // __has_attribute*
169	IdentifierInfo Ident__has_include; // __has_include*
170	IdentifierInfo Ident__has_include_next; // __has_include_next*
171	IdentifierInfo Ident__has_warning; // __has_warning*
172	IdentifierInfo Ident__is_identifier; // __is_identifier*
173	IdentifierInfo Ident__building_module; // __building_module*
174	IdentifierInfo Ident__MODULE__; // __MODULE__*
175	IdentifierInfo Ident__has_cpp_attribute; // __has_cpp_attribute*
176	IdentifierInfo Ident__has_c_attribute; // __has_c_attribute*
177	IdentifierInfo Ident__has_declspec; // __has_declspec_attribute*
178	IdentifierInfo Ident__is_target_arch; // __is_target_arch*
179	IdentifierInfo Ident__is_target_vendor; // __is_target_vendor*
180	IdentifierInfo Ident__is_target_os; // __is_target_os*
181	IdentifierInfo Ident__is_target_environment; // __is_target_environment*
182
183	SourceLocation DATELoc, TIMELoc;
184
185	// Next __COUNTER__ value, starts at 0.
186	unsigned CounterValue = `0`;
187
188	enum {
189	/// Maximum depth of \#includes.
190	MaxAllowedIncludeStackDepth = `200`
191	};
192
193	// State that is set before the preprocessor begins.
194	bool KeepComments : `1`;
195	bool KeepMacroComments : `1`;
196	bool SuppressIncludeNotFoundError : `1`;
197
198	// State that changes while the preprocessor runs:
199	bool InMacroArgs : `1`; // True if parsing fn macro invocation args.
200
201	/// Whether the preprocessor owns the header search object.
202	bool OwnsHeaderSearch : `1`;
203
204	/// True if macro expansion is disabled.
205	bool DisableMacroExpansion : `1`;
206
207	/// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
208	/// when parsing preprocessor directives.
209	bool MacroExpansionInDirectivesOverride : `1`;
210
211	class ResetMacroExpansionHelper;
212
213	/// Whether we have already loaded macros from the external source.
214	mutable bool ReadMacrosFromExternalSource : `1`;
215
216	/// True if pragmas are enabled.
217	bool PragmasEnabled : `1`;
218
219	/// True if the current build action is a preprocessing action.
220	bool PreprocessedOutput : `1`;
221
222	/// True if we are currently preprocessing a #if or #elif directive
223	bool ParsingIfOrElifDirective;
224
225	/// True if we are pre-expanding macro arguments.
226	bool InMacroArgPreExpansion;
227
228	/// Mapping/lookup information for all identifiers in
229	/// the program, including program keywords.
230	mutable IdentifierTable Identifiers;
231
232	/// This table contains all the selectors in the program.
233	///
234	/// Unlike IdentifierTable above, this table isn't* populated by the*
235	/// preprocessor. It is declared/expanded here because its role/lifetime is
236	/// conceptually similar to the IdentifierTable. In addition, the current
237	/// control flow (in clang::ParseAST()), make it convenient to put here.
238	///
239	/// FIXME: Make sure the lifetime of Identifiers/Selectors isn't* tied to*
240	/// the lifetime of the preprocessor.
241	SelectorTable Selectors;
242
243	/// Information about builtins.
244	Builtin::Context BuiltinInfo;
245
246	/// Tracks all of the pragmas that the client registered
247	/// with this preprocessor.
248	std::unique_ptr<PragmaNamespace> PragmaHandlers;
249
250	/// Pragma handlers of the original source is stored here during the
251	/// parsing of a model file.
252	std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
253
254	/// Tracks all of the comment handlers that the client registered
255	/// with this preprocessor.
256	std::vector<CommentHandler *> CommentHandlers;
257
258	/// True if we want to ignore EOF token and continue later on (thus
259	/// avoid tearing the Lexer and etc. down).
260	bool IncrementalProcessing = false;
261
262	/// The kind of translation unit we are processing.
263	TranslationUnitKind TUKind;
264
265	/// The code-completion handler.
266	CodeCompletionHandler *CodeComplete = nullptr;
267
268	/// The file that we're performing code-completion for, if any.
269	const FileEntry *CodeCompletionFile = nullptr;
270
271	/// The offset in file for the code-completion point.
272	unsigned CodeCompletionOffset = `0`;
273
274	/// The location for the code-completion point. This gets instantiated
275	/// when the CodeCompletionFile gets \#include'ed for preprocessing.
276	SourceLocation CodeCompletionLoc;
277
278	/// The start location for the file of the code-completion point.
279	///
280	/// This gets instantiated when the CodeCompletionFile gets \#include'ed
281	/// for preprocessing.
282	SourceLocation CodeCompletionFileLoc;
283
284	/// The source location of the \c import contextual keyword we just
285	/// lexed, if any.
286	SourceLocation ModuleImportLoc;
287
288	/// The module import path that we're currently processing.
289	SmallVector<std::pair<IdentifierInfo *, SourceLocation>, `2`> ModuleImportPath;
290
291	/// Whether the last token we lexed was an '@'.
292	bool LastTokenWasAt = false;
293
294	/// Whether the module import expects an identifier next. Otherwise,
295	/// it expects a '.' or ';'.
296	bool ModuleImportExpectsIdentifier = false;
297
298	/// The source location of the currently-active
299	/// \#pragma clang arc_cf_code_audited begin.
300	SourceLocation PragmaARCCFCodeAuditedLoc;
301
302	/// The source location of the currently-active
303	/// \#pragma clang assume_nonnull begin.
304	SourceLocation PragmaAssumeNonNullLoc;
305
306	/// True if we hit the code-completion point.
307	bool CodeCompletionReached = false;
308
309	/// The code completion token containing the information
310	/// on the stem that is to be code completed.
311	IdentifierInfo *CodeCompletionII = nullptr;
312
313	/// The directory that the main file should be considered to occupy,
314	/// if it does not correspond to a real file (as happens when building a
315	/// module).
316	const DirectoryEntry *MainFileDir = nullptr;
317
318	/// The number of bytes that we will initially skip when entering the
319	/// main file, along with a flag that indicates whether skipping this number
320	/// of bytes will place the lexer at the start of a line.
321	///
322	/// This is used when loading a precompiled preamble.
323	std::pair<int, bool> SkipMainFilePreamble;
324
325	public:
326	struct PreambleSkipInfo {
327	SourceLocation HashTokenLoc;
328	SourceLocation IfTokenLoc;
329	bool FoundNonSkipPortion;
330	bool FoundElse;
331	SourceLocation ElseLoc;
332
333	PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
334	bool FoundNonSkipPortion, bool FoundElse,
335	SourceLocation ElseLoc)
336	: HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
337	FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
338	ElseLoc(ElseLoc) {}
339	};
340
341	private:
342	friend class ASTReader;
343	friend class MacroArgs;
344
345	class PreambleConditionalStackStore {
346	enum State {
347	Off = `0`,
348	Recording = `1`,
349	Replaying = `2`,
350	};
351
352	public:
353	PreambleConditionalStackStore() = default;
354
355	void startRecording() { ConditionalStackState = Recording; }
356	void startReplaying() { ConditionalStackState = Replaying; }
357	bool isRecording() const { return ConditionalStackState == Recording; }
358	bool isReplaying() const { return ConditionalStackState == Replaying; }
359
360	ArrayRef<PPConditionalInfo> getStack() const {
361	return ConditionalStack;
362	}
363
364	void doneReplaying() {
365	ConditionalStack.clear();
366	ConditionalStackState = Off;
367	}
368
369	void setStack(ArrayRef<PPConditionalInfo> s) {
370	if (!isRecording() && !isReplaying())
371	return;
372	ConditionalStack.clear();
373	ConditionalStack.append(s.begin(), s.end());
374	}
375
376	bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
377
378	bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
379
380	void clearSkipInfo() { SkipInfo.reset(); }
381
382	llvm::Optional<PreambleSkipInfo> SkipInfo;
383
384	private:
385	SmallVector<PPConditionalInfo, `4`> ConditionalStack;
386	State ConditionalStackState = Off;
387	} PreambleConditionalStack;
388
389	/// The current top of the stack that we're lexing from if
390	/// not expanding a macro and we are lexing directly from source code.
391	///
392	/// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
393	std::unique_ptr<Lexer> CurLexer;
394
395	/// The current top of stack that we're lexing from if
396	/// not expanding from a macro and we are lexing from a PTH cache.
397	///
398	/// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
399	std::unique_ptr<PTHLexer> CurPTHLexer;
400
401	/// The current top of the stack what we're lexing from
402	/// if not expanding a macro.
403	///
404	/// This is an alias for either CurLexer or CurPTHLexer.
405	PreprocessorLexer *CurPPLexer = nullptr;
406
407	/// Used to find the current FileEntry, if CurLexer is non-null
408	/// and if applicable.
409	///
410	/// This allows us to implement \#include_next and find directory-specific
411	/// properties.
412	const DirectoryLookup *CurDirLookup = nullptr;
413
414	/// The current macro we are expanding, if we are expanding a macro.
415	///
416	/// One of CurLexer and CurTokenLexer must be null.
417	std::unique_ptr<TokenLexer> CurTokenLexer;
418
419	/// The kind of lexer we're currently working with.
420	enum CurLexerKind {
421	CLK_Lexer,
422	CLK_PTHLexer,
423	CLK_TokenLexer,
424	CLK_CachingLexer,
425	CLK_LexAfterModuleImport
426	} CurLexerKind = CLK_Lexer;
427
428	/// If the current lexer is for a submodule that is being built, this
429	/// is that submodule.
430	Module *CurLexerSubmodule = nullptr;
431
432	/// Keeps track of the stack of files currently
433	/// \#included, and macros currently being expanded from, not counting
434	/// CurLexer/CurTokenLexer.
435	struct IncludeStackInfo {
436	enum CurLexerKind CurLexerKind;
437	Module *TheSubmodule;
438	std::unique_ptr<Lexer> TheLexer;
439	std::unique_ptr<PTHLexer> ThePTHLexer;
440	PreprocessorLexer *ThePPLexer;
441	std::unique_ptr<TokenLexer> TheTokenLexer;
442	const DirectoryLookup *TheDirLookup;
443
444	// The following constructors are completely useless copies of the default
445	// versions, only needed to pacify MSVC.
446	IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
447	std::unique_ptr<Lexer> &&TheLexer,
448	std::unique_ptr<PTHLexer> &&ThePTHLexer,
449	PreprocessorLexer *ThePPLexer,
450	std::unique_ptr<TokenLexer> &&TheTokenLexer,
451	const DirectoryLookup *TheDirLookup)
452	: CurLexerKind(std::move(CurLexerKind)),
453	TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
454	ThePTHLexer(std::move(ThePTHLexer)),
455	ThePPLexer(std::move(ThePPLexer)),
456	TheTokenLexer(std::move(TheTokenLexer)),
457	TheDirLookup(std::move(TheDirLookup)) {}
458	};
459	std::vector<IncludeStackInfo> IncludeMacroStack;
460
461	/// Actions invoked when some preprocessor activity is
462	/// encountered (e.g. a file is \#included, etc).
463	std::unique_ptr<PPCallbacks> Callbacks;
464
465	struct MacroExpandsInfo {
466	Token Tok;
467	MacroDefinition MD;
468	SourceRange Range;
469
470	MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
471	: Tok(Tok), MD(MD), Range(Range) {}
472	};
473	SmallVector<MacroExpandsInfo, `2`> DelayedMacroExpandsCallbacks;
474
475	/// Information about a name that has been used to define a module macro.
476	struct ModuleMacroInfo {
477	/// The most recent macro directive for this identifier.
478	MacroDirective *MD;
479
480	/// The active module macros for this identifier.
481	llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
482
483	/// The generation number at which we last updated ActiveModuleMacros.
484	/// \see Preprocessor::VisibleModules.
485	unsigned ActiveModuleMacrosGeneration = `0`;
486
487	/// Whether this macro name is ambiguous.
488	bool IsAmbiguous = false;
489
490	/// The module macros that are overridden by this macro.
491	llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
492
493	ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
494	};
495
496	/// The state of a macro for an identifier.
497	class MacroState {
498	mutable llvm::PointerUnion<MacroDirective , ModuleMacroInfo > State;
499
500	ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
501	const IdentifierInfo II) const* {
502	if (II->isOutOfDate())
503	PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
504	// FIXME: Find a spare bit on IdentifierInfo and store a
505	// HasModuleMacros flag.
506	if (!II->hasMacroDefinition() \|\|
507	(!PP.getLangOpts().Modules &&
508	!PP.getLangOpts().ModulesLocalVisibility) \|\|
509	!PP.CurSubmoduleState->VisibleModules.getGeneration())
510	return nullptr;
511
512	auto Info = State.dyn_cast<ModuleMacroInfo>();
513	if (!Info) {
514	Info = new (PP.getPreprocessorAllocator())
515	ModuleMacroInfo(State.get<MacroDirective *>());
516	State = Info;
517	}
518
519	if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
520	Info->ActiveModuleMacrosGeneration)
521	PP.updateModuleMacroInfo(II, *Info);
522	return Info;
523	}
524
525	public:
526	MacroState() : MacroState(nullptr) {}
527	MacroState(MacroDirective *MD) : State(MD) {}
528
529	MacroState(MacroState &&O) noexcept : State(O.State) {
530	O.State = (MacroDirective *)nullptr;
531	}
532
533	MacroState &operator=(MacroState &&O) noexcept {
534	auto S = O.State;
535	O.State = (MacroDirective *)nullptr;
536	State = S;
537	return *this;
538	}
539
540	~MacroState() {
541	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
542	Info->~ModuleMacroInfo();
543	}
544
545	MacroDirective getLatest() const* {
546	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
547	return Info->MD;
548	return State.get<MacroDirective*>();
549	}
550
551	void setLatest(MacroDirective *MD) {
552	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
553	Info->MD = MD;
554	else
555	State = MD;
556	}
557
558	bool isAmbiguous(Preprocessor &PP, const IdentifierInfo II) const* {
559	auto *Info = getModuleInfo(PP, II);
560	return Info ? Info->IsAmbiguous : false;
561	}
562
563	ArrayRef<ModuleMacro *>
564	getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo II) const* {
565	if (auto *Info = getModuleInfo(PP, II))
566	return Info->ActiveModuleMacros;
567	return None;
568	}
569
570	MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
571	SourceManager &SourceMgr) const {
572	// FIXME: Incorporate module macros into the result of this.
573	if (auto *Latest = getLatest())
574	return Latest->findDirectiveAtLoc(Loc, SourceMgr);
575	return {};
576	}
577
578	void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
579	if (auto *Info = getModuleInfo(PP, II)) {
580	Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
581	Info->ActiveModuleMacros.begin(),
582	Info->ActiveModuleMacros.end());
583	Info->ActiveModuleMacros.clear();
584	Info->IsAmbiguous = false;
585	}
586	}
587
588	ArrayRef<ModuleMacro> getOverriddenMacros() const* {
589	if (auto Info = State.dyn_cast<ModuleMacroInfo>())
590	return Info->OverriddenMacros;
591	return None;
592	}
593
594	void setOverriddenMacros(Preprocessor &PP,
595	ArrayRef<ModuleMacro *> Overrides) {
596	auto Info = State.dyn_cast<ModuleMacroInfo>();
597	if (!Info) {
598	if (Overrides.empty())
599	return;
600	Info = new (PP.getPreprocessorAllocator())
601	ModuleMacroInfo(State.get<MacroDirective *>());
602	State = Info;
603	}
604	Info->OverriddenMacros.clear();
605	Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
606	Overrides.begin(), Overrides.end());
607	Info->ActiveModuleMacrosGeneration = `0`;
608	}
609	};
610
611	/// For each IdentifierInfo that was associated with a macro, we
612	/// keep a mapping to the history of all macro definitions and #undefs in
613	/// the reverse order (the latest one is in the head of the list).
614	///
615	/// This mapping lives within the \p CurSubmoduleState.
616	using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
617
618	struct SubmoduleState;
619
620	/// Information about a submodule that we're currently building.
621	struct BuildingSubmoduleInfo {
622	/// The module that we are building.
623	Module *M;
624
625	/// The location at which the module was included.
626	SourceLocation ImportLoc;
627
628	/// Whether we entered this submodule via a pragma.
629	bool IsPragma;
630
631	/// The previous SubmoduleState.
632	SubmoduleState *OuterSubmoduleState;
633
634	/// The number of pending module macro names when we started building this.
635	unsigned OuterPendingModuleMacroNames;
636
637	BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
638	SubmoduleState *OuterSubmoduleState,
639	unsigned OuterPendingModuleMacroNames)
640	: M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
641	OuterSubmoduleState(OuterSubmoduleState),
642	OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
643	};
644	SmallVector<BuildingSubmoduleInfo, `8`> BuildingSubmoduleStack;
645
646	/// Information about a submodule's preprocessor state.
647	struct SubmoduleState {
648	/// The macros for the submodule.
649	MacroMap Macros;
650
651	/// The set of modules that are visible within the submodule.
652	VisibleModuleSet VisibleModules;
653
654	// FIXME: CounterValue?
655	// FIXME: PragmaPushMacroInfo?
656	};
657	std::map<Module *, SubmoduleState> Submodules;
658
659	/// The preprocessor state for preprocessing outside of any submodule.
660	SubmoduleState NullSubmoduleState;
661
662	/// The current submodule state. Will be \p NullSubmoduleState if we're not
663	/// in a submodule.
664	SubmoduleState *CurSubmoduleState;
665
666	/// The set of known macros exported from modules.
667	llvm::FoldingSet<ModuleMacro> ModuleMacros;
668
669	/// The names of potential module macros that we've not yet processed.
670	llvm::SmallVector<const IdentifierInfo *, `32`> PendingModuleMacroNames;
671
672	/// The list of module macros, for each identifier, that are not overridden by
673	/// any other module macro.
674	llvm::DenseMap<const IdentifierInfo , llvm::TinyPtrVector<ModuleMacro >>
675	LeafModuleMacros;
676
677	/// Macros that we want to warn because they are not used at the end
678	/// of the translation unit.
679	///
680	/// We store just their SourceLocations instead of
681	/// something like MacroInfo. The benefit of this is that when we are*
682	/// deserializing from PCH, we don't need to deserialize identifier & macros
683	/// just so that we can report that they are unused, we just warn using
684	/// the SourceLocations of this set (that will be filled by the ASTReader).
685	/// We are using SmallPtrSet instead of a vector for faster removal.
686	using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, `32`>;
687	WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
688
689	/// A "freelist" of MacroArg objects that can be
690	/// reused for quick allocation.
691	MacroArgs *MacroArgCache = nullptr;
692
693	/// For each IdentifierInfo used in a \#pragma push_macro directive,
694	/// we keep a MacroInfo stack used to restore the previous macro value.
695	llvm::DenseMap<IdentifierInfo , std::vector<MacroInfo >>
696	PragmaPushMacroInfo;
697
698	// Various statistics we track for performance analysis.
699	unsigned NumDirectives = `0`;
700	unsigned NumDefined = `0`;
701	unsigned NumUndefined = `0`;
702	unsigned NumPragma = `0`;
703	unsigned NumIf = `0`;
704	unsigned NumElse = `0`;
705	unsigned NumEndif = `0`;
706	unsigned NumEnteredSourceFiles = `0`;
707	unsigned MaxIncludeStackDepth = `0`;
708	unsigned NumMacroExpanded = `0`;
709	unsigned NumFnMacroExpanded = `0`;
710	unsigned NumBuiltinMacroExpanded = `0`;
711	unsigned NumFastMacroExpanded = `0`;
712	unsigned NumTokenPaste = `0`;
713	unsigned NumFastTokenPaste = `0`;
714	unsigned NumSkipped = `0`;
715
716	/// The predefined macros that preprocessor should use from the
717	/// command line etc.
718	std::string Predefines;
719
720	/// The file ID for the preprocessor predefines.
721	FileID PredefinesFileID;
722
723	/// The file ID for the PCH through header.
724	FileID PCHThroughHeaderFileID;
725
726	/// Whether tokens are being skipped until the through header is seen.
727	bool SkippingUntilPCHThroughHeader = false;
728
729	/// \{
730	/// Cache of macro expanders to reduce malloc traffic.
731	enum { TokenLexerCacheSize = `8` };
732	unsigned NumCachedTokenLexers;
733	std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
734	/// \}
735
736	/// Keeps macro expanded tokens for TokenLexers.
737	//
738	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
739	/// going to lex in the cache and when it finishes the tokens are removed
740	/// from the end of the cache.
741	SmallVector<Token, `16`> MacroExpandedTokens;
742	std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
743
744	/// A record of the macro definitions and expansions that
745	/// occurred during preprocessing.
746	///
747	/// This is an optional side structure that can be enabled with
748	/// \c createPreprocessingRecord() prior to preprocessing.
749	PreprocessingRecord *Record = nullptr;
750
751	/// Cached tokens state.
752	using CachedTokensTy = SmallVector<Token, `1`>;
753
754	/// Cached tokens are stored here when we do backtracking or
755	/// lookahead. They are "lexed" by the CachingLex() method.
756	CachedTokensTy CachedTokens;
757
758	/// The position of the cached token that CachingLex() should
759	/// "lex" next.
760	///
761	/// If it points beyond the CachedTokens vector, it means that a normal
762	/// Lex() should be invoked.
763	CachedTokensTy::size_type CachedLexPos = `0`;
764
765	/// Stack of backtrack positions, allowing nested backtracks.
766	///
767	/// The EnableBacktrackAtThisPos() method pushes a position to
768	/// indicate where CachedLexPos should be set when the BackTrack() method is
769	/// invoked (at which point the last position is popped).
770	std::vector<CachedTokensTy::size_type> BacktrackPositions;
771
772	struct MacroInfoChain {
773	MacroInfo MI;
774	MacroInfoChain *Next;
775	};
776
777	/// MacroInfos are managed as a chain for easy disposal. This is the head
778	/// of that list.
779	MacroInfoChain *MIChainHead = nullptr;
780
781	void updateOutOfDateIdentifier(IdentifierInfo &II) const;
782
783	public:
784	Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
785	DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
786	MemoryBufferCache &PCMCache,
787	HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
788	IdentifierInfoLookup *IILookup = nullptr,
789	bool OwnsHeaderSearch = false,
790	TranslationUnitKind TUKind = TU_Complete);
791
792	~Preprocessor();
793
794	/// Initialize the preprocessor using information about the target.
795	///
796	/// \param Target is owned by the caller and must remain valid for the
797	/// lifetime of the preprocessor.
798	/// \param AuxTarget is owned by the caller and must remain valid for
799	/// the lifetime of the preprocessor.
800	void Initialize(const TargetInfo &Target,
801	const TargetInfo *AuxTarget = nullptr);
802
803	/// Initialize the preprocessor to parse a model file
804	///
805	/// To parse model files the preprocessor of the original source is reused to
806	/// preserver the identifier table. However to avoid some duplicate
807	/// information in the preprocessor some cleanup is needed before it is used
808	/// to parse model files. This method does that cleanup.
809	void InitializeForModelFile();
810
811	/// Cleanup after model file parsing
812	void FinalizeForModelFile();
813
814	/// Retrieve the preprocessor options used to initialize this
815	/// preprocessor.
816	PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
817
818	DiagnosticsEngine &getDiagnostics() const { return *Diags; }
819	void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
820
821	const LangOptions &getLangOpts() const { return LangOpts; }
822	const TargetInfo &getTargetInfo() const { return *Target; }
823	const TargetInfo getAuxTargetInfo() const* { return AuxTarget; }
824	FileManager &getFileManager() const { return FileMgr; }
825	SourceManager &getSourceManager() const { return SourceMgr; }
826	MemoryBufferCache &getPCMCache() const { return PCMCache; }
827	HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
828
829	IdentifierTable &getIdentifierTable() { return Identifiers; }
830	const IdentifierTable &getIdentifierTable() const { return Identifiers; }
831	SelectorTable &getSelectorTable() { return Selectors; }
832	Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
833	llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
834
835	void setPTHManager(PTHManager* pm);
836
837	PTHManager getPTHManager() { return* PTH.get(); }
838
839	void setExternalSource(ExternalPreprocessorSource *Source) {
840	ExternalSource = Source;
841	}
842
843	ExternalPreprocessorSource getExternalSource() const* {
844	return ExternalSource;
845	}
846
847	/// Retrieve the module loader associated with this preprocessor.
848	ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
849
850	bool hadModuleLoaderFatalFailure() const {
851	return TheModuleLoader.HadFatalFailure;
852	}
853
854	/// True if we are currently preprocessing a #if or #elif directive
855	bool isParsingIfOrElifDirective() const {
856	return ParsingIfOrElifDirective;
857	}
858
859	/// Control whether the preprocessor retains comments in output.
860	void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
861	this->KeepComments = KeepComments \| KeepMacroComments;
862	this->KeepMacroComments = KeepMacroComments;
863	}
864
865	bool getCommentRetentionState() const { return KeepComments; }
866
867	void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
868	bool getPragmasEnabled() const { return PragmasEnabled; }
869
870	void SetSuppressIncludeNotFoundError(bool Suppress) {
871	SuppressIncludeNotFoundError = Suppress;
872	}
873
874	bool GetSuppressIncludeNotFoundError() {
875	return SuppressIncludeNotFoundError;
876	}
877
878	/// Sets whether the preprocessor is responsible for producing output or if
879	/// it is producing tokens to be consumed by Parse and Sema.
880	void setPreprocessedOutput(bool IsPreprocessedOutput) {
881	PreprocessedOutput = IsPreprocessedOutput;
882	}
883
884	/// Returns true if the preprocessor is responsible for generating output,
885	/// false if it is producing tokens to be consumed by Parse and Sema.
886	bool isPreprocessedOutput() const { return PreprocessedOutput; }
887
888	/// Return true if we are lexing directly from the specified lexer.
889	bool isCurrentLexer(const PreprocessorLexer L) const* {
890	return CurPPLexer == L;
891	}
892
893	/// Return the current lexer being lexed from.
894	///
895	/// Note that this ignores any potentially active macro expansions and _Pragma
896	/// expansions going on at the time.
897	PreprocessorLexer getCurrentLexer() const* { return CurPPLexer; }
898
899	/// Return the current file lexer being lexed from.
900	///
901	/// Note that this ignores any potentially active macro expansions and _Pragma
902	/// expansions going on at the time.
903	PreprocessorLexer getCurrentFileLexer() const*;
904
905	/// Return the submodule owning the file being lexed. This may not be
906	/// the current module if we have changed modules since entering the file.
907	Module getCurrentLexerSubmodule() const* { return CurLexerSubmodule; }
908
909	/// Returns the FileID for the preprocessor predefines.
910	FileID getPredefinesFileID() const { return PredefinesFileID; }
911
912	/// \{
913	/// Accessors for preprocessor callbacks.
914	///
915	/// Note that this class takes ownership of any PPCallbacks object given to
916	/// it.
917	PPCallbacks getPPCallbacks() const* { return Callbacks.get(); }
918	void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
919	if (Callbacks)
920	C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
921	std::move(Callbacks));
922	Callbacks = std::move(C);
923	}
924	/// \}
925
926	bool isMacroDefined(StringRef Id) {
927	return isMacroDefined(&Identifiers.get(Id));
928	}
929	bool isMacroDefined(const IdentifierInfo *II) {
930	return II->hasMacroDefinition() &&
931	(!getLangOpts().Modules \|\| (bool)getMacroDefinition(II));
932	}
933
934	/// Determine whether II is defined as a macro within the module M,
935	/// if that is a module that we've already preprocessed. Does not check for
936	/// macros imported into M.
937	bool isMacroDefinedInLocalModule(const IdentifierInfo II, Module M) {
938	if (!II->hasMacroDefinition())
939	return false;
940	auto I = Submodules.find(M);
941	if (I == Submodules.end())
942	return false;
943	auto J = I->second.Macros.find(II);
944	if (J == I->second.Macros.end())
945	return false;
946	auto *MD = J->second.getLatest();
947	return MD && MD->isDefined();
948	}
949
950	MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
951	if (!II->hasMacroDefinition())
952	return {};
953
954	MacroState &S = CurSubmoduleState->Macros[II];
955	auto *MD = S.getLatest();
956	while (MD && isa<VisibilityMacroDirective>(MD))
957	MD = MD->getPrevious();
958	return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
959	S.getActiveModuleMacros(*this, II),
960	S.isAmbiguous(*this, II));
961	}
962
963	MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
964	SourceLocation Loc) {
965	if (!II->hadMacroDefinition())
966	return {};
967
968	MacroState &S = CurSubmoduleState->Macros[II];
969	MacroDirective::DefInfo DI;
970	if (auto *MD = S.getLatest())
971	DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
972	// FIXME: Compute the set of active module macros at the specified location.
973	return MacroDefinition(DI.getDirective(),
974	S.getActiveModuleMacros(*this, II),
975	S.isAmbiguous(*this, II));
976	}
977
978	/// Given an identifier, return its latest non-imported MacroDirective
979	/// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
980	MacroDirective getLocalMacroDirective(const* IdentifierInfo II) const* {
981	if (!II->hasMacroDefinition())
982	return nullptr;
983
984	auto *MD = getLocalMacroDirectiveHistory(II);
985	if (!MD \|\| MD->getDefinition().isUndefined())
986	return nullptr;
987
988	return MD;
989	}
990
991	const MacroInfo getMacroInfo(const* IdentifierInfo II) const* {
992	return const_cast<Preprocessor*>(this)->getMacroInfo(II);
993	}
994
995	MacroInfo getMacroInfo(const* IdentifierInfo *II) {
996	if (!II->hasMacroDefinition())
997	return nullptr;
998	if (auto MD = getMacroDefinition(II))
999	return MD.getMacroInfo();
1000	return nullptr;
1001	}
1002
1003	/// Given an identifier, return the latest non-imported macro
1004	/// directive for that identifier.
1005	///
1006	/// One can iterate over all previous macro directives from the most recent
1007	/// one.
1008	MacroDirective getLocalMacroDirectiveHistory(const* IdentifierInfo II) const*;
1009
1010	/// Add a directive to the macro directive history for this identifier.
1011	void appendMacroDirective(IdentifierInfo II, MacroDirective MD);
1012	DefMacroDirective appendDefMacroDirective(IdentifierInfo II, MacroInfo *MI,
1013	SourceLocation Loc) {
1014	DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1015	appendMacroDirective(II, MD);
1016	return MD;
1017	}
1018	DefMacroDirective appendDefMacroDirective(IdentifierInfo II,
1019	MacroInfo *MI) {
1020	return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1021	}
1022
1023	/// Set a MacroDirective that was loaded from a PCH file.
1024	void setLoadedMacroDirective(IdentifierInfo II, MacroDirective ED,
1025	MacroDirective *MD);
1026
1027	/// Register an exported macro for a module and identifier.
1028	ModuleMacro addModuleMacro(Module Mod, IdentifierInfo II, MacroInfo Macro,
1029	ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1030	ModuleMacro getModuleMacro(Module Mod, IdentifierInfo *II);
1031
1032	/// Get the list of leaf (non-overridden) module macros for a name.
1033	ArrayRef<ModuleMacro> getLeafModuleMacros(const* IdentifierInfo II) const* {
1034	if (II->isOutOfDate())
1035	updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1036	auto I = LeafModuleMacros.find(II);
1037	if (I != LeafModuleMacros.end())
1038	return I->second;
1039	return None;
1040	}
1041
1042	/// \{
1043	/// Iterators for the macro history table. Currently defined macros have
1044	/// IdentifierInfo::hasMacroDefinition() set and an empty
1045	/// MacroInfo::getUndefLoc() at the head of the list.
1046	using macro_iterator = MacroMap::const_iterator;
1047
1048	macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1049	macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1050
1051	llvm::iterator_range<macro_iterator>
1052	macros(bool IncludeExternalMacros = true) const {
1053	macro_iterator begin = macro_begin(IncludeExternalMacros);
1054	macro_iterator end = macro_end(IncludeExternalMacros);
1055	return llvm::make_range(begin, end);
1056	}
1057
1058	/// \}
1059
1060	/// Return the name of the macro defined before \p Loc that has
1061	/// spelling \p Tokens. If there are multiple macros with same spelling,
1062	/// return the last one defined.
1063	StringRef getLastMacroWithSpelling(SourceLocation Loc,
1064	ArrayRef<TokenValue> Tokens) const;
1065
1066	const std::string &getPredefines() const { return Predefines; }
1067
1068	/// Set the predefines for this Preprocessor.
1069	///
1070	/// These predefines are automatically injected when parsing the main file.
1071	void setPredefines(const char *P) { Predefines = P; }
1072	void setPredefines(StringRef P) { Predefines = P; }
1073
1074	/// Return information about the specified preprocessor
1075	/// identifier token.
1076	IdentifierInfo getIdentifierInfo(StringRef Name) const* {
1077	return &Identifiers.get(Name);
1078	}
1079
1080	/// Add the specified pragma handler to this preprocessor.
1081	///
1082	/// If \p Namespace is non-null, then it is a token required to exist on the
1083	/// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1084	void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1085	void AddPragmaHandler(PragmaHandler *Handler) {
1086	AddPragmaHandler(StringRef(), Handler);
1087	}
1088
1089	/// Remove the specific pragma handler from this preprocessor.
1090	///
1091	/// If \p Namespace is non-null, then it should be the namespace that
1092	/// \p Handler was added to. It is an error to remove a handler that
1093	/// has not been registered.
1094	void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1095	void RemovePragmaHandler(PragmaHandler *Handler) {
1096	RemovePragmaHandler(StringRef(), Handler);
1097	}
1098
1099	/// Install empty handlers for all pragmas (making them ignored).
1100	void IgnorePragmas();
1101
1102	/// Add the specified comment handler to the preprocessor.
1103	void addCommentHandler(CommentHandler *Handler);
1104
1105	/// Remove the specified comment handler.
1106	///
1107	/// It is an error to remove a handler that has not been registered.
1108	void removeCommentHandler(CommentHandler *Handler);
1109
1110	/// Set the code completion handler to the given object.
1111	void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1112	CodeComplete = &Handler;
1113	}
1114
1115	/// Retrieve the current code-completion handler.
1116	CodeCompletionHandler getCodeCompletionHandler() const* {
1117	return CodeComplete;
1118	}
1119
1120	/// Clear out the code completion handler.
1121	void clearCodeCompletionHandler() {
1122	CodeComplete = nullptr;
1123	}
1124
1125	/// Hook used by the lexer to invoke the "natural language" code
1126	/// completion point.
1127	void CodeCompleteNaturalLanguage();
1128
1129	/// Set the code completion token for filtering purposes.
1130	void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1131	CodeCompletionII = Filter;
1132	}
1133
1134	/// Get the code completion token for filtering purposes.
1135	StringRef getCodeCompletionFilter() {
1136	if (CodeCompletionII)
1137	return CodeCompletionII->getName();
1138	return {};
1139	}
1140
1141	/// Retrieve the preprocessing record, or NULL if there is no
1142	/// preprocessing record.
1143	PreprocessingRecord getPreprocessingRecord() const* { return Record; }
1144
1145	/// Create a new preprocessing record, which will keep track of
1146	/// all macro expansions, macro definitions, etc.
1147	void createPreprocessingRecord();
1148
1149	/// Returns true if the FileEntry is the PCH through header.
1150	bool isPCHThroughHeader(const FileEntry *File);
1151
1152	/// True if creating a PCH with a through header.
1153	bool creatingPCHWithThroughHeader();
1154
1155	/// True if using a PCH with a through header.
1156	bool usingPCHWithThroughHeader();
1157
1158	/// Skip tokens until after the #include of the through header.
1159	void SkipTokensUntilPCHThroughHeader();
1160
1161	/// Process directives while skipping until the through header is found.
1162	void HandleSkippedThroughHeaderDirective(Token &Result,
1163	SourceLocation HashLoc);
1164
1165	/// Enter the specified FileID as the main source file,
1166	/// which implicitly adds the builtin defines etc.
1167	void EnterMainSourceFile();
1168
1169	/// Inform the preprocessor callbacks that processing is complete.
1170	void EndSourceFile();
1171
1172	/// Add a source file to the top of the include stack and
1173	/// start lexing tokens from it instead of the current buffer.
1174	///
1175	/// Emits a diagnostic, doesn't enter the file, and returns true on error.
1176	bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
1177	SourceLocation Loc);
1178
1179	/// Add a Macro to the top of the include stack and start lexing
1180	/// tokens from it instead of the current buffer.
1181	///
1182	/// \param Args specifies the tokens input to a function-like macro.
1183	/// \param ILEnd specifies the location of the ')' for a function-like macro
1184	/// or the identifier for an object-like macro.
1185	void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1186	MacroArgs *Args);
1187
1188	/// Add a "macro" context to the top of the include stack,
1189	/// which will cause the lexer to start returning the specified tokens.
1190	///
1191	/// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1192	/// will not be subject to further macro expansion. Otherwise, these tokens
1193	/// will be re-macro-expanded when/if expansion is enabled.
1194	///
1195	/// If \p OwnsTokens is false, this method assumes that the specified stream
1196	/// of tokens has a permanent owner somewhere, so they do not need to be
1197	/// copied. If it is true, it assumes the array of tokens is allocated with
1198	/// \c new[] and the Preprocessor will delete[] it.
1199	private:
1200	void EnterTokenStream(const Token Toks, unsigned* NumToks,
1201	bool DisableMacroExpansion, bool OwnsTokens);
1202
1203	public:
1204	void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1205	bool DisableMacroExpansion) {
1206	EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1207	}
1208
1209	void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1210	EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1211	}
1212
1213	/// Pop the current lexer/macro exp off the top of the lexer stack.
1214	///
1215	/// This should only be used in situations where the current state of the
1216	/// top-of-stack lexer is known.
1217	void RemoveTopOfLexerStack();
1218
1219	/// From the point that this method is called, and until
1220	/// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1221	/// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1222	/// make the Preprocessor re-lex the same tokens.
1223	///
1224	/// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1225	/// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1226	/// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1227	///
1228	/// NOTE: DO NOT* forget to call either CommitBacktrackedTokens or Backtrack*
1229	/// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1230	/// tokens will continue indefinitely.
1231	///
1232	void EnableBacktrackAtThisPos();
1233
1234	/// Disable the last EnableBacktrackAtThisPos call.
1235	void CommitBacktrackedTokens();
1236
1237	struct CachedTokensRange {
1238	CachedTokensTy::size_type Begin, End;
1239	};
1240
1241	private:
1242	/// A range of cached tokens that should be erased after lexing
1243	/// when backtracking requires the erasure of such cached tokens.
1244	Optional<CachedTokensRange> CachedTokenRangeToErase;
1245
1246	public:
1247	/// Returns the range of cached tokens that were lexed since
1248	/// EnableBacktrackAtThisPos() was previously called.
1249	CachedTokensRange LastCachedTokenRange();
1250
1251	/// Erase the range of cached tokens that were lexed since
1252	/// EnableBacktrackAtThisPos() was previously called.
1253	void EraseCachedTokens(CachedTokensRange TokenRange);
1254
1255	/// Make Preprocessor re-lex the tokens that were lexed since
1256	/// EnableBacktrackAtThisPos() was previously called.
1257	void Backtrack();
1258
1259	/// True if EnableBacktrackAtThisPos() was called and
1260	/// caching of tokens is on.
1261	bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1262
1263	/// Lex the next token for this preprocessor.
1264	void Lex(Token &Result);
1265
1266	void LexAfterModuleImport(Token &Result);
1267
1268	void makeModuleVisible(Module *M, SourceLocation Loc);
1269
1270	SourceLocation getModuleImportLoc(Module M) const* {
1271	return CurSubmoduleState->VisibleModules.getImportLoc(M);
1272	}
1273
1274	/// Lex a string literal, which may be the concatenation of multiple
1275	/// string literals and may even come from macro expansion.
1276	/// \returns true on success, false if a error diagnostic has been generated.
1277	bool LexStringLiteral(Token &Result, std::string &String,
1278	const char *DiagnosticTag, bool AllowMacroExpansion) {
1279	if (AllowMacroExpansion)
1280	Lex(Result);
1281	else
1282	LexUnexpandedToken(Result);
1283	return FinishLexStringLiteral(Result, String, DiagnosticTag,
1284	AllowMacroExpansion);
1285	}
1286
1287	/// Complete the lexing of a string literal where the first token has
1288	/// already been lexed (see LexStringLiteral).
1289	bool FinishLexStringLiteral(Token &Result, std::string &String,
1290	const char *DiagnosticTag,
1291	bool AllowMacroExpansion);
1292
1293	/// Lex a token. If it's a comment, keep lexing until we get
1294	/// something not a comment.
1295	///
1296	/// This is useful in -E -C mode where comments would foul up preprocessor
1297	/// directive handling.
1298	void LexNonComment(Token &Result) {
1299	do
1300	Lex(Result);
1301	while (Result.getKind() == tok::comment);
1302	}
1303
1304	/// Just like Lex, but disables macro expansion of identifier tokens.
1305	void LexUnexpandedToken(Token &Result) {
1306	// Disable macro expansion.
1307	bool OldVal = DisableMacroExpansion;
1308	DisableMacroExpansion = true;
1309	// Lex the token.
1310	Lex(Result);
1311
1312	// Reenable it.
1313	DisableMacroExpansion = OldVal;
1314	}
1315
1316	/// Like LexNonComment, but this disables macro expansion of
1317	/// identifier tokens.
1318	void LexUnexpandedNonComment(Token &Result) {
1319	do
1320	LexUnexpandedToken(Result);
1321	while (Result.getKind() == tok::comment);
1322	}
1323
1324	/// Parses a simple integer literal to get its numeric value. Floating
1325	/// point literals and user defined literals are rejected. Used primarily to
1326	/// handle pragmas that accept integer arguments.
1327	bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1328
1329	/// Disables macro expansion everywhere except for preprocessor directives.
1330	void SetMacroExpansionOnlyInDirectives() {
1331	DisableMacroExpansion = true;
1332	MacroExpansionInDirectivesOverride = true;
1333	}
1334
1335	/// Peeks ahead N tokens and returns that token without consuming any
1336	/// tokens.
1337	///
1338	/// LookAhead(0) returns the next token that would be returned by Lex(),
1339	/// LookAhead(1) returns the token after it, etc. This returns normal
1340	/// tokens after phase 5. As such, it is equivalent to using
1341	/// 'Lex', not 'LexUnexpandedToken'.
1342	const Token &LookAhead(unsigned N) {
1343	if (CachedLexPos + N < CachedTokens.size())
1344	return CachedTokens[CachedLexPos+N];
1345	else
1346	return PeekAhead(N+`1`);
1347	}
1348
1349	/// When backtracking is enabled and tokens are cached,
1350	/// this allows to revert a specific number of tokens.
1351	///
1352	/// Note that the number of tokens being reverted should be up to the last
1353	/// backtrack position, not more.
1354	void RevertCachedTokens(unsigned N) {
1355	assert(isBacktrackEnabled() &&
1356	"Should only be called when tokens are cached for backtracking");
1357	assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1358	&& "Should revert tokens up to the last backtrack position, not more");
1359	assert(signed(CachedLexPos) - signed(N) >= `0` &&
1360	"Corrupted backtrack positions ?");
1361	CachedLexPos -= N;
1362	}
1363
1364	/// Enters a token in the token stream to be lexed next.
1365	///
1366	/// If BackTrack() is called afterwards, the token will remain at the
1367	/// insertion point.
1368	void EnterToken(const Token &Tok) {
1369	EnterCachingLexMode();
1370	CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1371	}
1372
1373	/// We notify the Preprocessor that if it is caching tokens (because
1374	/// backtrack is enabled) it should replace the most recent cached tokens
1375	/// with the given annotation token. This function has no effect if
1376	/// backtracking is not enabled.
1377	///
1378	/// Note that the use of this function is just for optimization, so that the
1379	/// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1380	/// invoked.
1381	void AnnotateCachedTokens(const Token &Tok) {
1382	assert(Tok.isAnnotation() && "Expected annotation token");
1383	if (CachedLexPos != `0` && isBacktrackEnabled())
1384	AnnotatePreviousCachedTokens(Tok);
1385	}
1386
1387	/// Get the location of the last cached token, suitable for setting the end
1388	/// location of an annotation token.
1389	SourceLocation getLastCachedTokenLocation() const {
1390	assert(CachedLexPos != `0`);
1391	return CachedTokens[CachedLexPos-`1`].getLastLoc();
1392	}
1393
1394	/// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1395	/// CachedTokens.
1396	bool IsPreviousCachedToken(const Token &Tok) const;
1397
1398	/// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1399	/// in \p NewToks.
1400	///
1401	/// Useful when a token needs to be split in smaller ones and CachedTokens
1402	/// most recent token must to be updated to reflect that.
1403	void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1404
1405	/// Replace the last token with an annotation token.
1406	///
1407	/// Like AnnotateCachedTokens(), this routine replaces an
1408	/// already-parsed (and resolved) token with an annotation
1409	/// token. However, this routine only replaces the last token with
1410	/// the annotation token; it does not affect any other cached
1411	/// tokens. This function has no effect if backtracking is not
1412	/// enabled.
1413	void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1414	assert(Tok.isAnnotation() && "Expected annotation token");
1415	if (CachedLexPos != `0` && isBacktrackEnabled())
1416	CachedTokens[CachedLexPos-`1`] = Tok;
1417	}
1418
1419	/// Enter an annotation token into the token stream.
1420	void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1421	void *AnnotationVal);
1422
1423	/// Update the current token to represent the provided
1424	/// identifier, in order to cache an action performed by typo correction.
1425	void TypoCorrectToken(const Token &Tok) {
1426	assert(Tok.getIdentifierInfo() && "Expected identifier token");
1427	if (CachedLexPos != `0` && isBacktrackEnabled())
1428	CachedTokens[CachedLexPos-`1`] = Tok;
1429	}
1430
1431	/// Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1432	/// CurTokenLexer pointers.
1433	void recomputeCurLexerKind();
1434
1435	/// Returns true if incremental processing is enabled
1436	bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1437
1438	/// Enables the incremental processing
1439	void enableIncrementalProcessing(bool value = true) {
1440	IncrementalProcessing = value;
1441	}
1442
1443	/// Specify the point at which code-completion will be performed.
1444	///
1445	/// \param File the file in which code completion should occur. If
1446	/// this file is included multiple times, code-completion will
1447	/// perform completion the first time it is included. If NULL, this
1448	/// function clears out the code-completion point.
1449	///
1450	/// \param Line the line at which code completion should occur
1451	/// (1-based).
1452	///
1453	/// \param Column the column at which code completion should occur
1454	/// (1-based).
1455	///
1456	/// \returns true if an error occurred, false otherwise.
1457	bool SetCodeCompletionPoint(const FileEntry *File,
1458	unsigned Line, unsigned Column);
1459
1460	/// Determine if we are performing code completion.
1461	bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1462
1463	/// Returns the location of the code-completion point.
1464	///
1465	/// Returns an invalid location if code-completion is not enabled or the file
1466	/// containing the code-completion point has not been lexed yet.
1467	SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1468
1469	/// Returns the start location of the file of code-completion point.
1470	///
1471	/// Returns an invalid location if code-completion is not enabled or the file
1472	/// containing the code-completion point has not been lexed yet.
1473	SourceLocation getCodeCompletionFileLoc() const {
1474	return CodeCompletionFileLoc;
1475	}
1476
1477	/// Returns true if code-completion is enabled and we have hit the
1478	/// code-completion point.
1479	bool isCodeCompletionReached() const { return CodeCompletionReached; }
1480
1481	/// Note that we hit the code-completion point.
1482	void setCodeCompletionReached() {
1483	assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1484	CodeCompletionReached = true;
1485	// Silence any diagnostics that occur after we hit the code-completion.
1486	getDiagnostics().setSuppressAllDiagnostics(true);
1487	}
1488
1489	/// The location of the currently-active \#pragma clang
1490	/// arc_cf_code_audited begin.
1491	///
1492	/// Returns an invalid location if there is no such pragma active.
1493	SourceLocation getPragmaARCCFCodeAuditedLoc() const {
1494	return PragmaARCCFCodeAuditedLoc;
1495	}
1496
1497	/// Set the location of the currently-active \#pragma clang
1498	/// arc_cf_code_audited begin. An invalid location ends the pragma.
1499	void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
1500	PragmaARCCFCodeAuditedLoc = Loc;
1501	}
1502
1503	/// The location of the currently-active \#pragma clang
1504	/// assume_nonnull begin.
1505	///
1506	/// Returns an invalid location if there is no such pragma active.
1507	SourceLocation getPragmaAssumeNonNullLoc() const {
1508	return PragmaAssumeNonNullLoc;
1509	}
1510
1511	/// Set the location of the currently-active \#pragma clang
1512	/// assume_nonnull begin. An invalid location ends the pragma.
1513	void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
1514	PragmaAssumeNonNullLoc = Loc;
1515	}
1516
1517	/// Set the directory in which the main file should be considered
1518	/// to have been found, if it is not a real file.
1519	void setMainFileDir(const DirectoryEntry *Dir) {
1520	MainFileDir = Dir;
1521	}
1522
1523	/// Instruct the preprocessor to skip part of the main source file.
1524	///
1525	/// \param Bytes The number of bytes in the preamble to skip.
1526	///
1527	/// \param StartOfLine Whether skipping these bytes puts the lexer at the
1528	/// start of a line.
1529	void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1530	SkipMainFilePreamble.first = Bytes;
1531	SkipMainFilePreamble.second = StartOfLine;
1532	}
1533
1534	/// Forwarding function for diagnostics. This emits a diagnostic at
1535	/// the specified Token's location, translating the token's start
1536	/// position in the current buffer into a SourcePosition object for rendering.
1537	DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1538	return Diags->Report(Loc, DiagID);
1539	}
1540
1541	DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1542	return Diags->Report(Tok.getLocation(), DiagID);
1543	}
1544
1545	/// Return the 'spelling' of the token at the given
1546	/// location; does not go up to the spelling location or down to the
1547	/// expansion location.
1548	///
1549	/// \param buffer A buffer which will be used only if the token requires
1550	/// "cleaning", e.g. if it contains trigraphs or escaped newlines
1551	/// \param invalid If non-null, will be set \c true if an error occurs.
1552	StringRef getSpelling(SourceLocation loc,
1553	SmallVectorImpl<char> &buffer,
1554	bool invalid = nullptr) const* {
1555	return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1556	}
1557
1558	/// Return the 'spelling' of the Tok token.
1559	///
1560	/// The spelling of a token is the characters used to represent the token in
1561	/// the source file after trigraph expansion and escaped-newline folding. In
1562	/// particular, this wants to get the true, uncanonicalized, spelling of
1563	/// things like digraphs, UCNs, etc.
1564	///
1565	/// \param Invalid If non-null, will be set \c true if an error occurs.
1566	std::string getSpelling(const Token &Tok, bool Invalid = nullptr) const* {
1567	return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1568	}
1569
1570	/// Get the spelling of a token into a preallocated buffer, instead
1571	/// of as an std::string.
1572	///
1573	/// The caller is required to allocate enough space for the token, which is
1574	/// guaranteed to be at least Tok.getLength() bytes long. The length of the
1575	/// actual result is returned.
1576	///
1577	/// Note that this method may do two possible things: it may either fill in
1578	/// the buffer specified with characters, or it may change the input pointer
1579	/// to point to a constant buffer with the data already in it (avoiding a
1580	/// copy). The caller is not allowed to modify the returned buffer pointer
1581	/// if an internal buffer is returned.
1582	unsigned getSpelling(const Token &Tok, const char *&Buffer,
1583	bool Invalid = nullptr) const* {
1584	return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1585	}
1586
1587	/// Get the spelling of a token into a SmallVector.
1588	///
1589	/// Note that the returned StringRef may not point to the
1590	/// supplied buffer if a copy can be avoided.
1591	StringRef getSpelling(const Token &Tok,
1592	SmallVectorImpl<char> &Buffer,
1593	bool Invalid = nullptr) const*;
1594
1595	/// Relex the token at the specified location.
1596	/// \returns true if there was a failure, false on success.
1597	bool getRawToken(SourceLocation Loc, Token &Result,
1598	bool IgnoreWhiteSpace = false) {
1599	return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1600	}
1601
1602	/// Given a Token \p Tok that is a numeric constant with length 1,
1603	/// return the character.
1604	char
1605	getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
1606	bool Invalid = nullptr) const* {
1607	assert(Tok.is(tok::numeric_constant) &&
1608	Tok.getLength() == `1` && "Called on unsupported token");
1609	assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1610
1611	// If the token is carrying a literal data pointer, just use it.
1612	if (const char *D = Tok.getLiteralData())
1613	return *D;
1614
1615	// Otherwise, fall back on getCharacterData, which is slower, but always
1616	// works.
1617	return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1618	}
1619
1620	/// Retrieve the name of the immediate macro expansion.
1621	///
1622	/// This routine starts from a source location, and finds the name of the
1623	/// macro responsible for its immediate expansion. It looks through any
1624	/// intervening macro argument expansions to compute this. It returns a
1625	/// StringRef that refers to the SourceManager-owned buffer of the source
1626	/// where that macro name is spelled. Thus, the result shouldn't out-live
1627	/// the SourceManager.
1628	StringRef getImmediateMacroName(SourceLocation Loc) {
1629	return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1630	}
1631
1632	/// Plop the specified string into a scratch buffer and set the
1633	/// specified token's location and length to it.
1634	///
1635	/// If specified, the source location provides a location of the expansion
1636	/// point of the token.
1637	void CreateString(StringRef Str, Token &Tok,
1638	SourceLocation ExpansionLocStart = SourceLocation(),
1639	SourceLocation ExpansionLocEnd = SourceLocation());
1640
1641	/// Split the first Length characters out of the token starting at TokLoc
1642	/// and return a location pointing to the split token. Re-lexing from the
1643	/// split token will return the split token rather than the original.
1644	SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1645
1646	/// Computes the source location just past the end of the
1647	/// token at this source location.
1648	///
1649	/// This routine can be used to produce a source location that
1650	/// points just past the end of the token referenced by \p Loc, and
1651	/// is generally used when a diagnostic needs to point just after a
1652	/// token where it expected something different that it received. If
1653	/// the returned source location would not be meaningful (e.g., if
1654	/// it points into a macro), this routine returns an invalid
1655	/// source location.
1656	///
1657	/// \param Offset an offset from the end of the token, where the source
1658	/// location should refer to. The default offset (0) produces a source
1659	/// location pointing just past the end of the token; an offset of 1 produces
1660	/// a source location pointing to the last character in the token, etc.
1661	SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = `0`) {
1662	return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1663	}
1664
1665	/// Returns true if the given MacroID location points at the first
1666	/// token of the macro expansion.
1667	///
1668	/// \param MacroBegin If non-null and function returns true, it is set to
1669	/// begin location of the macro.
1670	bool isAtStartOfMacroExpansion(SourceLocation loc,
1671	SourceLocation MacroBegin = nullptr) const* {
1672	return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1673	MacroBegin);
1674	}
1675
1676	/// Returns true if the given MacroID location points at the last
1677	/// token of the macro expansion.
1678	///
1679	/// \param MacroEnd If non-null and function returns true, it is set to
1680	/// end location of the macro.
1681	bool isAtEndOfMacroExpansion(SourceLocation loc,
1682	SourceLocation MacroEnd = nullptr) const* {
1683	return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1684	}
1685
1686	/// Print the token to stderr, used for debugging.
1687	void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1688	void DumpLocation(SourceLocation Loc) const;
1689	void DumpMacro(const MacroInfo &MI) const;
1690	void dumpMacroInfo(const IdentifierInfo *II);
1691
1692	/// Given a location that specifies the start of a
1693	/// token, return a new location that specifies a character within the token.
1694	SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
1695	unsigned Char) const {
1696	return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1697	}
1698
1699	/// Increment the counters for the number of token paste operations
1700	/// performed.
1701	///
1702	/// If fast was specified, this is a 'fast paste' case we handled.
1703	void IncrementPasteCounter(bool isFast) {
1704	if (isFast)
1705	++NumFastTokenPaste;
1706	else
1707	++NumTokenPaste;
1708	}
1709
1710	void PrintStats();
1711
1712	size_t getTotalMemory() const;
1713
1714	/// When the macro expander pastes together a comment (/##/) in Microsoft
1715	/// mode, this method handles updating the current state, returning the
1716	/// token on the next source line.
1717	void HandleMicrosoftCommentPaste(Token &Tok);
1718
1719	//===--------------------------------------------------------------------===//
1720	// Preprocessor callback methods. These are invoked by a lexer as various
1721	// directives and events are found.
1722
1723	/// Given a tok::raw_identifier token, look up the
1724	/// identifier information for the token and install it into the token,
1725	/// updating the token kind accordingly.
1726	IdentifierInfo LookUpIdentifierInfo(Token &Identifier) const*;
1727
1728	private:
1729	llvm::DenseMap<IdentifierInfo,unsigned*> PoisonReasons;
1730
1731	public:
1732	/// Specifies the reason for poisoning an identifier.
1733	///
1734	/// If that identifier is accessed while poisoned, then this reason will be
1735	/// used instead of the default "poisoned" diagnostic.
1736	void SetPoisonReason(IdentifierInfo II, unsigned* DiagID);
1737
1738	/// Display reason for poisoned identifier.
1739	void HandlePoisonedIdentifier(Token & Tok);
1740
1741	void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1742	if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1743	if(II->isPoisoned()) {
1744	HandlePoisonedIdentifier(Identifier);
1745	}
1746	}
1747	}
1748
1749	private:
1750	/// Identifiers used for SEH handling in Borland. These are only
1751	/// allowed in particular circumstances
1752	// __except block
1753	IdentifierInfo *Ident__exception_code,
1754	*Ident___exception_code,
1755	*Ident_GetExceptionCode;
1756	// __except filter expression
1757	IdentifierInfo *Ident__exception_info,
1758	*Ident___exception_info,
1759	*Ident_GetExceptionInfo;
1760	// __finally
1761	IdentifierInfo *Ident__abnormal_termination,
1762	*Ident___abnormal_termination,
1763	*Ident_AbnormalTermination;
1764
1765	const char *getCurLexerEndPos();
1766	void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1767
1768	public:
1769	void PoisonSEHIdentifiers(bool Poison = true); // Borland
1770
1771	/// Callback invoked when the lexer reads an identifier and has
1772	/// filled in the tokens IdentifierInfo member.
1773	///
1774	/// This callback potentially macro expands it or turns it into a named
1775	/// token (like 'for').
1776	///
1777	/// \returns true if we actually computed a token, false if we need to
1778	/// lex again.
1779	bool HandleIdentifier(Token &Identifier);
1780
1781	/// Callback invoked when the lexer hits the end of the current file.
1782	///
1783	/// This either returns the EOF token and returns true, or
1784	/// pops a level off the include stack and returns false, at which point the
1785	/// client should call lex again.
1786	bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1787
1788	/// Callback invoked when the current TokenLexer hits the end of its
1789	/// token stream.
1790	bool HandleEndOfTokenLexer(Token &Result);
1791
1792	/// Callback invoked when the lexer sees a # token at the start of a
1793	/// line.
1794	///
1795	/// This consumes the directive, modifies the lexer/preprocessor state, and
1796	/// advances the lexer(s) so that the next token read is the correct one.
1797	void HandleDirective(Token &Result);
1798
1799	/// Ensure that the next token is a tok::eod token.
1800	///
1801	/// If not, emit a diagnostic and consume up until the eod.
1802	/// If \p EnableMacros is true, then we consider macros that expand to zero
1803	/// tokens as being ok.
1804	void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1805
1806	/// Read and discard all tokens remaining on the current line until
1807	/// the tok::eod token is found.
1808	void DiscardUntilEndOfDirective();
1809
1810	/// Returns true if the preprocessor has seen a use of
1811	/// __DATE__ or __TIME__ in the file so far.
1812	bool SawDateOrTime() const {
1813	return DATELoc != SourceLocation() \|\| TIMELoc != SourceLocation();
1814	}
1815	unsigned getCounterValue() const { return CounterValue; }
1816	void setCounterValue(unsigned V) { CounterValue = V; }
1817
1818	/// Retrieves the module that we're currently building, if any.
1819	Module *getCurrentModule();
1820
1821	/// Allocate a new MacroInfo object with the provided SourceLocation.
1822	MacroInfo *AllocateMacroInfo(SourceLocation L);
1823
1824	/// Turn the specified lexer token into a fully checked and spelled
1825	/// filename, e.g. as an operand of \#include.
1826	///
1827	/// The caller is expected to provide a buffer that is large enough to hold
1828	/// the spelling of the filename, but is also expected to handle the case
1829	/// when this method decides to use a different buffer.
1830	///
1831	/// \returns true if the input filename was in <>'s or false if it was
1832	/// in ""'s.
1833	bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1834
1835	/// Given a "foo" or \<foo> reference, look up the indicated file.
1836	///
1837	/// Returns null on failure. \p isAngled indicates whether the file
1838	/// reference is for system \#include's or not (i.e. using <> instead of "").
1839	const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1840	bool isAngled, const DirectoryLookup *FromDir,
1841	const FileEntry *FromFile,
1842	const DirectoryLookup *&CurDir,
1843	SmallVectorImpl<char> *SearchPath,
1844	SmallVectorImpl<char> *RelativePath,
1845	ModuleMap::KnownHeader *SuggestedModule,
1846	bool *IsMapped, bool SkipCache = false);
1847
1848	/// Get the DirectoryLookup structure used to find the current
1849	/// FileEntry, if CurLexer is non-null and if applicable.
1850	///
1851	/// This allows us to implement \#include_next and find directory-specific
1852	/// properties.
1853	const DirectoryLookup GetCurDirLookup() { return* CurDirLookup; }
1854
1855	/// Return true if we're in the top-level file, not in a \#include.
1856	bool isInPrimaryFile() const;
1857
1858	/// Handle cases where the \#include name is expanded
1859	/// from a macro as multiple tokens, which need to be glued together.
1860	///
1861	/// This occurs for code like:
1862	/// \code
1863	/// \#define FOO <x/y.h>
1864	/// \#include FOO
1865	/// \endcode
1866	/// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1867	///
1868	/// This code concatenates and consumes tokens up to the '>' token. It
1869	/// returns false if the > was found, otherwise it returns true if it finds
1870	/// and consumes the EOD marker.
1871	bool ConcatenateIncludeName(SmallString<`128`> &FilenameBuffer,
1872	SourceLocation &End);
1873
1874	/// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1875	/// followed by EOD. Return true if the token is not a valid on-off-switch.
1876	bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1877
1878	bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1879	bool *ShadowFlag = nullptr);
1880
1881	void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1882	Module *LeaveSubmodule(bool ForPragma);
1883
1884	private:
1885	friend void TokenLexer::ExpandFunctionArguments();
1886
1887	void PushIncludeMacroStack() {
1888	assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1889	IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1890	std::move(CurLexer), std::move(CurPTHLexer),
1891	CurPPLexer, std::move(CurTokenLexer),
1892	CurDirLookup);
1893	CurPPLexer = nullptr;
1894	}
1895
1896	void PopIncludeMacroStack() {
1897	CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1898	CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1899	CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1900	CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1901	CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1902	CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1903	CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1904	IncludeMacroStack.pop_back();
1905	}
1906
1907	void PropagateLineStartLeadingSpaceInfo(Token &Result);
1908
1909	/// Determine whether we need to create module macros for #defines in the
1910	/// current context.
1911	bool needModuleMacros() const;
1912
1913	/// Update the set of active module macros and ambiguity flag for a module
1914	/// macro name.
1915	void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1916
1917	DefMacroDirective AllocateDefMacroDirective(MacroInfo MI,
1918	SourceLocation Loc);
1919	UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1920	VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1921	bool isPublic);
1922
1923	/// Lex and validate a macro name, which occurs after a
1924	/// \#define or \#undef.
1925	///
1926	/// \param MacroNameTok Token that represents the name defined or undefined.
1927	/// \param IsDefineUndef Kind if preprocessor directive.
1928	/// \param ShadowFlag Points to flag that is set if macro name shadows
1929	/// a keyword.
1930	///
1931	/// This emits a diagnostic, sets the token kind to eod,
1932	/// and discards the rest of the macro line if the macro name is invalid.
1933	void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1934	bool *ShadowFlag = nullptr);
1935
1936	/// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1937	/// entire line) of the macro's tokens and adds them to MacroInfo, and while
1938	/// doing so performs certain validity checks including (but not limited to):
1939	/// - # (stringization) is followed by a macro parameter
1940	/// \param MacroNameTok - Token that represents the macro name
1941	/// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1942	///
1943	/// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1944	/// returns a nullptr if an invalid sequence of tokens is encountered.
1945	MacroInfo *ReadOptionalMacroParameterListAndBody(
1946	const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1947
1948	/// The ( starting an argument list of a macro definition has just been read.
1949	/// Lex the rest of the parameters and the closing ), updating \p MI with
1950	/// what we learn and saving in \p LastTok the last token read.
1951	/// Return true if an error occurs parsing the arg list.
1952	bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1953
1954	/// We just read a \#if or related directive and decided that the
1955	/// subsequent tokens are in the \#if'd out portion of the
1956	/// file. Lex the rest of the file, until we see an \#endif. If \p
1957	/// FoundNonSkipPortion is true, then we have already emitted code for part of
1958	/// this \#if directive, so \#else/\#elif blocks should never be entered. If
1959	/// \p FoundElse is false, then \#else directives are ok, if not, then we have
1960	/// already seen one so a \#else directive is a duplicate. When this returns,
1961	/// the caller can lex the first valid token.
1962	void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1963	SourceLocation IfTokenLoc,
1964	bool FoundNonSkipPortion, bool FoundElse,
1965	SourceLocation ElseLoc = SourceLocation());
1966
1967	/// A fast PTH version of SkipExcludedConditionalBlock.
1968	void PTHSkipExcludedConditionalBlock();
1969
1970	/// Information about the result for evaluating an expression for a
1971	/// preprocessor directive.
1972	struct DirectiveEvalResult {
1973	/// Whether the expression was evaluated as true or not.
1974	bool Conditional;
1975
1976	/// True if the expression contained identifiers that were undefined.
1977	bool IncludedUndefinedIds;
1978	};
1979
1980	/// Evaluate an integer constant expression that may occur after a
1981	/// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1982	///
1983	/// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1984	DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1985
1986	/// Install the standard preprocessor pragmas:
1987	/// \#pragma GCC poison/system_header/dependency and \#pragma once.
1988	void RegisterBuiltinPragmas();
1989
1990	/// Register builtin macros such as __LINE__ with the identifier table.
1991	void RegisterBuiltinMacros();
1992
1993	/// If an identifier token is read that is to be expanded as a macro, handle
1994	/// it and return the next token as 'Tok'. If we lexed a token, return true;
1995	/// otherwise the caller should lex again.
1996	bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1997
1998	/// Cache macro expanded tokens for TokenLexers.
1999	//
2000	/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2001	/// going to lex in the cache and when it finishes the tokens are removed
2002	/// from the end of the cache.
2003	Token cacheMacroExpandedTokens(TokenLexer tokLexer,
2004	ArrayRef<Token> tokens);
2005
2006	void removeCachedMacroExpandedTokensOfLastLexer();
2007
2008	/// Determine whether the next preprocessor token to be
2009	/// lexed is a '('. If so, consume the token and return true, if not, this
2010	/// method should have no observable side-effect on the lexed tokens.
2011	bool isNextPPTokenLParen();
2012
2013	/// After reading "MACRO(", this method is invoked to read all of the formal
2014	/// arguments specified for the macro invocation. Returns null on error.
2015	MacroArgs ReadMacroCallArgumentList(Token &MacroName, MacroInfo MI,
2016	SourceLocation &ExpansionEnd);
2017
2018	/// If an identifier token is read that is to be expanded
2019	/// as a builtin macro, handle it and return the next token as 'Tok'.
2020	void ExpandBuiltinMacro(Token &Tok);
2021
2022	/// Read a \c _Pragma directive, slice it up, process it, then
2023	/// return the first token after the directive.
2024	/// This assumes that the \c _Pragma token has just been read into \p Tok.
2025	void Handle_Pragma(Token &Tok);
2026
2027	/// Like Handle_Pragma except the pragma text is not enclosed within
2028	/// a string literal.
2029	void HandleMicrosoft__pragma(Token &Tok);
2030
2031	/// Add a lexer to the top of the include stack and
2032	/// start lexing tokens from it instead of the current buffer.
2033	void EnterSourceFileWithLexer(Lexer TheLexer, const* DirectoryLookup *Dir);
2034
2035	/// Add a lexer to the top of the include stack and
2036	/// start getting tokens from it using the PTH cache.
2037	void EnterSourceFileWithPTH(PTHLexer PL, const* DirectoryLookup *Dir);
2038
2039	/// Set the FileID for the preprocessor predefines.
2040	void setPredefinesFileID(FileID FID) {
2041	assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2042	PredefinesFileID = FID;
2043	}
2044
2045	/// Set the FileID for the PCH through header.
2046	void setPCHThroughHeaderFileID(FileID FID);
2047
2048	/// Returns true if we are lexing from a file and not a
2049	/// pragma or a macro.
2050	static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2051	return L ? !L->isPragmaLexer() : P != nullptr;
2052	}
2053
2054	static bool IsFileLexer(const IncludeStackInfo& I) {
2055	return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2056	}
2057
2058	bool IsFileLexer() const {
2059	return IsFileLexer(CurLexer.get(), CurPPLexer);
2060	}
2061
2062	//===--------------------------------------------------------------------===//
2063	// Caching stuff.
2064	void CachingLex(Token &Result);
2065
2066	bool InCachingLexMode() const {
2067	// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2068	// that we are past EOF, not that we are in CachingLex mode.
2069	return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
2070	!IncludeMacroStack.empty();
2071	}
2072
2073	void EnterCachingLexMode();
2074
2075	void ExitCachingLexMode() {
2076	if (InCachingLexMode())
2077	RemoveTopOfLexerStack();
2078	}
2079
2080	const Token &PeekAhead(unsigned N);
2081	void AnnotatePreviousCachedTokens(const Token &Tok);
2082
2083	//===--------------------------------------------------------------------===//
2084	/// HandleDirective - implement the various preprocessor directives. These*
2085	/// should side-effect the current preprocessor object so that the next call
2086	/// to Lex() will return the appropriate token next.
2087	void HandleLineDirective();
2088	void HandleDigitDirective(Token &Tok);
2089	void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2090	void HandleIdentSCCSDirective(Token &Tok);
2091	void HandleMacroPublicDirective(Token &Tok);
2092	void HandleMacroPrivateDirective();
2093
2094	// File inclusion.
2095	void HandleIncludeDirective(SourceLocation HashLoc,
2096	Token &Tok,
2097	const DirectoryLookup *LookupFrom = nullptr,
2098	const FileEntry *LookupFromFile = nullptr,
2099	bool isImport = false);
2100	void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2101	void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2102	void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2103	void HandleMicrosoftImportDirective(Token &Tok);
2104
2105	public:
2106	/// Check that the given module is available, producing a diagnostic if not.
2107	/// \return \c true if the check failed (because the module is not available).
2108	/// \c false if the module appears to be usable.
2109	static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2110	const TargetInfo &TargetInfo,
2111	DiagnosticsEngine &Diags, Module *M);
2112
2113	// Module inclusion testing.
2114	/// Find the module that owns the source or header file that
2115	/// \p Loc points to. If the location is in a file that was included
2116	/// into a module, or is outside any module, returns nullptr.
2117	Module *getModuleForLocation(SourceLocation Loc);
2118
2119	/// We want to produce a diagnostic at location IncLoc concerning a
2120	/// missing module import.
2121	///
2122	/// \param IncLoc The location at which the missing import was detected.
2123	/// \param M The desired module.
2124	/// \param MLoc A location within the desired module at which some desired
2125	/// effect occurred (eg, where a desired entity was declared).
2126	///
2127	/// \return A file that can be #included to import a module containing MLoc.
2128	/// Null if no such file could be determined or if a #include is not
2129	/// appropriate.
2130	const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2131	Module *M,
2132	SourceLocation MLoc);
2133
2134	bool isRecordingPreamble() const {
2135	return PreambleConditionalStack.isRecording();
2136	}
2137
2138	bool hasRecordedPreamble() const {
2139	return PreambleConditionalStack.hasRecordedPreamble();
2140	}
2141
2142	ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2143	return PreambleConditionalStack.getStack();
2144	}
2145
2146	void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2147	PreambleConditionalStack.setStack(s);
2148	}
2149
2150	void setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s,
2151	llvm::Optional<PreambleSkipInfo> SkipInfo) {
2152	PreambleConditionalStack.startReplaying();
2153	PreambleConditionalStack.setStack(s);
2154	PreambleConditionalStack.SkipInfo = SkipInfo;
2155	}
2156
2157	llvm::Optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2158	return PreambleConditionalStack.SkipInfo;
2159	}
2160
2161	private:
2162	/// After processing predefined file, initialize the conditional stack from
2163	/// the preamble.
2164	void replayPreambleConditionalStack();
2165
2166	// Macro handling.
2167	void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
2168	void HandleUndefDirective();
2169
2170	// Conditional Inclusion.
2171	void HandleIfdefDirective(Token &Tok, const Token &HashToken,
2172	bool isIfndef, bool ReadAnyTokensBeforeDirective);
2173	void HandleIfDirective(Token &Tok, const Token &HashToken,
2174	bool ReadAnyTokensBeforeDirective);
2175	void HandleEndifDirective(Token &Tok);
2176	void HandleElseDirective(Token &Tok, const Token &HashToken);
2177	void HandleElifDirective(Token &Tok, const Token &HashToken);
2178
2179	// Pragmas.
2180	void HandlePragmaDirective(SourceLocation IntroducerLoc,
2181	PragmaIntroducerKind Introducer);
2182
2183	public:
2184	void HandlePragmaOnce(Token &OnceTok);
2185	void HandlePragmaMark();
2186	void HandlePragmaPoison();
2187	void HandlePragmaSystemHeader(Token &SysHeaderTok);
2188	void HandlePragmaDependency(Token &DependencyTok);
2189	void HandlePragmaPushMacro(Token &Tok);
2190	void HandlePragmaPopMacro(Token &Tok);
2191	void HandlePragmaIncludeAlias(Token &Tok);
2192	void HandlePragmaModuleBuild(Token &Tok);
2193	IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2194
2195	// Return true and store the first token only if any CommentHandler
2196	// has inserted some tokens and getCommentRetentionState() is false.
2197	bool HandleComment(Token &Token, SourceRange Comment);
2198
2199	/// A macro is used, update information about macros that need unused
2200	/// warnings.
2201	void markMacroAsUsed(MacroInfo *MI);
2202	};
2203
2204	/// Abstract base class that describes a handler that will receive
2205	/// source ranges for each of the comments encountered in the source file.
2206	class CommentHandler {
2207	public:
2208	virtual ~CommentHandler();
2209
2210	// The handler shall return true if it has pushed any tokens
2211	// to be read using e.g. EnterToken or EnterTokenStream.
2212	virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = `0`;
2213	};
2214
2215	/// Registry of pragma handlers added by plugins
2216	using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2217
2218	} // namespace clang
2219
2220	#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
2221

Warning: That file was not part of the compilation database. It may have many parsing errors.

Browse the source code of include/clang/Lex/Preprocessor.h