1//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines the clang::Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
17#include "clang/Basic/Builtins.h"
18#include "clang/Basic/Diagnostic.h"
19#include "clang/Basic/IdentifierTable.h"
20#include "clang/Basic/LLVM.h"
21#include "clang/Basic/LangOptions.h"
22#include "clang/Basic/Module.h"
23#include "clang/Basic/SourceLocation.h"
24#include "clang/Basic/SourceManager.h"
25#include "clang/Basic/TokenKinds.h"
26#include "clang/Lex/Lexer.h"
27#include "clang/Lex/MacroInfo.h"
28#include "clang/Lex/ModuleLoader.h"
29#include "clang/Lex/ModuleMap.h"
30#include "clang/Lex/PPCallbacks.h"
31#include "clang/Lex/Token.h"
32#include "clang/Lex/TokenLexer.h"
33#include "llvm/ADT/ArrayRef.h"
34#include "llvm/ADT/DenseMap.h"
35#include "llvm/ADT/FoldingSet.h"
36#include "llvm/ADT/None.h"
37#include "llvm/ADT/Optional.h"
38#include "llvm/ADT/PointerUnion.h"
39#include "llvm/ADT/STLExtras.h"
40#include "llvm/ADT/SmallPtrSet.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/ADT/StringRef.h"
43#include "llvm/ADT/TinyPtrVector.h"
44#include "llvm/ADT/iterator_range.h"
45#include "llvm/Support/Allocator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/Registry.h"
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <memory>
52#include <map>
53#include <string>
54#include <utility>
55#include <vector>
56
57namespace llvm {
58
59template<unsigned InternalLen> class SmallString;
60
61} // namespace llvm
62
63namespace clang {
64
65class CodeCompletionHandler;
66class CommentHandler;
67class DirectoryEntry;
68class DirectoryLookup;
69class ExternalPreprocessorSource;
70class FileEntry;
71class FileManager;
72class HeaderSearch;
73class MacroArgs;
74class MemoryBufferCache;
75class PragmaHandler;
76class PragmaNamespace;
77class PreprocessingRecord;
78class PreprocessorLexer;
79class PreprocessorOptions;
80class ScratchBuffer;
81class TargetInfo;
82
83/// Stores token information for comparing actual tokens with
84/// predefined values. Only handles simple tokens and identifiers.
85class TokenValue {
86 tok::TokenKind Kind;
87 IdentifierInfo *II;
88
89public:
90 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
91 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
92 assert(Kind != tok::identifier &&
93 "Identifiers should be created by TokenValue(IdentifierInfo *)");
94 assert(!tok::isLiteral(Kind) && "Literals are not supported.");
95 assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
96 }
97
98 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
99
100 bool operator==(const Token &Tok) const {
101 return Tok.getKind() == Kind &&
102 (!II || II == Tok.getIdentifierInfo());
103 }
104};
105
106/// Context in which macro name is used.
107enum MacroUse {
108 // other than #define or #undef
109 MU_Other = 0,
110
111 // macro name specified in #define
112 MU_Define = 1,
113
114 // macro name specified in #undef
115 MU_Undef = 2
116};
117
118/// Engages in a tight little dance with the lexer to efficiently
119/// preprocess tokens.
120///
121/// Lexers know only about tokens within a single source file, and don't
122/// know anything about preprocessor-level issues like the \#include stack,
123/// token expansion, etc.
124class Preprocessor {
125 friend class VAOptDefinitionContext;
126 friend class VariadicMacroScopeGuard;
127
128 std::shared_ptr<PreprocessorOptions> PPOpts;
129 DiagnosticsEngine *Diags;
130 LangOptions &LangOpts;
131 const TargetInfo *Target = nullptr;
132 const TargetInfo *AuxTarget = nullptr;
133 FileManager &FileMgr;
134 SourceManager &SourceMgr;
135 MemoryBufferCache &PCMCache;
136 std::unique_ptr<ScratchBuffer> ScratchBuf;
137 HeaderSearch &HeaderInfo;
138 ModuleLoader &TheModuleLoader;
139
140 /// External source of macros.
141 ExternalPreprocessorSource *ExternalSource;
142
143 /// A BumpPtrAllocator object used to quickly allocate and release
144 /// objects internal to the Preprocessor.
145 llvm::BumpPtrAllocator BP;
146
147 /// Identifiers for builtin macros and other builtins.
148 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
149 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
150 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
151 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
152 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
153 IdentifierInfo *Ident__COUNTER__; // __COUNTER__
154 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
155 IdentifierInfo *Ident__identifier; // __identifier
156 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
157 IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
158 IdentifierInfo *Ident__has_feature; // __has_feature
159 IdentifierInfo *Ident__has_extension; // __has_extension
160 IdentifierInfo *Ident__has_builtin; // __has_builtin
161 IdentifierInfo *Ident__has_attribute; // __has_attribute
162 IdentifierInfo *Ident__has_include; // __has_include
163 IdentifierInfo *Ident__has_include_next; // __has_include_next
164 IdentifierInfo *Ident__has_warning; // __has_warning
165 IdentifierInfo *Ident__is_identifier; // __is_identifier
166 IdentifierInfo *Ident__building_module; // __building_module
167 IdentifierInfo *Ident__MODULE__; // __MODULE__
168 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
169 IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
170 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
171 IdentifierInfo *Ident__is_target_arch; // __is_target_arch
172 IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
173 IdentifierInfo *Ident__is_target_os; // __is_target_os
174 IdentifierInfo *Ident__is_target_environment; // __is_target_environment
175
176 // Weak, only valid (and set) while InMacroArgs is true.
177 Token* ArgMacro;
178
179 SourceLocation DATELoc, TIMELoc;
180
181 // Next __COUNTER__ value, starts at 0.
182 unsigned CounterValue = 0;
183
184 enum {
185 /// Maximum depth of \#includes.
186 MaxAllowedIncludeStackDepth = 200
187 };
188
189 // State that is set before the preprocessor begins.
190 bool KeepComments : 1;
191 bool KeepMacroComments : 1;
192 bool SuppressIncludeNotFoundError : 1;
193
194 // State that changes while the preprocessor runs:
195 bool InMacroArgs : 1; // True if parsing fn macro invocation args.
196
197 /// Whether the preprocessor owns the header search object.
198 bool OwnsHeaderSearch : 1;
199
200 /// True if macro expansion is disabled.
201 bool DisableMacroExpansion : 1;
202
203 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
204 /// when parsing preprocessor directives.
205 bool MacroExpansionInDirectivesOverride : 1;
206
207 class ResetMacroExpansionHelper;
208
209 /// Whether we have already loaded macros from the external source.
210 mutable bool ReadMacrosFromExternalSource : 1;
211
212 /// True if pragmas are enabled.
213 bool PragmasEnabled : 1;
214
215 /// True if the current build action is a preprocessing action.
216 bool PreprocessedOutput : 1;
217
218 /// True if we are currently preprocessing a #if or #elif directive
219 bool ParsingIfOrElifDirective;
220
221 /// True if we are pre-expanding macro arguments.
222 bool InMacroArgPreExpansion;
223
224 /// Mapping/lookup information for all identifiers in
225 /// the program, including program keywords.
226 mutable IdentifierTable Identifiers;
227
228 /// This table contains all the selectors in the program.
229 ///
230 /// Unlike IdentifierTable above, this table *isn't* populated by the
231 /// preprocessor. It is declared/expanded here because its role/lifetime is
232 /// conceptually similar to the IdentifierTable. In addition, the current
233 /// control flow (in clang::ParseAST()), make it convenient to put here.
234 ///
235 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
236 /// the lifetime of the preprocessor.
237 SelectorTable Selectors;
238
239 /// Information about builtins.
240 Builtin::Context BuiltinInfo;
241
242 /// Tracks all of the pragmas that the client registered
243 /// with this preprocessor.
244 std::unique_ptr<PragmaNamespace> PragmaHandlers;
245
246 /// Pragma handlers of the original source is stored here during the
247 /// parsing of a model file.
248 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
249
250 /// Tracks all of the comment handlers that the client registered
251 /// with this preprocessor.
252 std::vector<CommentHandler *> CommentHandlers;
253
254 /// True if we want to ignore EOF token and continue later on (thus
255 /// avoid tearing the Lexer and etc. down).
256 bool IncrementalProcessing = false;
257
258 /// The kind of translation unit we are processing.
259 TranslationUnitKind TUKind;
260
261 /// The code-completion handler.
262 CodeCompletionHandler *CodeComplete = nullptr;
263
264 /// The file that we're performing code-completion for, if any.
265 const FileEntry *CodeCompletionFile = nullptr;
266
267 /// The offset in file for the code-completion point.
268 unsigned CodeCompletionOffset = 0;
269
270 /// The location for the code-completion point. This gets instantiated
271 /// when the CodeCompletionFile gets \#include'ed for preprocessing.
272 SourceLocation CodeCompletionLoc;
273
274 /// The start location for the file of the code-completion point.
275 ///
276 /// This gets instantiated when the CodeCompletionFile gets \#include'ed
277 /// for preprocessing.
278 SourceLocation CodeCompletionFileLoc;
279
280 /// The source location of the \c import contextual keyword we just
281 /// lexed, if any.
282 SourceLocation ModuleImportLoc;
283
284 /// The module import path that we're currently processing.
285 SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath;
286
287 /// Whether the last token we lexed was an '@'.
288 bool LastTokenWasAt = false;
289
290 /// Whether the module import expects an identifier next. Otherwise,
291 /// it expects a '.' or ';'.
292 bool ModuleImportExpectsIdentifier = false;
293
294 /// The source location of the currently-active
295 /// \#pragma clang arc_cf_code_audited begin.
296 SourceLocation PragmaARCCFCodeAuditedLoc;
297
298 /// The source location of the currently-active
299 /// \#pragma clang assume_nonnull begin.
300 SourceLocation PragmaAssumeNonNullLoc;
301
302 /// True if we hit the code-completion point.
303 bool CodeCompletionReached = false;
304
305 /// The code completion token containing the information
306 /// on the stem that is to be code completed.
307 IdentifierInfo *CodeCompletionII = nullptr;
308
309 /// Range for the code completion token.
310 SourceRange CodeCompletionTokenRange;
311
312 /// The directory that the main file should be considered to occupy,
313 /// if it does not correspond to a real file (as happens when building a
314 /// module).
315 const DirectoryEntry *MainFileDir = nullptr;
316
317 /// The number of bytes that we will initially skip when entering the
318 /// main file, along with a flag that indicates whether skipping this number
319 /// of bytes will place the lexer at the start of a line.
320 ///
321 /// This is used when loading a precompiled preamble.
322 std::pair<int, bool> SkipMainFilePreamble;
323
324 /// Whether we hit an error due to reaching max allowed include depth. Allows
325 /// to avoid hitting the same error over and over again.
326 bool HasReachedMaxIncludeDepth = false;
327
328public:
329 struct PreambleSkipInfo {
330 SourceLocation HashTokenLoc;
331 SourceLocation IfTokenLoc;
332 bool FoundNonSkipPortion;
333 bool FoundElse;
334 SourceLocation ElseLoc;
335
336 PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
337 bool FoundNonSkipPortion, bool FoundElse,
338 SourceLocation ElseLoc)
339 : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
340 FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
341 ElseLoc(ElseLoc) {}
342 };
343
344private:
345 friend class ASTReader;
346 friend class MacroArgs;
347
348 class PreambleConditionalStackStore {
349 enum State {
350 Off = 0,
351 Recording = 1,
352 Replaying = 2,
353 };
354
355 public:
356 PreambleConditionalStackStore() = default;
357
358 void startRecording() { ConditionalStackState = Recording; }
359 void startReplaying() { ConditionalStackState = Replaying; }
360 bool isRecording() const { return ConditionalStackState == Recording; }
361 bool isReplaying() const { return ConditionalStackState == Replaying; }
362
363 ArrayRef<PPConditionalInfo> getStack() const {
364 return ConditionalStack;
365 }
366
367 void doneReplaying() {
368 ConditionalStack.clear();
369 ConditionalStackState = Off;
370 }
371
372 void setStack(ArrayRef<PPConditionalInfo> s) {
373 if (!isRecording() && !isReplaying())
374 return;
375 ConditionalStack.clear();
376 ConditionalStack.append(s.begin(), s.end());
377 }
378
379 bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
380
381 bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
382
383 void clearSkipInfo() { SkipInfo.reset(); }
384
385 llvm::Optional<PreambleSkipInfo> SkipInfo;
386
387 private:
388 SmallVector<PPConditionalInfo, 4> ConditionalStack;
389 State ConditionalStackState = Off;
390 } PreambleConditionalStack;
391
392 /// The current top of the stack that we're lexing from if
393 /// not expanding a macro and we are lexing directly from source code.
394 ///
395 /// Only one of CurLexer, or CurTokenLexer will be non-null.
396 std::unique_ptr<Lexer> CurLexer;
397
398 /// The current top of the stack what we're lexing from
399 /// if not expanding a macro.
400 ///
401 /// This is an alias for CurLexer.
402 PreprocessorLexer *CurPPLexer = nullptr;
403
404 /// Used to find the current FileEntry, if CurLexer is non-null
405 /// and if applicable.
406 ///
407 /// This allows us to implement \#include_next and find directory-specific
408 /// properties.
409 const DirectoryLookup *CurDirLookup = nullptr;
410
411 /// The current macro we are expanding, if we are expanding a macro.
412 ///
413 /// One of CurLexer and CurTokenLexer must be null.
414 std::unique_ptr<TokenLexer> CurTokenLexer;
415
416 /// The kind of lexer we're currently working with.
417 enum CurLexerKind {
418 CLK_Lexer,
419 CLK_TokenLexer,
420 CLK_CachingLexer,
421 CLK_LexAfterModuleImport
422 } CurLexerKind = CLK_Lexer;
423
424 /// If the current lexer is for a submodule that is being built, this
425 /// is that submodule.
426 Module *CurLexerSubmodule = nullptr;
427
428 /// Keeps track of the stack of files currently
429 /// \#included, and macros currently being expanded from, not counting
430 /// CurLexer/CurTokenLexer.
431 struct IncludeStackInfo {
432 enum CurLexerKind CurLexerKind;
433 Module *TheSubmodule;
434 std::unique_ptr<Lexer> TheLexer;
435 PreprocessorLexer *ThePPLexer;
436 std::unique_ptr<TokenLexer> TheTokenLexer;
437 const DirectoryLookup *TheDirLookup;
438
439 // The following constructors are completely useless copies of the default
440 // versions, only needed to pacify MSVC.
441 IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
442 std::unique_ptr<Lexer> &&TheLexer,
443 PreprocessorLexer *ThePPLexer,
444 std::unique_ptr<TokenLexer> &&TheTokenLexer,
445 const DirectoryLookup *TheDirLookup)
446 : CurLexerKind(std::move(CurLexerKind)),
447 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
448 ThePPLexer(std::move(ThePPLexer)),
449 TheTokenLexer(std::move(TheTokenLexer)),
450 TheDirLookup(std::move(TheDirLookup)) {}
451 };
452 std::vector<IncludeStackInfo> IncludeMacroStack;
453
454 /// Actions invoked when some preprocessor activity is
455 /// encountered (e.g. a file is \#included, etc).
456 std::unique_ptr<PPCallbacks> Callbacks;
457
458 struct MacroExpandsInfo {
459 Token Tok;
460 MacroDefinition MD;
461 SourceRange Range;
462
463 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
464 : Tok(Tok), MD(MD), Range(Range) {}
465 };
466 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
467
468 /// Information about a name that has been used to define a module macro.
469 struct ModuleMacroInfo {
470 /// The most recent macro directive for this identifier.
471 MacroDirective *MD;
472
473 /// The active module macros for this identifier.
474 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
475
476 /// The generation number at which we last updated ActiveModuleMacros.
477 /// \see Preprocessor::VisibleModules.
478 unsigned ActiveModuleMacrosGeneration = 0;
479
480 /// Whether this macro name is ambiguous.
481 bool IsAmbiguous = false;
482
483 /// The module macros that are overridden by this macro.
484 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
485
486 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
487 };
488
489 /// The state of a macro for an identifier.
490 class MacroState {
491 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
492
493 ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
494 const IdentifierInfo *II) const {
495 if (II->isOutOfDate())
496 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
497 // FIXME: Find a spare bit on IdentifierInfo and store a
498 // HasModuleMacros flag.
499 if (!II->hasMacroDefinition() ||
500 (!PP.getLangOpts().Modules &&
501 !PP.getLangOpts().ModulesLocalVisibility) ||
502 !PP.CurSubmoduleState->VisibleModules.getGeneration())
503 return nullptr;
504
505 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
506 if (!Info) {
507 Info = new (PP.getPreprocessorAllocator())
508 ModuleMacroInfo(State.get<MacroDirective *>());
509 State = Info;
510 }
511
512 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
513 Info->ActiveModuleMacrosGeneration)
514 PP.updateModuleMacroInfo(II, *Info);
515 return Info;
516 }
517
518 public:
519 MacroState() : MacroState(nullptr) {}
520 MacroState(MacroDirective *MD) : State(MD) {}
521
522 MacroState(MacroState &&O) noexcept : State(O.State) {
523 O.State = (MacroDirective *)nullptr;
524 }
525
526 MacroState &operator=(MacroState &&O) noexcept {
527 auto S = O.State;
528 O.State = (MacroDirective *)nullptr;
529 State = S;
530 return *this;
531 }
532
533 ~MacroState() {
534 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
535 Info->~ModuleMacroInfo();
536 }
537
538 MacroDirective *getLatest() const {
539 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
540 return Info->MD;
541 return State.get<MacroDirective*>();
542 }
543
544 void setLatest(MacroDirective *MD) {
545 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
546 Info->MD = MD;
547 else
548 State = MD;
549 }
550
551 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
552 auto *Info = getModuleInfo(PP, II);
553 return Info ? Info->IsAmbiguous : false;
554 }
555
556 ArrayRef<ModuleMacro *>
557 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
558 if (auto *Info = getModuleInfo(PP, II))
559 return Info->ActiveModuleMacros;
560 return None;
561 }
562
563 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
564 SourceManager &SourceMgr) const {
565 // FIXME: Incorporate module macros into the result of this.
566 if (auto *Latest = getLatest())
567 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
568 return {};
569 }
570
571 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
572 if (auto *Info = getModuleInfo(PP, II)) {
573 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
574 Info->ActiveModuleMacros.begin(),
575 Info->ActiveModuleMacros.end());
576 Info->ActiveModuleMacros.clear();
577 Info->IsAmbiguous = false;
578 }
579 }
580
581 ArrayRef<ModuleMacro*> getOverriddenMacros() const {
582 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
583 return Info->OverriddenMacros;
584 return None;
585 }
586
587 void setOverriddenMacros(Preprocessor &PP,
588 ArrayRef<ModuleMacro *> Overrides) {
589 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
590 if (!Info) {
591 if (Overrides.empty())
592 return;
593 Info = new (PP.getPreprocessorAllocator())
594 ModuleMacroInfo(State.get<MacroDirective *>());
595 State = Info;
596 }
597 Info->OverriddenMacros.clear();
598 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
599 Overrides.begin(), Overrides.end());
600 Info->ActiveModuleMacrosGeneration = 0;
601 }
602 };
603
604 /// For each IdentifierInfo that was associated with a macro, we
605 /// keep a mapping to the history of all macro definitions and #undefs in
606 /// the reverse order (the latest one is in the head of the list).
607 ///
608 /// This mapping lives within the \p CurSubmoduleState.
609 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
610
611 struct SubmoduleState;
612
613 /// Information about a submodule that we're currently building.
614 struct BuildingSubmoduleInfo {
615 /// The module that we are building.
616 Module *M;
617
618 /// The location at which the module was included.
619 SourceLocation ImportLoc;
620
621 /// Whether we entered this submodule via a pragma.
622 bool IsPragma;
623
624 /// The previous SubmoduleState.
625 SubmoduleState *OuterSubmoduleState;
626
627 /// The number of pending module macro names when we started building this.
628 unsigned OuterPendingModuleMacroNames;
629
630 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
631 SubmoduleState *OuterSubmoduleState,
632 unsigned OuterPendingModuleMacroNames)
633 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
634 OuterSubmoduleState(OuterSubmoduleState),
635 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
636 };
637 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
638
639 /// Information about a submodule's preprocessor state.
640 struct SubmoduleState {
641 /// The macros for the submodule.
642 MacroMap Macros;
643
644 /// The set of modules that are visible within the submodule.
645 VisibleModuleSet VisibleModules;
646
647 // FIXME: CounterValue?
648 // FIXME: PragmaPushMacroInfo?
649 };
650 std::map<Module *, SubmoduleState> Submodules;
651
652 /// The preprocessor state for preprocessing outside of any submodule.
653 SubmoduleState NullSubmoduleState;
654
655 /// The current submodule state. Will be \p NullSubmoduleState if we're not
656 /// in a submodule.
657 SubmoduleState *CurSubmoduleState;
658
659 /// The set of known macros exported from modules.
660 llvm::FoldingSet<ModuleMacro> ModuleMacros;
661
662 /// The names of potential module macros that we've not yet processed.
663 llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
664
665 /// The list of module macros, for each identifier, that are not overridden by
666 /// any other module macro.
667 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
668 LeafModuleMacros;
669
670 /// Macros that we want to warn because they are not used at the end
671 /// of the translation unit.
672 ///
673 /// We store just their SourceLocations instead of
674 /// something like MacroInfo*. The benefit of this is that when we are
675 /// deserializing from PCH, we don't need to deserialize identifier & macros
676 /// just so that we can report that they are unused, we just warn using
677 /// the SourceLocations of this set (that will be filled by the ASTReader).
678 /// We are using SmallPtrSet instead of a vector for faster removal.
679 using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
680 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
681
682 /// A "freelist" of MacroArg objects that can be
683 /// reused for quick allocation.
684 MacroArgs *MacroArgCache = nullptr;
685
686 /// For each IdentifierInfo used in a \#pragma push_macro directive,
687 /// we keep a MacroInfo stack used to restore the previous macro value.
688 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
689 PragmaPushMacroInfo;
690
691 // Various statistics we track for performance analysis.
692 unsigned NumDirectives = 0;
693 unsigned NumDefined = 0;
694 unsigned NumUndefined = 0;
695 unsigned NumPragma = 0;
696 unsigned NumIf = 0;
697 unsigned NumElse = 0;
698 unsigned NumEndif = 0;
699 unsigned NumEnteredSourceFiles = 0;
700 unsigned MaxIncludeStackDepth = 0;
701 unsigned NumMacroExpanded = 0;
702 unsigned NumFnMacroExpanded = 0;
703 unsigned NumBuiltinMacroExpanded = 0;
704 unsigned NumFastMacroExpanded = 0;
705 unsigned NumTokenPaste = 0;
706 unsigned NumFastTokenPaste = 0;
707 unsigned NumSkipped = 0;
708
709 /// The predefined macros that preprocessor should use from the
710 /// command line etc.
711 std::string Predefines;
712
713 /// The file ID for the preprocessor predefines.
714 FileID PredefinesFileID;
715
716 /// The file ID for the PCH through header.
717 FileID PCHThroughHeaderFileID;
718
719 /// Whether tokens are being skipped until a #pragma hdrstop is seen.
720 bool SkippingUntilPragmaHdrStop = false;
721
722 /// Whether tokens are being skipped until the through header is seen.
723 bool SkippingUntilPCHThroughHeader = false;
724
725 /// \{
726 /// Cache of macro expanders to reduce malloc traffic.
727 enum { TokenLexerCacheSize = 8 };
728 unsigned NumCachedTokenLexers;
729 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
730 /// \}
731
732 /// Keeps macro expanded tokens for TokenLexers.
733 //
734 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
735 /// going to lex in the cache and when it finishes the tokens are removed
736 /// from the end of the cache.
737 SmallVector<Token, 16> MacroExpandedTokens;
738 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
739
740 /// A record of the macro definitions and expansions that
741 /// occurred during preprocessing.
742 ///
743 /// This is an optional side structure that can be enabled with
744 /// \c createPreprocessingRecord() prior to preprocessing.
745 PreprocessingRecord *Record = nullptr;
746
747 /// Cached tokens state.
748 using CachedTokensTy = SmallVector<Token, 1>;
749
750 /// Cached tokens are stored here when we do backtracking or
751 /// lookahead. They are "lexed" by the CachingLex() method.
752 CachedTokensTy CachedTokens;
753
754 /// The position of the cached token that CachingLex() should
755 /// "lex" next.
756 ///
757 /// If it points beyond the CachedTokens vector, it means that a normal
758 /// Lex() should be invoked.
759 CachedTokensTy::size_type CachedLexPos = 0;
760
761 /// Stack of backtrack positions, allowing nested backtracks.
762 ///
763 /// The EnableBacktrackAtThisPos() method pushes a position to
764 /// indicate where CachedLexPos should be set when the BackTrack() method is
765 /// invoked (at which point the last position is popped).
766 std::vector<CachedTokensTy::size_type> BacktrackPositions;
767
768 struct MacroInfoChain {
769 MacroInfo MI;
770 MacroInfoChain *Next;
771 };
772
773 /// MacroInfos are managed as a chain for easy disposal. This is the head
774 /// of that list.
775 MacroInfoChain *MIChainHead = nullptr;
776
777 void updateOutOfDateIdentifier(IdentifierInfo &II) const;
778
779public:
780 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
781 DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
782 MemoryBufferCache &PCMCache,
783 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
784 IdentifierInfoLookup *IILookup = nullptr,
785 bool OwnsHeaderSearch = false,
786 TranslationUnitKind TUKind = TU_Complete);
787
788 ~Preprocessor();
789
790 /// Initialize the preprocessor using information about the target.
791 ///
792 /// \param Target is owned by the caller and must remain valid for the
793 /// lifetime of the preprocessor.
794 /// \param AuxTarget is owned by the caller and must remain valid for
795 /// the lifetime of the preprocessor.
796 void Initialize(const TargetInfo &Target,
797 const TargetInfo *AuxTarget = nullptr);
798
799 /// Initialize the preprocessor to parse a model file
800 ///
801 /// To parse model files the preprocessor of the original source is reused to
802 /// preserver the identifier table. However to avoid some duplicate
803 /// information in the preprocessor some cleanup is needed before it is used
804 /// to parse model files. This method does that cleanup.
805 void InitializeForModelFile();
806
807 /// Cleanup after model file parsing
808 void FinalizeForModelFile();
809
810 /// Retrieve the preprocessor options used to initialize this
811 /// preprocessor.
812 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
813
814 DiagnosticsEngine &getDiagnostics() const { return *Diags; }
815 void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
816
817 const LangOptions &getLangOpts() const { return LangOpts; }
818 const TargetInfo &getTargetInfo() const { return *Target; }
819 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
820 FileManager &getFileManager() const { return FileMgr; }
821 SourceManager &getSourceManager() const { return SourceMgr; }
822 MemoryBufferCache &getPCMCache() const { return PCMCache; }
823 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
824
825 IdentifierTable &getIdentifierTable() { return Identifiers; }
826 const IdentifierTable &getIdentifierTable() const { return Identifiers; }
827 SelectorTable &getSelectorTable() { return Selectors; }
828 Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
829 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
830
831 void setExternalSource(ExternalPreprocessorSource *Source) {
832 ExternalSource = Source;
833 }
834
835 ExternalPreprocessorSource *getExternalSource() const {
836 return ExternalSource;
837 }
838
839 /// Retrieve the module loader associated with this preprocessor.
840 ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
841
842 bool hadModuleLoaderFatalFailure() const {
843 return TheModuleLoader.HadFatalFailure;
844 }
845
846 /// True if we are currently preprocessing a #if or #elif directive
847 bool isParsingIfOrElifDirective() const {
848 return ParsingIfOrElifDirective;
849 }
850
851 /// Control whether the preprocessor retains comments in output.
852 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
853 this->KeepComments = KeepComments | KeepMacroComments;
854 this->KeepMacroComments = KeepMacroComments;
855 }
856
857 bool getCommentRetentionState() const { return KeepComments; }
858
859 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
860 bool getPragmasEnabled() const { return PragmasEnabled; }
861
862 void SetSuppressIncludeNotFoundError(bool Suppress) {
863 SuppressIncludeNotFoundError = Suppress;
864 }
865
866 bool GetSuppressIncludeNotFoundError() {
867 return SuppressIncludeNotFoundError;
868 }
869
870 /// Sets whether the preprocessor is responsible for producing output or if
871 /// it is producing tokens to be consumed by Parse and Sema.
872 void setPreprocessedOutput(bool IsPreprocessedOutput) {
873 PreprocessedOutput = IsPreprocessedOutput;
874 }
875
876 /// Returns true if the preprocessor is responsible for generating output,
877 /// false if it is producing tokens to be consumed by Parse and Sema.
878 bool isPreprocessedOutput() const { return PreprocessedOutput; }
879
880 /// Return true if we are lexing directly from the specified lexer.
881 bool isCurrentLexer(const PreprocessorLexer *L) const {
882 return CurPPLexer == L;
883 }
884
885 /// Return the current lexer being lexed from.
886 ///
887 /// Note that this ignores any potentially active macro expansions and _Pragma
888 /// expansions going on at the time.
889 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
890
891 /// Return the current file lexer being lexed from.
892 ///
893 /// Note that this ignores any potentially active macro expansions and _Pragma
894 /// expansions going on at the time.
895 PreprocessorLexer *getCurrentFileLexer() const;
896
897 /// Return the submodule owning the file being lexed. This may not be
898 /// the current module if we have changed modules since entering the file.
899 Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
900
901 /// Returns the FileID for the preprocessor predefines.
902 FileID getPredefinesFileID() const { return PredefinesFileID; }
903
904 /// \{
905 /// Accessors for preprocessor callbacks.
906 ///
907 /// Note that this class takes ownership of any PPCallbacks object given to
908 /// it.
909 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
910 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
911 if (Callbacks)
912 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
913 std::move(Callbacks));
914 Callbacks = std::move(C);
915 }
916 /// \}
917
918 bool isMacroDefined(StringRef Id) {
919 return isMacroDefined(&Identifiers.get(Id));
920 }
921 bool isMacroDefined(const IdentifierInfo *II) {
922 return II->hasMacroDefinition() &&
923 (!getLangOpts().Modules || (bool)getMacroDefinition(II));
924 }
925
926 /// Determine whether II is defined as a macro within the module M,
927 /// if that is a module that we've already preprocessed. Does not check for
928 /// macros imported into M.
929 bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) {
930 if (!II->hasMacroDefinition())
931 return false;
932 auto I = Submodules.find(M);
933 if (I == Submodules.end())
934 return false;
935 auto J = I->second.Macros.find(II);
936 if (J == I->second.Macros.end())
937 return false;
938 auto *MD = J->second.getLatest();
939 return MD && MD->isDefined();
940 }
941
942 MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
943 if (!II->hasMacroDefinition())
944 return {};
945
946 MacroState &S = CurSubmoduleState->Macros[II];
947 auto *MD = S.getLatest();
948 while (MD && isa<VisibilityMacroDirective>(MD))
949 MD = MD->getPrevious();
950 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
951 S.getActiveModuleMacros(*this, II),
952 S.isAmbiguous(*this, II));
953 }
954
955 MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
956 SourceLocation Loc) {
957 if (!II->hadMacroDefinition())
958 return {};
959
960 MacroState &S = CurSubmoduleState->Macros[II];
961 MacroDirective::DefInfo DI;
962 if (auto *MD = S.getLatest())
963 DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
964 // FIXME: Compute the set of active module macros at the specified location.
965 return MacroDefinition(DI.getDirective(),
966 S.getActiveModuleMacros(*this, II),
967 S.isAmbiguous(*this, II));
968 }
969
970 /// Given an identifier, return its latest non-imported MacroDirective
971 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
972 MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const {
973 if (!II->hasMacroDefinition())
974 return nullptr;
975
976 auto *MD = getLocalMacroDirectiveHistory(II);
977 if (!MD || MD->getDefinition().isUndefined())
978 return nullptr;
979
980 return MD;
981 }
982
983 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
984 return const_cast<Preprocessor*>(this)->getMacroInfo(II);
985 }
986
987 MacroInfo *getMacroInfo(const IdentifierInfo *II) {
988 if (!II->hasMacroDefinition())
989 return nullptr;
990 if (auto MD = getMacroDefinition(II))
991 return MD.getMacroInfo();
992 return nullptr;
993 }
994
995 /// Given an identifier, return the latest non-imported macro
996 /// directive for that identifier.
997 ///
998 /// One can iterate over all previous macro directives from the most recent
999 /// one.
1000 MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1001
1002 /// Add a directive to the macro directive history for this identifier.
1003 void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1004 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
1005 SourceLocation Loc) {
1006 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1007 appendMacroDirective(II, MD);
1008 return MD;
1009 }
1010 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II,
1011 MacroInfo *MI) {
1012 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1013 }
1014
1015 /// Set a MacroDirective that was loaded from a PCH file.
1016 void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1017 MacroDirective *MD);
1018
1019 /// Register an exported macro for a module and identifier.
1020 ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1021 ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1022 ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1023
1024 /// Get the list of leaf (non-overridden) module macros for a name.
1025 ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
1026 if (II->isOutOfDate())
1027 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1028 auto I = LeafModuleMacros.find(II);
1029 if (I != LeafModuleMacros.end())
1030 return I->second;
1031 return None;
1032 }
1033
1034 /// \{
1035 /// Iterators for the macro history table. Currently defined macros have
1036 /// IdentifierInfo::hasMacroDefinition() set and an empty
1037 /// MacroInfo::getUndefLoc() at the head of the list.
1038 using macro_iterator = MacroMap::const_iterator;
1039
1040 macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1041 macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1042
1043 llvm::iterator_range<macro_iterator>
1044 macros(bool IncludeExternalMacros = true) const {
1045 macro_iterator begin = macro_begin(IncludeExternalMacros);
1046 macro_iterator end = macro_end(IncludeExternalMacros);
1047 return llvm::make_range(begin, end);
1048 }
1049
1050 /// \}
1051
1052 /// Return the name of the macro defined before \p Loc that has
1053 /// spelling \p Tokens. If there are multiple macros with same spelling,
1054 /// return the last one defined.
1055 StringRef getLastMacroWithSpelling(SourceLocation Loc,
1056 ArrayRef<TokenValue> Tokens) const;
1057
1058 const std::string &getPredefines() const { return Predefines; }
1059
1060 /// Set the predefines for this Preprocessor.
1061 ///
1062 /// These predefines are automatically injected when parsing the main file.
1063 void setPredefines(const char *P) { Predefines = P; }
1064 void setPredefines(StringRef P) { Predefines = P; }
1065
1066 /// Return information about the specified preprocessor
1067 /// identifier token.
1068 IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1069 return &Identifiers.get(Name);
1070 }
1071
1072 /// Add the specified pragma handler to this preprocessor.
1073 ///
1074 /// If \p Namespace is non-null, then it is a token required to exist on the
1075 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1076 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1077 void AddPragmaHandler(PragmaHandler *Handler) {
1078 AddPragmaHandler(StringRef(), Handler);
1079 }
1080
1081 /// Remove the specific pragma handler from this preprocessor.
1082 ///
1083 /// If \p Namespace is non-null, then it should be the namespace that
1084 /// \p Handler was added to. It is an error to remove a handler that
1085 /// has not been registered.
1086 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1087 void RemovePragmaHandler(PragmaHandler *Handler) {
1088 RemovePragmaHandler(StringRef(), Handler);
1089 }
1090
1091 /// Install empty handlers for all pragmas (making them ignored).
1092 void IgnorePragmas();
1093
1094 /// Add the specified comment handler to the preprocessor.
1095 void addCommentHandler(CommentHandler *Handler);
1096
1097 /// Remove the specified comment handler.
1098 ///
1099 /// It is an error to remove a handler that has not been registered.
1100 void removeCommentHandler(CommentHandler *Handler);
1101
1102 /// Set the code completion handler to the given object.
1103 void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1104 CodeComplete = &Handler;
1105 }
1106
1107 /// Retrieve the current code-completion handler.
1108 CodeCompletionHandler *getCodeCompletionHandler() const {
1109 return CodeComplete;
1110 }
1111
1112 /// Clear out the code completion handler.
1113 void clearCodeCompletionHandler() {
1114 CodeComplete = nullptr;
1115 }
1116
1117 /// Hook used by the lexer to invoke the "included file" code
1118 /// completion point.
1119 void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1120
1121 /// Hook used by the lexer to invoke the "natural language" code
1122 /// completion point.
1123 void CodeCompleteNaturalLanguage();
1124
1125 /// Set the code completion token for filtering purposes.
1126 void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1127 CodeCompletionII = Filter;
1128 }
1129
1130 /// Set the code completion token range for detecting replacement range later
1131 /// on.
1132 void setCodeCompletionTokenRange(const SourceLocation Start,
1133 const SourceLocation End) {
1134 CodeCompletionTokenRange = {Start, End};
1135 }
1136 SourceRange getCodeCompletionTokenRange() const {
1137 return CodeCompletionTokenRange;
1138 }
1139
1140 /// Get the code completion token for filtering purposes.
1141 StringRef getCodeCompletionFilter() {
1142 if (CodeCompletionII)
1143 return CodeCompletionII->getName();
1144 return {};
1145 }
1146
1147 /// Retrieve the preprocessing record, or NULL if there is no
1148 /// preprocessing record.
1149 PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1150
1151 /// Create a new preprocessing record, which will keep track of
1152 /// all macro expansions, macro definitions, etc.
1153 void createPreprocessingRecord();
1154
1155 /// Returns true if the FileEntry is the PCH through header.
1156 bool isPCHThroughHeader(const FileEntry *FE);
1157
1158 /// True if creating a PCH with a through header.
1159 bool creatingPCHWithThroughHeader();
1160
1161 /// True if using a PCH with a through header.
1162 bool usingPCHWithThroughHeader();
1163
1164 /// True if creating a PCH with a #pragma hdrstop.
1165 bool creatingPCHWithPragmaHdrStop();
1166
1167 /// True if using a PCH with a #pragma hdrstop.
1168 bool usingPCHWithPragmaHdrStop();
1169
1170 /// Skip tokens until after the #include of the through header or
1171 /// until after a #pragma hdrstop.
1172 void SkipTokensWhileUsingPCH();
1173
1174 /// Process directives while skipping until the through header or
1175 /// #pragma hdrstop is found.
1176 void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1177 SourceLocation HashLoc);
1178
1179 /// Enter the specified FileID as the main source file,
1180 /// which implicitly adds the builtin defines etc.
1181 void EnterMainSourceFile();
1182
1183 /// Inform the preprocessor callbacks that processing is complete.
1184 void EndSourceFile();
1185
1186 /// Add a source file to the top of the include stack and
1187 /// start lexing tokens from it instead of the current buffer.
1188 ///
1189 /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1190 bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1191 SourceLocation Loc);
1192
1193 /// Add a Macro to the top of the include stack and start lexing
1194 /// tokens from it instead of the current buffer.
1195 ///
1196 /// \param Args specifies the tokens input to a function-like macro.
1197 /// \param ILEnd specifies the location of the ')' for a function-like macro
1198 /// or the identifier for an object-like macro.
1199 void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1200 MacroArgs *Args);
1201
1202 /// Add a "macro" context to the top of the include stack,
1203 /// which will cause the lexer to start returning the specified tokens.
1204 ///
1205 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1206 /// will not be subject to further macro expansion. Otherwise, these tokens
1207 /// will be re-macro-expanded when/if expansion is enabled.
1208 ///
1209 /// If \p OwnsTokens is false, this method assumes that the specified stream
1210 /// of tokens has a permanent owner somewhere, so they do not need to be
1211 /// copied. If it is true, it assumes the array of tokens is allocated with
1212 /// \c new[] and the Preprocessor will delete[] it.
1213private:
1214 void EnterTokenStream(const Token *Toks, unsigned NumToks,
1215 bool DisableMacroExpansion, bool OwnsTokens);
1216
1217public:
1218 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1219 bool DisableMacroExpansion) {
1220 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1221 }
1222
1223 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1224 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1225 }
1226
1227 /// Pop the current lexer/macro exp off the top of the lexer stack.
1228 ///
1229 /// This should only be used in situations where the current state of the
1230 /// top-of-stack lexer is known.
1231 void RemoveTopOfLexerStack();
1232
1233 /// From the point that this method is called, and until
1234 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1235 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1236 /// make the Preprocessor re-lex the same tokens.
1237 ///
1238 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1239 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1240 /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1241 ///
1242 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1243 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1244 /// tokens will continue indefinitely.
1245 ///
1246 void EnableBacktrackAtThisPos();
1247
1248 /// Disable the last EnableBacktrackAtThisPos call.
1249 void CommitBacktrackedTokens();
1250
1251 struct CachedTokensRange {
1252 CachedTokensTy::size_type Begin, End;
1253 };
1254
1255private:
1256 /// A range of cached tokens that should be erased after lexing
1257 /// when backtracking requires the erasure of such cached tokens.
1258 Optional<CachedTokensRange> CachedTokenRangeToErase;
1259
1260public:
1261 /// Returns the range of cached tokens that were lexed since
1262 /// EnableBacktrackAtThisPos() was previously called.
1263 CachedTokensRange LastCachedTokenRange();
1264
1265 /// Erase the range of cached tokens that were lexed since
1266 /// EnableBacktrackAtThisPos() was previously called.
1267 void EraseCachedTokens(CachedTokensRange TokenRange);
1268
1269 /// Make Preprocessor re-lex the tokens that were lexed since
1270 /// EnableBacktrackAtThisPos() was previously called.
1271 void Backtrack();
1272
1273 /// True if EnableBacktrackAtThisPos() was called and
1274 /// caching of tokens is on.
1275 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1276
1277 /// Lex the next token for this preprocessor.
1278 void Lex(Token &Result);
1279
1280 void LexAfterModuleImport(Token &Result);
1281
1282 void makeModuleVisible(Module *M, SourceLocation Loc);
1283
1284 SourceLocation getModuleImportLoc(Module *M) const {
1285 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1286 }
1287
1288 /// Lex a string literal, which may be the concatenation of multiple
1289 /// string literals and may even come from macro expansion.
1290 /// \returns true on success, false if a error diagnostic has been generated.
1291 bool LexStringLiteral(Token &Result, std::string &String,
1292 const char *DiagnosticTag, bool AllowMacroExpansion) {
1293 if (AllowMacroExpansion)
1294 Lex(Result);
1295 else
1296 LexUnexpandedToken(Result);
1297 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1298 AllowMacroExpansion);
1299 }
1300
1301 /// Complete the lexing of a string literal where the first token has
1302 /// already been lexed (see LexStringLiteral).
1303 bool FinishLexStringLiteral(Token &Result, std::string &String,
1304 const char *DiagnosticTag,
1305 bool AllowMacroExpansion);
1306
1307 /// Lex a token. If it's a comment, keep lexing until we get
1308 /// something not a comment.
1309 ///
1310 /// This is useful in -E -C mode where comments would foul up preprocessor
1311 /// directive handling.
1312 void LexNonComment(Token &Result) {
1313 do
1314 Lex(Result);
1315 while (Result.getKind() == tok::comment);
1316 }
1317
1318 /// Just like Lex, but disables macro expansion of identifier tokens.
1319 void LexUnexpandedToken(Token &Result) {
1320 // Disable macro expansion.
1321 bool OldVal = DisableMacroExpansion;
1322 DisableMacroExpansion = true;
1323 // Lex the token.
1324 Lex(Result);
1325
1326 // Reenable it.
1327 DisableMacroExpansion = OldVal;
1328 }
1329
1330 /// Like LexNonComment, but this disables macro expansion of
1331 /// identifier tokens.
1332 void LexUnexpandedNonComment(Token &Result) {
1333 do
1334 LexUnexpandedToken(Result);
1335 while (Result.getKind() == tok::comment);
1336 }
1337
1338 /// Parses a simple integer literal to get its numeric value. Floating
1339 /// point literals and user defined literals are rejected. Used primarily to
1340 /// handle pragmas that accept integer arguments.
1341 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1342
1343 /// Disables macro expansion everywhere except for preprocessor directives.
1344 void SetMacroExpansionOnlyInDirectives() {
1345 DisableMacroExpansion = true;
1346 MacroExpansionInDirectivesOverride = true;
1347 }
1348
1349 /// Peeks ahead N tokens and returns that token without consuming any
1350 /// tokens.
1351 ///
1352 /// LookAhead(0) returns the next token that would be returned by Lex(),
1353 /// LookAhead(1) returns the token after it, etc. This returns normal
1354 /// tokens after phase 5. As such, it is equivalent to using
1355 /// 'Lex', not 'LexUnexpandedToken'.
1356 const Token &LookAhead(unsigned N) {
1357 if (CachedLexPos + N < CachedTokens.size())
1358 return CachedTokens[CachedLexPos+N];
1359 else
1360 return PeekAhead(N+1);
1361 }
1362
1363 /// When backtracking is enabled and tokens are cached,
1364 /// this allows to revert a specific number of tokens.
1365 ///
1366 /// Note that the number of tokens being reverted should be up to the last
1367 /// backtrack position, not more.
1368 void RevertCachedTokens(unsigned N) {
1369 assert(isBacktrackEnabled() &&
1370 "Should only be called when tokens are cached for backtracking");
1371 assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1372 && "Should revert tokens up to the last backtrack position, not more");
1373 assert(signed(CachedLexPos) - signed(N) >= 0 &&
1374 "Corrupted backtrack positions ?");
1375 CachedLexPos -= N;
1376 }
1377
1378 /// Enters a token in the token stream to be lexed next.
1379 ///
1380 /// If BackTrack() is called afterwards, the token will remain at the
1381 /// insertion point.
1382 void EnterToken(const Token &Tok) {
1383 EnterCachingLexMode();
1384 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1385 }
1386
1387 /// We notify the Preprocessor that if it is caching tokens (because
1388 /// backtrack is enabled) it should replace the most recent cached tokens
1389 /// with the given annotation token. This function has no effect if
1390 /// backtracking is not enabled.
1391 ///
1392 /// Note that the use of this function is just for optimization, so that the
1393 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1394 /// invoked.
1395 void AnnotateCachedTokens(const Token &Tok) {
1396 assert(Tok.isAnnotation() && "Expected annotation token");
1397 if (CachedLexPos != 0 && isBacktrackEnabled())
1398 AnnotatePreviousCachedTokens(Tok);
1399 }
1400
1401 /// Get the location of the last cached token, suitable for setting the end
1402 /// location of an annotation token.
1403 SourceLocation getLastCachedTokenLocation() const {
1404 assert(CachedLexPos != 0);
1405 return CachedTokens[CachedLexPos-1].getLastLoc();
1406 }
1407
1408 /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1409 /// CachedTokens.
1410 bool IsPreviousCachedToken(const Token &Tok) const;
1411
1412 /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1413 /// in \p NewToks.
1414 ///
1415 /// Useful when a token needs to be split in smaller ones and CachedTokens
1416 /// most recent token must to be updated to reflect that.
1417 void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1418
1419 /// Replace the last token with an annotation token.
1420 ///
1421 /// Like AnnotateCachedTokens(), this routine replaces an
1422 /// already-parsed (and resolved) token with an annotation
1423 /// token. However, this routine only replaces the last token with
1424 /// the annotation token; it does not affect any other cached
1425 /// tokens. This function has no effect if backtracking is not
1426 /// enabled.
1427 void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1428 assert(Tok.isAnnotation() && "Expected annotation token");
1429 if (CachedLexPos != 0 && isBacktrackEnabled())
1430 CachedTokens[CachedLexPos-1] = Tok;
1431 }
1432
1433 /// Enter an annotation token into the token stream.
1434 void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1435 void *AnnotationVal);
1436
1437 /// Update the current token to represent the provided
1438 /// identifier, in order to cache an action performed by typo correction.
1439 void TypoCorrectToken(const Token &Tok) {
1440 assert(Tok.getIdentifierInfo() && "Expected identifier token");
1441 if (CachedLexPos != 0 && isBacktrackEnabled())
1442 CachedTokens[CachedLexPos-1] = Tok;
1443 }
1444
1445 /// Recompute the current lexer kind based on the CurLexer/
1446 /// CurTokenLexer pointers.
1447 void recomputeCurLexerKind();
1448
1449 /// Returns true if incremental processing is enabled
1450 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1451
1452 /// Enables the incremental processing
1453 void enableIncrementalProcessing(bool value = true) {
1454 IncrementalProcessing = value;
1455 }
1456
1457 /// Specify the point at which code-completion will be performed.
1458 ///
1459 /// \param File the file in which code completion should occur. If
1460 /// this file is included multiple times, code-completion will
1461 /// perform completion the first time it is included. If NULL, this
1462 /// function clears out the code-completion point.
1463 ///
1464 /// \param Line the line at which code completion should occur
1465 /// (1-based).
1466 ///
1467 /// \param Column the column at which code completion should occur
1468 /// (1-based).
1469 ///
1470 /// \returns true if an error occurred, false otherwise.
1471 bool SetCodeCompletionPoint(const FileEntry *File,
1472 unsigned Line, unsigned Column);
1473
1474 /// Determine if we are performing code completion.
1475 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1476
1477 /// Returns the location of the code-completion point.
1478 ///
1479 /// Returns an invalid location if code-completion is not enabled or the file
1480 /// containing the code-completion point has not been lexed yet.
1481 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1482
1483 /// Returns the start location of the file of code-completion point.
1484 ///
1485 /// Returns an invalid location if code-completion is not enabled or the file
1486 /// containing the code-completion point has not been lexed yet.
1487 SourceLocation getCodeCompletionFileLoc() const {
1488 return CodeCompletionFileLoc;
1489 }
1490
1491 /// Returns true if code-completion is enabled and we have hit the
1492 /// code-completion point.
1493 bool isCodeCompletionReached() const { return CodeCompletionReached; }
1494
1495 /// Note that we hit the code-completion point.
1496 void setCodeCompletionReached() {
1497 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1498 CodeCompletionReached = true;
1499 // Silence any diagnostics that occur after we hit the code-completion.
1500 getDiagnostics().setSuppressAllDiagnostics(true);
1501 }
1502
1503 /// The location of the currently-active \#pragma clang
1504 /// arc_cf_code_audited begin.
1505 ///
1506 /// Returns an invalid location if there is no such pragma active.
1507 SourceLocation getPragmaARCCFCodeAuditedLoc() const {
1508 return PragmaARCCFCodeAuditedLoc;
1509 }
1510
1511 /// Set the location of the currently-active \#pragma clang
1512 /// arc_cf_code_audited begin. An invalid location ends the pragma.
1513 void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
1514 PragmaARCCFCodeAuditedLoc = Loc;
1515 }
1516
1517 /// The location of the currently-active \#pragma clang
1518 /// assume_nonnull begin.
1519 ///
1520 /// Returns an invalid location if there is no such pragma active.
1521 SourceLocation getPragmaAssumeNonNullLoc() const {
1522 return PragmaAssumeNonNullLoc;
1523 }
1524
1525 /// Set the location of the currently-active \#pragma clang
1526 /// assume_nonnull begin. An invalid location ends the pragma.
1527 void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
1528 PragmaAssumeNonNullLoc = Loc;
1529 }
1530
1531 /// Set the directory in which the main file should be considered
1532 /// to have been found, if it is not a real file.
1533 void setMainFileDir(const DirectoryEntry *Dir) {
1534 MainFileDir = Dir;
1535 }
1536
1537 /// Instruct the preprocessor to skip part of the main source file.
1538 ///
1539 /// \param Bytes The number of bytes in the preamble to skip.
1540 ///
1541 /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1542 /// start of a line.
1543 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1544 SkipMainFilePreamble.first = Bytes;
1545 SkipMainFilePreamble.second = StartOfLine;
1546 }
1547
1548 /// Forwarding function for diagnostics. This emits a diagnostic at
1549 /// the specified Token's location, translating the token's start
1550 /// position in the current buffer into a SourcePosition object for rendering.
1551 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1552 return Diags->Report(Loc, DiagID);
1553 }
1554
1555 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1556 return Diags->Report(Tok.getLocation(), DiagID);
1557 }
1558
1559 /// Return the 'spelling' of the token at the given
1560 /// location; does not go up to the spelling location or down to the
1561 /// expansion location.
1562 ///
1563 /// \param buffer A buffer which will be used only if the token requires
1564 /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1565 /// \param invalid If non-null, will be set \c true if an error occurs.
1566 StringRef getSpelling(SourceLocation loc,
1567 SmallVectorImpl<char> &buffer,
1568 bool *invalid = nullptr) const {
1569 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1570 }
1571
1572 /// Return the 'spelling' of the Tok token.
1573 ///
1574 /// The spelling of a token is the characters used to represent the token in
1575 /// the source file after trigraph expansion and escaped-newline folding. In
1576 /// particular, this wants to get the true, uncanonicalized, spelling of
1577 /// things like digraphs, UCNs, etc.
1578 ///
1579 /// \param Invalid If non-null, will be set \c true if an error occurs.
1580 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1581 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1582 }
1583
1584 /// Get the spelling of a token into a preallocated buffer, instead
1585 /// of as an std::string.
1586 ///
1587 /// The caller is required to allocate enough space for the token, which is
1588 /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1589 /// actual result is returned.
1590 ///
1591 /// Note that this method may do two possible things: it may either fill in
1592 /// the buffer specified with characters, or it may *change the input pointer*
1593 /// to point to a constant buffer with the data already in it (avoiding a
1594 /// copy). The caller is not allowed to modify the returned buffer pointer
1595 /// if an internal buffer is returned.
1596 unsigned getSpelling(const Token &Tok, const char *&Buffer,
1597 bool *Invalid = nullptr) const {
1598 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1599 }
1600
1601 /// Get the spelling of a token into a SmallVector.
1602 ///
1603 /// Note that the returned StringRef may not point to the
1604 /// supplied buffer if a copy can be avoided.
1605 StringRef getSpelling(const Token &Tok,
1606 SmallVectorImpl<char> &Buffer,
1607 bool *Invalid = nullptr) const;
1608
1609 /// Relex the token at the specified location.
1610 /// \returns true if there was a failure, false on success.
1611 bool getRawToken(SourceLocation Loc, Token &Result,
1612 bool IgnoreWhiteSpace = false) {
1613 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1614 }
1615
1616 /// Given a Token \p Tok that is a numeric constant with length 1,
1617 /// return the character.
1618 char
1619 getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
1620 bool *Invalid = nullptr) const {
1621 assert(Tok.is(tok::numeric_constant) &&
1622 Tok.getLength() == 1 && "Called on unsupported token");
1623 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1624
1625 // If the token is carrying a literal data pointer, just use it.
1626 if (const char *D = Tok.getLiteralData())
1627 return *D;
1628
1629 // Otherwise, fall back on getCharacterData, which is slower, but always
1630 // works.
1631 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1632 }
1633
1634 /// Retrieve the name of the immediate macro expansion.
1635 ///
1636 /// This routine starts from a source location, and finds the name of the
1637 /// macro responsible for its immediate expansion. It looks through any
1638 /// intervening macro argument expansions to compute this. It returns a
1639 /// StringRef that refers to the SourceManager-owned buffer of the source
1640 /// where that macro name is spelled. Thus, the result shouldn't out-live
1641 /// the SourceManager.
1642 StringRef getImmediateMacroName(SourceLocation Loc) {
1643 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1644 }
1645
1646 /// Plop the specified string into a scratch buffer and set the
1647 /// specified token's location and length to it.
1648 ///
1649 /// If specified, the source location provides a location of the expansion
1650 /// point of the token.
1651 void CreateString(StringRef Str, Token &Tok,
1652 SourceLocation ExpansionLocStart = SourceLocation(),
1653 SourceLocation ExpansionLocEnd = SourceLocation());
1654
1655 /// Split the first Length characters out of the token starting at TokLoc
1656 /// and return a location pointing to the split token. Re-lexing from the
1657 /// split token will return the split token rather than the original.
1658 SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1659
1660 /// Computes the source location just past the end of the
1661 /// token at this source location.
1662 ///
1663 /// This routine can be used to produce a source location that
1664 /// points just past the end of the token referenced by \p Loc, and
1665 /// is generally used when a diagnostic needs to point just after a
1666 /// token where it expected something different that it received. If
1667 /// the returned source location would not be meaningful (e.g., if
1668 /// it points into a macro), this routine returns an invalid
1669 /// source location.
1670 ///
1671 /// \param Offset an offset from the end of the token, where the source
1672 /// location should refer to. The default offset (0) produces a source
1673 /// location pointing just past the end of the token; an offset of 1 produces
1674 /// a source location pointing to the last character in the token, etc.
1675 SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
1676 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1677 }
1678
1679 /// Returns true if the given MacroID location points at the first
1680 /// token of the macro expansion.
1681 ///
1682 /// \param MacroBegin If non-null and function returns true, it is set to
1683 /// begin location of the macro.
1684 bool isAtStartOfMacroExpansion(SourceLocation loc,
1685 SourceLocation *MacroBegin = nullptr) const {
1686 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1687 MacroBegin);
1688 }
1689
1690 /// Returns true if the given MacroID location points at the last
1691 /// token of the macro expansion.
1692 ///
1693 /// \param MacroEnd If non-null and function returns true, it is set to
1694 /// end location of the macro.
1695 bool isAtEndOfMacroExpansion(SourceLocation loc,
1696 SourceLocation *MacroEnd = nullptr) const {
1697 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1698 }
1699
1700 /// Print the token to stderr, used for debugging.
1701 void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1702 void DumpLocation(SourceLocation Loc) const;
1703 void DumpMacro(const MacroInfo &MI) const;
1704 void dumpMacroInfo(const IdentifierInfo *II);
1705
1706 /// Given a location that specifies the start of a
1707 /// token, return a new location that specifies a character within the token.
1708 SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
1709 unsigned Char) const {
1710 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1711 }
1712
1713 /// Increment the counters for the number of token paste operations
1714 /// performed.
1715 ///
1716 /// If fast was specified, this is a 'fast paste' case we handled.
1717 void IncrementPasteCounter(bool isFast) {
1718 if (isFast)
1719 ++NumFastTokenPaste;
1720 else
1721 ++NumTokenPaste;
1722 }
1723
1724 void PrintStats();
1725
1726 size_t getTotalMemory() const;
1727
1728 /// When the macro expander pastes together a comment (/##/) in Microsoft
1729 /// mode, this method handles updating the current state, returning the
1730 /// token on the next source line.
1731 void HandleMicrosoftCommentPaste(Token &Tok);
1732
1733 //===--------------------------------------------------------------------===//
1734 // Preprocessor callback methods. These are invoked by a lexer as various
1735 // directives and events are found.
1736
1737 /// Given a tok::raw_identifier token, look up the
1738 /// identifier information for the token and install it into the token,
1739 /// updating the token kind accordingly.
1740 IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1741
1742private:
1743 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1744
1745public:
1746 /// Specifies the reason for poisoning an identifier.
1747 ///
1748 /// If that identifier is accessed while poisoned, then this reason will be
1749 /// used instead of the default "poisoned" diagnostic.
1750 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1751
1752 /// Display reason for poisoned identifier.
1753 void HandlePoisonedIdentifier(Token & Identifier);
1754
1755 void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1756 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1757 if(II->isPoisoned()) {
1758 HandlePoisonedIdentifier(Identifier);
1759 }
1760 }
1761 }
1762
1763private:
1764 /// Identifiers used for SEH handling in Borland. These are only
1765 /// allowed in particular circumstances
1766 // __except block
1767 IdentifierInfo *Ident__exception_code,
1768 *Ident___exception_code,
1769 *Ident_GetExceptionCode;
1770 // __except filter expression
1771 IdentifierInfo *Ident__exception_info,
1772 *Ident___exception_info,
1773 *Ident_GetExceptionInfo;
1774 // __finally
1775 IdentifierInfo *Ident__abnormal_termination,
1776 *Ident___abnormal_termination,
1777 *Ident_AbnormalTermination;
1778
1779 const char *getCurLexerEndPos();
1780 void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1781
1782public:
1783 void PoisonSEHIdentifiers(bool Poison = true); // Borland
1784
1785 /// Callback invoked when the lexer reads an identifier and has
1786 /// filled in the tokens IdentifierInfo member.
1787 ///
1788 /// This callback potentially macro expands it or turns it into a named
1789 /// token (like 'for').
1790 ///
1791 /// \returns true if we actually computed a token, false if we need to
1792 /// lex again.
1793 bool HandleIdentifier(Token &Identifier);
1794
1795 /// Callback invoked when the lexer hits the end of the current file.
1796 ///
1797 /// This either returns the EOF token and returns true, or
1798 /// pops a level off the include stack and returns false, at which point the
1799 /// client should call lex again.
1800 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1801
1802 /// Callback invoked when the current TokenLexer hits the end of its
1803 /// token stream.
1804 bool HandleEndOfTokenLexer(Token &Result);
1805
1806 /// Callback invoked when the lexer sees a # token at the start of a
1807 /// line.
1808 ///
1809 /// This consumes the directive, modifies the lexer/preprocessor state, and
1810 /// advances the lexer(s) so that the next token read is the correct one.
1811 void HandleDirective(Token &Result);
1812
1813 /// Ensure that the next token is a tok::eod token.
1814 ///
1815 /// If not, emit a diagnostic and consume up until the eod.
1816 /// If \p EnableMacros is true, then we consider macros that expand to zero
1817 /// tokens as being ok.
1818 void CheckEndOfDirective(const char *DirType, bool EnableMacros = false);
1819
1820 /// Read and discard all tokens remaining on the current line until
1821 /// the tok::eod token is found. Returns the range of the skipped tokens.
1822 SourceRange DiscardUntilEndOfDirective();
1823
1824 /// Returns true if the preprocessor has seen a use of
1825 /// __DATE__ or __TIME__ in the file so far.
1826 bool SawDateOrTime() const {
1827 return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1828 }
1829 unsigned getCounterValue() const { return CounterValue; }
1830 void setCounterValue(unsigned V) { CounterValue = V; }
1831
1832 /// Retrieves the module that we're currently building, if any.
1833 Module *getCurrentModule();
1834
1835 /// Allocate a new MacroInfo object with the provided SourceLocation.
1836 MacroInfo *AllocateMacroInfo(SourceLocation L);
1837
1838 /// Turn the specified lexer token into a fully checked and spelled
1839 /// filename, e.g. as an operand of \#include.
1840 ///
1841 /// The caller is expected to provide a buffer that is large enough to hold
1842 /// the spelling of the filename, but is also expected to handle the case
1843 /// when this method decides to use a different buffer.
1844 ///
1845 /// \returns true if the input filename was in <>'s or false if it was
1846 /// in ""'s.
1847 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1848
1849 /// Given a "foo" or \<foo> reference, look up the indicated file.
1850 ///
1851 /// Returns null on failure. \p isAngled indicates whether the file
1852 /// reference is for system \#include's or not (i.e. using <> instead of "").
1853 const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1854 bool isAngled, const DirectoryLookup *FromDir,
1855 const FileEntry *FromFile,
1856 const DirectoryLookup *&CurDir,
1857 SmallVectorImpl<char> *SearchPath,
1858 SmallVectorImpl<char> *RelativePath,
1859 ModuleMap::KnownHeader *SuggestedModule,
1860 bool *IsMapped, bool *IsFrameworkFound,
1861 bool SkipCache = false);
1862
1863 /// Get the DirectoryLookup structure used to find the current
1864 /// FileEntry, if CurLexer is non-null and if applicable.
1865 ///
1866 /// This allows us to implement \#include_next and find directory-specific
1867 /// properties.
1868 const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1869
1870 /// Return true if we're in the top-level file, not in a \#include.
1871 bool isInPrimaryFile() const;
1872
1873 /// Handle cases where the \#include name is expanded
1874 /// from a macro as multiple tokens, which need to be glued together.
1875 ///
1876 /// This occurs for code like:
1877 /// \code
1878 /// \#define FOO <x/y.h>
1879 /// \#include FOO
1880 /// \endcode
1881 /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1882 ///
1883 /// This code concatenates and consumes tokens up to the '>' token. It
1884 /// returns false if the > was found, otherwise it returns true if it finds
1885 /// and consumes the EOD marker.
1886 bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1887 SourceLocation &End);
1888
1889 /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1890 /// followed by EOD. Return true if the token is not a valid on-off-switch.
1891 bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1892
1893 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1894 bool *ShadowFlag = nullptr);
1895
1896 void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1897 Module *LeaveSubmodule(bool ForPragma);
1898
1899private:
1900 friend void TokenLexer::ExpandFunctionArguments();
1901
1902 void PushIncludeMacroStack() {
1903 assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1904 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1905 std::move(CurLexer), CurPPLexer,
1906 std::move(CurTokenLexer), CurDirLookup);
1907 CurPPLexer = nullptr;
1908 }
1909
1910 void PopIncludeMacroStack() {
1911 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1912 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1913 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1914 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1915 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1916 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1917 IncludeMacroStack.pop_back();
1918 }
1919
1920 void PropagateLineStartLeadingSpaceInfo(Token &Result);
1921
1922 /// Determine whether we need to create module macros for #defines in the
1923 /// current context.
1924 bool needModuleMacros() const;
1925
1926 /// Update the set of active module macros and ambiguity flag for a module
1927 /// macro name.
1928 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1929
1930 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1931 SourceLocation Loc);
1932 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1933 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1934 bool isPublic);
1935
1936 /// Lex and validate a macro name, which occurs after a
1937 /// \#define or \#undef.
1938 ///
1939 /// \param MacroNameTok Token that represents the name defined or undefined.
1940 /// \param IsDefineUndef Kind if preprocessor directive.
1941 /// \param ShadowFlag Points to flag that is set if macro name shadows
1942 /// a keyword.
1943 ///
1944 /// This emits a diagnostic, sets the token kind to eod,
1945 /// and discards the rest of the macro line if the macro name is invalid.
1946 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1947 bool *ShadowFlag = nullptr);
1948
1949 /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1950 /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1951 /// doing so performs certain validity checks including (but not limited to):
1952 /// - # (stringization) is followed by a macro parameter
1953 /// \param MacroNameTok - Token that represents the macro name
1954 /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1955 ///
1956 /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1957 /// returns a nullptr if an invalid sequence of tokens is encountered.
1958 MacroInfo *ReadOptionalMacroParameterListAndBody(
1959 const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1960
1961 /// The ( starting an argument list of a macro definition has just been read.
1962 /// Lex the rest of the parameters and the closing ), updating \p MI with
1963 /// what we learn and saving in \p LastTok the last token read.
1964 /// Return true if an error occurs parsing the arg list.
1965 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1966
1967 /// We just read a \#if or related directive and decided that the
1968 /// subsequent tokens are in the \#if'd out portion of the
1969 /// file. Lex the rest of the file, until we see an \#endif. If \p
1970 /// FoundNonSkipPortion is true, then we have already emitted code for part of
1971 /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1972 /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1973 /// already seen one so a \#else directive is a duplicate. When this returns,
1974 /// the caller can lex the first valid token.
1975 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
1976 SourceLocation IfTokenLoc,
1977 bool FoundNonSkipPortion, bool FoundElse,
1978 SourceLocation ElseLoc = SourceLocation());
1979
1980 /// Information about the result for evaluating an expression for a
1981 /// preprocessor directive.
1982 struct DirectiveEvalResult {
1983 /// Whether the expression was evaluated as true or not.
1984 bool Conditional;
1985
1986 /// True if the expression contained identifiers that were undefined.
1987 bool IncludedUndefinedIds;
1988
1989 /// The source range for the expression.
1990 SourceRange ExprRange;
1991 };
1992
1993 /// Evaluate an integer constant expression that may occur after a
1994 /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1995 ///
1996 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1997 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1998
1999 /// Install the standard preprocessor pragmas:
2000 /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2001 void RegisterBuiltinPragmas();
2002
2003 /// Register builtin macros such as __LINE__ with the identifier table.
2004 void RegisterBuiltinMacros();
2005
2006 /// If an identifier token is read that is to be expanded as a macro, handle
2007 /// it and return the next token as 'Tok'. If we lexed a token, return true;
2008 /// otherwise the caller should lex again.
2009 bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2010
2011 /// Cache macro expanded tokens for TokenLexers.
2012 //
2013 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2014 /// going to lex in the cache and when it finishes the tokens are removed
2015 /// from the end of the cache.
2016 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2017 ArrayRef<Token> tokens);
2018
2019 void removeCachedMacroExpandedTokensOfLastLexer();
2020
2021 /// Determine whether the next preprocessor token to be
2022 /// lexed is a '('. If so, consume the token and return true, if not, this
2023 /// method should have no observable side-effect on the lexed tokens.
2024 bool isNextPPTokenLParen();
2025
2026 /// After reading "MACRO(", this method is invoked to read all of the formal
2027 /// arguments specified for the macro invocation. Returns null on error.
2028 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2029 SourceLocation &MacroEnd);
2030
2031 /// If an identifier token is read that is to be expanded
2032 /// as a builtin macro, handle it and return the next token as 'Tok'.
2033 void ExpandBuiltinMacro(Token &Tok);
2034
2035 /// Read a \c _Pragma directive, slice it up, process it, then
2036 /// return the first token after the directive.
2037 /// This assumes that the \c _Pragma token has just been read into \p Tok.
2038 void Handle_Pragma(Token &Tok);
2039
2040 /// Like Handle_Pragma except the pragma text is not enclosed within
2041 /// a string literal.
2042 void HandleMicrosoft__pragma(Token &Tok);
2043
2044 /// Add a lexer to the top of the include stack and
2045 /// start lexing tokens from it instead of the current buffer.
2046 void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2047
2048 /// Set the FileID for the preprocessor predefines.
2049 void setPredefinesFileID(FileID FID) {
2050 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2051 PredefinesFileID = FID;
2052 }
2053
2054 /// Set the FileID for the PCH through header.
2055 void setPCHThroughHeaderFileID(FileID FID);
2056
2057 /// Returns true if we are lexing from a file and not a
2058 /// pragma or a macro.
2059 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2060 return L ? !L->isPragmaLexer() : P != nullptr;
2061 }
2062
2063 static bool IsFileLexer(const IncludeStackInfo& I) {
2064 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2065 }
2066
2067 bool IsFileLexer() const {
2068 return IsFileLexer(CurLexer.get(), CurPPLexer);
2069 }
2070
2071 //===--------------------------------------------------------------------===//
2072 // Caching stuff.
2073 void CachingLex(Token &Result);
2074
2075 bool InCachingLexMode() const {
2076 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2077 // that we are past EOF, not that we are in CachingLex mode.
2078 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2079 }
2080
2081 void EnterCachingLexMode();
2082
2083 void ExitCachingLexMode() {
2084 if (InCachingLexMode())
2085 RemoveTopOfLexerStack();
2086 }
2087
2088 const Token &PeekAhead(unsigned N);
2089 void AnnotatePreviousCachedTokens(const Token &Tok);
2090
2091 //===--------------------------------------------------------------------===//
2092 /// Handle*Directive - implement the various preprocessor directives. These
2093 /// should side-effect the current preprocessor object so that the next call
2094 /// to Lex() will return the appropriate token next.
2095 void HandleLineDirective();
2096 void HandleDigitDirective(Token &Tok);
2097 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2098 void HandleIdentSCCSDirective(Token &Tok);
2099 void HandleMacroPublicDirective(Token &Tok);
2100 void HandleMacroPrivateDirective();
2101
2102 // File inclusion.
2103 void HandleIncludeDirective(SourceLocation HashLoc,
2104 Token &Tok,
2105 const DirectoryLookup *LookupFrom = nullptr,
2106 const FileEntry *LookupFromFile = nullptr,
2107 bool isImport = false);
2108 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2109 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2110 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2111 void HandleMicrosoftImportDirective(Token &Tok);
2112
2113public:
2114 /// Check that the given module is available, producing a diagnostic if not.
2115 /// \return \c true if the check failed (because the module is not available).
2116 /// \c false if the module appears to be usable.
2117 static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2118 const TargetInfo &TargetInfo,
2119 DiagnosticsEngine &Diags, Module *M);
2120
2121 // Module inclusion testing.
2122 /// Find the module that owns the source or header file that
2123 /// \p Loc points to. If the location is in a file that was included
2124 /// into a module, or is outside any module, returns nullptr.
2125 Module *getModuleForLocation(SourceLocation Loc);
2126
2127 /// We want to produce a diagnostic at location IncLoc concerning a
2128 /// missing module import.
2129 ///
2130 /// \param IncLoc The location at which the missing import was detected.
2131 /// \param M The desired module.
2132 /// \param MLoc A location within the desired module at which some desired
2133 /// effect occurred (eg, where a desired entity was declared).
2134 ///
2135 /// \return A file that can be #included to import a module containing MLoc.
2136 /// Null if no such file could be determined or if a #include is not
2137 /// appropriate.
2138 const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2139 Module *M,
2140 SourceLocation MLoc);
2141
2142 bool isRecordingPreamble() const {
2143 return PreambleConditionalStack.isRecording();
2144 }
2145
2146 bool hasRecordedPreamble() const {
2147 return PreambleConditionalStack.hasRecordedPreamble();
2148 }
2149
2150 ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2151 return PreambleConditionalStack.getStack();
2152 }
2153
2154 void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2155 PreambleConditionalStack.setStack(s);
2156 }
2157
2158 void setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s,
2159 llvm::Optional<PreambleSkipInfo> SkipInfo) {
2160 PreambleConditionalStack.startReplaying();
2161 PreambleConditionalStack.setStack(s);
2162 PreambleConditionalStack.SkipInfo = SkipInfo;
2163 }
2164
2165 llvm::Optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2166 return PreambleConditionalStack.SkipInfo;
2167 }
2168
2169private:
2170 /// After processing predefined file, initialize the conditional stack from
2171 /// the preamble.
2172 void replayPreambleConditionalStack();
2173
2174 // Macro handling.
2175 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2176 void HandleUndefDirective();
2177
2178 // Conditional Inclusion.
2179 void HandleIfdefDirective(Token &Result, const Token &HashToken,
2180 bool isIfndef, bool ReadAnyTokensBeforeDirective);
2181 void HandleIfDirective(Token &IfToken, const Token &HashToken,
2182 bool ReadAnyTokensBeforeDirective);
2183 void HandleEndifDirective(Token &EndifToken);
2184 void HandleElseDirective(Token &Result, const Token &HashToken);
2185 void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2186
2187 // Pragmas.
2188 void HandlePragmaDirective(SourceLocation IntroducerLoc,
2189 PragmaIntroducerKind Introducer);
2190
2191public:
2192 void HandlePragmaOnce(Token &OnceTok);
2193 void HandlePragmaMark();
2194 void HandlePragmaPoison();
2195 void HandlePragmaSystemHeader(Token &SysHeaderTok);
2196 void HandlePragmaDependency(Token &DependencyTok);
2197 void HandlePragmaPushMacro(Token &Tok);
2198 void HandlePragmaPopMacro(Token &Tok);
2199 void HandlePragmaIncludeAlias(Token &Tok);
2200 void HandlePragmaModuleBuild(Token &Tok);
2201 void HandlePragmaHdrstop(Token &Tok);
2202 IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2203
2204 // Return true and store the first token only if any CommentHandler
2205 // has inserted some tokens and getCommentRetentionState() is false.
2206 bool HandleComment(Token &result, SourceRange Comment);
2207
2208 /// A macro is used, update information about macros that need unused
2209 /// warnings.
2210 void markMacroAsUsed(MacroInfo *MI);
2211};
2212
2213/// Abstract base class that describes a handler that will receive
2214/// source ranges for each of the comments encountered in the source file.
2215class CommentHandler {
2216public:
2217 virtual ~CommentHandler();
2218
2219 // The handler shall return true if it has pushed any tokens
2220 // to be read using e.g. EnterToken or EnterTokenStream.
2221 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2222};
2223
2224/// Registry of pragma handlers added by plugins
2225using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2226
2227} // namespace clang
2228
2229#endif // LLVM_CLANG_LEX_PREPROCESSOR_H
2230