1//===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper class to build precompiled preamble.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Frontend/PrecompiledPreamble.h"
14#include "clang/Basic/FileManager.h"
15#include "clang/Basic/LangStandard.h"
16#include "clang/Frontend/CompilerInstance.h"
17#include "clang/Frontend/CompilerInvocation.h"
18#include "clang/Frontend/FrontendActions.h"
19#include "clang/Frontend/FrontendOptions.h"
20#include "clang/Lex/HeaderSearch.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Preprocessor.h"
23#include "clang/Lex/PreprocessorOptions.h"
24#include "clang/Serialization/ASTWriter.h"
25#include "llvm/ADT/SmallString.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/Config/llvm-config.h"
29#include "llvm/Support/CrashRecoveryContext.h"
30#include "llvm/Support/FileSystem.h"
31#include "llvm/Support/Path.h"
32#include "llvm/Support/Process.h"
33#include "llvm/Support/VirtualFileSystem.h"
34#include <limits>
35#include <mutex>
36#include <utility>
37
38using namespace clang;
39
40namespace {
41
42StringRef getInMemoryPreamblePath() {
43#if defined(LLVM_ON_UNIX)
44 return "/__clang_tmp/___clang_inmemory_preamble___";
45#elif defined(_WIN32)
46 return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
47#else
48#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
49 return "/__clang_tmp/___clang_inmemory_preamble___";
50#endif
51}
52
53IntrusiveRefCntPtr<llvm::vfs::FileSystem>
54createVFSOverlayForPreamblePCH(StringRef PCHFilename,
55 std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
56 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
57 // We want only the PCH file from the real filesystem to be available,
58 // so we create an in-memory VFS with just that and overlay it on top.
59 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS(
60 new llvm::vfs::InMemoryFileSystem());
61 PCHFS->addFile(Path: PCHFilename, ModificationTime: 0, Buffer: std::move(PCHBuffer));
62 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> Overlay(
63 new llvm::vfs::OverlayFileSystem(VFS));
64 Overlay->pushOverlay(FS: PCHFS);
65 return Overlay;
66}
67
68class PreambleDependencyCollector : public DependencyCollector {
69public:
70 // We want to collect all dependencies for correctness. Avoiding the real
71 // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
72 // but there is no way to distinguish between those and the ones that can be
73 // spuriously added by '-isystem' (e.g. to suppress warnings from those
74 // headers).
75 bool needSystemDependencies() override { return true; }
76};
77
78// Collects files whose existence would invalidate the preamble.
79// Collecting *all* of these would make validating it too slow though, so we
80// just find all the candidates for 'file not found' diagnostics.
81//
82// A caveat that may be significant for generated files: we'll omit files under
83// search path entries whose roots don't exist when the preamble is built.
84// These are pruned by InitHeaderSearch and so we don't see the search path.
85// It would be nice to include them but we don't want to duplicate all the rest
86// of the InitHeaderSearch logic to reconstruct them.
87class MissingFileCollector : public PPCallbacks {
88 llvm::StringSet<> &Out;
89 const HeaderSearch &Search;
90 const SourceManager &SM;
91
92public:
93 MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
94 const SourceManager &SM)
95 : Out(Out), Search(Search), SM(SM) {}
96
97 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
98 StringRef FileName, bool IsAngled,
99 CharSourceRange FilenameRange,
100 OptionalFileEntryRef File, StringRef SearchPath,
101 StringRef RelativePath, const Module *SuggestedModule,
102 bool ModuleImported,
103 SrcMgr::CharacteristicKind FileType) override {
104 // File is std::nullopt if it wasn't found.
105 // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
106 if (File)
107 return;
108
109 // If it's a rare absolute include, we know the full path already.
110 if (llvm::sys::path::is_absolute(path: FileName)) {
111 Out.insert(key: FileName);
112 return;
113 }
114
115 // Reconstruct the filenames that would satisfy this directive...
116 llvm::SmallString<256> Buf;
117 auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
118 Buf = DE.getName();
119 llvm::sys::path::append(path&: Buf, a: FileName);
120 llvm::sys::path::remove_dots(path&: Buf, /*remove_dot_dot=*/true);
121 Out.insert(key: Buf);
122 };
123 // ...relative to the including file.
124 if (!IsAngled) {
125 if (OptionalFileEntryRef IncludingFile =
126 SM.getFileEntryRefForID(FID: SM.getFileID(SpellingLoc: IncludeTok.getLocation())))
127 if (IncludingFile->getDir())
128 NotFoundRelativeTo(IncludingFile->getDir());
129 }
130 // ...relative to the search paths.
131 for (const auto &Dir : llvm::make_range(
132 x: IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
133 y: Search.search_dir_end())) {
134 // No support for frameworks or header maps yet.
135 if (Dir.isNormalDir())
136 NotFoundRelativeTo(*Dir.getDirRef());
137 }
138 }
139};
140
141/// Keeps a track of files to be deleted in destructor.
142class TemporaryFiles {
143public:
144 // A static instance to be used by all clients.
145 static TemporaryFiles &getInstance();
146
147private:
148 // Disallow constructing the class directly.
149 TemporaryFiles() = default;
150 // Disallow copy.
151 TemporaryFiles(const TemporaryFiles &) = delete;
152
153public:
154 ~TemporaryFiles();
155
156 /// Adds \p File to a set of tracked files.
157 void addFile(StringRef File);
158
159 /// Remove \p File from disk and from the set of tracked files.
160 void removeFile(StringRef File);
161
162private:
163 std::mutex Mutex;
164 llvm::StringSet<> Files;
165};
166
167TemporaryFiles &TemporaryFiles::getInstance() {
168 static TemporaryFiles Instance;
169 return Instance;
170}
171
172TemporaryFiles::~TemporaryFiles() {
173 std::lock_guard<std::mutex> Guard(Mutex);
174 for (const auto &File : Files)
175 llvm::sys::fs::remove(path: File.getKey());
176}
177
178void TemporaryFiles::addFile(StringRef File) {
179 std::lock_guard<std::mutex> Guard(Mutex);
180 auto IsInserted = Files.insert(key: File).second;
181 (void)IsInserted;
182 assert(IsInserted && "File has already been added");
183}
184
185void TemporaryFiles::removeFile(StringRef File) {
186 std::lock_guard<std::mutex> Guard(Mutex);
187 auto WasPresent = Files.erase(Key: File);
188 (void)WasPresent;
189 assert(WasPresent && "File was not tracked");
190 llvm::sys::fs::remove(path: File);
191}
192
193// A temp file that would be deleted on destructor call. If destructor is not
194// called for any reason, the file will be deleted at static objects'
195// destruction.
196// An assertion will fire if two TempPCHFiles are created with the same name,
197// so it's not intended to be used outside preamble-handling.
198class TempPCHFile {
199public:
200 // A main method used to construct TempPCHFile.
201 static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
202 // FIXME: This is a hack so that we can override the preamble file during
203 // crash-recovery testing, which is the only case where the preamble files
204 // are not necessarily cleaned up.
205 if (const char *TmpFile = ::getenv(name: "CINDEXTEST_PREAMBLE_FILE"))
206 return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
207
208 llvm::SmallString<128> File;
209 // Using the versions of createTemporaryFile() and
210 // createUniqueFile() with a file descriptor guarantees
211 // that we would never get a race condition in a multi-threaded setting
212 // (i.e., multiple threads getting the same temporary path).
213 int FD;
214 std::error_code EC;
215 if (StoragePath.empty())
216 EC = llvm::sys::fs::createTemporaryFile(Prefix: "preamble", Suffix: "pch", ResultFD&: FD, ResultPath&: File);
217 else {
218 llvm::SmallString<128> TempPath = StoragePath;
219 // Use the same filename model as fs::createTemporaryFile().
220 llvm::sys::path::append(path&: TempPath, a: "preamble-%%%%%%.pch");
221 namespace fs = llvm::sys::fs;
222 // Use the same owner-only file permissions as fs::createTemporaryFile().
223 EC = fs::createUniqueFile(Model: TempPath, ResultFD&: FD, ResultPath&: File, Flags: fs::OF_None,
224 Mode: fs::owner_read | fs::owner_write);
225 }
226 if (EC)
227 return nullptr;
228 // We only needed to make sure the file exists, close the file right away.
229 llvm::sys::Process::SafelyCloseFileDescriptor(FD);
230 return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
231 }
232
233 TempPCHFile &operator=(const TempPCHFile &) = delete;
234 TempPCHFile(const TempPCHFile &) = delete;
235 ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(File: FilePath); };
236
237 /// A path where temporary file is stored.
238 llvm::StringRef getFilePath() const { return FilePath; };
239
240private:
241 TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
242 TemporaryFiles::getInstance().addFile(File: this->FilePath);
243 }
244
245 std::string FilePath;
246};
247
248class PrecompilePreambleAction : public ASTFrontendAction {
249public:
250 PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
251 PreambleCallbacks &Callbacks)
252 : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
253 Callbacks(Callbacks) {}
254
255 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
256 StringRef InFile) override;
257
258 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
259
260 void setEmittedPreamblePCH(ASTWriter &Writer) {
261 if (FileOS) {
262 *FileOS << Buffer->Data;
263 // Make sure it hits disk now.
264 FileOS.reset();
265 }
266
267 this->HasEmittedPreamblePCH = true;
268 Callbacks.AfterPCHEmitted(Writer);
269 }
270
271 bool BeginSourceFileAction(CompilerInstance &CI) override {
272 assert(CI.getLangOpts().CompilingPCH);
273 return ASTFrontendAction::BeginSourceFileAction(CI);
274 }
275
276 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
277 bool hasCodeCompletionSupport() const override { return false; }
278 bool hasASTFileSupport() const override { return false; }
279 TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
280
281private:
282 friend class PrecompilePreambleConsumer;
283
284 bool HasEmittedPreamblePCH = false;
285 std::shared_ptr<PCHBuffer> Buffer;
286 bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
287 std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
288 PreambleCallbacks &Callbacks;
289};
290
291class PrecompilePreambleConsumer : public PCHGenerator {
292public:
293 PrecompilePreambleConsumer(PrecompilePreambleAction &Action, Preprocessor &PP,
294 InMemoryModuleCache &ModuleCache,
295 StringRef isysroot,
296 std::shared_ptr<PCHBuffer> Buffer)
297 : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
298 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
299 /*AllowASTWithErrors=*/true),
300 Action(Action) {}
301
302 bool HandleTopLevelDecl(DeclGroupRef DG) override {
303 Action.Callbacks.HandleTopLevelDecl(DG);
304 return true;
305 }
306
307 void HandleTranslationUnit(ASTContext &Ctx) override {
308 PCHGenerator::HandleTranslationUnit(Ctx);
309 if (!hasEmittedPCH())
310 return;
311 Action.setEmittedPreamblePCH(getWriter());
312 }
313
314 bool shouldSkipFunctionBody(Decl *D) override {
315 return Action.Callbacks.shouldSkipFunctionBody(D);
316 }
317
318private:
319 PrecompilePreambleAction &Action;
320};
321
322std::unique_ptr<ASTConsumer>
323PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
324 StringRef InFile) {
325 std::string Sysroot;
326 if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
327 return nullptr;
328
329 if (WritePCHFile) {
330 std::string OutputFile; // unused
331 FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
332 if (!FileOS)
333 return nullptr;
334 }
335
336 if (!CI.getFrontendOpts().RelocatablePCH)
337 Sysroot.clear();
338
339 return std::make_unique<PrecompilePreambleConsumer>(
340 args&: *this, args&: CI.getPreprocessor(), args&: CI.getModuleCache(), args&: Sysroot, args&: Buffer);
341}
342
343template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
344 if (!Val)
345 return false;
346 Output = std::move(*Val);
347 return true;
348}
349
350} // namespace
351
352PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
353 const llvm::MemoryBufferRef &Buffer,
354 unsigned MaxLines) {
355 return Lexer::ComputePreamble(Buffer: Buffer.getBuffer(), LangOpts, MaxLines);
356}
357
358class PrecompiledPreamble::PCHStorage {
359public:
360 static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
361 assert(File);
362 std::unique_ptr<PCHStorage> S(new PCHStorage());
363 S->File = std::move(File);
364 return S;
365 }
366 static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
367 std::unique_ptr<PCHStorage> S(new PCHStorage());
368 S->Memory = std::move(Buf);
369 return S;
370 }
371
372 enum class Kind { InMemory, TempFile };
373 Kind getKind() const {
374 if (Memory)
375 return Kind::InMemory;
376 if (File)
377 return Kind::TempFile;
378 llvm_unreachable("Neither Memory nor File?");
379 }
380 llvm::StringRef filePath() const {
381 assert(getKind() == Kind::TempFile);
382 return File->getFilePath();
383 }
384 llvm::StringRef memoryContents() const {
385 assert(getKind() == Kind::InMemory);
386 return StringRef(Memory->Data.data(), Memory->Data.size());
387 }
388
389 // Shrink in-memory buffers to fit.
390 // This incurs a copy, but preambles tend to be long-lived.
391 // Only safe to call once nothing can alias the buffer.
392 void shrink() {
393 if (!Memory)
394 return;
395 Memory->Data = decltype(Memory->Data)(Memory->Data);
396 }
397
398private:
399 PCHStorage() = default;
400 PCHStorage(const PCHStorage &) = delete;
401 PCHStorage &operator=(const PCHStorage &) = delete;
402
403 std::shared_ptr<PCHBuffer> Memory;
404 std::unique_ptr<TempPCHFile> File;
405};
406
407PrecompiledPreamble::~PrecompiledPreamble() = default;
408PrecompiledPreamble::PrecompiledPreamble(PrecompiledPreamble &&) = default;
409PrecompiledPreamble &
410PrecompiledPreamble::operator=(PrecompiledPreamble &&) = default;
411
412llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
413 const CompilerInvocation &Invocation,
414 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
415 DiagnosticsEngine &Diagnostics,
416 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
417 std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
418 StringRef StoragePath, PreambleCallbacks &Callbacks) {
419 assert(VFS && "VFS is null");
420
421 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
422 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
423 PreprocessorOptions &PreprocessorOpts =
424 PreambleInvocation->getPreprocessorOpts();
425
426 std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
427 std::unique_ptr<PCHStorage> Storage;
428 if (StoreInMemory) {
429 Storage = PCHStorage::inMemory(Buf: Buffer);
430 } else {
431 // Create a temporary file for the precompiled preamble. In rare
432 // circumstances, this can fail.
433 std::unique_ptr<TempPCHFile> PreamblePCHFile =
434 TempPCHFile::create(StoragePath);
435 if (!PreamblePCHFile)
436 return BuildPreambleError::CouldntCreateTempFile;
437 Storage = PCHStorage::file(File: std::move(PreamblePCHFile));
438 }
439
440 // Save the preamble text for later; we'll need to compare against it for
441 // subsequent reparses.
442 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
443 MainFileBuffer->getBufferStart() +
444 Bounds.Size);
445 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
446
447 // Tell the compiler invocation to generate a temporary precompiled header.
448 FrontendOpts.ProgramAction = frontend::GeneratePCH;
449 FrontendOpts.OutputFile = std::string(
450 StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
451 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
452 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
453 // Inform preprocessor to record conditional stack when building the preamble.
454 PreprocessorOpts.GeneratePreamble = true;
455
456 // Create the compiler instance to use for building the precompiled preamble.
457 std::unique_ptr<CompilerInstance> Clang(
458 new CompilerInstance(std::move(PCHContainerOps)));
459
460 // Recover resources if we crash before exiting this method.
461 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
462 Clang.get());
463
464 Clang->setInvocation(std::move(PreambleInvocation));
465 Clang->setDiagnostics(&Diagnostics);
466
467 // Create the target instance.
468 if (!Clang->createTarget())
469 return BuildPreambleError::CouldntCreateTargetInfo;
470
471 if (Clang->getFrontendOpts().Inputs.size() != 1 ||
472 Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
473 InputKind::Source ||
474 Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
475 Language::LLVM_IR) {
476 return BuildPreambleError::BadInputs;
477 }
478
479 // Clear out old caches and data.
480 Diagnostics.Reset();
481 ProcessWarningOptions(Diags&: Diagnostics, Opts: Clang->getDiagnosticOpts());
482
483 VFS =
484 createVFSFromCompilerInvocation(CI: Clang->getInvocation(), Diags&: Diagnostics, BaseFS: VFS);
485
486 // Create a file manager object to provide access to and cache the filesystem.
487 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
488
489 // Create the source manager.
490 Clang->setSourceManager(
491 new SourceManager(Diagnostics, Clang->getFileManager()));
492
493 auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
494 Clang->addDependencyCollector(Listener: PreambleDepCollector);
495
496 Clang->getLangOpts().CompilingPCH = true;
497
498 // Remap the main source file to the preamble buffer.
499 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
500 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
501 InputData: MainFileBuffer->getBuffer().slice(Start: 0, End: Bounds.Size), BufferName: MainFilePath);
502 if (PreprocessorOpts.RetainRemappedFileBuffers) {
503 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
504 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: PreambleInputBuffer.get());
505 } else {
506 // In that case, remapped buffer will be deleted by CompilerInstance on
507 // BeginSourceFile, so we call release() to avoid double deletion.
508 PreprocessorOpts.addRemappedFile(From: MainFilePath,
509 To: PreambleInputBuffer.release());
510 }
511
512 auto Act = std::make_unique<PrecompilePreambleAction>(
513 args: std::move(Buffer),
514 /*WritePCHFile=*/args: Storage->getKind() == PCHStorage::Kind::TempFile,
515 args&: Callbacks);
516 if (!Act->BeginSourceFile(CI&: *Clang.get(), Input: Clang->getFrontendOpts().Inputs[0]))
517 return BuildPreambleError::BeginSourceFileFailed;
518
519 // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
520 // referenced in the callback.
521 Callbacks.BeforeExecute(CI&: *Clang);
522
523 std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
524 Callbacks.createPPCallbacks();
525 if (DelegatedPPCallbacks)
526 Clang->getPreprocessor().addPPCallbacks(C: std::move(DelegatedPPCallbacks));
527 if (auto CommentHandler = Callbacks.getCommentHandler())
528 Clang->getPreprocessor().addCommentHandler(Handler: CommentHandler);
529 llvm::StringSet<> MissingFiles;
530 Clang->getPreprocessor().addPPCallbacks(
531 C: std::make_unique<MissingFileCollector>(
532 args&: MissingFiles, args&: Clang->getPreprocessor().getHeaderSearchInfo(),
533 args&: Clang->getSourceManager()));
534
535 if (llvm::Error Err = Act->Execute())
536 return errorToErrorCode(Err: std::move(Err));
537
538 // Run the callbacks.
539 Callbacks.AfterExecute(CI&: *Clang);
540
541 Act->EndSourceFile();
542
543 if (!Act->hasEmittedPreamblePCH())
544 return BuildPreambleError::CouldntEmitPCH;
545 Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
546
547 // Keep track of all of the files that the source manager knows about,
548 // so we can verify whether they have changed or not.
549 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
550
551 SourceManager &SourceMgr = Clang->getSourceManager();
552 for (auto &Filename : PreambleDepCollector->getDependencies()) {
553 auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
554 if (!MaybeFile ||
555 MaybeFile == SourceMgr.getFileEntryRefForID(FID: SourceMgr.getMainFileID()))
556 continue;
557 auto File = *MaybeFile;
558 if (time_t ModTime = File.getModificationTime()) {
559 FilesInPreamble[File.getName()] =
560 PrecompiledPreamble::PreambleFileHash::createForFile(Size: File.getSize(),
561 ModTime);
562 } else {
563 llvm::MemoryBufferRef Buffer =
564 SourceMgr.getMemoryBufferForFileOrFake(File);
565 FilesInPreamble[File.getName()] =
566 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
567 }
568 }
569
570 // Shrinking the storage requires extra temporary memory.
571 // Destroying clang first reduces peak memory usage.
572 CICleanup.unregister();
573 Clang.reset();
574 Storage->shrink();
575 return PrecompiledPreamble(
576 std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
577 std::move(FilesInPreamble), std::move(MissingFiles));
578}
579
580PreambleBounds PrecompiledPreamble::getBounds() const {
581 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
582}
583
584std::size_t PrecompiledPreamble::getSize() const {
585 switch (Storage->getKind()) {
586 case PCHStorage::Kind::InMemory:
587 return Storage->memoryContents().size();
588 case PCHStorage::Kind::TempFile: {
589 uint64_t Result;
590 if (llvm::sys::fs::file_size(Path: Storage->filePath(), Result))
591 return 0;
592
593 assert(Result <= std::numeric_limits<std::size_t>::max() &&
594 "file size did not fit into size_t");
595 return Result;
596 }
597 }
598 llvm_unreachable("Unhandled storage kind");
599}
600
601bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
602 const llvm::MemoryBufferRef &MainFileBuffer,
603 PreambleBounds Bounds,
604 llvm::vfs::FileSystem &VFS) const {
605
606 assert(
607 Bounds.Size <= MainFileBuffer.getBufferSize() &&
608 "Buffer is too large. Bounds were calculated from a different buffer?");
609
610 auto PreambleInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
611 PreprocessorOptions &PreprocessorOpts =
612 PreambleInvocation->getPreprocessorOpts();
613
614 // We've previously computed a preamble. Check whether we have the same
615 // preamble now that we did before, and that there's enough space in
616 // the main-file buffer within the precompiled preamble to fit the
617 // new main file.
618 if (PreambleBytes.size() != Bounds.Size ||
619 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
620 !std::equal(first1: PreambleBytes.begin(), last1: PreambleBytes.end(),
621 first2: MainFileBuffer.getBuffer().begin()))
622 return false;
623 // The preamble has not changed. We may be able to re-use the precompiled
624 // preamble.
625
626 // Check that none of the files used by the preamble have changed.
627 // First, make a record of those files that have been overridden via
628 // remapping or unsaved_files.
629 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
630 llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
631 for (const auto &R : PreprocessorOpts.RemappedFiles) {
632 llvm::vfs::Status Status;
633 if (!moveOnNoError(Val: VFS.status(Path: R.second), Output&: Status)) {
634 // If we can't stat the file we're remapping to, assume that something
635 // horrible happened.
636 return false;
637 }
638 // If a mapped file was previously missing, then it has changed.
639 llvm::SmallString<128> MappedPath(R.first);
640 if (!VFS.makeAbsolute(Path&: MappedPath))
641 OverriddenAbsPaths.insert(key: MappedPath);
642
643 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
644 Size: Status.getSize(), ModTime: llvm::sys::toTimeT(TP: Status.getLastModificationTime()));
645 }
646
647 // OverridenFileBuffers tracks only the files not found in VFS.
648 llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
649 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
650 const PrecompiledPreamble::PreambleFileHash PreambleHash =
651 PreambleFileHash::createForMemoryBuffer(Buffer: RB.second->getMemBufferRef());
652 llvm::vfs::Status Status;
653 if (moveOnNoError(Val: VFS.status(Path: RB.first), Output&: Status))
654 OverriddenFiles[Status.getUniqueID()] = PreambleHash;
655 else
656 OverridenFileBuffers[RB.first] = PreambleHash;
657
658 llvm::SmallString<128> MappedPath(RB.first);
659 if (!VFS.makeAbsolute(Path&: MappedPath))
660 OverriddenAbsPaths.insert(key: MappedPath);
661 }
662
663 // Check whether anything has changed.
664 for (const auto &F : FilesInPreamble) {
665 auto OverridenFileBuffer = OverridenFileBuffers.find(Key: F.first());
666 if (OverridenFileBuffer != OverridenFileBuffers.end()) {
667 // The file's buffer was remapped and the file was not found in VFS.
668 // Check whether it matches up with the previous mapping.
669 if (OverridenFileBuffer->second != F.second)
670 return false;
671 continue;
672 }
673
674 llvm::vfs::Status Status;
675 if (!moveOnNoError(Val: VFS.status(Path: F.first()), Output&: Status)) {
676 // If the file's buffer is not remapped and we can't stat it,
677 // assume that something horrible happened.
678 return false;
679 }
680
681 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
682 OverriddenFiles.find(x: Status.getUniqueID());
683 if (Overridden != OverriddenFiles.end()) {
684 // This file was remapped; check whether the newly-mapped file
685 // matches up with the previous mapping.
686 if (Overridden->second != F.second)
687 return false;
688 continue;
689 }
690
691 // Neither the file's buffer nor the file itself was remapped;
692 // check whether it has changed on disk.
693 if (Status.getSize() != uint64_t(F.second.Size) ||
694 llvm::sys::toTimeT(TP: Status.getLastModificationTime()) !=
695 F.second.ModTime)
696 return false;
697 }
698 for (const auto &F : MissingFiles) {
699 // A missing file may be "provided" by an override buffer or file.
700 if (OverriddenAbsPaths.count(Key: F.getKey()))
701 return false;
702 // If a file previously recorded as missing exists as a regular file, then
703 // consider the preamble out-of-date.
704 if (auto Status = VFS.status(Path: F.getKey())) {
705 if (Status->isRegularFile())
706 return false;
707 }
708 }
709 return true;
710}
711
712void PrecompiledPreamble::AddImplicitPreamble(
713 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
714 llvm::MemoryBuffer *MainFileBuffer) const {
715 PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
716 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
717}
718
719void PrecompiledPreamble::OverridePreamble(
720 CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
721 llvm::MemoryBuffer *MainFileBuffer) const {
722 auto Bounds = ComputePreambleBounds(LangOpts: CI.getLangOpts(), Buffer: *MainFileBuffer, MaxLines: 0);
723 configurePreamble(Bounds, CI, VFS, MainFileBuffer);
724}
725
726PrecompiledPreamble::PrecompiledPreamble(
727 std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
728 bool PreambleEndsAtStartOfLine,
729 llvm::StringMap<PreambleFileHash> FilesInPreamble,
730 llvm::StringSet<> MissingFiles)
731 : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
732 MissingFiles(std::move(MissingFiles)),
733 PreambleBytes(std::move(PreambleBytes)),
734 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
735 assert(this->Storage != nullptr);
736}
737
738PrecompiledPreamble::PreambleFileHash
739PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
740 time_t ModTime) {
741 PreambleFileHash Result;
742 Result.Size = Size;
743 Result.ModTime = ModTime;
744 Result.MD5 = {};
745 return Result;
746}
747
748PrecompiledPreamble::PreambleFileHash
749PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
750 const llvm::MemoryBufferRef &Buffer) {
751 PreambleFileHash Result;
752 Result.Size = Buffer.getBufferSize();
753 Result.ModTime = 0;
754
755 llvm::MD5 MD5Ctx;
756 MD5Ctx.update(Str: Buffer.getBuffer().data());
757 MD5Ctx.final(Result&: Result.MD5);
758
759 return Result;
760}
761
762void PrecompiledPreamble::configurePreamble(
763 PreambleBounds Bounds, CompilerInvocation &CI,
764 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
765 llvm::MemoryBuffer *MainFileBuffer) const {
766 assert(VFS);
767
768 auto &PreprocessorOpts = CI.getPreprocessorOpts();
769
770 // Remap main file to point to MainFileBuffer.
771 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
772 PreprocessorOpts.addRemappedFile(From: MainFilePath, To: MainFileBuffer);
773
774 // Configure ImpicitPCHInclude.
775 PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
776 PreprocessorOpts.PrecompiledPreambleBytes.second =
777 Bounds.PreambleEndsAtStartOfLine;
778 PreprocessorOpts.DisablePCHOrModuleValidation =
779 DisableValidationForModuleKind::PCH;
780
781 // Don't bother generating the long version of the predefines buffer.
782 // The preamble is going to overwrite it anyway.
783 PreprocessorOpts.UsePredefines = false;
784
785 setupPreambleStorage(Storage: *Storage, PreprocessorOpts, VFS);
786}
787
788void PrecompiledPreamble::setupPreambleStorage(
789 const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
790 IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS) {
791 if (Storage.getKind() == PCHStorage::Kind::TempFile) {
792 llvm::StringRef PCHPath = Storage.filePath();
793 PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
794
795 // Make sure we can access the PCH file even if we're using a VFS
796 IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS =
797 llvm::vfs::getRealFileSystem();
798 if (VFS == RealFS || VFS->exists(Path: PCHPath))
799 return;
800 auto Buf = RealFS->getBufferForFile(Name: PCHPath);
801 if (!Buf) {
802 // We can't read the file even from RealFS, this is clearly an error,
803 // but we'll just leave the current VFS as is and let clang's code
804 // figure out what to do with missing PCH.
805 return;
806 }
807
808 // We have a slight inconsistency here -- we're using the VFS to
809 // read files, but the PCH was generated in the real file system.
810 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(*Buf), VFS);
811 } else {
812 assert(Storage.getKind() == PCHStorage::Kind::InMemory);
813 // For in-memory preamble, we have to provide a VFS overlay that makes it
814 // accessible.
815 StringRef PCHPath = getInMemoryPreamblePath();
816 PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
817
818 auto Buf = llvm::MemoryBuffer::getMemBuffer(
819 InputData: Storage.memoryContents(), BufferName: PCHPath, /*RequiresNullTerminator=*/false);
820 VFS = createVFSOverlayForPreamblePCH(PCHFilename: PCHPath, PCHBuffer: std::move(Buf), VFS);
821 }
822}
823
824void PreambleCallbacks::BeforeExecute(CompilerInstance &CI) {}
825void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
826void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
827void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
828std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
829 return nullptr;
830}
831CommentHandler *PreambleCallbacks::getCommentHandler() { return nullptr; }
832
833static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
834
835std::error_code clang::make_error_code(BuildPreambleError Error) {
836 return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
837}
838
839const char *BuildPreambleErrorCategory::name() const noexcept {
840 return "build-preamble.error";
841}
842
843std::string BuildPreambleErrorCategory::message(int condition) const {
844 switch (static_cast<BuildPreambleError>(condition)) {
845 case BuildPreambleError::CouldntCreateTempFile:
846 return "Could not create temporary file for PCH";
847 case BuildPreambleError::CouldntCreateTargetInfo:
848 return "CreateTargetInfo() return null";
849 case BuildPreambleError::BeginSourceFileFailed:
850 return "BeginSourceFile() return an error";
851 case BuildPreambleError::CouldntEmitPCH:
852 return "Could not emit PCH";
853 case BuildPreambleError::BadInputs:
854 return "Command line arguments must contain exactly one source file";
855 }
856 llvm_unreachable("unexpected BuildPreambleError");
857}
858

source code of clang/lib/Frontend/PrecompiledPreamble.cpp