1//===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines the virtual file system interface vfs::FileSystem.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_VIRTUALFILESYSTEM_H
15#define LLVM_SUPPORT_VIRTUALFILESYSTEM_H
16
17#include "llvm/ADT/IntrusiveRefCntPtr.h"
18#include "llvm/ADT/STLFunctionalExtras.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Support/Chrono.h"
22#include "llvm/Support/Errc.h"
23#include "llvm/Support/Error.h"
24#include "llvm/Support/ErrorOr.h"
25#include "llvm/Support/ExtensibleRTTI.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/Path.h"
28#include "llvm/Support/SourceMgr.h"
29#include <cassert>
30#include <cstdint>
31#include <ctime>
32#include <memory>
33#include <optional>
34#include <stack>
35#include <string>
36#include <system_error>
37#include <utility>
38#include <vector>
39
40namespace llvm {
41
42class MemoryBuffer;
43class MemoryBufferRef;
44class Twine;
45
46namespace vfs {
47
48/// The result of a \p status operation.
49class Status {
50 std::string Name;
51 llvm::sys::fs::UniqueID UID;
52 llvm::sys::TimePoint<> MTime;
53 uint32_t User;
54 uint32_t Group;
55 uint64_t Size;
56 llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::status_error;
57 llvm::sys::fs::perms Perms;
58
59public:
60 /// Whether this entity has an external path different from the virtual path,
61 /// and the external path is exposed by leaking it through the abstraction.
62 /// For example, a RedirectingFileSystem will set this for paths where
63 /// UseExternalName is true.
64 ///
65 /// FIXME: Currently the external path is exposed by replacing the virtual
66 /// path in this Status object. Instead, we should leave the path in the
67 /// Status intact (matching the requested virtual path) - see
68 /// FileManager::getFileRef for how we plan to fix this.
69 bool ExposesExternalVFSPath = false;
70
71 Status() = default;
72 Status(const llvm::sys::fs::file_status &Status);
73 Status(const Twine &Name, llvm::sys::fs::UniqueID UID,
74 llvm::sys::TimePoint<> MTime, uint32_t User, uint32_t Group,
75 uint64_t Size, llvm::sys::fs::file_type Type,
76 llvm::sys::fs::perms Perms);
77
78 /// Get a copy of a Status with a different size.
79 static Status copyWithNewSize(const Status &In, uint64_t NewSize);
80 /// Get a copy of a Status with a different name.
81 static Status copyWithNewName(const Status &In, const Twine &NewName);
82 static Status copyWithNewName(const llvm::sys::fs::file_status &In,
83 const Twine &NewName);
84
85 /// Returns the name that should be used for this file or directory.
86 StringRef getName() const { return Name; }
87
88 /// @name Status interface from llvm::sys::fs
89 /// @{
90 llvm::sys::fs::file_type getType() const { return Type; }
91 llvm::sys::fs::perms getPermissions() const { return Perms; }
92 llvm::sys::TimePoint<> getLastModificationTime() const { return MTime; }
93 llvm::sys::fs::UniqueID getUniqueID() const { return UID; }
94 uint32_t getUser() const { return User; }
95 uint32_t getGroup() const { return Group; }
96 uint64_t getSize() const { return Size; }
97 /// @}
98 /// @name Status queries
99 /// These are static queries in llvm::sys::fs.
100 /// @{
101 bool equivalent(const Status &Other) const;
102 bool isDirectory() const;
103 bool isRegularFile() const;
104 bool isOther() const;
105 bool isSymlink() const;
106 bool isStatusKnown() const;
107 bool exists() const;
108 /// @}
109};
110
111/// Represents an open file.
112class File {
113public:
114 /// Destroy the file after closing it (if open).
115 /// Sub-classes should generally call close() inside their destructors. We
116 /// cannot do that from the base class, since close is virtual.
117 virtual ~File();
118
119 /// Get the status of the file.
120 virtual llvm::ErrorOr<Status> status() = 0;
121
122 /// Get the name of the file
123 virtual llvm::ErrorOr<std::string> getName() {
124 if (auto Status = status())
125 return Status->getName().str();
126 else
127 return Status.getError();
128 }
129
130 /// Get the contents of the file as a \p MemoryBuffer.
131 virtual llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
132 getBuffer(const Twine &Name, int64_t FileSize = -1,
133 bool RequiresNullTerminator = true, bool IsVolatile = false) = 0;
134
135 /// Closes the file.
136 virtual std::error_code close() = 0;
137
138 // Get the same file with a different path.
139 static ErrorOr<std::unique_ptr<File>>
140 getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P);
141
142protected:
143 // Set the file's underlying path.
144 virtual void setPath(const Twine &Path) {}
145};
146
147/// A member of a directory, yielded by a directory_iterator.
148/// Only information available on most platforms is included.
149class directory_entry {
150 std::string Path;
151 llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::type_unknown;
152
153public:
154 directory_entry() = default;
155 directory_entry(std::string Path, llvm::sys::fs::file_type Type)
156 : Path(std::move(Path)), Type(Type) {}
157
158 llvm::StringRef path() const { return Path; }
159 llvm::sys::fs::file_type type() const { return Type; }
160};
161
162namespace detail {
163
164/// An interface for virtual file systems to provide an iterator over the
165/// (non-recursive) contents of a directory.
166struct DirIterImpl {
167 virtual ~DirIterImpl();
168
169 /// Sets \c CurrentEntry to the next entry in the directory on success,
170 /// to directory_entry() at end, or returns a system-defined \c error_code.
171 virtual std::error_code increment() = 0;
172
173 directory_entry CurrentEntry;
174};
175
176} // namespace detail
177
178/// An input iterator over the entries in a virtual path, similar to
179/// llvm::sys::fs::directory_iterator.
180class directory_iterator {
181 std::shared_ptr<detail::DirIterImpl> Impl; // Input iterator semantics on copy
182
183public:
184 directory_iterator(std::shared_ptr<detail::DirIterImpl> I)
185 : Impl(std::move(I)) {
186 assert(Impl.get() != nullptr && "requires non-null implementation");
187 if (Impl->CurrentEntry.path().empty())
188 Impl.reset(); // Normalize the end iterator to Impl == nullptr.
189 }
190
191 /// Construct an 'end' iterator.
192 directory_iterator() = default;
193
194 /// Equivalent to operator++, with an error code.
195 directory_iterator &increment(std::error_code &EC) {
196 assert(Impl && "attempting to increment past end");
197 EC = Impl->increment();
198 if (Impl->CurrentEntry.path().empty())
199 Impl.reset(); // Normalize the end iterator to Impl == nullptr.
200 return *this;
201 }
202
203 const directory_entry &operator*() const { return Impl->CurrentEntry; }
204 const directory_entry *operator->() const { return &Impl->CurrentEntry; }
205
206 bool operator==(const directory_iterator &RHS) const {
207 if (Impl && RHS.Impl)
208 return Impl->CurrentEntry.path() == RHS.Impl->CurrentEntry.path();
209 return !Impl && !RHS.Impl;
210 }
211 bool operator!=(const directory_iterator &RHS) const {
212 return !(*this == RHS);
213 }
214};
215
216class FileSystem;
217
218namespace detail {
219
220/// Keeps state for the recursive_directory_iterator.
221struct RecDirIterState {
222 std::stack<directory_iterator, std::vector<directory_iterator>> Stack;
223 bool HasNoPushRequest = false;
224};
225
226} // end namespace detail
227
228/// An input iterator over the recursive contents of a virtual path,
229/// similar to llvm::sys::fs::recursive_directory_iterator.
230class recursive_directory_iterator {
231 FileSystem *FS;
232 std::shared_ptr<detail::RecDirIterState>
233 State; // Input iterator semantics on copy.
234
235public:
236 recursive_directory_iterator(FileSystem &FS, const Twine &Path,
237 std::error_code &EC);
238
239 /// Construct an 'end' iterator.
240 recursive_directory_iterator() = default;
241
242 /// Equivalent to operator++, with an error code.
243 recursive_directory_iterator &increment(std::error_code &EC);
244
245 const directory_entry &operator*() const { return *State->Stack.top(); }
246 const directory_entry *operator->() const { return &*State->Stack.top(); }
247
248 bool operator==(const recursive_directory_iterator &Other) const {
249 return State == Other.State; // identity
250 }
251 bool operator!=(const recursive_directory_iterator &RHS) const {
252 return !(*this == RHS);
253 }
254
255 /// Gets the current level. Starting path is at level 0.
256 int level() const {
257 assert(!State->Stack.empty() &&
258 "Cannot get level without any iteration state");
259 return State->Stack.size() - 1;
260 }
261
262 void no_push() { State->HasNoPushRequest = true; }
263};
264
265/// The virtual file system interface.
266class FileSystem : public llvm::ThreadSafeRefCountedBase<FileSystem>,
267 public RTTIExtends<FileSystem, RTTIRoot> {
268public:
269 static const char ID;
270 virtual ~FileSystem();
271
272 /// Get the status of the entry at \p Path, if one exists.
273 virtual llvm::ErrorOr<Status> status(const Twine &Path) = 0;
274
275 /// Get a \p File object for the file at \p Path, if one exists.
276 virtual llvm::ErrorOr<std::unique_ptr<File>>
277 openFileForRead(const Twine &Path) = 0;
278
279 /// This is a convenience method that opens a file, gets its content and then
280 /// closes the file.
281 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
282 getBufferForFile(const Twine &Name, int64_t FileSize = -1,
283 bool RequiresNullTerminator = true, bool IsVolatile = false);
284
285 /// Get a directory_iterator for \p Dir.
286 /// \note The 'end' iterator is directory_iterator().
287 virtual directory_iterator dir_begin(const Twine &Dir,
288 std::error_code &EC) = 0;
289
290 /// Set the working directory. This will affect all following operations on
291 /// this file system and may propagate down for nested file systems.
292 virtual std::error_code setCurrentWorkingDirectory(const Twine &Path) = 0;
293
294 /// Get the working directory of this file system.
295 virtual llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const = 0;
296
297 /// Gets real path of \p Path e.g. collapse all . and .. patterns, resolve
298 /// symlinks. For real file system, this uses `llvm::sys::fs::real_path`.
299 /// This returns errc::operation_not_permitted if not implemented by subclass.
300 virtual std::error_code getRealPath(const Twine &Path,
301 SmallVectorImpl<char> &Output);
302
303 /// Check whether \p Path exists. By default this uses \c status(), but
304 /// filesystems may provide a more efficient implementation if available.
305 virtual bool exists(const Twine &Path);
306
307 /// Is the file mounted on a local filesystem?
308 virtual std::error_code isLocal(const Twine &Path, bool &Result);
309
310 /// Make \a Path an absolute path.
311 ///
312 /// Makes \a Path absolute using the current directory if it is not already.
313 /// An empty \a Path will result in the current directory.
314 ///
315 /// /absolute/path => /absolute/path
316 /// relative/../path => <current-directory>/relative/../path
317 ///
318 /// \param Path A path that is modified to be an absolute path.
319 /// \returns success if \a path has been made absolute, otherwise a
320 /// platform-specific error_code.
321 virtual std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const;
322
323 enum class PrintType { Summary, Contents, RecursiveContents };
324 void print(raw_ostream &OS, PrintType Type = PrintType::Contents,
325 unsigned IndentLevel = 0) const {
326 printImpl(OS, Type, IndentLevel);
327 }
328
329 using VisitCallbackTy = llvm::function_ref<void(FileSystem &)>;
330 virtual void visitChildFileSystems(VisitCallbackTy Callback) {}
331 void visit(VisitCallbackTy Callback) {
332 Callback(*this);
333 visitChildFileSystems(Callback);
334 }
335
336#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
337 LLVM_DUMP_METHOD void dump() const;
338#endif
339
340protected:
341 virtual void printImpl(raw_ostream &OS, PrintType Type,
342 unsigned IndentLevel) const {
343 printIndent(OS, IndentLevel);
344 OS << "FileSystem\n";
345 }
346
347 void printIndent(raw_ostream &OS, unsigned IndentLevel) const {
348 for (unsigned i = 0; i < IndentLevel; ++i)
349 OS << " ";
350 }
351};
352
353/// Gets an \p vfs::FileSystem for the 'real' file system, as seen by
354/// the operating system.
355/// The working directory is linked to the process's working directory.
356/// (This is usually thread-hostile).
357IntrusiveRefCntPtr<FileSystem> getRealFileSystem();
358
359/// Create an \p vfs::FileSystem for the 'real' file system, as seen by
360/// the operating system.
361/// It has its own working directory, independent of (but initially equal to)
362/// that of the process.
363std::unique_ptr<FileSystem> createPhysicalFileSystem();
364
365/// A file system that allows overlaying one \p AbstractFileSystem on top
366/// of another.
367///
368/// Consists of a stack of >=1 \p FileSystem objects, which are treated as being
369/// one merged file system. When there is a directory that exists in more than
370/// one file system, the \p OverlayFileSystem contains a directory containing
371/// the union of their contents. The attributes (permissions, etc.) of the
372/// top-most (most recently added) directory are used. When there is a file
373/// that exists in more than one file system, the file in the top-most file
374/// system overrides the other(s).
375class OverlayFileSystem : public RTTIExtends<OverlayFileSystem, FileSystem> {
376 using FileSystemList = SmallVector<IntrusiveRefCntPtr<FileSystem>, 1>;
377
378 /// The stack of file systems, implemented as a list in order of
379 /// their addition.
380 FileSystemList FSList;
381
382public:
383 static const char ID;
384 OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> Base);
385
386 /// Pushes a file system on top of the stack.
387 void pushOverlay(IntrusiveRefCntPtr<FileSystem> FS);
388
389 llvm::ErrorOr<Status> status(const Twine &Path) override;
390 bool exists(const Twine &Path) override;
391 llvm::ErrorOr<std::unique_ptr<File>>
392 openFileForRead(const Twine &Path) override;
393 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
394 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
395 std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
396 std::error_code isLocal(const Twine &Path, bool &Result) override;
397 std::error_code getRealPath(const Twine &Path,
398 SmallVectorImpl<char> &Output) override;
399
400 using iterator = FileSystemList::reverse_iterator;
401 using const_iterator = FileSystemList::const_reverse_iterator;
402 using reverse_iterator = FileSystemList::iterator;
403 using const_reverse_iterator = FileSystemList::const_iterator;
404 using range = iterator_range<iterator>;
405 using const_range = iterator_range<const_iterator>;
406
407 /// Get an iterator pointing to the most recently added file system.
408 iterator overlays_begin() { return FSList.rbegin(); }
409 const_iterator overlays_begin() const { return FSList.rbegin(); }
410
411 /// Get an iterator pointing one-past the least recently added file system.
412 iterator overlays_end() { return FSList.rend(); }
413 const_iterator overlays_end() const { return FSList.rend(); }
414
415 /// Get an iterator pointing to the least recently added file system.
416 reverse_iterator overlays_rbegin() { return FSList.begin(); }
417 const_reverse_iterator overlays_rbegin() const { return FSList.begin(); }
418
419 /// Get an iterator pointing one-past the most recently added file system.
420 reverse_iterator overlays_rend() { return FSList.end(); }
421 const_reverse_iterator overlays_rend() const { return FSList.end(); }
422
423 range overlays_range() { return llvm::reverse(C&: FSList); }
424 const_range overlays_range() const { return llvm::reverse(C: FSList); }
425
426protected:
427 void printImpl(raw_ostream &OS, PrintType Type,
428 unsigned IndentLevel) const override;
429 void visitChildFileSystems(VisitCallbackTy Callback) override;
430};
431
432/// By default, this delegates all calls to the underlying file system. This
433/// is useful when derived file systems want to override some calls and still
434/// proxy other calls.
435class ProxyFileSystem : public RTTIExtends<ProxyFileSystem, FileSystem> {
436public:
437 static const char ID;
438 explicit ProxyFileSystem(IntrusiveRefCntPtr<FileSystem> FS)
439 : FS(std::move(FS)) {}
440
441 llvm::ErrorOr<Status> status(const Twine &Path) override {
442 return FS->status(Path);
443 }
444 bool exists(const Twine &Path) override { return FS->exists(Path); }
445 llvm::ErrorOr<std::unique_ptr<File>>
446 openFileForRead(const Twine &Path) override {
447 return FS->openFileForRead(Path);
448 }
449 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {
450 return FS->dir_begin(Dir, EC);
451 }
452 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
453 return FS->getCurrentWorkingDirectory();
454 }
455 std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
456 return FS->setCurrentWorkingDirectory(Path);
457 }
458 std::error_code getRealPath(const Twine &Path,
459 SmallVectorImpl<char> &Output) override {
460 return FS->getRealPath(Path, Output);
461 }
462 std::error_code isLocal(const Twine &Path, bool &Result) override {
463 return FS->isLocal(Path, Result);
464 }
465
466protected:
467 FileSystem &getUnderlyingFS() const { return *FS; }
468 void visitChildFileSystems(VisitCallbackTy Callback) override {
469 if (FS) {
470 Callback(*FS);
471 FS->visitChildFileSystems(Callback);
472 }
473 }
474
475private:
476 IntrusiveRefCntPtr<FileSystem> FS;
477
478 virtual void anchor() override;
479};
480
481namespace detail {
482
483class InMemoryDirectory;
484class InMemoryNode;
485
486struct NewInMemoryNodeInfo {
487 llvm::sys::fs::UniqueID DirUID;
488 StringRef Path;
489 StringRef Name;
490 time_t ModificationTime;
491 std::unique_ptr<llvm::MemoryBuffer> Buffer;
492 uint32_t User;
493 uint32_t Group;
494 llvm::sys::fs::file_type Type;
495 llvm::sys::fs::perms Perms;
496
497 Status makeStatus() const;
498};
499
500class NamedNodeOrError {
501 ErrorOr<std::pair<llvm::SmallString<128>, const detail::InMemoryNode *>>
502 Value;
503
504public:
505 NamedNodeOrError(llvm::SmallString<128> Name,
506 const detail::InMemoryNode *Node)
507 : Value(std::make_pair(x&: Name, y&: Node)) {}
508 NamedNodeOrError(std::error_code EC) : Value(EC) {}
509 NamedNodeOrError(llvm::errc EC) : Value(EC) {}
510
511 StringRef getName() const { return (*Value).first; }
512 explicit operator bool() const { return static_cast<bool>(Value); }
513 operator std::error_code() const { return Value.getError(); }
514 std::error_code getError() const { return Value.getError(); }
515 const detail::InMemoryNode *operator*() const { return (*Value).second; }
516};
517
518} // namespace detail
519
520/// An in-memory file system.
521class InMemoryFileSystem : public RTTIExtends<InMemoryFileSystem, FileSystem> {
522 std::unique_ptr<detail::InMemoryDirectory> Root;
523 std::string WorkingDirectory;
524 bool UseNormalizedPaths = true;
525
526public:
527 static const char ID;
528
529private:
530 using MakeNodeFn = llvm::function_ref<std::unique_ptr<detail::InMemoryNode>(
531 detail::NewInMemoryNodeInfo)>;
532
533 /// Create node with \p MakeNode and add it into this filesystem at \p Path.
534 bool addFile(const Twine &Path, time_t ModificationTime,
535 std::unique_ptr<llvm::MemoryBuffer> Buffer,
536 std::optional<uint32_t> User, std::optional<uint32_t> Group,
537 std::optional<llvm::sys::fs::file_type> Type,
538 std::optional<llvm::sys::fs::perms> Perms, MakeNodeFn MakeNode);
539
540 /// Looks up the in-memory node for the path \p P.
541 /// If \p FollowFinalSymlink is true, the returned node is guaranteed to
542 /// not be a symlink and its path may differ from \p P.
543 detail::NamedNodeOrError lookupNode(const Twine &P, bool FollowFinalSymlink,
544 size_t SymlinkDepth = 0) const;
545
546 class DirIterator;
547
548public:
549 explicit InMemoryFileSystem(bool UseNormalizedPaths = true);
550 ~InMemoryFileSystem() override;
551
552 /// Add a file containing a buffer or a directory to the VFS with a
553 /// path. The VFS owns the buffer. If present, User, Group, Type
554 /// and Perms apply to the newly-created file or directory.
555 /// \return true if the file or directory was successfully added,
556 /// false if the file or directory already exists in the file system with
557 /// different contents.
558 bool addFile(const Twine &Path, time_t ModificationTime,
559 std::unique_ptr<llvm::MemoryBuffer> Buffer,
560 std::optional<uint32_t> User = std::nullopt,
561 std::optional<uint32_t> Group = std::nullopt,
562 std::optional<llvm::sys::fs::file_type> Type = std::nullopt,
563 std::optional<llvm::sys::fs::perms> Perms = std::nullopt);
564
565 /// Add a hard link to a file.
566 ///
567 /// Here hard links are not intended to be fully equivalent to the classical
568 /// filesystem. Both the hard link and the file share the same buffer and
569 /// status (and thus have the same UniqueID). Because of this there is no way
570 /// to distinguish between the link and the file after the link has been
571 /// added.
572 ///
573 /// The \p Target path must be an existing file or a hardlink. The
574 /// \p NewLink file must not have been added before. The \p Target
575 /// path must not be a directory. The \p NewLink node is added as a hard
576 /// link which points to the resolved file of \p Target node.
577 /// \return true if the above condition is satisfied and hardlink was
578 /// successfully created, false otherwise.
579 bool addHardLink(const Twine &NewLink, const Twine &Target);
580
581 /// Arbitrary max depth to search through symlinks. We can get into problems
582 /// if a link links to a link that links back to the link, for example.
583 static constexpr size_t MaxSymlinkDepth = 16;
584
585 /// Add a symbolic link. Unlike a HardLink, because \p Target doesn't need
586 /// to refer to a file (or refer to anything, as it happens). Also, an
587 /// in-memory directory for \p Target isn't automatically created.
588 bool
589 addSymbolicLink(const Twine &NewLink, const Twine &Target,
590 time_t ModificationTime,
591 std::optional<uint32_t> User = std::nullopt,
592 std::optional<uint32_t> Group = std::nullopt,
593 std::optional<llvm::sys::fs::perms> Perms = std::nullopt);
594
595 /// Add a buffer to the VFS with a path. The VFS does not own the buffer.
596 /// If present, User, Group, Type and Perms apply to the newly-created file
597 /// or directory.
598 /// \return true if the file or directory was successfully added,
599 /// false if the file or directory already exists in the file system with
600 /// different contents.
601 bool addFileNoOwn(const Twine &Path, time_t ModificationTime,
602 const llvm::MemoryBufferRef &Buffer,
603 std::optional<uint32_t> User = std::nullopt,
604 std::optional<uint32_t> Group = std::nullopt,
605 std::optional<llvm::sys::fs::file_type> Type = std::nullopt,
606 std::optional<llvm::sys::fs::perms> Perms = std::nullopt);
607
608 std::string toString() const;
609
610 /// Return true if this file system normalizes . and .. in paths.
611 bool useNormalizedPaths() const { return UseNormalizedPaths; }
612
613 llvm::ErrorOr<Status> status(const Twine &Path) override;
614 llvm::ErrorOr<std::unique_ptr<File>>
615 openFileForRead(const Twine &Path) override;
616 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
617
618 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
619 return WorkingDirectory;
620 }
621 /// Canonicalizes \p Path by combining with the current working
622 /// directory and normalizing the path (e.g. remove dots). If the current
623 /// working directory is not set, this returns errc::operation_not_permitted.
624 ///
625 /// This doesn't resolve symlinks as they are not supported in in-memory file
626 /// system.
627 std::error_code getRealPath(const Twine &Path,
628 SmallVectorImpl<char> &Output) override;
629 std::error_code isLocal(const Twine &Path, bool &Result) override;
630 std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
631
632protected:
633 void printImpl(raw_ostream &OS, PrintType Type,
634 unsigned IndentLevel) const override;
635};
636
637/// Get a globally unique ID for a virtual file or directory.
638llvm::sys::fs::UniqueID getNextVirtualUniqueID();
639
640/// Gets a \p FileSystem for a virtual file system described in YAML
641/// format.
642std::unique_ptr<FileSystem>
643getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
644 llvm::SourceMgr::DiagHandlerTy DiagHandler,
645 StringRef YAMLFilePath, void *DiagContext = nullptr,
646 IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
647
648struct YAMLVFSEntry {
649 template <typename T1, typename T2>
650 YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false)
651 : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)),
652 IsDirectory(IsDirectory) {}
653 std::string VPath;
654 std::string RPath;
655 bool IsDirectory = false;
656};
657
658class RedirectingFSDirIterImpl;
659class RedirectingFileSystemParser;
660
661/// A virtual file system parsed from a YAML file.
662///
663/// Currently, this class allows creating virtual files and directories. Virtual
664/// files map to existing external files in \c ExternalFS, and virtual
665/// directories may either map to existing directories in \c ExternalFS or list
666/// their contents in the form of other virtual directories and/or files.
667///
668/// The basic structure of the parsed file is:
669/// \verbatim
670/// {
671/// 'version': <version number>,
672/// <optional configuration>
673/// 'roots': [
674/// <directory entries>
675/// ]
676/// }
677/// \endverbatim
678/// The roots may be absolute or relative. If relative they will be made
679/// absolute against either current working directory or the directory where
680/// the Overlay YAML file is located, depending on the 'root-relative'
681/// configuration.
682///
683/// All configuration options are optional.
684/// 'case-sensitive': <boolean, default=(true for Posix, false for Windows)>
685/// 'use-external-names': <boolean, default=true>
686/// 'root-relative': <string, one of 'cwd' or 'overlay-dir', default='cwd'>
687/// 'overlay-relative': <boolean, default=false>
688/// 'fallthrough': <boolean, default=true, deprecated - use 'redirecting-with'
689/// instead>
690/// 'redirecting-with': <string, one of 'fallthrough', 'fallback', or
691/// 'redirect-only', default='fallthrough'>
692///
693/// To clarify, 'root-relative' option will prepend the current working
694/// directory, or the overlay directory to the 'roots->name' field only if
695/// 'roots->name' is a relative path. On the other hand, when 'overlay-relative'
696/// is set to 'true', external paths will always be prepended with the overlay
697/// directory, even if external paths are not relative paths. The
698/// 'root-relative' option has no interaction with the 'overlay-relative'
699/// option.
700///
701/// Virtual directories that list their contents are represented as
702/// \verbatim
703/// {
704/// 'type': 'directory',
705/// 'name': <string>,
706/// 'contents': [ <file or directory entries> ]
707/// }
708/// \endverbatim
709/// The default attributes for such virtual directories are:
710/// \verbatim
711/// MTime = now() when created
712/// Perms = 0777
713/// User = Group = 0
714/// Size = 0
715/// UniqueID = unspecified unique value
716/// \endverbatim
717/// When a path prefix matches such a directory, the next component in the path
718/// is matched against the entries in the 'contents' array.
719///
720/// Re-mapped directories, on the other hand, are represented as
721/// /// \verbatim
722/// {
723/// 'type': 'directory-remap',
724/// 'name': <string>,
725/// 'use-external-name': <boolean>, # Optional
726/// 'external-contents': <path to external directory>
727/// }
728/// \endverbatim
729/// and inherit their attributes from the external directory. When a path
730/// prefix matches such an entry, the unmatched components are appended to the
731/// 'external-contents' path, and the resulting path is looked up in the
732/// external file system instead.
733///
734/// Re-mapped files are represented as
735/// \verbatim
736/// {
737/// 'type': 'file',
738/// 'name': <string>,
739/// 'use-external-name': <boolean>, # Optional
740/// 'external-contents': <path to external file>
741/// }
742/// \endverbatim
743/// Their attributes and file contents are determined by looking up the file at
744/// their 'external-contents' path in the external file system.
745///
746/// For 'file', 'directory' and 'directory-remap' entries the 'name' field may
747/// contain multiple path components (e.g. /path/to/file). However, any
748/// directory in such a path that contains more than one child must be uniquely
749/// represented by a 'directory' entry.
750///
751/// When the 'use-external-name' field is set, calls to \a vfs::File::status()
752/// give the external (remapped) filesystem name instead of the name the file
753/// was accessed by. This is an intentional leak through the \a
754/// RedirectingFileSystem abstraction layer. It enables clients to discover
755/// (and use) the external file location when communicating with users or tools
756/// that don't use the same VFS overlay.
757///
758/// FIXME: 'use-external-name' causes behaviour that's inconsistent with how
759/// "real" filesystems behave. Maybe there should be a separate channel for
760/// this information.
761class RedirectingFileSystem
762 : public RTTIExtends<RedirectingFileSystem, vfs::FileSystem> {
763public:
764 static const char ID;
765 enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File };
766 enum NameKind { NK_NotSet, NK_External, NK_Virtual };
767
768 /// The type of redirection to perform.
769 enum class RedirectKind {
770 /// Lookup the redirected path first (ie. the one specified in
771 /// 'external-contents') and if that fails "fallthrough" to a lookup of the
772 /// originally provided path.
773 Fallthrough,
774 /// Lookup the provided path first and if that fails, "fallback" to a
775 /// lookup of the redirected path.
776 Fallback,
777 /// Only lookup the redirected path, do not lookup the originally provided
778 /// path.
779 RedirectOnly
780 };
781
782 /// The type of relative path used by Roots.
783 enum class RootRelativeKind {
784 /// The roots are relative to the current working directory.
785 CWD,
786 /// The roots are relative to the directory where the Overlay YAML file
787 // locates.
788 OverlayDir
789 };
790
791 /// A single file or directory in the VFS.
792 class Entry {
793 EntryKind Kind;
794 std::string Name;
795
796 public:
797 Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
798 virtual ~Entry() = default;
799
800 StringRef getName() const { return Name; }
801 EntryKind getKind() const { return Kind; }
802 };
803
804 /// A directory in the vfs with explicitly specified contents.
805 class DirectoryEntry : public Entry {
806 std::vector<std::unique_ptr<Entry>> Contents;
807 Status S;
808
809 public:
810 /// Constructs a directory entry with explicitly specified contents.
811 DirectoryEntry(StringRef Name, std::vector<std::unique_ptr<Entry>> Contents,
812 Status S)
813 : Entry(EK_Directory, Name), Contents(std::move(Contents)),
814 S(std::move(S)) {}
815
816 /// Constructs an empty directory entry.
817 DirectoryEntry(StringRef Name, Status S)
818 : Entry(EK_Directory, Name), S(std::move(S)) {}
819
820 Status getStatus() { return S; }
821
822 void addContent(std::unique_ptr<Entry> Content) {
823 Contents.push_back(x: std::move(Content));
824 }
825
826 Entry *getLastContent() const { return Contents.back().get(); }
827
828 using iterator = decltype(Contents)::iterator;
829
830 iterator contents_begin() { return Contents.begin(); }
831 iterator contents_end() { return Contents.end(); }
832
833 static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
834 };
835
836 /// A file or directory in the vfs that is mapped to a file or directory in
837 /// the external filesystem.
838 class RemapEntry : public Entry {
839 std::string ExternalContentsPath;
840 NameKind UseName;
841
842 protected:
843 RemapEntry(EntryKind K, StringRef Name, StringRef ExternalContentsPath,
844 NameKind UseName)
845 : Entry(K, Name), ExternalContentsPath(ExternalContentsPath),
846 UseName(UseName) {}
847
848 public:
849 StringRef getExternalContentsPath() const { return ExternalContentsPath; }
850
851 /// Whether to use the external path as the name for this file or directory.
852 bool useExternalName(bool GlobalUseExternalName) const {
853 return UseName == NK_NotSet ? GlobalUseExternalName
854 : (UseName == NK_External);
855 }
856
857 NameKind getUseName() const { return UseName; }
858
859 static bool classof(const Entry *E) {
860 switch (E->getKind()) {
861 case EK_DirectoryRemap:
862 [[fallthrough]];
863 case EK_File:
864 return true;
865 case EK_Directory:
866 return false;
867 }
868 llvm_unreachable("invalid entry kind");
869 }
870 };
871
872 /// A directory in the vfs that maps to a directory in the external file
873 /// system.
874 class DirectoryRemapEntry : public RemapEntry {
875 public:
876 DirectoryRemapEntry(StringRef Name, StringRef ExternalContentsPath,
877 NameKind UseName)
878 : RemapEntry(EK_DirectoryRemap, Name, ExternalContentsPath, UseName) {}
879
880 static bool classof(const Entry *E) {
881 return E->getKind() == EK_DirectoryRemap;
882 }
883 };
884
885 /// A file in the vfs that maps to a file in the external file system.
886 class FileEntry : public RemapEntry {
887 public:
888 FileEntry(StringRef Name, StringRef ExternalContentsPath, NameKind UseName)
889 : RemapEntry(EK_File, Name, ExternalContentsPath, UseName) {}
890
891 static bool classof(const Entry *E) { return E->getKind() == EK_File; }
892 };
893
894 /// Represents the result of a path lookup into the RedirectingFileSystem.
895 struct LookupResult {
896 /// Chain of parent directory entries for \c E.
897 llvm::SmallVector<Entry *, 32> Parents;
898
899 /// The entry the looked-up path corresponds to.
900 Entry *E;
901
902 private:
903 /// When the found Entry is a DirectoryRemapEntry, stores the path in the
904 /// external file system that the looked-up path in the virtual file system
905 // corresponds to.
906 std::optional<std::string> ExternalRedirect;
907
908 public:
909 LookupResult(Entry *E, sys::path::const_iterator Start,
910 sys::path::const_iterator End);
911
912 /// If the found Entry maps the input path to a path in the external
913 /// file system (i.e. it is a FileEntry or DirectoryRemapEntry), returns
914 /// that path.
915 std::optional<StringRef> getExternalRedirect() const {
916 if (isa<DirectoryRemapEntry>(Val: E))
917 return StringRef(*ExternalRedirect);
918 if (auto *FE = dyn_cast<FileEntry>(Val: E))
919 return FE->getExternalContentsPath();
920 return std::nullopt;
921 }
922
923 /// Get the (canonical) path of the found entry. This uses the as-written
924 /// path components from the VFS specification.
925 void getPath(llvm::SmallVectorImpl<char> &Path) const;
926 };
927
928private:
929 friend class RedirectingFSDirIterImpl;
930 friend class RedirectingFileSystemParser;
931
932 /// Canonicalize path by removing ".", "..", "./", components. This is
933 /// a VFS request, do not bother about symlinks in the path components
934 /// but canonicalize in order to perform the correct entry search.
935 std::error_code makeCanonicalForLookup(SmallVectorImpl<char> &Path) const;
936
937 /// Get the File status, or error, from the underlying external file system.
938 /// This returns the status with the originally requested name, while looking
939 /// up the entry using a potentially different path.
940 ErrorOr<Status> getExternalStatus(const Twine &LookupPath,
941 const Twine &OriginalPath) const;
942
943 /// Make \a Path an absolute path.
944 ///
945 /// Makes \a Path absolute using the \a WorkingDir if it is not already.
946 ///
947 /// /absolute/path => /absolute/path
948 /// relative/../path => <WorkingDir>/relative/../path
949 ///
950 /// \param WorkingDir A path that will be used as the base Dir if \a Path
951 /// is not already absolute.
952 /// \param Path A path that is modified to be an absolute path.
953 /// \returns success if \a path has been made absolute, otherwise a
954 /// platform-specific error_code.
955 std::error_code makeAbsolute(StringRef WorkingDir,
956 SmallVectorImpl<char> &Path) const;
957
958 // In a RedirectingFileSystem, keys can be specified in Posix or Windows
959 // style (or even a mixture of both), so this comparison helper allows
960 // slashes (representing a root) to match backslashes (and vice versa). Note
961 // that, other than the root, path components should not contain slashes or
962 // backslashes.
963 bool pathComponentMatches(llvm::StringRef lhs, llvm::StringRef rhs) const {
964 if ((CaseSensitive ? lhs.equals(RHS: rhs) : lhs.equals_insensitive(RHS: rhs)))
965 return true;
966 return (lhs == "/" && rhs == "\\") || (lhs == "\\" && rhs == "/");
967 }
968
969 /// The root(s) of the virtual file system.
970 std::vector<std::unique_ptr<Entry>> Roots;
971
972 /// The current working directory of the file system.
973 std::string WorkingDirectory;
974
975 /// The file system to use for external references.
976 IntrusiveRefCntPtr<FileSystem> ExternalFS;
977
978 /// This represents the directory path that the YAML file is located.
979 /// This will be prefixed to each 'external-contents' if IsRelativeOverlay
980 /// is set. This will also be prefixed to each 'roots->name' if RootRelative
981 /// is set to RootRelativeKind::OverlayDir and the path is relative.
982 std::string OverlayFileDir;
983
984 /// @name Configuration
985 /// @{
986
987 /// Whether to perform case-sensitive comparisons.
988 ///
989 /// Currently, case-insensitive matching only works correctly with ASCII.
990 bool CaseSensitive = is_style_posix(S: sys::path::Style::native);
991
992 /// IsRelativeOverlay marks whether a OverlayFileDir path must
993 /// be prefixed in every 'external-contents' when reading from YAML files.
994 bool IsRelativeOverlay = false;
995
996 /// Whether to use to use the value of 'external-contents' for the
997 /// names of files. This global value is overridable on a per-file basis.
998 bool UseExternalNames = true;
999
1000 /// True if this FS has redirected a lookup. This does not include
1001 /// fallthrough.
1002 mutable bool HasBeenUsed = false;
1003
1004 /// Used to enable or disable updating `HasBeenUsed`.
1005 bool UsageTrackingActive = false;
1006
1007 /// Determines the lookups to perform, as well as their order. See
1008 /// \c RedirectKind for details.
1009 RedirectKind Redirection = RedirectKind::Fallthrough;
1010
1011 /// Determine the prefix directory if the roots are relative paths. See
1012 /// \c RootRelativeKind for details.
1013 RootRelativeKind RootRelative = RootRelativeKind::CWD;
1014 /// @}
1015
1016 RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS);
1017
1018 /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly recursing
1019 /// into the contents of \p From if it is a directory. Returns a LookupResult
1020 /// giving the matched entry and, if that entry is a FileEntry or
1021 /// DirectoryRemapEntry, the path it redirects to in the external file system.
1022 ErrorOr<LookupResult>
1023 lookupPathImpl(llvm::sys::path::const_iterator Start,
1024 llvm::sys::path::const_iterator End, Entry *From,
1025 llvm::SmallVectorImpl<Entry *> &Entries) const;
1026
1027 /// Get the status for a path with the provided \c LookupResult.
1028 ErrorOr<Status> status(const Twine &LookupPath, const Twine &OriginalPath,
1029 const LookupResult &Result);
1030
1031public:
1032 /// Looks up \p Path in \c Roots and returns a LookupResult giving the
1033 /// matched entry and, if the entry was a FileEntry or DirectoryRemapEntry,
1034 /// the path it redirects to in the external file system.
1035 ErrorOr<LookupResult> lookupPath(StringRef Path) const;
1036
1037 /// Parses \p Buffer, which is expected to be in YAML format and
1038 /// returns a virtual file system representing its contents.
1039 static std::unique_ptr<RedirectingFileSystem>
1040 create(std::unique_ptr<MemoryBuffer> Buffer,
1041 SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
1042 void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
1043
1044 /// Redirect each of the remapped files from first to second.
1045 static std::unique_ptr<RedirectingFileSystem>
1046 create(ArrayRef<std::pair<std::string, std::string>> RemappedFiles,
1047 bool UseExternalNames, FileSystem &ExternalFS);
1048
1049 ErrorOr<Status> status(const Twine &Path) override;
1050 bool exists(const Twine &Path) override;
1051 ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
1052
1053 std::error_code getRealPath(const Twine &Path,
1054 SmallVectorImpl<char> &Output) override;
1055
1056 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
1057
1058 std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
1059
1060 std::error_code isLocal(const Twine &Path, bool &Result) override;
1061
1062 std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const override;
1063
1064 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
1065
1066 void setOverlayFileDir(StringRef PrefixDir);
1067
1068 StringRef getOverlayFileDir() const;
1069
1070 /// Sets the redirection kind to \c Fallthrough if true or \c RedirectOnly
1071 /// otherwise. Will removed in the future, use \c setRedirection instead.
1072 void setFallthrough(bool Fallthrough);
1073
1074 void setRedirection(RedirectingFileSystem::RedirectKind Kind);
1075
1076 std::vector<llvm::StringRef> getRoots() const;
1077
1078 bool hasBeenUsed() const { return HasBeenUsed; };
1079 void clearHasBeenUsed() { HasBeenUsed = false; }
1080
1081 void setUsageTrackingActive(bool Active) { UsageTrackingActive = Active; }
1082
1083 void printEntry(raw_ostream &OS, Entry *E, unsigned IndentLevel = 0) const;
1084
1085protected:
1086 void printImpl(raw_ostream &OS, PrintType Type,
1087 unsigned IndentLevel) const override;
1088 void visitChildFileSystems(VisitCallbackTy Callback) override;
1089};
1090
1091/// Collect all pairs of <virtual path, real path> entries from the
1092/// \p YAMLFilePath. This is used by the module dependency collector to forward
1093/// the entries into the reproducer output VFS YAML file.
1094void collectVFSFromYAML(
1095 std::unique_ptr<llvm::MemoryBuffer> Buffer,
1096 llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
1097 SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
1098 void *DiagContext = nullptr,
1099 IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
1100
1101class YAMLVFSWriter {
1102 std::vector<YAMLVFSEntry> Mappings;
1103 std::optional<bool> IsCaseSensitive;
1104 std::optional<bool> IsOverlayRelative;
1105 std::optional<bool> UseExternalNames;
1106 std::string OverlayDir;
1107
1108 void addEntry(StringRef VirtualPath, StringRef RealPath, bool IsDirectory);
1109
1110public:
1111 YAMLVFSWriter() = default;
1112
1113 void addFileMapping(StringRef VirtualPath, StringRef RealPath);
1114 void addDirectoryMapping(StringRef VirtualPath, StringRef RealPath);
1115
1116 void setCaseSensitivity(bool CaseSensitive) {
1117 IsCaseSensitive = CaseSensitive;
1118 }
1119
1120 void setUseExternalNames(bool UseExtNames) { UseExternalNames = UseExtNames; }
1121
1122 void setOverlayDir(StringRef OverlayDirectory) {
1123 IsOverlayRelative = true;
1124 OverlayDir.assign(str: OverlayDirectory.str());
1125 }
1126
1127 const std::vector<YAMLVFSEntry> &getMappings() const { return Mappings; }
1128
1129 void write(llvm::raw_ostream &OS);
1130};
1131
1132} // namespace vfs
1133} // namespace llvm
1134
1135#endif // LLVM_SUPPORT_VIRTUALFILESYSTEM_H
1136

source code of llvm/include/llvm/Support/VirtualFileSystem.h