1//===- ExternalASTSource.h - Abstract External AST Interface ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ExternalASTSource interface, which enables
10// construction of AST nodes from some external source.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_AST_EXTERNALASTSOURCE_H
15#define LLVM_CLANG_AST_EXTERNALASTSOURCE_H
16
17#include "clang/AST/CharUnits.h"
18#include "clang/AST/DeclBase.h"
19#include "clang/Basic/LLVM.h"
20#include "clang/Basic/Module.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/IntrusiveRefCntPtr.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/PointerUnion.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/Support/PointerLikeTypeTraits.h"
31#include <cassert>
32#include <cstddef>
33#include <cstdint>
34#include <iterator>
35#include <string>
36#include <utility>
37
38namespace clang {
39
40class ASTConsumer;
41class ASTContext;
42class CXXBaseSpecifier;
43class CXXCtorInitializer;
44class CXXRecordDecl;
45class DeclarationName;
46class FieldDecl;
47class IdentifierInfo;
48class NamedDecl;
49class ObjCInterfaceDecl;
50class RecordDecl;
51class Selector;
52class Stmt;
53class TagDecl;
54
55/// Abstract interface for external sources of AST nodes.
56///
57/// External AST sources provide AST nodes constructed from some
58/// external source, such as a precompiled header. External AST
59/// sources can resolve types and declarations from abstract IDs into
60/// actual type and declaration nodes, and read parts of declaration
61/// contexts.
62class ExternalASTSource : public RefCountedBase<ExternalASTSource> {
63 friend class ExternalSemaSource;
64
65 /// Generation number for this external AST source. Must be increased
66 /// whenever we might have added new redeclarations for existing decls.
67 uint32_t CurrentGeneration = 0;
68
69 /// Whether this AST source also provides information for
70 /// semantic analysis.
71 bool SemaSource = false;
72
73public:
74 ExternalASTSource() = default;
75 virtual ~ExternalASTSource();
76
77 /// RAII class for safely pairing a StartedDeserializing call
78 /// with FinishedDeserializing.
79 class Deserializing {
80 ExternalASTSource *Source;
81
82 public:
83 explicit Deserializing(ExternalASTSource *source) : Source(source) {
84 assert(Source);
85 Source->StartedDeserializing();
86 }
87
88 ~Deserializing() {
89 Source->FinishedDeserializing();
90 }
91 };
92
93 /// Get the current generation of this AST source. This number
94 /// is incremented each time the AST source lazily extends an existing
95 /// entity.
96 uint32_t getGeneration() const { return CurrentGeneration; }
97
98 /// Resolve a declaration ID into a declaration, potentially
99 /// building a new declaration.
100 ///
101 /// This method only needs to be implemented if the AST source ever
102 /// passes back decl sets as VisibleDeclaration objects.
103 ///
104 /// The default implementation of this method is a no-op.
105 virtual Decl *GetExternalDecl(uint32_t ID);
106
107 /// Resolve a selector ID into a selector.
108 ///
109 /// This operation only needs to be implemented if the AST source
110 /// returns non-zero for GetNumKnownSelectors().
111 ///
112 /// The default implementation of this method is a no-op.
113 virtual Selector GetExternalSelector(uint32_t ID);
114
115 /// Returns the number of selectors known to the external AST
116 /// source.
117 ///
118 /// The default implementation of this method is a no-op.
119 virtual uint32_t GetNumExternalSelectors();
120
121 /// Resolve the offset of a statement in the decl stream into
122 /// a statement.
123 ///
124 /// This operation is meant to be used via a LazyOffsetPtr. It only
125 /// needs to be implemented if the AST source uses methods like
126 /// FunctionDecl::setLazyBody when building decls.
127 ///
128 /// The default implementation of this method is a no-op.
129 virtual Stmt *GetExternalDeclStmt(uint64_t Offset);
130
131 /// Resolve the offset of a set of C++ constructor initializers in
132 /// the decl stream into an array of initializers.
133 ///
134 /// The default implementation of this method is a no-op.
135 virtual CXXCtorInitializer **GetExternalCXXCtorInitializers(uint64_t Offset);
136
137 /// Resolve the offset of a set of C++ base specifiers in the decl
138 /// stream into an array of specifiers.
139 ///
140 /// The default implementation of this method is a no-op.
141 virtual CXXBaseSpecifier *GetExternalCXXBaseSpecifiers(uint64_t Offset);
142
143 /// Update an out-of-date identifier.
144 virtual void updateOutOfDateIdentifier(IdentifierInfo &II) {}
145
146 /// Find all declarations with the given name in the given context,
147 /// and add them to the context by calling SetExternalVisibleDeclsForName
148 /// or SetNoExternalVisibleDeclsForName.
149 /// \return \c true if any declarations might have been found, \c false if
150 /// we definitely have no declarations with tbis name.
151 ///
152 /// The default implementation of this method is a no-op returning \c false.
153 virtual bool
154 FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name);
155
156 /// Ensures that the table of all visible declarations inside this
157 /// context is up to date.
158 ///
159 /// The default implementation of this function is a no-op.
160 virtual void completeVisibleDeclsMap(const DeclContext *DC);
161
162 /// Retrieve the module that corresponds to the given module ID.
163 virtual Module *getModule(unsigned ID) { return nullptr; }
164
165 /// Determine whether D comes from a PCH which was built with a corresponding
166 /// object file.
167 virtual bool DeclIsFromPCHWithObjectFile(const Decl *D) { return false; }
168
169 /// Abstracts clang modules and precompiled header files and holds
170 /// everything needed to generate debug info for an imported module
171 /// or PCH.
172 class ASTSourceDescriptor {
173 StringRef PCHModuleName;
174 StringRef Path;
175 StringRef ASTFile;
176 ASTFileSignature Signature;
177 const Module *ClangModule = nullptr;
178
179 public:
180 ASTSourceDescriptor() = default;
181 ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile,
182 ASTFileSignature Signature)
183 : PCHModuleName(std::move(Name)), Path(std::move(Path)),
184 ASTFile(std::move(ASTFile)), Signature(Signature) {}
185 ASTSourceDescriptor(const Module &M);
186
187 std::string getModuleName() const;
188 StringRef getPath() const { return Path; }
189 StringRef getASTFile() const { return ASTFile; }
190 ASTFileSignature getSignature() const { return Signature; }
191 const Module *getModuleOrNull() const { return ClangModule; }
192 };
193
194 /// Return a descriptor for the corresponding module, if one exists.
195 virtual llvm::Optional<ASTSourceDescriptor> getSourceDescriptor(unsigned ID);
196
197 enum ExtKind { EK_Always, EK_Never, EK_ReplyHazy };
198
199 virtual ExtKind hasExternalDefinitions(const Decl *D);
200
201 /// Finds all declarations lexically contained within the given
202 /// DeclContext, after applying an optional filter predicate.
203 ///
204 /// \param IsKindWeWant a predicate function that returns true if the passed
205 /// declaration kind is one we are looking for.
206 ///
207 /// The default implementation of this method is a no-op.
208 virtual void
209 FindExternalLexicalDecls(const DeclContext *DC,
210 llvm::function_ref<bool(Decl::Kind)> IsKindWeWant,
211 SmallVectorImpl<Decl *> &Result);
212
213 /// Finds all declarations lexically contained within the given
214 /// DeclContext.
215 void FindExternalLexicalDecls(const DeclContext *DC,
216 SmallVectorImpl<Decl *> &Result) {
217 FindExternalLexicalDecls(DC, [](Decl::Kind) { return true; }, Result);
218 }
219
220 /// Get the decls that are contained in a file in the Offset/Length
221 /// range. \p Length can be 0 to indicate a point at \p Offset instead of
222 /// a range.
223 virtual void FindFileRegionDecls(FileID File, unsigned Offset,
224 unsigned Length,
225 SmallVectorImpl<Decl *> &Decls);
226
227 /// Gives the external AST source an opportunity to complete
228 /// the redeclaration chain for a declaration. Called each time we
229 /// need the most recent declaration of a declaration after the
230 /// generation count is incremented.
231 virtual void CompleteRedeclChain(const Decl *D);
232
233 /// Gives the external AST source an opportunity to complete
234 /// an incomplete type.
235 virtual void CompleteType(TagDecl *Tag);
236
237 /// Gives the external AST source an opportunity to complete an
238 /// incomplete Objective-C class.
239 ///
240 /// This routine will only be invoked if the "externally completed" bit is
241 /// set on the ObjCInterfaceDecl via the function
242 /// \c ObjCInterfaceDecl::setExternallyCompleted().
243 virtual void CompleteType(ObjCInterfaceDecl *Class);
244
245 /// Loads comment ranges.
246 virtual void ReadComments();
247
248 /// Notify ExternalASTSource that we started deserialization of
249 /// a decl or type so until FinishedDeserializing is called there may be
250 /// decls that are initializing. Must be paired with FinishedDeserializing.
251 ///
252 /// The default implementation of this method is a no-op.
253 virtual void StartedDeserializing();
254
255 /// Notify ExternalASTSource that we finished the deserialization of
256 /// a decl or type. Must be paired with StartedDeserializing.
257 ///
258 /// The default implementation of this method is a no-op.
259 virtual void FinishedDeserializing();
260
261 /// Function that will be invoked when we begin parsing a new
262 /// translation unit involving this external AST source.
263 ///
264 /// The default implementation of this method is a no-op.
265 virtual void StartTranslationUnit(ASTConsumer *Consumer);
266
267 /// Print any statistics that have been gathered regarding
268 /// the external AST source.
269 ///
270 /// The default implementation of this method is a no-op.
271 virtual void PrintStats();
272
273 /// Perform layout on the given record.
274 ///
275 /// This routine allows the external AST source to provide an specific
276 /// layout for a record, overriding the layout that would normally be
277 /// constructed. It is intended for clients who receive specific layout
278 /// details rather than source code (such as LLDB). The client is expected
279 /// to fill in the field offsets, base offsets, virtual base offsets, and
280 /// complete object size.
281 ///
282 /// \param Record The record whose layout is being requested.
283 ///
284 /// \param Size The final size of the record, in bits.
285 ///
286 /// \param Alignment The final alignment of the record, in bits.
287 ///
288 /// \param FieldOffsets The offset of each of the fields within the record,
289 /// expressed in bits. All of the fields must be provided with offsets.
290 ///
291 /// \param BaseOffsets The offset of each of the direct, non-virtual base
292 /// classes. If any bases are not given offsets, the bases will be laid
293 /// out according to the ABI.
294 ///
295 /// \param VirtualBaseOffsets The offset of each of the virtual base classes
296 /// (either direct or not). If any bases are not given offsets, the bases will be laid
297 /// out according to the ABI.
298 ///
299 /// \returns true if the record layout was provided, false otherwise.
300 virtual bool layoutRecordType(
301 const RecordDecl *Record, uint64_t &Size, uint64_t &Alignment,
302 llvm::DenseMap<const FieldDecl *, uint64_t> &FieldOffsets,
303 llvm::DenseMap<const CXXRecordDecl *, CharUnits> &BaseOffsets,
304 llvm::DenseMap<const CXXRecordDecl *, CharUnits> &VirtualBaseOffsets);
305
306 //===--------------------------------------------------------------------===//
307 // Queries for performance analysis.
308 //===--------------------------------------------------------------------===//
309
310 struct MemoryBufferSizes {
311 size_t malloc_bytes;
312 size_t mmap_bytes;
313
314 MemoryBufferSizes(size_t malloc_bytes, size_t mmap_bytes)
315 : malloc_bytes(malloc_bytes), mmap_bytes(mmap_bytes) {}
316 };
317
318 /// Return the amount of memory used by memory buffers, breaking down
319 /// by heap-backed versus mmap'ed memory.
320 MemoryBufferSizes getMemoryBufferSizes() const {
321 MemoryBufferSizes sizes(0, 0);
322 getMemoryBufferSizes(sizes);
323 return sizes;
324 }
325
326 virtual void getMemoryBufferSizes(MemoryBufferSizes &sizes) const;
327
328protected:
329 static DeclContextLookupResult
330 SetExternalVisibleDeclsForName(const DeclContext *DC,
331 DeclarationName Name,
332 ArrayRef<NamedDecl*> Decls);
333
334 static DeclContextLookupResult
335 SetNoExternalVisibleDeclsForName(const DeclContext *DC,
336 DeclarationName Name);
337
338 /// Increment the current generation.
339 uint32_t incrementGeneration(ASTContext &C);
340};
341
342/// A lazy pointer to an AST node (of base type T) that resides
343/// within an external AST source.
344///
345/// The AST node is identified within the external AST source by a
346/// 63-bit offset, and can be retrieved via an operation on the
347/// external AST source itself.
348template<typename T, typename OffsT, T* (ExternalASTSource::*Get)(OffsT Offset)>
349struct LazyOffsetPtr {
350 /// Either a pointer to an AST node or the offset within the
351 /// external AST source where the AST node can be found.
352 ///
353 /// If the low bit is clear, a pointer to the AST node. If the low
354 /// bit is set, the upper 63 bits are the offset.
355 mutable uint64_t Ptr = 0;
356
357public:
358 LazyOffsetPtr() = default;
359 explicit LazyOffsetPtr(T *Ptr) : Ptr(reinterpret_cast<uint64_t>(Ptr)) {}
360
361 explicit LazyOffsetPtr(uint64_t Offset) : Ptr((Offset << 1) | 0x01) {
362 assert((Offset << 1 >> 1) == Offset && "Offsets must require < 63 bits");
363 if (Offset == 0)
364 Ptr = 0;
365 }
366
367 LazyOffsetPtr &operator=(T *Ptr) {
368 this->Ptr = reinterpret_cast<uint64_t>(Ptr);
369 return *this;
370 }
371
372 LazyOffsetPtr &operator=(uint64_t Offset) {
373 assert((Offset << 1 >> 1) == Offset && "Offsets must require < 63 bits");
374 if (Offset == 0)
375 Ptr = 0;
376 else
377 Ptr = (Offset << 1) | 0x01;
378
379 return *this;
380 }
381
382 /// Whether this pointer is non-NULL.
383 ///
384 /// This operation does not require the AST node to be deserialized.
385 explicit operator bool() const { return Ptr != 0; }
386
387 /// Whether this pointer is non-NULL.
388 ///
389 /// This operation does not require the AST node to be deserialized.
390 bool isValid() const { return Ptr != 0; }
391
392 /// Whether this pointer is currently stored as an offset.
393 bool isOffset() const { return Ptr & 0x01; }
394
395 /// Retrieve the pointer to the AST node that this lazy pointer points to.
396 ///
397 /// \param Source the external AST source.
398 ///
399 /// \returns a pointer to the AST node.
400 T* get(ExternalASTSource *Source) const {
401 if (isOffset()) {
402 assert(Source &&
403 "Cannot deserialize a lazy pointer without an AST source");
404 Ptr = reinterpret_cast<uint64_t>((Source->*Get)(Ptr >> 1));
405 }
406 return reinterpret_cast<T*>(Ptr);
407 }
408};
409
410/// A lazy value (of type T) that is within an AST node of type Owner,
411/// where the value might change in later generations of the external AST
412/// source.
413template<typename Owner, typename T, void (ExternalASTSource::*Update)(Owner)>
414struct LazyGenerationalUpdatePtr {
415 /// A cache of the value of this pointer, in the most recent generation in
416 /// which we queried it.
417 struct LazyData {
418 ExternalASTSource *ExternalSource;
419 uint32_t LastGeneration = 0;
420 T LastValue;
421
422 LazyData(ExternalASTSource *Source, T Value)
423 : ExternalSource(Source), LastValue(Value) {}
424 };
425
426 // Our value is represented as simply T if there is no external AST source.
427 using ValueType = llvm::PointerUnion<T, LazyData*>;
428 ValueType Value;
429
430 LazyGenerationalUpdatePtr(ValueType V) : Value(V) {}
431
432 // Defined in ASTContext.h
433 static ValueType makeValue(const ASTContext &Ctx, T Value);
434
435public:
436 explicit LazyGenerationalUpdatePtr(const ASTContext &Ctx, T Value = T())
437 : Value(makeValue(Ctx, Value)) {}
438
439 /// Create a pointer that is not potentially updated by later generations of
440 /// the external AST source.
441 enum NotUpdatedTag { NotUpdated };
442 LazyGenerationalUpdatePtr(NotUpdatedTag, T Value = T())
443 : Value(Value) {}
444
445 /// Forcibly set this pointer (which must be lazy) as needing updates.
446 void markIncomplete() {
447 Value.template get<LazyData *>()->LastGeneration = 0;
448 }
449
450 /// Set the value of this pointer, in the current generation.
451 void set(T NewValue) {
452 if (auto *LazyVal = Value.template dyn_cast<LazyData *>()) {
453 LazyVal->LastValue = NewValue;
454 return;
455 }
456 Value = NewValue;
457 }
458
459 /// Set the value of this pointer, for this and all future generations.
460 void setNotUpdated(T NewValue) { Value = NewValue; }
461
462 /// Get the value of this pointer, updating its owner if necessary.
463 T get(Owner O) {
464 if (auto *LazyVal = Value.template dyn_cast<LazyData *>()) {
465 if (LazyVal->LastGeneration != LazyVal->ExternalSource->getGeneration()) {
466 LazyVal->LastGeneration = LazyVal->ExternalSource->getGeneration();
467 (LazyVal->ExternalSource->*Update)(O);
468 }
469 return LazyVal->LastValue;
470 }
471 return Value.template get<T>();
472 }
473
474 /// Get the most recently computed value of this pointer without updating it.
475 T getNotUpdated() const {
476 if (auto *LazyVal = Value.template dyn_cast<LazyData *>())
477 return LazyVal->LastValue;
478 return Value.template get<T>();
479 }
480
481 void *getOpaqueValue() { return Value.getOpaqueValue(); }
482 static LazyGenerationalUpdatePtr getFromOpaqueValue(void *Ptr) {
483 return LazyGenerationalUpdatePtr(ValueType::getFromOpaqueValue(Ptr));
484 }
485};
486
487} // namespace clang
488
489/// Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be
490/// placed into a PointerUnion.
491namespace llvm {
492
493template<typename Owner, typename T,
494 void (clang::ExternalASTSource::*Update)(Owner)>
495struct PointerLikeTypeTraits<
496 clang::LazyGenerationalUpdatePtr<Owner, T, Update>> {
497 using Ptr = clang::LazyGenerationalUpdatePtr<Owner, T, Update>;
498
499 static void *getAsVoidPointer(Ptr P) { return P.getOpaqueValue(); }
500 static Ptr getFromVoidPointer(void *P) { return Ptr::getFromOpaqueValue(P); }
501
502 enum {
503 NumLowBitsAvailable = PointerLikeTypeTraits<T>::NumLowBitsAvailable - 1
504 };
505};
506
507} // namespace llvm
508
509namespace clang {
510
511/// Represents a lazily-loaded vector of data.
512///
513/// The lazily-loaded vector of data contains data that is partially loaded
514/// from an external source and partially added by local translation. The
515/// items loaded from the external source are loaded lazily, when needed for
516/// iteration over the complete vector.
517template<typename T, typename Source,
518 void (Source::*Loader)(SmallVectorImpl<T>&),
519 unsigned LoadedStorage = 2, unsigned LocalStorage = 4>
520class LazyVector {
521 SmallVector<T, LoadedStorage> Loaded;
522 SmallVector<T, LocalStorage> Local;
523
524public:
525 /// Iteration over the elements in the vector.
526 ///
527 /// In a complete iteration, the iterator walks the range [-M, N),
528 /// where negative values are used to indicate elements
529 /// loaded from the external source while non-negative values are used to
530 /// indicate elements added via \c push_back().
531 /// However, to provide iteration in source order (for, e.g., chained
532 /// precompiled headers), dereferencing the iterator flips the negative
533 /// values (corresponding to loaded entities), so that position -M
534 /// corresponds to element 0 in the loaded entities vector, position -M+1
535 /// corresponds to element 1 in the loaded entities vector, etc. This
536 /// gives us a reasonably efficient, source-order walk.
537 ///
538 /// We define this as a wrapping iterator around an int. The
539 /// iterator_adaptor_base class forwards the iterator methods to basic integer
540 /// arithmetic.
541 class iterator
542 : public llvm::iterator_adaptor_base<
543 iterator, int, std::random_access_iterator_tag, T, int, T *, T &> {
544 friend class LazyVector;
545
546 LazyVector *Self;
547
548 iterator(LazyVector *Self, int Position)
549 : iterator::iterator_adaptor_base(Position), Self(Self) {}
550
551 bool isLoaded() const { return this->I < 0; }
552
553 public:
554 iterator() : iterator(nullptr, 0) {}
555
556 typename iterator::reference operator*() const {
557 if (isLoaded())
558 return Self->Loaded.end()[this->I];
559 return Self->Local.begin()[this->I];
560 }
561 };
562
563 iterator begin(Source *source, bool LocalOnly = false) {
564 if (LocalOnly)
565 return iterator(this, 0);
566
567 if (source)
568 (source->*Loader)(Loaded);
569 return iterator(this, -(int)Loaded.size());
570 }
571
572 iterator end() {
573 return iterator(this, Local.size());
574 }
575
576 void push_back(const T& LocalValue) {
577 Local.push_back(LocalValue);
578 }
579
580 void erase(iterator From, iterator To) {
581 if (From.isLoaded() && To.isLoaded()) {
582 Loaded.erase(&*From, &*To);
583 return;
584 }
585
586 if (From.isLoaded()) {
587 Loaded.erase(&*From, Loaded.end());
588 From = begin(nullptr, true);
589 }
590
591 Local.erase(&*From, &*To);
592 }
593};
594
595/// A lazy pointer to a statement.
596using LazyDeclStmtPtr =
597 LazyOffsetPtr<Stmt, uint64_t, &ExternalASTSource::GetExternalDeclStmt>;
598
599/// A lazy pointer to a declaration.
600using LazyDeclPtr =
601 LazyOffsetPtr<Decl, uint32_t, &ExternalASTSource::GetExternalDecl>;
602
603/// A lazy pointer to a set of CXXCtorInitializers.
604using LazyCXXCtorInitializersPtr =
605 LazyOffsetPtr<CXXCtorInitializer *, uint64_t,
606 &ExternalASTSource::GetExternalCXXCtorInitializers>;
607
608/// A lazy pointer to a set of CXXBaseSpecifiers.
609using LazyCXXBaseSpecifiersPtr =
610 LazyOffsetPtr<CXXBaseSpecifier, uint64_t,
611 &ExternalASTSource::GetExternalCXXBaseSpecifiers>;
612
613} // namespace clang
614
615#endif // LLVM_CLANG_AST_EXTERNALASTSOURCE_H
616