1//===--- CrossTranslationUnit.h - -------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides an interface to load binary AST dumps on demand. This
10// feature can be utilized for tools that require cross translation unit
11// support.
12//
13//===----------------------------------------------------------------------===//
14#ifndef LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H
15#define LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H
16
17#include "clang/AST/ASTImporterSharedState.h"
18#include "clang/Analysis/MacroExpansionContext.h"
19#include "clang/Basic/LLVM.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/Optional.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/StringMap.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/Path.h"
26
27namespace clang {
28class CompilerInstance;
29class ASTContext;
30class ASTImporter;
31class ASTUnit;
32class DeclContext;
33class FunctionDecl;
34class VarDecl;
35class NamedDecl;
36class TranslationUnitDecl;
37
38namespace cross_tu {
39
40enum class index_error_code {
41 unspecified = 1,
42 missing_index_file,
43 invalid_index_format,
44 multiple_definitions,
45 missing_definition,
46 failed_import,
47 failed_to_get_external_ast,
48 failed_to_generate_usr,
49 triple_mismatch,
50 lang_mismatch,
51 lang_dialect_mismatch,
52 load_threshold_reached,
53 invocation_list_ambiguous,
54 invocation_list_file_not_found,
55 invocation_list_empty,
56 invocation_list_wrong_format,
57 invocation_list_lookup_unsuccessful
58};
59
60class IndexError : public llvm::ErrorInfo<IndexError> {
61public:
62 static char ID;
63 IndexError(index_error_code C) : Code(C), LineNo(0) {}
64 IndexError(index_error_code C, std::string FileName, int LineNo = 0)
65 : Code(C), FileName(std::move(FileName)), LineNo(LineNo) {}
66 IndexError(index_error_code C, std::string FileName, std::string TripleToName,
67 std::string TripleFromName)
68 : Code(C), FileName(std::move(FileName)),
69 TripleToName(std::move(TripleToName)),
70 TripleFromName(std::move(TripleFromName)) {}
71 void log(raw_ostream &OS) const override;
72 std::error_code convertToErrorCode() const override;
73 index_error_code getCode() const { return Code; }
74 int getLineNum() const { return LineNo; }
75 std::string getFileName() const { return FileName; }
76 std::string getTripleToName() const { return TripleToName; }
77 std::string getTripleFromName() const { return TripleFromName; }
78
79private:
80 index_error_code Code;
81 std::string FileName;
82 int LineNo;
83 std::string TripleToName;
84 std::string TripleFromName;
85};
86
87/// This function parses an index file that determines which
88/// translation unit contains which definition. The IndexPath is not prefixed
89/// with CTUDir, so an absolute path is expected for consistent results.
90///
91/// The index file format is the following:
92/// each line consists of an USR and a filepath separated by a space.
93///
94/// \return Returns a map where the USR is the key and the filepath is the value
95/// or an error.
96llvm::Expected<llvm::StringMap<std::string>>
97parseCrossTUIndex(StringRef IndexPath);
98
99std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
100
101using InvocationListTy = llvm::StringMap<llvm::SmallVector<std::string, 32>>;
102/// Parse the YAML formatted invocation list file content \p FileContent.
103/// The format is expected to be a mapping from from absolute source file
104/// paths in the filesystem to a list of command-line parts, which
105/// constitute the invocation needed to compile that file. That invocation
106/// will be used to produce the AST of the TU.
107llvm::Expected<InvocationListTy> parseInvocationList(
108 StringRef FileContent,
109 llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix);
110
111// Returns true if the variable or any field of a record variable is const.
112bool containsConst(const VarDecl *VD, const ASTContext &ACtx);
113
114/// This class is used for tools that requires cross translation
115/// unit capability.
116///
117/// This class can load definitions from external AST sources.
118/// The loaded definition will be merged back to the original AST using the
119/// AST Importer.
120/// In order to use this class, an index file is required that describes
121/// the locations of the AST files for each definition.
122///
123/// Note that this class also implements caching.
124class CrossTranslationUnitContext {
125public:
126 CrossTranslationUnitContext(CompilerInstance &CI);
127 ~CrossTranslationUnitContext();
128
129 /// This function loads a function or variable definition from an
130 /// external AST file and merges it into the original AST.
131 ///
132 /// This method should only be used on functions that have no definitions or
133 /// variables that have no initializer in
134 /// the current translation unit. A function definition with the same
135 /// declaration will be looked up in the index file which should be in the
136 /// \p CrossTUDir directory, called \p IndexName. In case the declaration is
137 /// found in the index the corresponding AST will be loaded and the
138 /// definition will be merged into the original AST using the AST Importer.
139 ///
140 /// \return The declaration with the definition will be returned.
141 /// If no suitable definition is found in the index file or multiple
142 /// definitions found error will be returned.
143 ///
144 /// Note that the AST files should also be in the \p CrossTUDir.
145 llvm::Expected<const FunctionDecl *>
146 getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir,
147 StringRef IndexName, bool DisplayCTUProgress = false);
148 llvm::Expected<const VarDecl *>
149 getCrossTUDefinition(const VarDecl *VD, StringRef CrossTUDir,
150 StringRef IndexName, bool DisplayCTUProgress = false);
151
152 /// This function loads a definition from an external AST file.
153 ///
154 /// A definition with the same declaration will be looked up in the
155 /// index file which should be in the \p CrossTUDir directory, called
156 /// \p IndexName. In case the declaration is found in the index the
157 /// corresponding AST will be loaded. If the number of TUs imported
158 /// reaches \p CTULoadTreshold, no loading is performed.
159 ///
160 /// \return Returns a pointer to the ASTUnit that contains the definition of
161 /// the looked up name or an Error.
162 /// The returned pointer is never a nullptr.
163 ///
164 /// Note that the AST files should also be in the \p CrossTUDir.
165 llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName,
166 StringRef CrossTUDir,
167 StringRef IndexName,
168 bool DisplayCTUProgress = false);
169
170 /// This function merges a definition from a separate AST Unit into
171 /// the current one which was created by the compiler instance that
172 /// was passed to the constructor.
173 ///
174 /// \return Returns the resulting definition or an error.
175 llvm::Expected<const FunctionDecl *> importDefinition(const FunctionDecl *FD,
176 ASTUnit *Unit);
177 llvm::Expected<const VarDecl *> importDefinition(const VarDecl *VD,
178 ASTUnit *Unit);
179
180 /// Get a name to identify a named decl.
181 static llvm::Optional<std::string> getLookupName(const NamedDecl *ND);
182
183 /// Emit diagnostics for the user for potential configuration errors.
184 void emitCrossTUDiagnostics(const IndexError &IE);
185
186 /// Returns the MacroExpansionContext for the imported TU to which the given
187 /// source-location corresponds.
188 /// \p ToLoc Source location in the imported-to AST.
189 /// \note If any error happens such as \p ToLoc is a non-imported
190 /// source-location, empty is returned.
191 /// \note Macro expansion tracking for imported TUs is not implemented yet.
192 /// It returns empty unconditionally.
193 llvm::Optional<clang::MacroExpansionContext>
194 getMacroExpansionContextForSourceLocation(
195 const clang::SourceLocation &ToLoc) const;
196
197private:
198 void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU);
199 ASTImporter &getOrCreateASTImporter(ASTUnit *Unit);
200 template <typename T>
201 llvm::Expected<const T *> getCrossTUDefinitionImpl(const T *D,
202 StringRef CrossTUDir,
203 StringRef IndexName,
204 bool DisplayCTUProgress);
205 template <typename T>
206 const T *findDefInDeclContext(const DeclContext *DC,
207 StringRef LookupName);
208 template <typename T>
209 llvm::Expected<const T *> importDefinitionImpl(const T *D, ASTUnit *Unit);
210
211 using ImporterMapTy =
212 llvm::DenseMap<TranslationUnitDecl *, std::unique_ptr<ASTImporter>>;
213
214 ImporterMapTy ASTUnitImporterMap;
215
216 ASTContext &Context;
217 std::shared_ptr<ASTImporterSharedState> ImporterSharedSt;
218
219 using LoadResultTy = llvm::Expected<std::unique_ptr<ASTUnit>>;
220
221 /// Loads ASTUnits from AST-dumps or source-files.
222 class ASTLoader {
223 public:
224 ASTLoader(CompilerInstance &CI, StringRef CTUDir,
225 StringRef InvocationListFilePath);
226
227 /// Load the ASTUnit by its identifier found in the index file. If the
228 /// indentifier is suffixed with '.ast' it is considered a dump. Otherwise
229 /// it is treated as source-file, and on-demand parsed. Relative paths are
230 /// prefixed with CTUDir.
231 LoadResultTy load(StringRef Identifier);
232
233 /// Lazily initialize the invocation list information, which is needed for
234 /// on-demand parsing.
235 llvm::Error lazyInitInvocationList();
236
237 private:
238 /// The style used for storage and lookup of filesystem paths.
239 /// Defaults to posix.
240 const llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix;
241
242 /// Loads an AST from a pch-dump.
243 LoadResultTy loadFromDump(StringRef Identifier);
244 /// Loads an AST from a source-file.
245 LoadResultTy loadFromSource(StringRef Identifier);
246
247 CompilerInstance &CI;
248 StringRef CTUDir;
249 /// The path to the file containing the invocation list, which is in YAML
250 /// format, and contains a mapping from source files to compiler invocations
251 /// that produce the AST used for analysis.
252 StringRef InvocationListFilePath;
253 /// In case of on-demand parsing, the invocations for parsing the source
254 /// files is stored.
255 llvm::Optional<InvocationListTy> InvocationList;
256 };
257
258 /// Maintain number of AST loads and check for reaching the load limit.
259 class ASTLoadGuard {
260 public:
261 ASTLoadGuard(unsigned Limit) : Limit(Limit) {}
262
263 /// Indicates, whether a new load operation is permitted, it is within the
264 /// threshold.
265 operator bool() const { return Count < Limit; }
266
267 /// Tell that a new AST was loaded successfully.
268 void indicateLoadSuccess() { ++Count; }
269
270 private:
271 /// The number of ASTs actually imported.
272 unsigned Count{0u};
273 /// The limit (threshold) value for number of loaded ASTs.
274 const unsigned Limit;
275 };
276
277 /// Storage and load of ASTUnits, cached access, and providing searchability
278 /// are the concerns of ASTUnitStorage class.
279 class ASTUnitStorage {
280 public:
281 ASTUnitStorage(CompilerInstance &CI);
282 /// Loads an ASTUnit for a function.
283 ///
284 /// \param FunctionName USR name of the function.
285 /// \param CrossTUDir Path to the directory used to store CTU related files.
286 /// \param IndexName Name of the file inside \p CrossTUDir which maps
287 /// function USR names to file paths. These files contain the corresponding
288 /// AST-dumps.
289 /// \param DisplayCTUProgress Display a message about loading new ASTs.
290 ///
291 /// \return An Expected instance which contains the ASTUnit pointer or the
292 /// error occured during the load.
293 llvm::Expected<ASTUnit *> getASTUnitForFunction(StringRef FunctionName,
294 StringRef CrossTUDir,
295 StringRef IndexName,
296 bool DisplayCTUProgress);
297 /// Identifies the path of the file which can be used to load the ASTUnit
298 /// for a given function.
299 ///
300 /// \param FunctionName USR name of the function.
301 /// \param CrossTUDir Path to the directory used to store CTU related files.
302 /// \param IndexName Name of the file inside \p CrossTUDir which maps
303 /// function USR names to file paths. These files contain the corresponding
304 /// AST-dumps.
305 ///
306 /// \return An Expected instance containing the filepath.
307 llvm::Expected<std::string> getFileForFunction(StringRef FunctionName,
308 StringRef CrossTUDir,
309 StringRef IndexName);
310
311 private:
312 llvm::Error ensureCTUIndexLoaded(StringRef CrossTUDir, StringRef IndexName);
313 llvm::Expected<ASTUnit *> getASTUnitForFile(StringRef FileName,
314 bool DisplayCTUProgress);
315
316 template <typename... T> using BaseMapTy = llvm::StringMap<T...>;
317 using OwningMapTy = BaseMapTy<std::unique_ptr<clang::ASTUnit>>;
318 using NonOwningMapTy = BaseMapTy<clang::ASTUnit *>;
319
320 OwningMapTy FileASTUnitMap;
321 NonOwningMapTy NameASTUnitMap;
322
323 using IndexMapTy = BaseMapTy<std::string>;
324 IndexMapTy NameFileMap;
325
326 /// Loads the AST based on the identifier found in the index.
327 ASTLoader Loader;
328
329 /// Limit the number of loaded ASTs. It is used to limit the memory usage
330 /// of the CrossTranslationUnitContext. The ASTUnitStorage has the
331 /// information whether the AST to load is actually loaded or returned from
332 /// cache. This information is needed to maintain the counter.
333 ASTLoadGuard LoadGuard;
334 };
335
336 ASTUnitStorage ASTStorage;
337};
338
339} // namespace cross_tu
340} // namespace clang
341
342#endif // LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H
343