1/****************************************************************************
2 * Copyright (C) 2012-2016 Woboq GmbH
3 * Olivier Goffart <contact at woboq.com>
4 * https://woboq.com/codebrowser.html
5 *
6 * This file is part of the Woboq Code Browser.
7 *
8 * Commercial License Usage:
9 * Licensees holding valid commercial licenses provided by Woboq may use
10 * this file in accordance with the terms contained in a written agreement
11 * between the licensee and Woboq.
12 * For further information see https://woboq.com/codebrowser.html
13 *
14 * Alternatively, this work may be used under a Creative Commons
15 * Attribution-NonCommercial-ShareAlike 3.0 (CC-BY-NC-SA 3.0) License.
16 * http://creativecommons.org/licenses/by-nc-sa/3.0/deed.en_US
17 * This license does not allow you to use the code browser to assist the
18 * development of your commercial software. If you intent to do so, consider
19 * purchasing a commercial licence.
20 ****************************************************************************/
21
22
23#include "llvm/Support/CommandLine.h"
24#include "clang/Frontend/FrontendActions.h"
25#include "clang/Tooling/JSONCompilationDatabase.h"
26#include "clang/Tooling/Tooling.h"
27#include "clang/AST/ASTContext.h"
28
29#include <clang/Frontend/CompilerInstance.h>
30#include <llvm/Support/Path.h>
31#include <llvm/ADT/StringSwitch.h>
32
33#include <iostream>
34#include <fstream>
35#include <limits>
36#include <stdexcept>
37#include "annotator.h"
38#include "stringbuilder.h"
39#include "browserastvisitor.h"
40#include "preprocessorcallback.h"
41#include "projectmanager.h"
42#include "filesystem.h"
43#include "compat.h"
44#include <ctime>
45
46#include "embedded_includes.h"
47
48namespace cl = llvm::cl;
49
50cl::opt<std::string> BuildPath(
51 "b",
52 cl::value_desc("compile_commands.json"),
53 cl::desc("Path to the compilation database (compile_commands.json) If this argument is not passed, the compilation arguments can be passed on the command line after '--'"),
54 cl::Optional);
55
56cl::list<std::string> SourcePaths(
57 cl::Positional,
58 cl::desc("<sources>* [-- <compile command>]"),
59 cl::ZeroOrMore);
60
61cl::opt<std::string> OutputPath(
62 "o",
63 cl::value_desc("output path"),
64 cl::desc("Output directory where the generated files will be put"),
65 cl::Required);
66
67cl::list<std::string> ProjectPaths(
68 "p",
69 cl::value_desc("<project>:<path>[:<revision>]"),
70 cl::desc("Project specification: The name of the project, the absolute path of the source code, and the revision separated by colons. Example: -p projectname:/path/to/source/code:0.3beta"),
71 cl::ZeroOrMore);
72
73
74cl::list<std::string> ExternalProjectPaths(
75 "e",
76 cl::value_desc("<project>:<path>:<url>"),
77 cl::desc("Reference to an external project. Example: -e clang/include/clang:/opt/llvm/include/clang/:https://code.woboq.org/llvm"),
78 cl::ZeroOrMore);
79
80cl::opt<std::string> DataPath(
81 "d",
82 cl::value_desc("data path"),
83 cl::desc("Data url where all the javascript and css files are found. Can be absolute, or relative to the output directory. Defaults to ../data"),
84 cl::Optional);
85
86cl::opt<bool> ProcessAllSources(
87 "a",
88 cl::desc("Process all files from the compile_commands.json. If this argument is passed, the list of sources does not need to be passed"));
89
90cl::extrahelp extra(
91
92R"(
93
94EXAMPLES:
95
96Simple generation without compile command or project (compile command specified inline)
97 codebrowser_generator -o ~/public_html/code -d https://code.woboq.org/data $PWD -- -std=c++14 -I/opt/llvm/include
98
99With a project
100 codebrowser_generator -b $PWD/compile_commands.js -a -p codebrowser:$PWD -o ~/public_html/code
101)");
102
103#if 1
104std::string locationToString(clang::SourceLocation loc, clang::SourceManager& sm) {
105 clang::PresumedLoc fixed = sm.getPresumedLoc(loc);
106 if (!fixed.isValid())
107 return "???";
108 return (llvm::Twine(fixed.getFilename()) + ":" + llvm::Twine(fixed.getLine())).str();
109}
110#endif
111
112enum class DatabaseType {
113 InDatabase,
114 NotInDatabase,
115 ProcessFullDirectory
116};
117
118struct BrowserDiagnosticClient : clang::DiagnosticConsumer {
119 Annotator &annotator;
120 BrowserDiagnosticClient(Annotator &fm) : annotator(fm) {}
121
122 virtual void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic& Info) override {
123 std::string clas;
124 llvm::SmallString<1000> diag;
125 Info.FormatDiagnostic(diag);
126
127 switch(DiagLevel) {
128 case clang::DiagnosticsEngine::Fatal:
129 std::cerr << "FATAL ";
130 LLVM_FALLTHROUGH;
131 case clang::DiagnosticsEngine::Error:
132 std::cerr << "Error: " << locationToString(Info.getLocation(), annotator.getSourceMgr())
133 << ": " << diag.c_str() << std::endl;
134 clas = "error";
135 break;
136 case clang::DiagnosticsEngine::Warning:
137 clas = "warning";
138 break;
139 default:
140 return;
141 }
142 clang::SourceRange Range = Info.getLocation();
143 annotator.reportDiagnostic(Range, diag.c_str(), clas);
144 }
145};
146
147class BrowserASTConsumer : public clang::ASTConsumer
148{
149 clang::CompilerInstance &ci;
150 Annotator annotator;
151 DatabaseType WasInDatabase;
152public:
153 BrowserASTConsumer(clang::CompilerInstance &ci, ProjectManager &projectManager, DatabaseType WasInDatabase)
154 : clang::ASTConsumer(), ci(ci), annotator(projectManager), WasInDatabase(WasInDatabase)
155 {
156 //ci.getLangOpts().DelayedTemplateParsing = (true);
157 ci.getPreprocessor().enableIncrementalProcessing();
158 }
159 virtual ~BrowserASTConsumer() {
160 ci.getDiagnostics().setClient(new clang::IgnoringDiagConsumer, true);
161 }
162
163 virtual void Initialize(clang::ASTContext& Ctx) override {
164 annotator.setSourceMgr(Ctx.getSourceManager(), Ctx.getLangOpts());
165 annotator.setMangleContext(Ctx.createMangleContext());
166 ci.getPreprocessor().addPPCallbacks(maybe_unique(new PreprocessorCallback(
167 annotator, ci.getPreprocessor(), WasInDatabase == DatabaseType::ProcessFullDirectory)));
168 ci.getDiagnostics().setClient(new BrowserDiagnosticClient(annotator), true);
169 ci.getDiagnostics().setErrorLimit(0);
170 }
171
172 virtual bool HandleTopLevelDecl(clang::DeclGroupRef D) override {
173 if (ci.getDiagnostics().hasFatalErrorOccurred()) {
174 // Reset errors: (Hack to ignore the fatal errors.)
175 ci.getDiagnostics().Reset();
176 // When there was fatal error, processing the warnings may cause crashes
177 ci.getDiagnostics().setIgnoreAllWarnings(true);
178 }
179 return true;
180 }
181
182 virtual void HandleTranslationUnit(clang::ASTContext& Ctx) override {
183
184 /* if (PP.getDiagnostics().hasErrorOccurred())
185 return;*/
186 ci.getPreprocessor().getDiagnostics().getClient();
187
188
189 BrowserASTVisitor v(annotator);
190 v.TraverseDecl(Ctx.getTranslationUnitDecl());
191
192
193 annotator.generate(ci.getSema(), WasInDatabase != DatabaseType::NotInDatabase);
194 }
195
196 virtual bool shouldSkipFunctionBody(clang::Decl *D) override {
197 return !annotator.shouldProcess(
198 clang::FullSourceLoc(D->getLocation(),annotator.getSourceMgr())
199 .getExpansionLoc().getFileID());
200 }
201};
202
203class BrowserAction : public clang::ASTFrontendAction {
204 static std::set<std::string> processed;
205 DatabaseType WasInDatabase;
206protected:
207#if CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR <= 5
208 virtual clang::ASTConsumer *
209#else
210 virtual std::unique_ptr<clang::ASTConsumer>
211#endif
212 CreateASTConsumer(clang::CompilerInstance &CI,
213 llvm::StringRef InFile) override {
214 if (processed.count(InFile.str())) {
215 std::cerr << "Skipping already processed " << InFile.str()<< std::endl;
216 return nullptr;
217 }
218 processed.insert(InFile.str());
219
220 CI.getFrontendOpts().SkipFunctionBodies = true;
221
222 return maybe_unique(new BrowserASTConsumer(CI, *projectManager, WasInDatabase));
223 }
224
225public:
226 BrowserAction(DatabaseType WasInDatabase = DatabaseType::InDatabase) : WasInDatabase(WasInDatabase) {}
227 virtual bool hasCodeCompletionSupport() const override { return true; }
228 static ProjectManager *projectManager;
229};
230
231
232std::set<std::string> BrowserAction::processed;
233ProjectManager *BrowserAction::projectManager = nullptr;
234
235static bool proceedCommand(std::vector<std::string> command, llvm::StringRef Directory,
236 llvm::StringRef file, clang::FileManager *FM, DatabaseType WasInDatabase) {
237 // This code change all the paths to be absolute paths
238 // FIXME: it is a bit fragile.
239 bool previousIsDashI = false;
240 bool previousNeedsMacro = false;
241 bool hasNoStdInc = false;
242 for(std::string &A : command) {
243 if (previousIsDashI && !A.empty() && A[0] != '/') {
244 A = Directory % "/" % A;
245 previousIsDashI = false;
246 continue;
247 } else if (A == "-I") {
248 previousIsDashI = true;
249 continue;
250 } else if (A == "-nostdinc") {
251 hasNoStdInc = true;
252 continue;
253 } else if (A == "-U" || A == "-D") {
254 previousNeedsMacro = true;
255 continue;
256 }
257 if (previousNeedsMacro) {
258 previousNeedsMacro = false;
259 continue;
260 }
261 previousIsDashI = false;
262 if (A.empty()) continue;
263 if (llvm::StringRef(A).startswith("-I") && A[2] != '/') {
264 A = "-I" % Directory % "/" % llvm::StringRef(A).substr(2);
265 continue;
266 }
267 if (A[0] == '-' || A[0] == '/') continue;
268 std::string PossiblePath = Directory % "/" % A;
269 if (llvm::sys::fs::exists(PossiblePath))
270 A = PossiblePath;
271 }
272
273#if CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR < 6
274 auto Ajust = [&](clang::tooling::ArgumentsAdjuster &&aj) { command = aj.Adjust(command); };
275 Ajust(clang::tooling::ClangSyntaxOnlyAdjuster());
276 Ajust(clang::tooling::ClangStripOutputAdjuster());
277#elif CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR < 8
278 command = clang::tooling::getClangSyntaxOnlyAdjuster()(command);
279 command = clang::tooling::getClangStripOutputAdjuster()(command);
280#else
281 command = clang::tooling::getClangSyntaxOnlyAdjuster()(command, file);
282 command = clang::tooling::getClangStripOutputAdjuster()(command, file);
283#endif
284
285 if (!hasNoStdInc) {
286#ifndef _WIN32
287 command.push_back("-isystem");
288#else
289 command.push_back("-I");
290#endif
291
292 command.push_back("/builtins");
293 }
294
295 command.push_back("-Qunused-arguments");
296 command.push_back("-Wno-unknown-warning-option");
297 clang::tooling::ToolInvocation Inv(command, new BrowserAction(WasInDatabase), FM);
298
299 if (!hasNoStdInc) {
300 // Map the builtins includes
301 const EmbeddedFile *f = EmbeddedFiles;
302 while (f->filename) {
303 Inv.mapVirtualFile(f->filename, {f->content , f->size } );
304 f++;
305 }
306 }
307 bool result = Inv.run();
308 if (!result) {
309 std::cerr << "Error: The file was not recognized as source code: " << file.str() << std::endl;
310 }
311 return result;
312}
313
314int main(int argc, const char **argv) {
315 std::string ErrorMessage;
316 std::unique_ptr<clang::tooling::CompilationDatabase> Compilations(
317 clang::tooling::FixedCompilationDatabase::loadFromCommandLine(argc, argv
318#if CLANG_VERSION_MAJOR >= 5
319 , ErrorMessage
320#endif
321 ));
322 if (!ErrorMessage.empty()) {
323 std::cerr << ErrorMessage << std::endl;
324 ErrorMessage = {};
325 }
326
327 llvm::cl::ParseCommandLineOptions(argc, argv);
328
329#ifdef _WIN32
330 make_forward_slashes(OutputPath._Get_data()._Myptr());
331#endif
332
333 ProjectManager projectManager(OutputPath, DataPath);
334 for(std::string &s : ProjectPaths) {
335 auto colonPos = s.find(':');
336 if (colonPos >= s.size()) {
337 std::cerr << "fail to parse project option : " << s << std::endl;
338 continue;
339 }
340 auto secondColonPos = s.find(':', colonPos+1);
341 ProjectInfo info { s.substr(0, colonPos), s.substr(colonPos+1, secondColonPos - colonPos -1),
342 secondColonPos < s.size() ? s.substr(secondColonPos + 1) : std::string() };
343 if (!projectManager.addProject(std::move(info))) {
344 std::cerr << "invalid project directory for : " << s << std::endl;
345 }
346 }
347 for(std::string &s : ExternalProjectPaths) {
348 auto colonPos = s.find(':');
349 if (colonPos >= s.size()) {
350 std::cerr << "fail to parse project option : " << s << std::endl;
351 continue;
352 }
353 auto secondColonPos = s.find(':', colonPos+1);
354 if (secondColonPos >= s.size()) {
355 std::cerr << "fail to parse project option : " << s << std::endl;
356 continue;
357 }
358 ProjectInfo info { s.substr(0, colonPos), s.substr(colonPos+1, secondColonPos - colonPos -1),
359 ProjectInfo::External };
360 info.external_root_url = s.substr(secondColonPos + 1);
361 if (!projectManager.addProject(std::move(info))) {
362 std::cerr << "invalid project directory for : " << s << std::endl;
363 }
364 }
365 BrowserAction::projectManager = &projectManager;
366
367
368 if (!Compilations && llvm::sys::fs::exists(BuildPath)) {
369 if (llvm::sys::fs::is_directory(BuildPath)) {
370 Compilations = std::unique_ptr<clang::tooling::CompilationDatabase>(
371 clang::tooling::CompilationDatabase::loadFromDirectory(BuildPath, ErrorMessage));
372 } else {
373 Compilations = std::unique_ptr<clang::tooling::CompilationDatabase>(
374 clang::tooling::JSONCompilationDatabase::loadFromFile(BuildPath, ErrorMessage
375#if CLANG_VERSION_MAJOR >= 4
376 , clang::tooling::JSONCommandLineSyntax::AutoDetect
377#endif
378 ));
379 }
380 if (!Compilations && !ErrorMessage.empty()) {
381 std::cerr << ErrorMessage << std::endl;
382 }
383 }
384
385 if (!Compilations) {
386 std::cerr << "Could not load compilationdatabase. "
387 "Please use the -b option to a path containing a compile_commands.json, or use "
388 "'--' followed by the compilation commands." << std::endl;
389 return EXIT_FAILURE;
390 }
391
392 bool IsProcessingAllDirectory = false;
393 std::vector<std::string> DirContents;
394 std::vector<std::string> AllFiles = Compilations->getAllFiles();
395 std::sort(AllFiles.begin(), AllFiles.end());
396 llvm::ArrayRef<std::string> Sources = SourcePaths;
397 if (Sources.empty() && ProcessAllSources) {
398 // Because else the order is too random
399 Sources = AllFiles;
400 } else if (ProcessAllSources) {
401 std::cerr << "Cannot use both sources and '-a'" << std::endl;
402 return EXIT_FAILURE;
403 } else if (Sources.size() == 1 && llvm::sys::fs::is_directory(Sources.front())) {
404#if CLANG_VERSION_MAJOR != 3 || CLANG_VERSION_MINOR >= 5
405 // A directory was passed, process all the files in that directory
406 llvm::SmallString<128> DirName;
407 llvm::sys::path::native(Sources.front(), DirName);
408 while (DirName.endswith("/"))
409 DirName.pop_back();
410 std::error_code EC;
411 for (llvm::sys::fs::recursive_directory_iterator it(DirName.str(), EC), DirEnd;
412 it != DirEnd && !EC; it.increment(EC)) {
413 if (llvm::sys::path::filename(it->path()).startswith(".")) {
414 it.no_push();
415 continue;
416 }
417 DirContents.push_back(it->path());
418 }
419 Sources = DirContents;
420 IsProcessingAllDirectory = true;
421 if (EC) {
422 std::cerr << "Error reading the directory: " << EC.message() << std::endl;
423 return EXIT_FAILURE;
424 }
425
426 if (ProjectPaths.empty()) {
427 ProjectInfo info { llvm::sys::path::filename(DirName), DirName.str() };
428 projectManager.addProject(std::move(info));
429 }
430#else
431 std::cerr << "Passing directory is only implemented with llvm >= 3.5" << std::endl;
432 return EXIT_FAILURE;
433#endif
434 }
435
436 if (Sources.empty()) {
437 std::cerr << "No source files. Please pass source files as argument, or use '-a'" << std::endl;
438 return EXIT_FAILURE;
439 }
440 if (ProjectPaths.empty() && !IsProcessingAllDirectory) {
441 std::cerr << "You must specify a project name and directory with '-p name:directory'" << std::endl;
442 return EXIT_FAILURE;
443 }
444
445 clang::FileManager FM({"."});
446 FM.Retain();
447 int Progress = 0;
448
449 std::vector<std::string> NotInDB;
450
451 for (const auto &it : Sources) {
452 std::string file = clang::tooling::getAbsolutePath(it);
453 Progress++;
454
455 if (it.empty() || it == "-")
456 continue;
457
458 llvm::SmallString<256> filename;
459 canonicalize(file, filename);
460
461 if (auto project = projectManager.projectForFile(filename)) {
462 if (!projectManager.shouldProcess(filename, project)) {
463 std::cerr << "Sources: Skipping already processed " << filename.c_str() << std::endl;
464 continue;
465 }
466 } else {
467 std::cerr << "Sources: Skipping file not included by any project " << filename.c_str() << std::endl;
468 continue;
469 }
470
471 bool isHeader = llvm::StringSwitch<bool>(llvm::sys::path::extension(filename))
472 .Cases(".h", ".H", ".hh", ".hpp", true)
473 .Default(false);
474
475 auto compileCommandsForFile = Compilations->getCompileCommands(file);
476 if (!compileCommandsForFile.empty() && !isHeader) {
477 std::cerr << '[' << (100 * Progress / Sources.size()) << "%] Processing " << file << "\n";
478 proceedCommand(compileCommandsForFile.front().CommandLine,
479 compileCommandsForFile.front().Directory, file, &FM,
480 IsProcessingAllDirectory ? DatabaseType::ProcessFullDirectory : DatabaseType::InDatabase);
481 } else {
482 // TODO: Try to find a command line for a file in the same path
483 std::cerr << "Delayed " << file << "\n";
484 Progress--;
485 NotInDB.push_back(filename.str());
486 continue;
487 }
488
489 }
490
491 for (const auto &it : NotInDB) {
492 std::string file = clang::tooling::getAbsolutePath(it);
493 Progress++;
494
495 if (auto project = projectManager.projectForFile(file)) {
496 if (!projectManager.shouldProcess(file, project)) {
497 std::cerr << "NotInDB: Skipping already processed " << file.c_str() << std::endl;
498 continue;
499 }
500 } else {
501 std::cerr << "NotInDB: Skipping file not included by any project " << file.c_str() << std::endl;
502 continue;
503 }
504
505 llvm::StringRef similar;
506
507 auto compileCommandsForFile = Compilations->getCompileCommands(file);
508 std::string fileForCommands = file;
509 if (compileCommandsForFile.empty()) {
510 // Find the element with the bigger prefix
511 auto lower = std::lower_bound(AllFiles.cbegin(), AllFiles.cend(), file);
512 if (lower == AllFiles.cend())
513 lower = AllFiles.cbegin();
514 compileCommandsForFile = Compilations->getCompileCommands(*lower);
515 fileForCommands = *lower;
516 }
517
518 bool success = false;
519 if (!compileCommandsForFile.empty()) {
520 std::cerr << '[' << (100 * Progress / Sources.size()) << "%] Processing " << file << "\n";
521 auto command = compileCommandsForFile.front().CommandLine;
522 std::replace(command.begin(), command.end(), fileForCommands, it);
523 if (llvm::StringRef(file).endswith(".qdoc")) {
524 command.insert(command.begin() + 1, "-xc++");
525 // include the header for this .qdoc file
526 command.push_back("-include");
527 command.push_back(llvm::StringRef(file).substr(0, file.size() - 5) % ".h");
528 }
529 success = proceedCommand(std::move(command), compileCommandsForFile.front().Directory,
530 file, &FM,
531 IsProcessingAllDirectory ? DatabaseType::ProcessFullDirectory : DatabaseType::NotInDatabase);
532 } else {
533 std::cerr << "Could not find commands for " << file << "\n";
534 }
535
536 if (!success && !IsProcessingAllDirectory) {
537 ProjectInfo *projectinfo = projectManager.projectForFile(file);
538 if (!projectinfo)
539 continue;
540 if (!projectManager.shouldProcess(file, projectinfo))
541 continue;
542
543 auto now = std::time(0);
544 auto tm = localtime(&now);
545 char buf[80];
546 std::strftime(buf, sizeof(buf), "%Y-%b-%d", tm);
547
548 std::string footer = "Generated on <em>" % std::string(buf) % "</em>"
549 % " from project " % projectinfo->name % "</a>";
550 if (!projectinfo->revision.empty())
551 footer %= " revision <em>" % projectinfo->revision % "</em>";
552
553#if CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR <= 4
554 llvm::OwningPtr<llvm::MemoryBuffer> Buf;
555 if (!llvm::MemoryBuffer::getFile(file, Buf))
556 continue;
557#else
558 auto B = llvm::MemoryBuffer::getFile(file);
559 if (!B)
560 continue;
561 std::unique_ptr<llvm::MemoryBuffer> Buf = std::move(B.get());
562#endif
563
564 std::string fn = projectinfo->name % "/" % llvm::StringRef(file).substr(projectinfo->source_path.size());
565
566 Generator g;
567 g.generate(projectManager.outputPrefix, projectManager.dataPath, fn,
568 Buf->getBufferStart(), Buf->getBufferEnd(), footer,
569 "Warning: This file is not a C or C++ file. It does not have highlighting.",
570 std::set<std::string>());
571
572 std::ofstream fileIndex;
573 fileIndex.open(projectManager.outputPrefix + "/otherIndex", std::ios::app);
574 if (!fileIndex)
575 continue;
576 fileIndex << fn << '\n';
577 }
578 }
579}
580
581