[libclang][reproducers] Add compilation caching support to reproducers for explicitly-built modules. #11960
+130
−11
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -694,6 +694,9 @@ struct DependencyScannerReproducerOptions { | |
| std::optional<std::string> WorkingDirectory; | ||
| std::optional<std::string> ReproducerLocation; | ||
| bool UseUniqueReproducerName; | ||
| CASOptions CASOpts; | ||
| std::shared_ptr<cas::ObjectStore> CAS; | ||
| std::shared_ptr<cas::ActionCache> Cache; | ||
|
|
||
| DependencyScannerReproducerOptions(int argc, const char *const *argv, | ||
| const char *ModuleName, | ||
|
|
@@ -750,6 +753,20 @@ clang_experimental_DependencyScannerReproducerOptions_create( | |
| UseUniqueReproducerName}); | ||
| } | ||
|
|
||
| void clang_experimental_DependencyScannerReproducerOptions_setCASOptions( | ||
| CXDependencyScannerReproducerOptions CXOptions, CXCASDatabases CDBs, | ||
| CXCASOptions CASOpts) { | ||
| DependencyScannerReproducerOptions &Opts = *unwrap(CXOptions); | ||
| if (CDBs) { | ||
| cas::WrappedCASDatabases &DBs = *cas::unwrap(CDBs); | ||
| Opts.CASOpts = DBs.CASOpts; | ||
| Opts.CAS = DBs.CAS; | ||
| Opts.Cache = DBs.Cache; | ||
| } | ||
| if (CASOpts) | ||
| Opts.CASOpts = *cas::unwrap(CASOpts); | ||
| } | ||
|
|
||
| void clang_experimental_DependencyScannerReproducerOptions_dispose( | ||
| CXDependencyScannerReproducerOptions Options) { | ||
| delete unwrap(Options); | ||
|
|
@@ -773,11 +790,18 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| return Report(CXError_InvalidArguments) | ||
| << "non-unique reproducer is allowed only in a custom location"; | ||
|
|
||
| std::shared_ptr<llvm::cas::ObjectStore> UpstreamCAS = Opts.CAS; | ||
| bool IsReproducerCASBased(UpstreamCAS); | ||
| DependencyScanningService DepsService( | ||
| ScanningMode::DependencyDirectivesScan, | ||
| IsReproducerCASBased ? ScanningOutputFormat::FullIncludeTree | ||
| : ScanningOutputFormat::Full, | ||
| Opts.CASOpts, UpstreamCAS, Opts.Cache); | ||
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = | ||
| llvm::vfs::createPhysicalFileSystem(); | ||
| if (UpstreamCAS) | ||
| FS = llvm::cas::createCASProvidingFileSystem(UpstreamCAS, std::move(FS)); | ||
| DependencyScanningTool DepsTool(DepsService, FS); | ||
|
|
||
| llvm::SmallString<128> ReproScriptPath; | ||
| int ScriptFD; | ||
|
|
@@ -844,7 +868,7 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| // it is easier to run the reproducer with a different compiler and to | ||
| // simplify running an individual command manually. | ||
| std::string ReproExecutable = "\"${CLANG:-" + Opts.BuildArgs.front() + "}\""; | ||
| auto PrintArguments = [IsReproducerCASBased, &ReproExecutable, &FileCacheName, | ||
| &ClangOpts](llvm::raw_fd_ostream &OS, | ||
| ArrayRef<std::string> Arguments, | ||
| bool RedirectOutput) { | ||
|
|
@@ -856,7 +880,8 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| ClangOpts.ParseArgs(CharArgs, MissingArgIndex, MissingArgCount, | ||
| llvm::opt::Visibility(options::CC1Option)); | ||
|
|
||
| // CAS-based reproducer doesn't use VFS overlays. | ||
| bool DidAddVFSOverlay = IsReproducerCASBased; | ||
| OS << ReproExecutable; | ||
| for (const llvm::opt::Arg *Arg : ParsedArgs) { | ||
| const llvm::opt::Option &Opt = Arg->getOption(); | ||
|
|
@@ -866,6 +891,10 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| DidAddVFSOverlay = true; | ||
| } | ||
| } | ||
| if (Opt.matches(options::OPT_fcas_path)) { | ||
| OS << " -fcas-path \"" << FileCacheName << "/cas\""; | ||
| continue; | ||
| } | ||
| bool IsOutputArg = Opt.matches(options::OPT_o) || | ||
| Opt.matches(options::OPT_dependency_file); | ||
| llvm::opt::ArgStringList OutArgs; | ||
|
|
@@ -899,6 +928,60 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| auto RealFS = llvm::vfs::getRealFileSystem(); | ||
| RealFS->setCurrentWorkingDirectory(*Opts.WorkingDirectory); | ||
|
|
||
| if (IsReproducerCASBased) { | ||
| SmallString<128> CASPath = FileCachePath; | ||
| llvm::sys::path::append(CASPath, "cas"); | ||
| clang::CASOptions ReproducerCASOpts; | ||
| ReproducerCASOpts.CASPath = CASPath.str(); | ||
| ReproducerCASOpts.PluginPath = Opts.CASOpts.PluginPath; | ||
| ReproducerCASOpts.PluginOptions = Opts.CASOpts.PluginOptions; | ||
| auto DBsOrErr = ReproducerCASOpts.getOrCreateDatabases(); | ||
| if (!DBsOrErr) | ||
| return ReportFailure() << "failed to create a CAS database\n" | ||
| << toString(DBsOrErr.takeError()); | ||
| std::shared_ptr<llvm::cas::ObjectStore> ReproCAS = DBsOrErr->first; | ||
|
|
||
| auto transplantCASIncludeTree = | ||
| [UpstreamCAS, ReproCAS]( | ||
| const std::optional<std::string> &IncludeTreeID) -> llvm::Error { | ||
| if (!IncludeTreeID.has_value()) | ||
| // Missing `IncludeTreeID` likely indicates a problem but ignore it, so | ||
| // can capture enough data to reproduce it later. | ||
| return llvm::Error::success(); | ||
| auto IDOrErr = UpstreamCAS->parseID(*IncludeTreeID); | ||
| if (!IDOrErr) | ||
| return llvm::make_error<llvm::StringError>( | ||
| "failure to parse include tree id '" + *IncludeTreeID + | ||
| "':" + toString(IDOrErr.takeError()), | ||
| llvm::inconvertibleErrorCode()); | ||
| std::optional<cas::ObjectRef> UpstreamRef = | ||
| UpstreamCAS->getReference(*IDOrErr); | ||
| if (!UpstreamRef.has_value()) | ||
| return llvm::make_error<llvm::StringError>( | ||
| "missing include tree with ID '" + *IncludeTreeID + | ||
| "' in the provided CAS object storage", | ||
| llvm::inconvertibleErrorCode()); | ||
| auto ReproRefOrErr = ReproCAS->importObject(*UpstreamCAS, *UpstreamRef); | ||
| if (!ReproRefOrErr) | ||
| return llvm::make_error<llvm::StringError>( | ||
| "failure to import an include tree with id '" + *IncludeTreeID + | ||
| "':" + toString(ReproRefOrErr.takeError()), | ||
| llvm::inconvertibleErrorCode()); | ||
| return llvm::Error::success(); | ||
| }; | ||
|
|
||
| if (auto Err = transplantCASIncludeTree(TU.IncludeTreeID)) | ||
| return ReportFailure() | ||
| << "failed to transplant a translation unit include tree due to " | ||
| << toString(std::move(Err)); | ||
| for (const ModuleDeps &ModuleDep : TU.ModuleGraph) { | ||
| if (auto Err = transplantCASIncludeTree(ModuleDep.IncludeTreeID)) | ||
| return ReportFailure() | ||
| << "failed to transplant a module '" + ModuleDep.ID.ModuleName + | ||
| "' include tree due to " | ||
| << toString(std::move(Err)); | ||
| } | ||
|
Comment on lines
+973
to
+983
Author
There was a problem hiding this comment. I've made this code not as involved as in Swift reproducers on purpose as I don't have corresponding use cases. @cachemeifyoucan do you have any use cases in mind that I should address? There was a problem hiding this comment. What use case are you talking about here?
Author
There was a problem hiding this comment. Thanks for the explanation. I didn't know about this difference between swift and clang. I don't have a specific use case in mind. I was concerned my test case wasn't representative enough and worked by accident. But seems like my approach is sufficient (both in testing and in code). |
||
| } else { | ||
| SmallString<128> VFSCachePath = FileCachePath; | ||
| llvm::sys::path::append(VFSCachePath, "vfs"); | ||
| std::string VFSCachePathStr = VFSCachePath.str().str(); | ||
|
|
@@ -919,6 +1002,7 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( | |
| llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); | ||
| if (FileCollector.writeMapping(VFSOverlayPath)) | ||
| return ReportFailure() << "failed to write a VFS overlay mapping"; | ||
| } | ||
|
|
||
| return Report(CXError_Success) | ||
| << "Created a reproducer. Sources and associated run script(s) are " | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't know if it is worth it (or common) to check the presence of an include tree in a generated CAS.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For swift, I just find a way to execute the reproducer.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the suggestion, it seems to be a more reliable way to check the logic.