Merge pull request #81494 from cachemeifyoucan/eng/PR-remove-clang-module-cas-fs

[Caching] Remove CASFS based clang module implementation

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -520,7 +520,7 @@ REMARK(matching_output_produced,none,
 
 // Caching related diagnostics
 ERROR(error_caching_no_cas_fs, none,
-    "caching is enabled without -cas-fs option, input is not immutable", ())
+    "caching is enabled without CAS file-system options, input is not immutable", ())
 ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef))
 
 REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef))

include/swift/AST/ModuleDependencies.h

@@ -1000,7 +1000,7 @@ using ModuleDependenciesKindMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+  SwiftDependencyTracker(std::shared_ptr<llvm::cas::ObjectStore> CAS,
                          llvm::PrefixMapper *Mapper,
                          const CompilerInvocation &CI);
 
@@ -1009,7 +1009,8 @@ class SwiftDependencyTracker {
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
-  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS;
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
   llvm::PrefixMapper *Mapper;
 
   struct FileEntry {
@@ -1035,10 +1036,7 @@ class SwiftDependencyScanningService {
       ClangScanningService;
 
   /// CachingOnDiskFileSystem for dependency tracking.
-  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
-
-  /// If use clang include tree.
-  bool UseClangIncludeTree = false;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> CacheFS;
 
   /// CAS Instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
@@ -1076,8 +1074,8 @@ class SwiftDependencyScanningService {
     return *SharedFilesystemCache;
   }
 
-  llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
-    assert(CacheFS && "Expect CachingOnDiskFileSystem");
+  llvm::vfs::FileSystem &getSharedCachingFS() const {
+    assert(CacheFS && "Expect a CASFileSystem");
     return *CacheFS;
   }
 
@@ -1088,20 +1086,17 @@ class SwiftDependencyScanningService {
 
   std::optional<SwiftDependencyTracker>
   createSwiftDependencyTracker(const CompilerInvocation &CI) {
-    if (!CacheFS)
+    if (!CAS)
       return std::nullopt;
 
-    return SwiftDependencyTracker(*CacheFS, Mapper.get(), CI);
+    return SwiftDependencyTracker(CAS, Mapper.get(), CI);
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
-    if (UseClangIncludeTree)
+    if (CAS)
       return llvm::cas::createCASProvidingFileSystem(
           CAS, llvm::vfs::createPhysicalFileSystem());
 
-    if (CacheFS)
-      return CacheFS->createProxyFS();
-
     return llvm::vfs::createPhysicalFileSystem();
   }
 

include/swift/Basic/CASOptions.h

@@ -37,9 +37,6 @@ class CASOptions final {
   /// CASOptions
   clang::CASOptions CASOpts;
 
-  /// CASFS Root.
-  std::vector<std::string> CASFSRootIDs;
-
   /// Clang Include Trees.
   std::string ClangIncludeTree;
 
@@ -62,9 +59,8 @@ class CASOptions final {
   /// Check to see if a CASFileSystem is required.
   bool requireCASFS() const {
     return EnableCaching &&
-           (!CASFSRootIDs.empty() || !ClangIncludeTree.empty() ||
-            !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
-            !BridgingHeaderPCHCacheKey.empty());
+           (!ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() ||
+            !InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty());
   }
 
   /// Return a hash code of any components from these options that should

include/swift/Basic/LangOptions.h

@@ -1094,12 +1094,6 @@ namespace swift {
     /// built and provided to the compiler invocation.
     bool DisableImplicitClangModules = false;
 
-    /// Enable ClangIncludeTree for explicit module builds scanning.
-    bool UseClangIncludeTree = false;
-
-    /// Using ClangIncludeTreeRoot for compilation.
-    bool HasClangIncludeTreeRoot = false;
-
     /// Whether the dependency scanner should construct all swift-frontend
     /// invocations directly from clang cc1 args.
     bool ClangImporterDirectCC1Scan = false;

include/swift/ClangImporter/ClangImporter.h

@@ -474,11 +474,6 @@ class ClangImporter final : public ClangModuleLoader {
   /// Reads the original source file name from PCH.
   std::string getOriginalSourceFile(StringRef PCHFilename);
 
-  /// Add clang dependency file names.
-  ///
-  /// \param files The list of file to append dependencies to.
-  void addClangInvovcationDependencies(std::vector<std::string> &files);
-
   /// Makes a temporary replica of the ClangImporter's CompilerInstance, reads a
   /// module map into the replica and emits a PCM file for one of the modules it
   /// declares. Delegates to clang for everything except construction of the

include/swift/Frontend/CachingUtils.h

@@ -70,7 +70,7 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
     llvm::StringRef Filename = "");
 
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
-createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
+createCASFileSystem(llvm::cas::ObjectStore &CAS,
                     const std::string &IncludeTreeRoot,
                     const std::string &IncludeTreeFileList);
 

include/swift/Option/FrontendOptions.td

@@ -1454,9 +1454,6 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
 def no_clang_include_tree: Flag<["-"], "no-clang-include-tree">,
   HelpText<"Do not use clang include tree, fallback to use CAS filesystem to build clang modules">;
 
-def cas_fs: Separate<["-"], "cas-fs">,
-  HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
-
 def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
   HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
 def clang_include_tree_filelist: Separate<["-"], "clang-include-tree-filelist">,

lib/AST/ModuleDependencies.cpp

@@ -25,7 +25,6 @@
 #include "swift/Strings.h"
 #include "clang/CAS/IncludeTree.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
-#include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -602,13 +601,16 @@ swift::dependencies::registerBackDeployLibraries(
 }
 
 SwiftDependencyTracker::SwiftDependencyTracker(
-    llvm::cas::CachingOnDiskFileSystem &FS, llvm::PrefixMapper *Mapper,
+    std::shared_ptr<llvm::cas::ObjectStore> CAS, llvm::PrefixMapper *Mapper,
     const CompilerInvocation &CI)
-    : FS(FS.createProxyFS()), Mapper(Mapper) {
+    : CAS(CAS), Mapper(Mapper) {
   auto &SearchPathOpts = CI.getSearchPathOptions();
 
+  FS = llvm::cas::createCASProvidingFileSystem(
+      CAS, llvm::vfs::createPhysicalFileSystem());
+
   auto addCommonFile = [&](StringRef path) {
-    auto file = FS.openFileForRead(path);
+    auto file = FS->openFileForRead(path);
     if (!file)
       return;
     auto status = (*file)->status();
@@ -679,7 +681,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
   llvm::SmallVector<clang::cas::IncludeTree::FileList::FileEntry> Files;
   for (auto &file : TrackedFiles) {
     auto includeTreeFile = clang::cas::IncludeTree::File::create(
-        FS->getCAS(), file.first, file.second.FileRef);
+        *CAS, file.first, file.second.FileRef);
     if (!includeTreeFile) {
       return llvm::createStringError("CASFS createTree failed for " +
                                      file.first + ": " +
@@ -691,7 +693,7 @@ SwiftDependencyTracker::createTreeFromDependencies() {
   }
 
   auto includeTreeList =
-      clang::cas::IncludeTree::FileList::create(FS->getCAS(), Files, {});
+      clang::cas::IncludeTree::FileList::create(*CAS, Files, {});
   if (!includeTreeList)
     return llvm::createStringError("casfs include-tree filelist error: " +
                                    toString(includeTreeList.takeError()));
@@ -719,14 +721,8 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
   CAS = Instance.getSharedCASInstance();
   ActionCache = Instance.getSharedCacheInstance();
 
-  auto CachingFS =
-      llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
-  if (!CachingFS) {
-    Instance.getDiags().diagnose(SourceLoc(), diag::error_cas_fs_creation,
-                                 toString(CachingFS.takeError()));
-    return true;
-  }
-  CacheFS = std::move(*CachingFS);
+  CacheFS = llvm::cas::createCASProvidingFileSystem(
+      CAS, llvm::vfs::createPhysicalFileSystem());
 
   // Setup prefix mapping.
   auto &ScannerPrefixMapper =
@@ -744,19 +740,14 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
     Mapper->sort();
   }
 
-  UseClangIncludeTree =
-      Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
   const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
-      UseClangIncludeTree
-          ? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree
-          : clang::tooling::dependencies::ScanningOutputFormat::FullTree;
+      clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree;
 
   ClangScanningService.emplace(
       clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
-      ClangScanningFormat,
-      Instance.getInvocation().getCASOptions().CASOpts,
+      ClangScanningFormat, Instance.getInvocation().getCASOptions().CASOpts,
       Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(),
-      UseClangIncludeTree ? nullptr : CacheFS,
+      /*CachingOnDiskFileSystem=*/nullptr,
       // The current working directory optimization (off by default)
       // should not impact CAS. We set the optization to all to be
       // consistent with the non-CAS case.

lib/ClangImporter/ClangImporter.cpp

@@ -1038,27 +1038,6 @@ std::string ClangImporter::getOriginalSourceFile(StringRef PCHFilename) {
       Impl.Instance->getPCHContainerReader(), Impl.Instance->getDiagnostics());
 }
 
-void ClangImporter::addClangInvovcationDependencies(
-    std::vector<std::string> &files) {
-  auto addFiles = [&files](const auto &F) {
-    files.insert(files.end(), F.begin(), F.end());
-  };
-  auto &invocation = *Impl.Invocation;
-  // FIXME: Add file dependencies that are not accounted. The long term solution
-  // is to do a dependency scanning for clang importer and use that directly.
-  SmallVector<std::string, 4> HeaderMapFileNames;
-  Impl.Instance->getPreprocessor().getHeaderSearchInfo().getHeaderMapFileNames(
-      HeaderMapFileNames);
-  addFiles(HeaderMapFileNames);
-  addFiles(invocation.getHeaderSearchOpts().VFSOverlayFiles);
-  // FIXME: Should not depend on working directory. Build system/swift driver
-  // should not pass working directory here but if that option is passed,
-  // repect that and add that into CASFS.
-  auto CWD = invocation.getFileSystemOpts().WorkingDir;
-  if (!CWD.empty())
-    files.push_back(CWD);
-}
-
 std::optional<std::string>
 ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions,
                               StringRef SwiftPCHHash, bool &isExplicit) {
@@ -1186,7 +1165,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
                clang::frontend::ActionKind::GenerateModule ||
            CI->getFrontendOpts().ProgramAction ==
                clang::frontend::ActionKind::GeneratePCH) &&
-          ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
+          !ctx.CASOpts.ClangIncludeTree.empty()) {
         CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree;
         CI->getFrontendOpts().Inputs.clear();
       }
@@ -1246,7 +1225,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
 
   std::vector<std::string> FilteredModuleMapFiles;
   for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) {
-    if (ctx.ClangImporterOpts.UseClangIncludeTree) {
+    if (ctx.CASOpts.HasImmutableFileSystem) {
       // There is no need to add any module map file here. Issue a warning and
       // drop the option.
       Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile);
@@ -2731,7 +2710,6 @@ ClangImporter::Implementation::Implementation(
           !ctx.ClangImporterOpts.BridgingHeader.empty()),
       DisableOverlayModules(ctx.ClangImporterOpts.DisableOverlayModules),
       EnableClangSPI(ctx.ClangImporterOpts.EnableClangSPI),
-      UseClangIncludeTree(ctx.ClangImporterOpts.UseClangIncludeTree),
       IsReadingBridgingPCH(false),
       CurrentVersion(ImportNameVersion::fromOptions(ctx.LangOpts)),
       Walker(DiagnosticWalker(*this)), BuffersForDiagnostics(ctx.SourceMgr),
@@ -4258,11 +4236,9 @@ ClangImporter::getSwiftExplicitModuleDirectCC1Args() const {
   auto &CGOpts = instance.getCodeGenOpts();
   CGOpts.DebugCompilationDir.clear();
 
-  if (Impl.SwiftContext.ClangImporterOpts.UseClangIncludeTree) {
     // FileSystemOptions.
-    auto &FSOpts = instance.getFileSystemOpts();
-    FSOpts.WorkingDir.clear();
-  }
+  auto &FSOpts = instance.getFileSystemOpts();
+  FSOpts.WorkingDir.clear();
 
   if (!Impl.SwiftContext.SearchPathOpts.ScannerPrefixMapper.empty()) {
     // Remap all the paths if requested.

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -147,8 +147,8 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
     // Swift frontend option for input file path (Foo.modulemap).
     swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
 
-    // Handle VFSOverlay. If include tree is used, there is no need for overlay.
-    if (!ctx.ClangImporterOpts.UseClangIncludeTree) {
+    // Handle VFSOverlay. If caching is enabled, there is no need for overlay.
+    if (!ctx.CASOpts.EnableCaching) {
       for (auto &overlay : ctx.SearchPathOpts.VFSOverlayFiles) {
         swiftArgs.push_back("-vfsoverlay");
         swiftArgs.push_back(remapPath(overlay));
@@ -217,12 +217,6 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
     ctx.CASOpts.enumerateCASConfigurationFlags(
         [&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
 
-    if (!RootID.empty()) {
-      swiftArgs.push_back("-no-clang-include-tree");
-      swiftArgs.push_back("-cas-fs");
-      swiftArgs.push_back(RootID);
-    }
-
     if (!IncludeTree.empty()) {
       swiftArgs.push_back("-clang-include-tree-root");
       swiftArgs.push_back(IncludeTree);
@@ -324,11 +318,6 @@ void ClangImporter::getBridgingHeaderOptions(
     swiftArgs.push_back("-clang-include-tree-root");
     swiftArgs.push_back(*Tree);
   }
-  if (auto CASFS = deps.CASFileSystemRootID) {
-    swiftArgs.push_back("-no-clang-include-tree");
-    swiftArgs.push_back("-cas-fs");
-    swiftArgs.push_back(*CASFS);
-  }
 }
 
 ModuleDependencyVector

lib/ClangImporter/ImporterImpl.h

@@ -478,7 +478,6 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation
   const bool BridgingHeaderExplicitlyRequested;
   const bool DisableOverlayModules;
   const bool EnableClangSPI;
-  const bool UseClangIncludeTree;
 
   bool IsReadingBridgingPCH;
   llvm::SmallVector<clang::serialization::SubmoduleID, 2> PCHImportedSubmodules;

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -595,14 +595,6 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
   auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
        {}, buildCommands, {}, {}, {});
 
-  if (ScanASTContext.CASOpts.EnableCaching) {
-    std::vector<std::string> clangDependencyFiles;
-    clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
-    llvm::for_each(clangDependencyFiles, [&](std::string &file) {
-      mainDependencies.addAuxiliaryFile(file);
-    });
-  }
-
   llvm::StringSet<> alreadyAddedModules;
   // Compute Implicit dependencies of the main module
   {

lib/DependencyScan/ScanDependencies.cpp

@@ -316,8 +316,6 @@ class ExplicitModuleDependencyResolver {
     }
 
     // Collect CAS deppendencies from clang modules.
-    if (!clangDepDetails.CASFileSystemRootID.empty())
-      rootIDs.push_back(clangDepDetails.CASFileSystemRootID);
     if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) {
       if (addIncludeTree(clangDepDetails.CASClangIncludeTreeRootID))
         return true;
@@ -521,11 +519,6 @@ class ExplicitModuleDependencyResolver {
     if (resolvingDepInfo.isSwiftInterfaceModule() ||
         resolvingDepInfo.isSwiftSourceModule()) {
       // Update with casfs option.
-      for (auto rootID : rootIDs) {
-        commandline.push_back("-cas-fs");
-        commandline.push_back(rootID);
-      }
-
       if (computeCASFileSystem(dependencyInfoCopy))
         return true;
     }
@@ -679,7 +672,6 @@ class ExplicitModuleDependencyResolver {
   const ModuleDependencyInfo &resolvingDepInfo;
 
   std::optional<SwiftDependencyTracker> tracker;
-  std::vector<std::string> rootIDs;
   std::vector<llvm::cas::ObjectRef> fileListRefs;
   std::vector<std::string> commandline;
   std::vector<std::string> bridgingHeaderBuildCmd;