From d4f430df6a354f989e58fcc8c24e501f9480bb93 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 21 Jul 2025 11:16:33 -0700 Subject: [PATCH 1/4] [IncludeTree] Do not pass compilation directories When using include-tree, all the relative paths references should be turned into absolute path already. There is no need to pass options like `-fdebug-compilation-dir` for debug info generation as it makes cache hit less likely when two identical compilations where performed with different compilation directories. --- clang/lib/CodeGen/CGDebugInfo.cpp | 4 ++++ .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 3 +++ clang/test/ClangScanDeps/include-tree-preserve-pch-path.c | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 44ee60d734ce7..e0ae8181139d6 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3325,6 +3325,10 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod, std::string Remapped = remapDIPath(Path); StringRef Relative(Remapped); StringRef CompDir = TheCU->getDirectory(); + // If compilation is not empty, return the path as it is. + if (CompDir.empty()) + return Remapped; + if (Relative.consume_front(CompDir)) Relative.consume_front(llvm::sys::path::get_separator()); diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 9e3377381a489..0612430ea2fde 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -103,6 +103,9 @@ void tooling::dependencies::configureInvocationForCaching( // Clear APINotes options. CI.getAPINotesOpts().ModuleSearchPaths = {}; + // Reset debug/coverage compilation dir as include-tree uses absolute path. + CodeGenOpts.DebugCompilationDir.clear(); + CodeGenOpts.CoverageCompilationDir.clear(); // Update output paths, and clear working directory. auto CWD = FileSystemOpts.WorkingDir; updateRelativePath(FrontendOpts.OutputFile, CWD); diff --git a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c index c9331dea2dc51..65e3f156cdaeb 100644 --- a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c +++ b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c @@ -1,5 +1,6 @@ // REQUIRES: ondisk_cas +// RUN: rm -rf %t // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json @@ -10,9 +11,10 @@ // CHECK: "-fmodule-format=obj" // CHECK: "-dwarf-ext-refs" +// CHECK-NOT: -fdebug-compilation-dir // RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp -// RUN: %clang @%t/pch.rsp +// RUN: %clang @%t/pch.rsp -Rcompile-job-cache // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_tu.json // RUN: FileCheck %s -input-file %t/deps_tu.json -DPREFIX=%/t @@ -21,7 +23,9 @@ // RUN: %clang @%t/tu.rsp // RUN: cat %t/tu.ll | FileCheck %s -check-prefix=LLVMIR -DPREFIX=%/t -// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "prefix.pch" +// LLVMIR: !DIFile(filename: "[[PREFIX]]/tu.c", directory: "") +// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "[[PREFIX]]/prefix.pch" +// LLVMIR: !DIFile(filename: "prefix.h", directory: "") // Extract include-tree casid // RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid From 512458bbeccd06cb75b10bed1e235fa6cd665c4d Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 23 Jul 2025 16:10:42 -0700 Subject: [PATCH 2/4] [CAS] Move CASConfiguration into LLVM Move CASConfiguration into LLVM and add a JSON based configuration file to help constructing CAS from a init file `.cas-config`. --- clang/include/clang/CAS/CASOptions.h | 65 +++-------- clang/lib/CAS/CASOptions.cpp | 45 ++------ llvm/include/llvm/CAS/CASConfiguration.h | 79 +++++++++++++ llvm/lib/CAS/CASConfiguration.cpp | 122 ++++++++++++++++++++ llvm/lib/CAS/CMakeLists.txt | 1 + llvm/unittests/CAS/CASConfigurationTest.cpp | 63 ++++++++++ llvm/unittests/CAS/CMakeLists.txt | 1 + 7 files changed, 289 insertions(+), 87 deletions(-) create mode 100644 llvm/include/llvm/CAS/CASConfiguration.h create mode 100644 llvm/lib/CAS/CASConfiguration.cpp create mode 100644 llvm/unittests/CAS/CASConfigurationTest.cpp diff --git a/clang/include/clang/CAS/CASOptions.h b/clang/include/clang/CAS/CASOptions.h index 63f5a1f6d36c6..8799d040cee54 100644 --- a/clang/include/clang/CAS/CASOptions.h +++ b/clang/include/clang/CAS/CASOptions.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_CAS_CASOPTIONS_H #include "llvm/ADT/SmallVector.h" +#include "llvm/CAS/CASConfiguration.h" #include "llvm/Support/Error.h" #include #include @@ -30,52 +31,6 @@ namespace clang { class DiagnosticsEngine; -/// Base class for options configuring which CAS to use. Separated for the -/// fields where we don't need special move/copy logic. -/// -/// TODO: Add appropriate options once we support plugins. -class CASConfiguration { -public: - enum CASKind { - UnknownCAS, - InMemoryCAS, - OnDiskCAS, - }; - - /// Kind of CAS to use. - CASKind getKind() const { - return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS; - } - - /// Path to a persistent backing store on-disk. This is optional, although \a - /// CASFileSystemRootID is unlikely to work without it. - /// - /// - "" means there is none; falls back to in-memory. - /// - "auto" is an alias for an automatically chosen location in the user's - /// system cache. - std::string CASPath; - - std::string PluginPath; - /// Each entry is a (, ) pair. - std::vector> PluginOptions; - - friend bool operator==(const CASConfiguration &LHS, - const CASConfiguration &RHS) { - return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath && - LHS.PluginOptions == RHS.PluginOptions; - } - friend bool operator!=(const CASConfiguration &LHS, - const CASConfiguration &RHS) { - return !(LHS == RHS); - } - -private: - /// Whether the configuration has been "frozen", in order to hide the kind of - /// CAS that's in use. - bool IsFrozen = false; - friend class CASOptions; -}; - /// Options configuring which CAS to use. User-accessible fields should be /// defined in CASConfiguration to enable caching a CAS instance. /// @@ -87,8 +42,18 @@ class CASConfiguration { /// clang::createVFSFromCompilerInvocation() uses the same CAS instance that /// the rest of the compiler job does, without updating all callers. Probably /// it would be better to update all callers and remove it from here. -class CASOptions : public CASConfiguration { +class CASOptions : public llvm::cas::CASConfiguration { public: + enum CASKind { + UnknownCAS, + InMemoryCAS, + OnDiskCAS, + }; + + /// Kind of CAS to use. + CASKind getKind() const { + return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS; + } /// Get a CAS & ActionCache defined by the options above. Future calls will /// return the same instances... unless the configuration has changed, in /// which case new ones will be created. @@ -117,8 +82,6 @@ class CASOptions : public CASConfiguration { /// default on-disk CAS, otherwise this is a noop. void ensurePersistentCAS(); - void getResolvedCASPath(llvm::SmallVectorImpl &Result) const; - private: /// Initialize Cached CAS and ActionCache. llvm::Error initCache() const; @@ -133,6 +96,10 @@ class CASOptions : public CASConfiguration { CASConfiguration Config; }; mutable CachedCAS Cache; + + /// Whether the configuration has been "frozen", in order to hide the kind of + /// CAS that's in use. + bool IsFrozen = false; }; } // end namespace clang diff --git a/clang/lib/CAS/CASOptions.cpp b/clang/lib/CAS/CASOptions.cpp index e9d940d5cf2be..381fde3a24820 100644 --- a/clang/lib/CAS/CASOptions.cpp +++ b/clang/lib/CAS/CASOptions.cpp @@ -22,7 +22,7 @@ std::pair, std::shared_ptr> CASOptions::getOrCreateDatabases(DiagnosticsEngine &Diags, bool CreateEmptyDBsOnFailure) const { - if (Cache.Config.IsFrozen) + if (IsFrozen) return {Cache.CAS, Cache.AC}; if (auto E = initCache()) @@ -44,7 +44,7 @@ CASOptions::getOrCreateDatabases() const { } void CASOptions::freezeConfig(DiagnosticsEngine &Diags) { - if (Cache.Config.IsFrozen) + if (IsFrozen) return; // Make sure the cache is initialized. @@ -57,7 +57,7 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) { // scheduled/executed at a level that has access to the configuration. auto &CurrentConfig = static_cast(*this); CurrentConfig = CASConfiguration(); - CurrentConfig.IsFrozen = Cache.Config.IsFrozen = true; + IsFrozen = true; if (Cache.CAS) { // Set the CASPath to the hash schema, since that leaks through CASContext's @@ -90,41 +90,10 @@ llvm::Error CASOptions::initCache() const { Cache.Config = CurrentConfig; StringRef CASPath = Cache.Config.CASPath; - if (!PluginPath.empty()) { - std::pair, std::shared_ptr> DBs; - if (llvm::Error E = - createPluginCASDatabases(PluginPath, CASPath, PluginOptions) - .moveInto(DBs)) { - return E; - } - std::tie(Cache.CAS, Cache.AC) = std::move(DBs); - return llvm::Error::success(); - } - - if (CASPath.empty()) { - Cache.CAS = llvm::cas::createInMemoryCAS(); - Cache.AC = llvm::cas::createInMemoryActionCache(); - return llvm::Error::success(); - } - - SmallString<256> PathBuf; - getResolvedCASPath(PathBuf); - if (CASPath == "auto") { - getDefaultOnDiskCASPath(PathBuf); - CASPath = PathBuf; - } - std::pair, std::unique_ptr> DBs; - if (llvm::Error E = createOnDiskUnifiedCASDatabases(CASPath).moveInto(DBs)) - return E; + auto DBs = Cache.Config.createDatabases(); + if (!DBs) + return DBs.takeError(); - std::tie(Cache.CAS, Cache.AC) = std::move(DBs); + std::tie(Cache.CAS, Cache.AC) = std::move(*DBs); return llvm::Error::success(); } - -void CASOptions::getResolvedCASPath(SmallVectorImpl &Result) const { - if (CASPath == "auto") { - getDefaultOnDiskCASPath(Result); - } else { - Result.assign(CASPath.begin(), CASPath.end()); - } -} diff --git a/llvm/include/llvm/CAS/CASConfiguration.h b/llvm/include/llvm/CAS/CASConfiguration.h new file mode 100644 index 0000000000000..c07b7b546b7cf --- /dev/null +++ b/llvm/include/llvm/CAS/CASConfiguration.h @@ -0,0 +1,79 @@ +//===- CASOptions.h - Options for configuring the CAS -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Defines the llvm::cas::CASConfiguration interface. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_CASCONFIGURATION_H +#define LLVM_CAS_CASCONFIGURATION_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" +#include +#include + +namespace llvm::cas { + +class ActionCache; +class ObjectStore; + +/// Base class for options configuring which CAS to use. +class CASConfiguration { +public: + /// Path to a persistent backing store on-disk. + /// + /// - "" means there is none; falls back to in-memory. + /// - "auto" is an alias for an automatically chosen location in the user's + /// system cache. + std::string CASPath; + /// Path to the CAS plugin library. + std::string PluginPath; + /// Each entry is a (, ) pair. + std::vector> PluginOptions; + + friend bool operator==(const CASConfiguration &LHS, + const CASConfiguration &RHS) { + return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath && + LHS.PluginOptions == RHS.PluginOptions; + } + friend bool operator!=(const CASConfiguration &LHS, + const CASConfiguration &RHS) { + return !(LHS == RHS); + } + + // Get resolved CASPath. + void getResolvedCASPath(llvm::SmallVectorImpl &Result) const; + + // Create CASDatabase from the CASConfiguration. + llvm::Expected, + std::shared_ptr>> + createDatabases() const; + + /// Write CAS configuration file. + void writeConfigurationFile(raw_ostream &OS) const; + + /// Create CASConfiguration from config file content. + static llvm::Expected + createFromConfig(llvm::StringRef Content); + + /// Create CASConfiguration from recurively search config file from a path. + /// + /// Returns the path to configuration file and its corresponding + /// CASConfiguration. + static std::optional> + createFromSearchConfigFile( + StringRef Path, + llvm::IntrusiveRefCntPtr VFS = nullptr); +}; + +} // namespace llvm::cas + +#endif diff --git a/llvm/lib/CAS/CASConfiguration.cpp b/llvm/lib/CAS/CASConfiguration.cpp new file mode 100644 index 0000000000000..c816f20ed5e4b --- /dev/null +++ b/llvm/lib/CAS/CASConfiguration.cpp @@ -0,0 +1,122 @@ +//===- CASConfiguration.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CAS/CASConfiguration.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/BuiltinUnifiedCASDatabases.h" +#include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/JSON.h" + +using namespace llvm; +using namespace llvm::cas; + +void CASConfiguration::getResolvedCASPath( + llvm::SmallVectorImpl &Result) const { + if (CASPath == "auto") { + getDefaultOnDiskCASPath(Result); + } else { + Result.assign(CASPath.begin(), CASPath.end()); + } +} + +Expected, std::shared_ptr>> +CASConfiguration::createDatabases() const { + if (!PluginPath.empty()) + return createPluginCASDatabases(PluginPath, CASPath, PluginOptions); + + if (CASPath.empty()) { + return std::pair(createInMemoryCAS(), createInMemoryActionCache()); + } + + SmallString<128> PathBuf; + getResolvedCASPath(PathBuf); + + std::pair, std::unique_ptr> DBs; + return createOnDiskUnifiedCASDatabases(PathBuf); +} + +void CASConfiguration::writeConfigurationFile(raw_ostream &OS) const { + using namespace llvm::json; + Object Root; + Root["CASPath"] = CASPath; + Root["PluginPath"] = PluginPath; + + Array PlugOpts; + for (const auto &Opt : PluginOptions) { + Object Entry; + Entry[Opt.first] = Opt.second; + PlugOpts.emplace_back(std::move(Entry)); + } + Root["PluginOptions"] = std::move(PlugOpts); + + OS << formatv("{0:2}", Value(std::move(Root))); +} + +Expected +CASConfiguration::createFromConfig(StringRef Content) { + auto Parsed = json::parse(Content); + if (!Parsed) + return Parsed.takeError(); + + CASConfiguration Config; + auto *Root = Parsed->getAsObject(); + if (!Root) + return createStringError( + "CASConfiguration file error: top level object missing"); + + if (auto CASPath = Root->getString("CASPath")) + Config.CASPath = *CASPath; + + if (auto PluginPath = Root->getString("PluginPath")) + Config.PluginPath = *PluginPath; + + if (auto *Opts = Root->getArray("PluginOptions")) { + for (auto &Opt : *Opts) { + if (auto *Arg = Opt.getAsObject()) { + for (auto &Entry : *Arg) { + if (auto V = Entry.second.getAsString()) + Config.PluginOptions.emplace_back(Entry.first.str(), *V); + } + } + } + } + + return Config; +} + +std::optional> +CASConfiguration::createFromSearchConfigFile( + StringRef Path, IntrusiveRefCntPtr VFS) { + if (!VFS) + VFS = vfs::getRealFileSystem(); + + while (!Path.empty()) { + SmallString<256> ConfigPath(Path); + sys::path::append(ConfigPath, ".cas-config"); + auto File = VFS->openFileForRead(ConfigPath); + if (!File || !*File) { + Path = sys::path::parent_path(Path); + continue; + } + + auto Buffer = (*File)->getBuffer(ConfigPath); + if (!Buffer || !*Buffer) { + Path = sys::path::parent_path(Path); + continue; + } + + auto Config = createFromConfig((*Buffer)->getBuffer()); + if (!Config) { + consumeError(Config.takeError()); + Path = sys::path::parent_path(Path); + continue; + } + return std::pair{ConfigPath.str().str(), *Config}; + } + return std::nullopt; +} diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt index ca70b0bfee9f0..2f647033a1196 100644 --- a/llvm/lib/CAS/CMakeLists.txt +++ b/llvm/lib/CAS/CMakeLists.txt @@ -7,6 +7,7 @@ add_llvm_component_library(LLVMCAS ActionCaches.cpp BuiltinCAS.cpp BuiltinUnifiedCASDatabases.cpp + CASConfiguration.cpp CASFileSystem.cpp CASNodeSchema.cpp CASOutputBackend.cpp diff --git a/llvm/unittests/CAS/CASConfigurationTest.cpp b/llvm/unittests/CAS/CASConfigurationTest.cpp new file mode 100644 index 0000000000000..0416ab799e215 --- /dev/null +++ b/llvm/unittests/CAS/CASConfigurationTest.cpp @@ -0,0 +1,63 @@ +//===- CASConfiguration.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CAS/CASConfiguration.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::cas; + +TEST(CASConfigurationTest, roundTrips) { + auto roundTripConfig = [](CASConfiguration &Config) { + std::string Serialized; + raw_string_ostream OS(Serialized); + Config.writeConfigurationFile(OS); + + std::optional NewConfig; + ASSERT_THAT_ERROR( + CASConfiguration::createFromConfig(Serialized).moveInto(NewConfig), + Succeeded()); + ASSERT_TRUE(Config == *NewConfig); + }; + + CASConfiguration Config; + roundTripConfig(Config); + + Config.CASPath = "/tmp"; + roundTripConfig(Config); + + Config.PluginPath = "/test.plug"; + roundTripConfig(Config); + + Config.PluginOptions.emplace_back("a", "b"); + roundTripConfig(Config); + + Config.PluginOptions.emplace_back("c", "d"); + roundTripConfig(Config); +} + +TEST(CASConfigurationTest, configFileSearch) { + auto VFS = makeIntrusiveRefCnt(); + ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + + // Add an empty file. + VFS->addFile("/a/b/c/.cas-config", 0, + llvm::MemoryBuffer::getMemBufferCopy("")); + ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + + VFS->addFile("/a/b/c/d/.cas-config", 0, + llvm::MemoryBuffer::getMemBufferCopy("{\"CASPath\": \"/tmp\"}")); + CASConfiguration Config; + Config.CASPath = "/tmp"; + auto NewConfig = + CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS); + ASSERT_TRUE(NewConfig); + ASSERT_TRUE(NewConfig->first == "/a/b/c/d/.cas-config"); + ASSERT_TRUE(Config == NewConfig->second); +} diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt index c7f65479e8e1c..c6fb564dd4621 100644 --- a/llvm/unittests/CAS/CMakeLists.txt +++ b/llvm/unittests/CAS/CMakeLists.txt @@ -18,6 +18,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_unittest(CASTests ActionCacheTest.cpp BuiltinUnifiedCASDatabasesTest.cpp + CASConfigurationTest.cpp CASFileSystemTest.cpp CASTestConfig.cpp CASOutputBackendTest.cpp From 82f4013da6b8ce8985a244e3edc2d6ab860794e0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 24 Jul 2025 11:23:04 -0700 Subject: [PATCH 3/4] Use CASID to encode splitDwarfFilename --- .../include/clang/Basic/ASTSourceDescriptor.h | 6 +- .../include/clang/Basic/DiagnosticCASKinds.td | 6 +- clang/include/clang/Basic/Module.h | 10 + .../include/clang/Frontend/CompilerInstance.h | 4 +- clang/include/clang/Serialization/ASTReader.h | 10 + .../clang/Serialization/InMemoryModuleCache.h | 11 +- .../include/clang/Serialization/ModuleFile.h | 3 + clang/lib/Basic/ASTSourceDescriptor.cpp | 1 + clang/lib/CodeGen/CGDebugInfo.cpp | 5 + .../CodeGen/ObjectFilePCHContainerWriter.cpp | 4 +- clang/lib/Frontend/CompilerInstance.cpp | 102 +++++++-- clang/lib/Frontend/FrontendAction.cpp | 18 +- clang/lib/Serialization/ASTReader.cpp | 44 +++- clang/lib/Serialization/ASTWriter.cpp | 8 +- .../lib/Serialization/InMemoryModuleCache.cpp | 13 +- .../DependencyScanningWorker.cpp | 10 +- .../IncludeTreeActionController.cpp | 7 +- .../DependencyScanning/ModuleDepCollector.cpp | 18 +- .../DependencyScanning/ScanAndUpdateArgs.cpp | 12 +- clang/test/CAS/depscan-include-tree.c | 4 +- .../ClangScanDeps/cas-fs-multiple-commands.c | 12 +- clang/test/ClangScanDeps/gmodules.c | 215 ++++++++++++++++++ .../include-tree-multiple-commands.c | 12 +- .../include-tree-preserve-pch-path.c | 21 +- .../ClangScanDeps/modules-cas-context-hash.c | 6 +- .../modules-cas-fs-prefix-mapping.c | 8 +- .../modules-cas-trees-with-pch.c | 10 +- clang/test/ClangScanDeps/modules-cas-trees.c | 8 +- .../modules-include-tree-dependency-file.c | 2 +- ...include-tree-pch-common-stale-prefix-map.c | 192 ++++++++++++++++ .../modules-include-tree-prefix-map.c | 16 +- .../modules-include-tree-with-pch.c | 8 +- .../test/ClangScanDeps/modules-include-tree.c | 16 +- .../modules-pch-cas-fs-prefix-mapping.c | 16 +- llvm/tools/dsymutil/BinaryHolder.cpp | 44 +++- llvm/tools/dsymutil/BinaryHolder.h | 9 +- llvm/tools/dsymutil/Options.td | 5 + llvm/tools/dsymutil/dsymutil.cpp | 21 +- 38 files changed, 763 insertions(+), 154 deletions(-) create mode 100644 clang/test/ClangScanDeps/gmodules.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c diff --git a/clang/include/clang/Basic/ASTSourceDescriptor.h b/clang/include/clang/Basic/ASTSourceDescriptor.h index 175e0551db765..c0ce5486d4d4a 100644 --- a/clang/include/clang/Basic/ASTSourceDescriptor.h +++ b/clang/include/clang/Basic/ASTSourceDescriptor.h @@ -30,14 +30,15 @@ class ASTSourceDescriptor { StringRef Path; StringRef ASTFile; ASTFileSignature Signature; + StringRef CASID; Module *ClangModule = nullptr; public: ASTSourceDescriptor() = default; ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile, - ASTFileSignature Signature) + ASTFileSignature Signature, StringRef CASID) : PCHModuleName(std::move(Name)), Path(std::move(Path)), - ASTFile(std::move(ASTFile)), Signature(Signature) {} + ASTFile(std::move(ASTFile)), Signature(Signature), CASID(CASID) {} ASTSourceDescriptor(Module &M); std::string getModuleName() const; @@ -45,6 +46,7 @@ class ASTSourceDescriptor { StringRef getASTFile() const { return ASTFile; } ASTFileSignature getSignature() const { return Signature; } Module *getModuleOrNull() const { return ClangModule; } + StringRef getCASID() const { return CASID; } }; } // namespace clang diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index 7dd533c17d53d..f37b134859ff7 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -23,8 +23,10 @@ def err_cas_depscan_daemon_connection: Error< def err_cas_depscan_failed: Error< "CAS-based dependency scan failed: %0">, DefaultFatal; def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal; -def err_cas_unloadable_module : Error< - "module file '%0' not found: unloadable module cache key %1: %2">, DefaultFatal; +def err_cas_unloadable_module + : Error<"module file '%0' not found: unloadable %select{casid|module cache " + "key}1 %2: %3">, + DefaultFatal; def err_cas_missing_module : Error< "module file '%0' not found: missing module cache key %1: %2">, DefaultFatal; def err_cas_missing_root_id : Error< diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index cddf9d2b81ce8..eacab0e106924 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -274,6 +274,9 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. std::optional ModuleCacheKey; + /// The \c CASID for the loaded module, otherwise empty. + std::string CASID; + /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -773,6 +776,13 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } + StringRef getCASID() const { return getTopLevelModule()->CASID; } + + void setCASID(std::string ID) { + assert(getCASID().empty() || getCASID() == ID); + getTopLevelModule()->CASID = std::move(ID); + } + /// Retrieve the umbrella directory as written. std::optional getUmbrellaDirAsWritten() const { if (const auto *Dir = std::get_if(&Umbrella)) diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 6728cb82d545c..05d3f06236c08 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -999,8 +999,8 @@ class CompilerInstance : public ModuleLoader { /// "-fmodule-file-cache-key", or an imported pcm file. Used in diagnostics. /// /// \returns true on failure. - bool addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider); + bool addCachedModuleFile(StringRef Path, StringRef CASID, StringRef Provider, + bool IsKey); ModuleCache &getModuleCache() const { return *ModCache; } diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index a38572531533b..2843dd1f4f740 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -286,6 +286,14 @@ class ASTReaderListener { return false; } + /// Called for each module CASID. + /// + /// \returns true to indicate the key cannot be loaded. + virtual bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) { + return false; + } + /// Indicates that a particular module file extension has been read. virtual void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) {} @@ -342,6 +350,8 @@ class ChainedASTReaderListener : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) override; }; diff --git a/clang/include/clang/Serialization/InMemoryModuleCache.h b/clang/include/clang/Serialization/InMemoryModuleCache.h index fc3ba334fc64d..8aa3a48db6542 100644 --- a/clang/include/clang/Serialization/InMemoryModuleCache.h +++ b/clang/include/clang/Serialization/InMemoryModuleCache.h @@ -30,6 +30,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { struct PCM { std::unique_ptr Buffer; + std::string CASID; + /// Track whether this PCM is known to be good (either built or /// successfully imported by a CompilerInstance/ASTReader using this /// cache). @@ -38,6 +40,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { PCM() = default; PCM(std::unique_ptr Buffer) : Buffer(std::move(Buffer)) {} + + PCM(std::unique_ptr Buffer, llvm::StringRef CASID) + : Buffer(std::move(Buffer)), CASID(CASID.str()) {} }; /// Cache of buffers. @@ -64,7 +69,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// \post state is Tentative /// \return a reference to the buffer as a convenience. llvm::MemoryBuffer &addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer); + std::unique_ptr Buffer, + llvm::StringRef CASID = ""); /// Store a just-built PCM under the Filename. /// @@ -90,6 +96,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// Get a pointer to the pCM if it exists; else nullptr. llvm::MemoryBuffer *lookupPCM(llvm::StringRef Filename) const; + /// Get the PCM if it exits; else nullptr. + const PCM *lookup(llvm::StringRef Filename) const; + /// Check whether the PCM is final and has been shown to work. /// /// \return true iff state is Final. diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 4eeed6a65dad1..c9dd1c8bb1985 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -147,6 +147,9 @@ class ModuleFile { /// The \c ActionCache key for this module, or empty. std::string ModuleCacheKey; + /// The \c CASID for the module, or empty. + std::string CASID; + /// The CAS filesystem root ID for implicit modules built with the dependency /// scanner, or empty. std::string CASFileSystemRootID; diff --git a/clang/lib/Basic/ASTSourceDescriptor.cpp b/clang/lib/Basic/ASTSourceDescriptor.cpp index 8072c08a51d3a..6cecc8347d911 100644 --- a/clang/lib/Basic/ASTSourceDescriptor.cpp +++ b/clang/lib/Basic/ASTSourceDescriptor.cpp @@ -21,6 +21,7 @@ ASTSourceDescriptor::ASTSourceDescriptor(Module &M) Path = M.Directory->getName(); if (auto File = M.getASTFile()) ASTFile = File->getName(); + CASID = M.getCASID(); } std::string ASTSourceDescriptor::getModuleName() const { diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index e0ae8181139d6..71f8fd3e8edf9 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3355,6 +3355,11 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod, PCM = Mod.getPath(); } llvm::sys::path::append(PCM, Mod.getASTFile()); + + // FIXME: Prefer CASID if exists. + if (!Mod.getCASID().empty()) + PCM = Mod.getCASID(); + DIB.createCompileUnit( TheCU->getSourceLanguage(), // TODO: Support "Source" from external AST providers? diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp index 8b54552623872..0b20fc00df716 100644 --- a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp +++ b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp @@ -191,8 +191,8 @@ class PCHContainerGenerator : public ASTConsumer { // Prepare CGDebugInfo to emit debug info for a clang module. auto *DI = Builder->getModuleDebugInfo(); StringRef ModuleName = llvm::sys::path::filename(MainFileName); - DI->setPCHDescriptor( - {ModuleName, "", OutputFileName, ASTFileSignature::createDISentinel()}); + DI->setPCHDescriptor({ModuleName, "", OutputFileName, + ASTFileSignature::createDISentinel(), /*CASID=*/""}); DI->setModuleMap(MMap); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 45987996515a2..70fed8166b34d 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -636,6 +636,8 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; private: bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, @@ -2589,25 +2591,51 @@ void CompilerInstance::setExternalSemaSource( ExternalSemaSrc = std::move(ESS); } -static bool addCachedModuleFileToInMemoryCache( - StringRef Path, StringRef CacheKey, StringRef Provider, - cas::ObjectStore &CAS, cas::ActionCache &Cache, - ModuleCache &ModCache, DiagnosticsEngine &Diags) { +static bool addCachedModuleFileToInMemoryCache(StringRef Path, + cas::ObjectStore &CAS, + cas::ObjectRef Object, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + // FIXME: We wait to materialize each module file before proceeding, which + // introduces latency for a network CAS. Instead we should collect all the + // module keys and materialize them concurrently using \c getProxyFuture, for + // better network utilization. + auto OutputProxy = CAS.getProxy(Object); + if (!OutputProxy) { + Diags.Report(diag::err_cas_unloadable_module) + << Path << 0 << CAS.getID(Object).toString() << OutputProxy.takeError(); + return true; + } + + auto PCMID = OutputProxy->getID().toString(); + if (const auto *PCM = ModCache.getInMemoryModuleCache().lookup(Path)) { + // If the CASID in the module cache differs, return error. + if (!PCM->CASID.empty() && PCM->CASID != PCMID) + return true; - if (ModCache.getInMemoryModuleCache().lookupPCM(Path)) return false; + } + + ModCache.getInMemoryModuleCache().addPCM(Path, OutputProxy->getMemoryBuffer(), + OutputProxy->getID().toString()); + return false; +} +static bool addCachedModuleFileToInMemoryCacheFromKey( + StringRef Path, StringRef CacheKey, StringRef Provider, + cas::ObjectStore &CAS, cas::ActionCache &Cache, ModuleCache &ModCache, + DiagnosticsEngine &Diags) { auto ID = CAS.parseID(CacheKey); if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << ID.takeError(); + << Path << 1 << CacheKey << ID.takeError(); return true; } auto Value = Cache.get(*ID); if (!Value) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << Value.takeError(); + << Path << 1 << CacheKey << Value.takeError(); return true; } if (!*Value) { @@ -2628,7 +2656,7 @@ static bool addCachedModuleFileToInMemoryCache( auto ValueRef = CAS.getReference(**Value); if (!ValueRef) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << "result module cannot be loaded from CAS"; + << Path << 1 << CacheKey << "result module cannot be loaded from CAS"; return true; } @@ -2637,44 +2665,68 @@ static bool addCachedModuleFileToInMemoryCache( cas::CompileJobResultSchema Schema(CAS); if (llvm::Error E = Schema.load(*ValueRef).moveInto(Result)) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << std::move(E); + << Path << 1 << CacheKey << std::move(E); return true; } auto Output = Result->getOutput(cas::CompileJobCacheResult::OutputKind::MainOutput); if (!Output) llvm::report_fatal_error("missing main output"); - // FIXME: We wait to materialize each module file before proceeding, which - // introduces latency for a network CAS. Instead we should collect all the - // module keys and materialize them concurrently using \c getProxyFuture, for - // better network utilization. - auto OutputProxy = CAS.getProxy(Output->Object); - if (!OutputProxy) { + + return addCachedModuleFileToInMemoryCache(Path, CAS, Output->Object, ModCache, + Diags); +} + +static bool addCachedModuleFileToInMemoryCacheFromID(StringRef Filename, + cas::ObjectStore &CAS, + StringRef CASID, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + auto ID = CAS.parseID(CASID); + if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << OutputProxy.takeError(); + << Filename << 0 << CASID << ID.takeError(); return true; } + auto ModuleRef = CAS.getReference(*ID); + if (!ModuleRef) { + Diags.Report(diag::err_cas_unloadable_module) + << Filename << 1 << CASID << "does not exist in CAS"; - ModCache.getInMemoryModuleCache().addPCM(Path, - OutputProxy->getMemoryBuffer()); - return false; + return true; + } + + return addCachedModuleFileToInMemoryCache(Filename, CAS, *ModuleRef, ModCache, + Diags); } -bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider) { - return addCachedModuleFileToInMemoryCache( - Path, CacheKey, Provider, getOrCreateObjectStore(), - getOrCreateActionCache(), getModuleCache(), getDiagnostics()); +bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CASID, + StringRef Provider, bool IsKey) { + if (IsKey) + return addCachedModuleFileToInMemoryCacheFromKey( + Path, CASID, Provider, getOrCreateObjectStore(), + getOrCreateActionCache(), getModuleCache(), getDiagnostics()); + + return addCachedModuleFileToInMemoryCacheFromID( + Path, getOrCreateObjectStore(), CASID, getModuleCache(), + getDiagnostics()); } bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { // FIXME: add name/path of the importing module? - return addCachedModuleFileToInMemoryCache( + return addCachedModuleFileToInMemoryCacheFromKey( Filename, CacheKey, "imported module", CAS, Cache, ModCache, Diags); } +bool CompileCacheASTReaderHelper::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return addCachedModuleFileToInMemoryCacheFromID(Filename, CAS, CASID, + ModCache, Diags); +} + /// Verify that ID is in the CAS. Otherwise the module cache probably was /// created with a different CAS. bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 23ee2eea56f8e..0d02902d83e25 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1258,7 +1258,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // Provide any modules from the action cache. for (const auto &KeyPair : CI.getFrontendOpts().ModuleCacheKeys) if (CI.addCachedModuleFile(KeyPair.first, KeyPair.second, - "-fmodule-file-cache-key")) + "-fmodule-file-cache-key", + /*IsKey=*/true)) return false; @@ -1322,9 +1323,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener); DeleteDeserialListener = true; } + std::string PCHCASID; if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty() || IncludeTreePCH) { - StringRef PCHPath; + std::string PCHPath; DisableValidationForModuleKind DisableValidation; std::unique_ptr PCHBuffer; if (IncludeTreePCH) { @@ -1333,7 +1335,11 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, if (llvm::Error E = IncludeTreePCH->getMemoryBuffer().moveInto(PCHBuffer)) return reportError(std::move(E)); - PCHPath = PCHBuffer->getBufferIdentifier(); + auto PCHFile = IncludeTreePCH->getContents(); + if (!PCHFile) + return reportError(PCHFile.takeError()); + PCHCASID = PCHFile->getID().toString(); + PCHPath = PCHCASID; } else { PCHPath = CI.getPreprocessorOpts().ImplicitPCHInclude; DisableValidation = @@ -1345,6 +1351,12 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener, std::move(PCHBuffer)); if (!CI.getASTContext().getExternalSource()) return false; + + // After loading PCH, set its CASID for content. + if (!PCHCASID.empty()) { + auto &MF = CI.getASTReader()->getModuleManager().getPrimaryModule(); + MF.CASID = PCHCASID; + } } // If modules are enabled, create the AST reader before creating // any builtins, so that all declarations know that they might be diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 14346f63f0906..42da13236b1ea 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -276,6 +276,12 @@ bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, return First->readModuleCacheKey(ModuleName, Filename, CacheKey) || Second->readModuleCacheKey(ModuleName, Filename, CacheKey); } +bool ChainedASTReaderListener::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return First->readModuleCASID(ModuleName, Filename, CASID) || + Second->readModuleCASID(ModuleName, Filename, CASID); +} void ChainedASTReaderListener::readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) { @@ -3383,7 +3389,7 @@ ASTReader::ReadControlBlock(ModuleFile &F, std::string ImportedFile; std::string StoredFile; bool IgnoreImportedByNote = false; - StringRef ImportedCacheKey; + StringRef ImportedCASKey; // For prebuilt and explicit modules first consult the file map for // an override. Note that here we don't search prebuilt module @@ -3431,18 +3437,30 @@ ASTReader::ReadControlBlock(ModuleFile &F, } } - ImportedCacheKey = ReadStringBlob(Record, Idx, Blob); + ImportedCASKey = ReadStringBlob(Record, Idx, Blob); } - if (!ImportedCacheKey.empty()) { - if (!Listener || Listener->readModuleCacheKey( - ImportedName, ImportedFile, ImportedCacheKey)) { + if (!ImportedCASKey.empty()) { + if (!Listener) { Diag(diag::err_ast_file_not_found) << moduleKindForDiagnostic(ImportedKind) << ImportedFile << true - << ("missing or unloadable module cache key" + ImportedCacheKey) + << ("missing listener to read CAS reference" + ImportedCASKey) .str(); return Failure; } + if (ImportedKind == MK_ImplicitModule) { + if (Listener->readModuleCacheKey(ImportedName, ImportedFile, + ImportedCASKey)) { + // If the module cache key failed to read, treat the module as + // out-of-date so it can be rebuilt. + return OutOfDate; + } + } else { + if (Listener->readModuleCASID(ImportedName, ImportedFile, + ImportedCASKey)) { + return OutOfDate; + } + } } // If our client can't cope with us being out of date, we can't cope with @@ -6151,7 +6169,7 @@ bool ASTReader::readASTFileControlBlock( auto Filename = ResolveImportedPath(PathBuf, FilenameStr, ModuleDir); StringRef CacheKey = ReadStringBlob(Record, Idx, Blob); if (!CacheKey.empty()) - Listener.readModuleCacheKey(ModuleName, *Filename, CacheKey); + Listener.readModuleCASID(ModuleName, *Filename, CacheKey); Listener.visitImport(ModuleName, *Filename); break; } @@ -6372,8 +6390,16 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, F.DidReadTopLevelSubmodule = true; CurrentModule->setASTFile(F.File); CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; - if (!F.ModuleCacheKey.empty()) + if (!F.ModuleCacheKey.empty()) { CurrentModule->setModuleCacheKey(F.ModuleCacheKey); + const auto *PCM = + ModuleMgr.getModuleCache().getInMemoryModuleCache().lookup( + F.FileName); + if (PCM && !PCM->CASID.empty()) { + CurrentModule->setCASID(PCM->CASID); + F.CASID = PCM->CASID; + } + } } CurrentModule->Kind = Kind; @@ -9981,7 +10007,7 @@ std::optional ASTReader::getSourceDescriptor(unsigned ID) { StringRef FileName = llvm::sys::path::filename(MF.FileName); return ASTSourceDescriptor(ModuleName, llvm::sys::path::parent_path(MF.FileName), - FileName, MF.Signature); + FileName, MF.Signature, MF.CASID); } return std::nullopt; } diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index be0ebcfa81b16..e496c660475c4 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1660,7 +1660,13 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) { llvm::append_range(Blob, M.Signature); AddPathBlob(M.FileName, Record, Blob); - AddStringBlob(M.ModuleCacheKey, Record, Blob); + if (M.Kind == serialization::MK_ImplicitModule) + AddStringBlob(M.ModuleCacheKey, Record, Blob); + else { + assert(!(M.CASID.empty() && !M.ModuleCacheKey.empty()) && + "should not have module cache key without CASID"); + AddStringBlob(M.CASID, Record, Blob); + } } Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); diff --git a/clang/lib/Serialization/InMemoryModuleCache.cpp b/clang/lib/Serialization/InMemoryModuleCache.cpp index d35fa2a807f4d..556f9453a1351 100644 --- a/clang/lib/Serialization/InMemoryModuleCache.cpp +++ b/clang/lib/Serialization/InMemoryModuleCache.cpp @@ -23,8 +23,9 @@ InMemoryModuleCache::getPCMState(llvm::StringRef Filename) const { llvm::MemoryBuffer & InMemoryModuleCache::addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer) { - auto Insertion = PCMs.insert(std::make_pair(Filename, std::move(Buffer))); + std::unique_ptr Buffer, + llvm::StringRef CASID) { + auto Insertion = PCMs.try_emplace(Filename, PCM{std::move(Buffer), CASID}); assert(Insertion.second && "Already has a PCM"); return *Insertion.first->second.Buffer; } @@ -48,6 +49,14 @@ InMemoryModuleCache::lookupPCM(llvm::StringRef Filename) const { return I->second.Buffer.get(); } +const InMemoryModuleCache::PCM * +InMemoryModuleCache::lookup(llvm::StringRef Filename) const { + auto I = PCMs.find(Filename); + if (I == PCMs.end()) + return nullptr; + return &I->second; +} + bool InMemoryModuleCache::isPCMFinal(llvm::StringRef Filename) const { return getPCMState(Filename) == Final; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 0420a83191922..a15ccc93b5b94 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -208,7 +208,15 @@ class PrebuiltModuleListener : public ASTReaderListener { CI.getFrontendOpts().ModuleCacheKeys.emplace_back(std::string(Filename), std::string(CacheKey)); // FIXME: add name/path of the importing module? - return CI.addCachedModuleFile(Filename, CacheKey, "imported module"); + return CI.addCachedModuleFile(Filename, CacheKey, "imported module", + /*IsKey=*/true); + } + + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override { + // FIXME: add name/path of the importing module? + return CI.addCachedModuleFile(Filename, CASID, "imported module", + /*IsKey=*/false); } private: diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index afee5dbf93aa0..334c7ef41f1d8 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -698,12 +698,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, if (!CASContents) return llvm::errorCodeToError(CASContents.getError()); - StringRef PCHFilename = ""; - if (NewInvocation.getFrontendOpts().IncludeTreePreservePCHPath) - PCHFilename = PPOpts.ImplicitPCHInclude; - - auto PCHFile = - cas::IncludeTree::File::create(DB, PCHFilename, **CASContents); + auto PCHFile = cas::IncludeTree::File::create(DB, "", **CASContents); if (!PCHFile) return PCHFile.takeError(); PCHRef = PCHFile->getRef(); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index a81f07de8f968..a76552865faa3 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -385,10 +385,14 @@ ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs( // Report the prebuilt modules this module uses. for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) { - CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); - if (PrebuiltModule.ModuleCacheKey) + if (PrebuiltModule.ModuleCacheKey) { + // canonicalize the PCM path if using CAS. + auto PCMFile = llvm::sys::path::filename(PrebuiltModule.PCMFile); + CI.getMutFrontendOpts().ModuleFiles.push_back(PCMFile.str()); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back( - PrebuiltModule.PCMFile, *PrebuiltModule.ModuleCacheKey); + PCMFile, *PrebuiltModule.ModuleCacheKey); + } else + CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); } // Add module file inputs from dependencies. @@ -443,9 +447,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else @@ -462,9 +468,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 0612430ea2fde..12f9d46c308cd 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -93,13 +93,6 @@ void tooling::dependencies::configureInvocationForCaching( PPOpts.MacroIncludes.clear(); PPOpts.Includes.clear(); } - if (!FrontendOpts.IncludeTreePreservePCHPath) { - // Disable `-gmodules` to avoid debug info referencing a non-existent PCH - // filename. - // FIXME: we should also allow -gmodules if there is no PCH involved. - CodeGenOpts.DebugTypeExtRefs = false; - HSOpts.ModuleFormat = "raw"; - } // Clear APINotes options. CI.getAPINotesOpts().ModuleSearchPaths = {}; @@ -113,6 +106,9 @@ void tooling::dependencies::configureInvocationForCaching( updateRelativePath(CI.getDiagnosticOpts().DiagnosticLogFile, CWD); updateRelativePath(CI.getDependencyOutputOpts().OutputFile, CWD); FileSystemOpts.WorkingDir.clear(); + + // IncludeTree always use absolute path. Do not set debug comp dir. + CodeGenOpts.DebugCompilationDir.clear(); break; } case CachingInputKind::FileSystemRoot: { @@ -213,8 +209,6 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, mapInPlaceAll(FrontendOpts.ASTMergeFiles); Mapper.mapInPlace(FrontendOpts.OverrideRecordLayoutsFile); Mapper.mapInPlace(FrontendOpts.StatsFile); - for (auto &[Path, _] : FrontendOpts.ModuleCacheKeys) - Mapper.mapInPlace(Path); // Filesystem options. Mapper.mapInPlace(FileSystemOpts.WorkingDir); diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index a75fb2e0f8b72..628993b6ca0a2 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -14,6 +14,8 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-fmodule-format=obj" +// CHECK: "-dwarf-ext-refs" // CHECK: "-coverage-data-file=[[PREFIX]]/t.gcda" // CHECK: "-coverage-notes-file=[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" @@ -21,8 +23,6 @@ // SHOULD-NOT: "-I" // SHOULD-NOT: "[[PREFIX]]/t.c" // SHOULD-NOT: "-D" -// SHOULD-NOT: "-dwarf-ext-refs" -// SHOULD-NOT: "-fmodule-format=obj" // RUN: FileCheck %s -input-file %t/inline.d -check-prefix=DEPS -DPREFIX=%t diff --git a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c index 28bf3857cf334..09e0ad5a330f4 100644 --- a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c +++ b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c @@ -109,12 +109,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK: "[[PREFIX]]/tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[PREFIX]]/tu.c" @@ -139,12 +139,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[PREFIX]]{{.}}tu.c" // CHECK-NEXT: } @@ -217,7 +217,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -229,7 +229,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-fs // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c new file mode 100644 index 0000000000000..6f5c827e4f246 --- /dev/null +++ b/clang/test/ClangScanDeps/gmodules.c @@ -0,0 +1,215 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +/// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +/// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: %clang @%t/pch.rsp + +/// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +/// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: %clang @%t/tu.rsp + +/// Check debug info is correct. +// RUN: %clang %t/tu.o -o %t/a.out +// RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// WARN-NOT: warning: + +// RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF + +/// Check module in a different directory and compare output. +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch_2.json +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Top > %t/Top_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Left > %t/Left_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --tu-index 0 > %t/pch_2.rsp +// RUN: %clang @%t/Top_2.rsp +// RUN: %clang @%t/Left_2.rsp +// RUN: %clang @%t/pch_2.rsp -o %t/prefix_2.pch +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_2.json +// RUN: %deps-to-rsp %t/deps_2.json --module-name Right > %t/Right_2.rsp +// RUN: %deps-to-rsp %t/deps_2.json --tu-index 0 > %t/tu_2.rsp +// RUN: %clang @%t/Right_2.rsp +// RUN: %clang @%t/tu_2.rsp + +// Check all the types are available +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Prefix") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" + + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file= +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -c -o DIR/tu.o DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" +struct Prefix { struct Top top; }; + +//--- tu.c +#include "Right.h" + +int main(void) { + struct Left _left; + struct Right _right; + struct Top _top; + struct Prefix _prefix; +} diff --git a/clang/test/ClangScanDeps/include-tree-multiple-commands.c b/clang/test/ClangScanDeps/include-tree-multiple-commands.c index e13a42f8d7727..a3868d948907b 100644 --- a/clang/test/ClangScanDeps/include-tree-multiple-commands.c +++ b/clang/test/ClangScanDeps/include-tree-multiple-commands.c @@ -104,12 +104,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK-NOT: "{{.*}}tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[SRC]]/tu.c" @@ -137,12 +137,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[SRC]]/tu.c" // CHECK-NEXT: } @@ -218,7 +218,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -231,7 +231,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-include-tree // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c index 65e3f156cdaeb..e755792c6b1fa 100644 --- a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c +++ b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c @@ -4,7 +4,6 @@ // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: sed -e "s|DIR|%/t|g" %t/cdb_no_preserve.json.template > %t/cdb_no_preserve.json // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_pch.json // RUN: FileCheck %s -input-file %t/deps_pch.json -DPREFIX=%/t @@ -24,38 +23,24 @@ // RUN: cat %t/tu.ll | FileCheck %s -check-prefix=LLVMIR -DPREFIX=%/t // LLVMIR: !DIFile(filename: "[[PREFIX]]/tu.c", directory: "") -// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "[[PREFIX]]/prefix.pch" +// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "llvmcas://{{.*}}" // LLVMIR: !DIFile(filename: "prefix.h", directory: "") // Extract include-tree casid // RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid // RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -check-prefix=INCLUDE_TREE -DPREFIX=%/t -// INCLUDE_TREE: (PCH) [[PREFIX]]/prefix.pch llvmcas:// - -// RUN: clang-scan-deps -compilation-database %t/cdb_no_preserve.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_no_preserve.json -// RUN: FileCheck %s -input-file %t/deps_no_preserve.json -DPREFIX=%/t -check-prefix=NO_PRESERVE - -// Note: "raw" is the default format, so it will not show up in the arguments. -// NO_PRESERVE-NOT: "-fmodule-format= -// NO_PRESERVE-NOT: "-dwarf-ext-refs" +// INCLUDE_TREE: (PCH) llvmcas:// //--- cdb_pch.json.template [{ "directory": "DIR", - "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g -Xclang -finclude-tree-preserve-pch-path", + "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g", "file": "DIR/prefix.h" }] //--- cdb.json.template -[{ - "directory": "DIR", - "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g -Xclang -finclude-tree-preserve-pch-path", - "file": "DIR/tu.c" -}] - -//--- cdb_no_preserve.json.template [{ "directory": "DIR", "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g", diff --git a/clang/test/ClangScanDeps/modules-cas-context-hash.c b/clang/test/ClangScanDeps/modules-cas-context-hash.c index b80709d5acda1..ff2f98746ca5b 100644 --- a/clang/test/ClangScanDeps/modules-cas-context-hash.c +++ b/clang/test/ClangScanDeps/modules-cas-context-hash.c @@ -33,7 +33,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[PREFIX]]/outputs/[[HASH]]/Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] @@ -57,7 +57,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[PREFIX]]/outputs/[[HASH]]/Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] @@ -105,4 +105,4 @@ module Mod { header "Mod.h" } //--- Mod.h //--- tu.c -#include "Mod.h" \ No newline at end of file +#include "Mod.h" diff --git a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c index 24190438079f4..f9defac1b5d75 100644 --- a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c @@ -66,7 +66,7 @@ // CHECK: "-o" // CHECK: "[[PREFIX]]/modules/{{.*}}/A-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}/B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// CHECK: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -75,7 +75,7 @@ // CHECK: "/^sdk" // CHECK: "-resource-dir" // CHECK: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=B=/^modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm" +// CHECK: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // CHECK: "-isystem" // CHECK: "/^tc/lib/clang/{{.*}}/include" // CHECK: "-internal-externc-isystem" @@ -142,7 +142,7 @@ // CHECK: "/^src" // CHECK: "-fmodule-map-file=/^src/module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}A-{{.*}}.pcm" +// CHECK: "{{.*}}A-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -151,7 +151,7 @@ // CHECK: "/^sdk" // CHECK: "-resource-dir" // CHECK: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=A=/^modules/{{.*}}/A-{{.*}}.pcm" +// CHECK: "-fmodule-file=A=A-{{.*}}.pcm" // CHECK: "-isystem" // CHECK: "/^tc/lib/clang/{{.*}}/include" // CHECK: "-internal-externc-isystem" diff --git a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c index d936e618ba770..5682293038946 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c +++ b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c @@ -64,11 +64,11 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[A_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[A_PCM:.*outputs.*A-.*\.pcm]]" +// PCH-NEXT: "{{.*}}[[A_PCM:A-.*\.pcm]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: "-fmodule-file-cache-key" -// PCH: "[[B_PCM:.*outputs.*B-.*\.pcm]]" +// PCH: "[[B_PCM:B-.*\.pcm]]" // PCH: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // PCH: "-fmodule-file={{(B=)?}}[[B_PCM]]" // PCH: ] @@ -88,7 +88,7 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[B_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[B_PCM]]" +// PCH-NEXT: "{{.*}}[[B_PCM]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: ] @@ -140,10 +140,10 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[C_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[C_PCM:.*outputs.*C-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[C_PCM:C-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:.*outputs.*B-.*\.pcm]]" +// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:B-.*\.pcm]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[B_PCM]]" // CHECK: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index 2077b697c6c13..1d1bd1de839d1 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -78,11 +78,11 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[LEFT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[LEFT_PCM:.*outputs.*Left-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[LEFT_PCM:Left-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[TOP_PCM:.*outputs.*Top-.*\.pcm]]" +// CHECK: "{{.*}}[[TOP_PCM:Top-.*\.pcm]]" // CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] @@ -107,7 +107,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[RIGHT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[RIGHT_PCM:.*outputs.*Right-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[RIGHT_PCM:Right-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" @@ -132,7 +132,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[TOP_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[TOP_PCM]]" +// CHECK-NEXT: "{{.*}}[[TOP_PCM]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c index fa5c123975401..084b0f5837fd0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c +++ b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c @@ -25,7 +25,7 @@ // DOT: digraph "dependencies" // DOT-DAG: [[TU:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}tu.m"]; // DOT-DAG: [[HEADER:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}A.h"]; -// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}Mod-{{.*}}.pcm"]; +// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="Mod-{{.*}}.pcm"]; // DOT-DAG: [[TU]] -> [[HEADER]] // DOT-DAG: [[HEADER]] -> [[PCM]] diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c new file mode 100644 index 0000000000000..f15c4244f2fd8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c @@ -0,0 +1,192 @@ +// Test that modifications to a common header (imported from both a PCH and a TU) +// cause rebuilds of dependent modules imported from the TU on incremental build. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module mod_common { header "mod_common.h" } +module mod_tu { header "mod_tu.h" } +module mod_tu_extra { header "mod_tu_extra.h" } + +//--- mod_common.h +#define MOD_COMMON_MACRO 0 +//--- mod_tu.h +#include "mod_common.h" +#if MOD_COMMON_MACRO +#include "mod_tu_extra.h" +#endif + +//--- mod_tu_extra.h + +//--- prefix.h +#include "mod_common.h" + +//--- tu.c +#include "mod_tu.h" + +// Clean: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Clean: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_clean.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_clean.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Clean: scan the TU. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_clean.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-CLEAN -DPREFIX=%/t +// CHECK-TU-CLEAN: { +// CHECK-TU-CLEAN-NEXT: "modules": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [], +// CHECK-TU-CLEAN-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "link-libraries": [], +// CHECK-TU-CLEAN-NEXT: "name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "translation-units": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "commands": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "module-name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "executable": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ] +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN: ] +// CHECK-TU-CLEAN: } + +// Clean: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_clean.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_clean.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp + +// Incremental: modify the common module. +// RUN: sleep 1 +// RUN: echo "#define MOD_COMMON_MACRO 1" > %t/mod_common.h +// RUN: echo "#define MOD_COMMON_MACRO_NEW 1" >> %t/mod_common.h + +// Incremental: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Incremental: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Incremental: scan the TU. This needs to invalidate modules imported from the +// TU that depend on modules imported from the PCH and discover the +// new dependency on 'mod_tu_extra'. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_incremental.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-INCREMENTAL -DPREFIX=%/t +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// CHECK-TU-INCREMENTAL: { +// CHECK-TU-INCREMENTAL-NEXT: "modules": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: }, +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu_extra.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "translation-units": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "commands": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "executable": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ] +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL: ] +// CHECK-TU-INCREMENTAL: } + +// Incremental: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu_extra > %t/mod_tu_extra.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu_extra.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c index 8eb7da7be4bea..46531cf8c1e89 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -132,9 +132,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -167,9 +167,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -261,13 +261,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=/^modules/{{.*}}/Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c index d8efe277c35ab..656f46c61856b 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -53,9 +53,9 @@ // CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -91,11 +91,11 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" -// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index de61673523e53..63d7af7bbec4f 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -116,9 +116,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -151,9 +151,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -273,13 +273,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c index aee657ce58e3d..0572d2bb4a0df 100644 --- a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c @@ -96,7 +96,7 @@ // PCH: "-o" // PCH: "[[PREFIX]]/modules/{{.*}}/A-{{.*}}.pcm" // PCH: "-fmodule-file-cache-key" -// PCH: "/^modules/{{.*}}/B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// PCH: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c" @@ -105,7 +105,7 @@ // PCH: "/^sdk" // PCH: "-resource-dir" // PCH: "/^tc/lib/clang/{{.*}}" -// PCH: "-fmodule-file=B=/^modules/{{.*}}/B-[[B_CONTEXT_HASH]].pcm" +// PCH: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // PCH: "-isystem" // PCH: "/^tc/lib/clang/{{.*}}/include" // PCH: "-internal-externc-isystem" @@ -172,7 +172,7 @@ // PCH: "/^src" // PCH: "-fmodule-map-file=/^src/module.modulemap" // PCH: "-fmodule-file-cache-key" -// PCH: "/^modules/{{.*}}A-{{.*}}.pcm" +// PCH: "{{.*}}A-{{.*}}.pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c-header" @@ -181,7 +181,7 @@ // PCH: "/^sdk" // PCH: "-resource-dir" // PCH: "/^tc/lib/clang/{{.*}}" -// PCH: "-fmodule-file=A=/^modules/{{.*}}/A-{{.*}}.pcm" +// PCH: "-fmodule-file=A=A-{{.*}}.pcm" // PCH: "-isystem" // PCH: "/^tc/lib/clang/{{.*}}/include" // PCH: "-internal-externc-isystem" @@ -208,9 +208,9 @@ // CHECK-NEXT: "/^src" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=/^modules/{{.*}}/B-{{.*}}.pcm" +// CHECK: "-fmodule-file=B-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}/B-{{.*}}.pcm" +// CHECK: "B-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -244,7 +244,7 @@ // CHECK-NEXT: "/^src" // CHECK: "-fmodule-map-file=/^src/module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "/^modules/{{.*}}C-{{.*}}.pcm" +// CHECK: "C-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -253,7 +253,7 @@ // CHECK-NEXT: "/^sdk" // CHECK: "-resource-dir" // CHECK-NEXT: "/^tc/lib/clang/{{.*}}" -// CHECK: "-fmodule-file=C=/^modules/{{.*}}/C-{{.*}}.pcm" +// CHECK: "-fmodule-file=C=C-{{.*}}.pcm" // CHECK: "-isystem" // CHECK-NEXT: "/^sdk/usr/local/include" // CHECK: "-isystem" diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp index 58751bddd3ba7..ed226681aea74 100644 --- a/llvm/tools/dsymutil/BinaryHolder.cpp +++ b/llvm/tools/dsymutil/BinaryHolder.cpp @@ -42,8 +42,9 @@ getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem, } BinaryHolder::BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts) - : VFS(VFS), Opts(Opts) {} + BinaryHolder::Options Opts, + std::shared_ptr CAS) + : VFS(VFS), CAS(std::move(CAS)), Opts(Opts) {} Error BinaryHolder::ArchiveEntry::load(IntrusiveRefCntPtr VFS, StringRef Filename, @@ -144,6 +145,37 @@ Error BinaryHolder::ObjectEntry::load(IntrusiveRefCntPtr VFS, return Error::success(); } +bool BinaryHolder::ObjectEntry::load(cas::ObjectStore &CAS, + StringRef PossibleID, Options Opts) { + auto ID = CAS.parseID(PossibleID); + if (!ID) { + consumeError(ID.takeError()); + return false; + } + + if (Opts.Verbose) + WithColor::note() << "try load cas object " << PossibleID << "\n"; + + auto Ref = CAS.getProxy(*ID); + if (!Ref) { + WithColor::warning() << "failed to load CAS object " << PossibleID << "\n"; + return false; + } + + // Only module files should be loaded here. Always not fat. + auto ObjectBuf = Ref->getMemoryBuffer(); + auto ErrOrObjectFile = + object::ObjectFile::createObjectFile(ObjectBuf->getMemBufferRef()); + if (!ErrOrObjectFile) { + WithColor::warning() << "malformed object loaded via " << PossibleID + << "\n"; + return false; + } + + Objects.push_back(std::move(*ErrOrObjectFile)); + return true; +} + std::vector BinaryHolder::ObjectEntry::getObjects() const { std::vector Result; @@ -265,9 +297,11 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { ObjectRefCounter[Filename]++; if (!ObjectCache.count(Filename)) { auto OE = std::make_unique(); - auto Err = OE->load(VFS, Filename, Timestamp, Opts); - if (Err) - return std::move(Err); + if (!(CAS && OE->load(*CAS, Filename, Opts))) { + auto Err = OE->load(VFS, Filename, Timestamp, Opts); + if (Err) + return std::move(Err); + } ObjectCache[Filename] = std::move(OE); } diff --git a/llvm/tools/dsymutil/BinaryHolder.h b/llvm/tools/dsymutil/BinaryHolder.h index cb5bd95978144..6a438deaa4efe 100644 --- a/llvm/tools/dsymutil/BinaryHolder.h +++ b/llvm/tools/dsymutil/BinaryHolder.h @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Object/Archive.h" #include "llvm/Object/Error.h" #include "llvm/Object/MachOUniversal.h" @@ -46,7 +47,8 @@ class BinaryHolder { }; BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts = {}); + BinaryHolder::Options Opts = {}, + std::shared_ptr CAS = nullptr); // Forward declarations for friend declaration. class ObjectEntry; @@ -67,6 +69,9 @@ class BinaryHolder { Error load(IntrusiveRefCntPtr VFS, StringRef Filename, TimestampTy Timestamp, BinaryHolder::Options = {}); + bool load(cas::ObjectStore &CAS, StringRef Filename, + BinaryHolder::Options = {}); + /// Access all owned ObjectFiles. std::vector getObjects() const; @@ -150,6 +155,8 @@ class BinaryHolder { /// Virtual File System instance. IntrusiveRefCntPtr VFS; + std::shared_ptr CAS; + Options Opts; }; diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td index ad35e55e33b12..31ad0ade4fb3b 100644 --- a/llvm/tools/dsymutil/Options.td +++ b/llvm/tools/dsymutil/Options.td @@ -218,3 +218,8 @@ def dsym_search_path: Separate<["-", "--"], "D">, MetaVarName<"">, HelpText<"Specify a directory that contain dSYM files to search for.">, Group; + +def cas : Separate<["-", "--"], "cas">, + MetaVarName<"">, + HelpText<"Specify CAS directory to load CAS references from.">, + Group; diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp index 913077eb0b06d..b77c946f337b3 100644 --- a/llvm/tools/dsymutil/dsymutil.cpp +++ b/llvm/tools/dsymutil/dsymutil.cpp @@ -23,6 +23,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/BuiltinUnifiedCASDatabases.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFVerifier.h" @@ -120,6 +123,7 @@ struct DsymutilOptions { DWARFVerify Verify = DWARFVerify::Default; ReproducerMode ReproMode = ReproducerMode::GenerateOnCrash; dsymutil::LinkOptions LinkOpts; + std::string CASPath; }; /// Return a list of input files. This function has logic for dealing with the @@ -397,6 +401,9 @@ static Expected getOptions(opt::InputArgList &Args) { for (auto *SearchPath : Args.filtered(OPT_dsym_search_path)) Options.LinkOpts.DSYMSearchPaths.push_back(SearchPath->getValue()); + if (opt::Arg *CASPath = Args.getLastArg(OPT_cas)) + Options.CASPath = CASPath->getValue(); + if (Error E = verifyOptions(Options)) return std::move(E); return Options; @@ -666,12 +673,24 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) { return EXIT_FAILURE; } + // Create CAS. Only support unified database for now. + std::shared_ptr CAS; + if (!Options.CASPath.empty()) { + auto DB = cas::createOnDiskUnifiedCASDatabases(Options.CASPath); + if (!DB) { + WithColor::error() << "failed to open CAS: " << toString(DB.takeError()) + << "\n"; + return EXIT_FAILURE; + } + CAS = std::move(DB->first); + } + for (auto &InputFile : Options.InputFiles) { // Shared a single binary holder for all the link steps. BinaryHolder::Options BinOpts; BinOpts.Verbose = Options.LinkOpts.Verbose; BinOpts.Warn = !Options.NoObjectTimestamp; - BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts); + BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts, CAS); // Dump the symbol table for each input file and requested arch if (Options.DumpStab) { From 2bbc40598e3170486af7831cb946862aee68d637 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 24 Jul 2025 13:29:19 -0700 Subject: [PATCH 4/4] dsymutil auto infer cas-path --- clang/test/ClangScanDeps/gmodules.c | 13 ++++++++++++- llvm/tools/dsymutil/BinaryHolder.cpp | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c index 6f5c827e4f246..674a6aa671435 100644 --- a/clang/test/ClangScanDeps/gmodules.c +++ b/clang/test/ClangScanDeps/gmodules.c @@ -4,6 +4,7 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json // RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json +// RUN: sed "s|DIR|%/t|g" %t/cas-config.template > %t/.cas-config /// Scan PCH // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ @@ -36,6 +37,7 @@ /// Check debug info is correct. // RUN: %clang %t/tu.o -o %t/a.out // RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// RUN: dsymutil %t/a.out -o %t/a2.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty // WARN-NOT: warning: // RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF @@ -58,7 +60,11 @@ // RUN: %deps-to-rsp %t/deps_2.json --module-name Right > %t/Right_2.rsp // RUN: %deps-to-rsp %t/deps_2.json --tu-index 0 > %t/tu_2.rsp // RUN: %clang @%t/Right_2.rsp -// RUN: %clang @%t/tu_2.rsp +// RUN: %clang @%t/tu_2.rsp -o %t/tu_2.o + +/// Diff all outputs +// RUN: diff %t/prefix.h.pch %t/prefix_2.pch +// RUN: diff %t/tu.o %t/tu_2.o // Check all the types are available // DWARF: DW_TAG_compile_unit @@ -213,3 +219,8 @@ int main(void) { struct Top _top; struct Prefix _prefix; } + +//--- cas-config.template +{ + "CASPath": "DIR/cas" +} diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp index ed226681aea74..af04248fac226 100644 --- a/llvm/tools/dsymutil/BinaryHolder.cpp +++ b/llvm/tools/dsymutil/BinaryHolder.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "BinaryHolder.h" +#include "llvm/CAS/CASConfiguration.h" #include "llvm/Object/MachO.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" @@ -263,6 +264,22 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename, return *(MemberCache[Key] = std::move(OE)); } +static std::shared_ptr searchAndCreateCAS(StringRef Path) { + auto Config = cas::CASConfiguration::createFromSearchConfigFile(Path); + if (!Config) + return nullptr; + + auto DB = Config->second.createDatabases(); + if (!DB) { + consumeError(DB.takeError()); + return nullptr; + } + + WithColor::note() << "create CAS using configuration: " << Config->first + << "'\n"; + return DB->first; +} + Expected BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { if (Opts.Verbose) @@ -297,6 +314,9 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { ObjectRefCounter[Filename]++; if (!ObjectCache.count(Filename)) { auto OE = std::make_unique(); + if (!CAS) + CAS = searchAndCreateCAS(Filename); + if (!(CAS && OE->load(*CAS, Filename, Opts))) { auto Err = OE->load(VFS, Filename, Timestamp, Opts); if (Err)