diff --git a/infra/indexer/CMakeLists.txt b/infra/indexer/CMakeLists.txt index 0c9288a04fa8..8e996cb99f58 100644 --- a/infra/indexer/CMakeLists.txt +++ b/infra/indexer/CMakeLists.txt @@ -15,7 +15,8 @@ set(CMAKE_EXE_LINKER_FLAGS "-fuse-ld=lld -lc++") include(FetchContent) FetchContent_Declare( absl - URL https://github.com/abseil/abseil-cpp/archive/d9e4955c65cd4367dd6bf46f4ccb8cd3d100540b.zip + URL https://github.com/abseil/abseil-cpp/archive/987c57f325f7fa8472fa84e1f885f7534d391b0d.zip + # https://github.com/abseil/abseil-cpp/releases/tag/20250814.0 ) FetchContent_MakeAvailable(absl) diff --git a/infra/indexer/index/file_copier.cc b/infra/indexer/index/file_copier.cc index 9a4673d6ed24..1d68efccc83b 100644 --- a/infra/indexer/index/file_copier.cc +++ b/infra/indexer/index/file_copier.cc @@ -41,10 +41,16 @@ void PreparePath(std::string& path) { FileCopier::FileCopier(absl::string_view base_path, absl::string_view index_path, - const std::vector& extra_paths) + const std::vector& extra_paths, + Behavior behavior) : base_path_(base_path), extra_paths_(extra_paths), - index_path_(index_path) { + index_path_(index_path), + behavior_(behavior) { + if (behavior_ == Behavior::kNoOp) { + return; + } + PreparePath(base_path_); for (std::string& extra_path : extra_paths_) { PreparePath(extra_path); @@ -52,7 +58,7 @@ FileCopier::FileCopier(absl::string_view base_path, } std::string FileCopier::AbsoluteToIndexPath(absl::string_view path) const { - CHECK(path.starts_with("/")) << "Absolute path expected"; + CHECK(path.starts_with("/")) << "Absolute path expected: " << path; std::string result = std::string(path); if (!base_path_.empty() && absl::StartsWith(path, base_path_)) { @@ -70,12 +76,20 @@ std::string FileCopier::AbsoluteToIndexPath(absl::string_view path) const { } void FileCopier::RegisterIndexedFile(absl::string_view index_path) { - absl::MutexLock lock(&mutex_); + if (behavior_ == Behavior::kNoOp) { + return; + } + + absl::MutexLock lock(mutex_); indexed_files_.emplace(index_path); } void FileCopier::CopyIndexedFiles() { - absl::MutexLock lock(&mutex_); + if (behavior_ == Behavior::kNoOp) { + return; + } + + absl::MutexLock lock(mutex_); for (const std::string& indexed_path : indexed_files_) { std::filesystem::path src_path = indexed_path; @@ -101,9 +115,14 @@ void FileCopier::CopyIndexedFiles() { << dst_path.parent_path() << " (error: " << error_code.message() << ")"; - QCHECK(std::filesystem::copy_file(src_path, dst_path, error_code)) - << "Failed to copy file " << src_path << " to " << dst_path - << " (error: " << error_code.message() << ")"; + using std::filesystem::copy_options; + const copy_options options = behavior_ == Behavior::kOverwriteExistingFiles + ? copy_options::overwrite_existing + : copy_options::none; + std::filesystem::copy_file(src_path, dst_path, options, error_code); + QCHECK(!error_code) << "Failed to copy file " << src_path << " to " + << dst_path << " (error: " << error_code.message() + << ")"; } } } // namespace indexer diff --git a/infra/indexer/index/file_copier.h b/infra/indexer/index/file_copier.h index c010512803c5..080f90b7c9fe 100644 --- a/infra/indexer/index/file_copier.h +++ b/infra/indexer/index/file_copier.h @@ -32,8 +32,11 @@ namespace indexer { // all InMemoryIndexes for the current project. class FileCopier { public: + enum class Behavior { kNoOp, kFailOnExistingFiles, kOverwriteExistingFiles }; + FileCopier(absl::string_view base_path, absl::string_view index_path, - const std::vector& extra_paths); + const std::vector& extra_paths, + Behavior behavior = Behavior::kFailOnExistingFiles); FileCopier(const FileCopier&) = delete; // Takes an absolute path. Rewrites this path into the representation it will @@ -51,6 +54,7 @@ class FileCopier { std::string base_path_; std::vector extra_paths_; const std::filesystem::path index_path_; + const Behavior behavior_; absl::Mutex mutex_; absl::flat_hash_set indexed_files_ ABSL_GUARDED_BY(mutex_); diff --git a/infra/indexer/main.cc b/infra/indexer/main.cc index 013793dae09f..a1fced1e5216 100644 --- a/infra/indexer/main.cc +++ b/infra/indexer/main.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include // NOLINT #include #include @@ -24,7 +22,6 @@ #include "init.h" #include "indexer/frontend/frontend.h" #include "indexer/index/file_copier.h" -#include "indexer/index/in_memory_index.h" #include "indexer/index/sqlite.h" #include "indexer/merge_queue.h" #include "absl/flags/flag.h" @@ -39,7 +36,9 @@ ABSL_FLAG(std::string, source_dir, "", "Source directory"); ABSL_FLAG(std::string, build_dir, "", "Build directory"); ABSL_FLAG(std::string, index_dir, "", - "Output index file directory (should be empty if it exists)"); + "Output index file directory (should be empty if it exists for a new " + "index or contain the old index for a --delta); required unless" + "--database_only is specified"); ABSL_FLAG(std::vector, extra_dirs, {"/"}, "Additional source directory/-ies (comma-separated)"); ABSL_FLAG(std::string, dry_run_regex, "", @@ -51,10 +50,26 @@ ABSL_FLAG(int, merge_queue_size, 16, "Length of merge queues"); ABSL_FLAG(bool, enable_expensive_checks, false, "Enable expensive database integrity checks"); ABSL_FLAG(bool, ignore_indexing_errors, false, "Ignore indexing errors"); +ABSL_FLAG(bool, delta, false, + "Has no effect with --database_only (which can however be directly " + "pointed to the delta database filename for the same effect). Expects" + " --index_dir to point to an existing database. Stores a delta" + " database alongside it only for the translation units in" + "`compile_commands.json` (for incremental indexing). IMPORTANT: The " + "latter should contain all the translation units dependent on any " + "files changed since the original index was built. Source files that " + "get indexed are copied to --index_dir (again)"); +ABSL_FLAG(std::string, database_only, "", + "Do not copy source files, only build the index database at the given" + " location (--index_dir is not effective in that case)"); + +static constexpr char kIndexDbName[] = "db.sqlite"; +static constexpr char kDeltaDbName[] = "delta.sqlite"; int main(int argc, char** argv) { - using oss_fuzz::indexer::InMemoryIndex; + using oss_fuzz::indexer::FileCopier; using oss_fuzz::indexer::MergeQueue; + using oss_fuzz::indexer::SaveAsSqlite; using clang::tooling::AllTUsToolExecutor; using clang::tooling::CompilationDatabase; @@ -62,14 +77,15 @@ int main(int argc, char** argv) { // When running inside a container, we cannot drop privileges. InitGoogleExceptChangeRootAndUser(argv[0], &argc, &argv, true); #else - InitGoogle(absl::NullSafeStringView(argv[0]), &argc, &argv, true); + InitGoogle(argv[0], &argc, &argv, true); #endif const std::string& source_dir = absl::GetFlag(FLAGS_source_dir); const std::string& build_dir = absl::GetFlag(FLAGS_build_dir); const std::string& index_dir = absl::GetFlag(FLAGS_index_dir); + const std::string& database_only = absl::GetFlag(FLAGS_database_only); const std::vector& extra_dirs = absl::GetFlag(FLAGS_extra_dirs); - auto index_path = std::filesystem::path(index_dir) / "db.sqlite"; + const bool delta = absl::GetFlag(FLAGS_delta); #ifndef NDEBUG LOG(ERROR) << "indexer is built without optimisations. Use 'blaze run -c opt'" @@ -88,7 +104,25 @@ int main(int argc, char** argv) { clang::tooling::Filter.setValue(dry_run_regex); } - oss_fuzz::indexer::FileCopier file_copier(source_dir, index_dir, extra_dirs); + std::string index_path = database_only; + FileCopier::Behavior behavior = FileCopier::Behavior::kNoOp; + if (database_only.empty()) { + if (delta) { + index_path = std::filesystem::path(index_dir) / kDeltaDbName; + behavior = FileCopier::Behavior::kOverwriteExistingFiles; + } else { + index_path = std::filesystem::path(index_dir) / kIndexDbName; + if (std::filesystem::exists(index_path)) { + LOG(ERROR) << "Index database '" << index_path + << "' already exists. Either specify an empty --index_dir or" + "use --delta or --database_only"; + return 1; + } + behavior = FileCopier::Behavior::kFailOnExistingFiles; + } + } + + FileCopier file_copier(source_dir, index_dir, extra_dirs, behavior); std::unique_ptr merge_queue = MergeQueue::Create( absl::GetFlag(FLAGS_merge_queues), absl::GetFlag(FLAGS_merge_queue_size));