[Index] Add an option to compress the record and unit files

include/swift/Frontend/FrontendOptions.h

@@ -135,6 +135,9 @@ class FrontendOptions {
 
   /// Include local definitions/references in the index data.
   bool IndexIncludeLocals = false;
+
+  /// Whether to compress the record and unit files in the index store.
+  bool IndexStoreCompress;
 
   bool SerializeDebugInfoSIL = false;
   /// If building a module from interface, ignore compiler flags

include/swift/Index/IndexRecord.h

@@ -56,7 +56,7 @@ namespace index {
 bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
                     StringRef indexStorePath, bool indexClangModules,
                     bool indexSystemModules, bool skipStdlib,
-                    bool includeLocals, bool isDebugCompilation,
+                    bool includeLocals, bool compress, bool isDebugCompilation,
                     bool isExplicitModuleBuild, StringRef targetTriple,
                     const DependencyTracker &dependencyTracker,
                     const PathRemapper &pathRemapper);
@@ -98,7 +98,7 @@ bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
 bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens,
                     StringRef moduleUnitToken, StringRef indexStorePath,
                     bool indexClangModules, bool indexSystemModules,
-                    bool skipStdlib, bool includeLocals,
+                    bool skipStdlib, bool includeLocals, bool compress,
                     bool isDebugCompilation, bool isExplicitModuleBuild,
                     StringRef targetTriple,
                     const DependencyTracker &dependencyTracker,

include/swift/Option/Options.td

@@ -1685,6 +1685,10 @@ def index_file_path : Separate<["-"], "index-file-path">,
 def index_store_path : Separate<["-"], "index-store-path">,
   Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">,
   HelpText<"Store indexing data to <path>">;
+
+def index_store_compress : Flag<["-"], "index-store-compress">,
+  Flags<[FrontendOption]>,
+  HelpText<"Compress the unit and record files in the index store">;
 
 def index_unit_output_path : Separate<["-"], "index-unit-output-path">,
   Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">,

lib/FrontendTool/FrontendTool.cpp

@@ -1884,6 +1884,7 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
                                  opts.IndexSystemModules,
                                  opts.IndexIgnoreStdlib,
                                  opts.IndexIncludeLocals,
+                                 opts.IndexStoreCompress,
                                  isDebugCompilation,
                                  opts.DisableImplicitModules,
                                  Invocation.getTargetTriple(),
@@ -1903,6 +1904,7 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
                                  opts.IndexSystemModules,
                                  opts.IndexIgnoreStdlib,
                                  opts.IndexIncludeLocals,
+                                 opts.IndexStoreCompress,
                                  isDebugCompilation,
                                  opts.DisableImplicitModules,
                                  Invocation.getTargetTriple(),

lib/Index/IndexRecord.cpp

@@ -311,9 +311,9 @@ StringRef StdlibGroupsIndexRecordingConsumer::findGroupForSymbol(const IndexSymb
 }
 
 static bool writeRecord(SymbolTracker &record, std::string Filename,
-                        std::string indexStorePath, DiagnosticEngine *diags,
+                        std::string indexStorePath, bool compress, DiagnosticEngine *diags,
                         std::string &outRecordFile) {
-  IndexRecordWriter recordWriter(indexStorePath);
+  IndexRecordWriter recordWriter(indexStorePath, compress);
   std::string error;
   auto result = recordWriter.beginRecord(
       Filename, record.hashRecord(), error, &outRecordFile);
@@ -360,25 +360,25 @@ static bool writeRecord(SymbolTracker &record, std::string Filename,
 
 static std::unique_ptr<IndexRecordingConsumer>
 makeRecordingConsumer(std::string Filename, std::string indexStorePath,
-                      bool includeLocals, DiagnosticEngine *diags,
+                      bool includeLocals, bool compress, DiagnosticEngine *diags,
                       std::string *outRecordFile,
                       bool *outFailed) {
   return std::make_unique<IndexRecordingConsumer>(includeLocals,
                                                   [=](SymbolTracker &record) {
-    *outFailed = writeRecord(record, Filename, indexStorePath, diags,
+    *outFailed = writeRecord(record, Filename, indexStorePath, compress, diags,
                              *outRecordFile);
   });
 }
 
 static bool
 recordSourceFile(SourceFile *SF, StringRef indexStorePath,
-                 bool includeLocals, DiagnosticEngine &diags,
+                 bool includeLocals, bool compress, DiagnosticEngine &diags,
                  llvm::function_ref<void(StringRef, StringRef)> callback) {
   std::string recordFile;
   bool failed = false;
   auto consumer =
       makeRecordingConsumer(SF->getFilename().str(), indexStorePath.str(),
-                            includeLocals, &diags, &recordFile, &failed);
+                            includeLocals, compress, &diags, &recordFile, &failed);
   indexSourceFile(SF, *consumer);
 
   if (!failed && !recordFile.empty())
@@ -418,6 +418,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  bool indexSystemModules,
                                  bool skipStdlib,
                                  bool includeLocals,
+                                 bool compress,
                                  bool explicitModulebuild,
                                  StringRef targetTriple,
                                  const clang::CompilerInstance &clangCI,
@@ -432,6 +433,7 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
                                   bool indexSystemModules,
                                   bool skipStdlib,
                                   bool includeLocals,
+                                  bool compress,
                                   bool explicitModuleBuild,
                                   StringRef targetTriple,
                                   const clang::CompilerInstance &clangCI,
@@ -508,6 +510,7 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
                                              indexClangModules,
                                              indexSystemModules, skipStdlib,
                                              includeLocals,
+                                             compress,
                                              explicitModuleBuild,
                                              targetTriple,
                                              clangCI, diags,
@@ -551,6 +554,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  bool indexSystemModules,
                                  bool skipStdlib,
                                  bool includeLocals,
+                                 bool compress,
                                  bool explicitModuleBuild,
                                  StringRef targetTriple,
                                  const clang::CompilerInstance &clangCI,
@@ -632,7 +636,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
     std::string recordFile;
     bool failed = false;
     auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(),
-                                          includeLocals, &diags, &recordFile, &failed);
+                                          includeLocals, compress, &diags, &recordFile, &failed);
     indexModule(module, *consumer);
 
     if (failed)
@@ -676,7 +680,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
       std::string outRecordFile;
       failed =
           failed || writeRecord(tracker, std::string(fileNameWithGroup.str()),
-                                indexStorePath.str(), &diags, outRecordFile);
+                                indexStorePath.str(), compress, &diags, outRecordFile);
       if (failed)
         return false;
       records.emplace_back(outRecordFile, moduleName.str().str());
@@ -698,7 +702,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   auto clangRemapper = pathRemapper.asClangPathRemapper();
 
   IndexUnitWriter unitWriter(
-      fileMgr, indexStorePath, "swift", swiftVersion, filename, moduleName,
+      fileMgr, indexStorePath, "swift", swiftVersion, compress, filename, moduleName,
       /*MainFile=*/{}, isSystem, /*IsModuleUnit=*/true, isDebugCompilation,
       targetTriple, sysrootPath, clangRemapper, getModuleInfoFromOpaqueModule);
 
@@ -718,7 +722,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
                                        ModuleDecl::ImportFilterKind::Default});
   StringScratchSpace moduleNameScratch;
   addModuleDependencies(imports, indexStorePath, indexClangModules,
-                        indexSystemModules, skipStdlib, includeLocals,
+                        indexSystemModules, skipStdlib, includeLocals, compress,
                         explicitModuleBuild,
                         targetTriple, clangCI, diags, unitWriter,
                         moduleNameScratch, pathRemapper, initialFile);
@@ -735,7 +739,7 @@ static bool
 recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
                      StringRef indexStorePath, bool indexClangModules,
                      bool indexSystemModules, bool skipStdlib,
-                     bool includeLocals, bool isDebugCompilation,
+                     bool includeLocals, bool compress, bool isDebugCompilation,
                      bool isExplicitModuleBuild, StringRef targetTriple,
                      ArrayRef<clang::FileEntryRef> fileDependencies,
                      const clang::CompilerInstance &clangCI,
@@ -754,7 +758,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
   StringRef swiftVersion;
   StringRef sysrootPath = clangCI.getHeaderSearchOpts().Sysroot;
   IndexUnitWriter unitWriter(
-      fileMgr, indexStorePath, "swift", swiftVersion, indexUnitToken,
+      fileMgr, indexStorePath, "swift", swiftVersion, compress, indexUnitToken,
       module->getNameStr(), *mainFile, isSystem,
       /*isModuleUnit=*/false, isDebugCompilation, targetTriple, sysrootPath,
       clangRemapper, getModuleInfoFromOpaqueModule);
@@ -765,7 +769,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
                                         ModuleDecl::getImportFilterLocal());
   StringScratchSpace moduleNameScratch;
   addModuleDependencies(imports, indexStorePath, indexClangModules,
-                        indexSystemModules, skipStdlib, includeLocals,
+                        indexSystemModules, skipStdlib, includeLocals, compress,
                         isExplicitModuleBuild, targetTriple, clangCI, diags,
                         unitWriter, moduleNameScratch, pathRemapper,
                         primarySourceFile);
@@ -774,7 +778,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
   for (auto F : fileDependencies)
     unitWriter.addFileDependency(F, /*FIXME:isSystem=*/false, /*Module=*/nullptr);
 
-  recordSourceFile(primarySourceFile, indexStorePath, includeLocals, diags,
+  recordSourceFile(primarySourceFile, indexStorePath, includeLocals, compress, diags,
                    [&](StringRef recordFile, StringRef filename) {
                      if (auto file = fileMgr.getFileRef(filename)) {
                        unitWriter.addRecordFile(
@@ -823,6 +827,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
                            bool indexSystemModules,
                            bool skipStdlib,
                            bool includeLocals,
+                           bool compress,
                            bool isDebugCompilation,
                            bool isExplicitModuleBuild,
                            StringRef targetTriple,
@@ -840,7 +845,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
 
   return recordSourceFileUnit(primarySourceFile, indexUnitToken,
                               indexStorePath, indexClangModules,
-                              indexSystemModules, skipStdlib, includeLocals,
+                              indexSystemModules, skipStdlib, includeLocals, compress,
                               isDebugCompilation, isExplicitModuleBuild,
                               targetTriple, {},
                               clangCI, pathRemapper, diags);
@@ -854,6 +859,7 @@ bool index::indexAndRecord(ModuleDecl *module,
                            bool indexSystemModules,
                            bool skipStdlib,
                            bool includeLocals,
+                           bool compress,
                            bool isDebugCompilation,
                            bool isExplicitModuleBuild,
                            StringRef targetTriple,
@@ -879,7 +885,7 @@ bool index::indexAndRecord(ModuleDecl *module,
       }
       if (recordSourceFileUnit(SF, indexUnitTokens[unitIndex],
                                indexStorePath, indexClangModules,
-                               indexSystemModules, skipStdlib, includeLocals,
+                               indexSystemModules, skipStdlib, includeLocals, compress,
                                isDebugCompilation, isExplicitModuleBuild,
                                targetTriple, {},
                                clangCI, pathRemapper, diags))

test/Index/Store/index_compress.swift

@@ -0,0 +1,6 @@
+// RUN: rm -rf %t
+// RUN: %target-swift-frontend -index-store-path %t/idx -index-store-compress -o %t.o -typecheck %s
+// RUN: c-index-test core -print-record %t/idx | %FileCheck %s
+
+func foo() {}
+// CHECK: [[@LINE-1]]:6 | function/Swift | s:4main3fooyyF | Def | rel: 0

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -922,6 +922,7 @@ struct IndexStoreOptions {
   bool IgnoreStdlib = false;
   bool DisableImplicitModules = false;
   bool IncludeLocals = false;
+  bool Compress = false;
 };
 
 struct IndexStoreInfo{};

tools/SourceKit/lib/SwiftLang/SwiftIndexing.cpp

@@ -403,6 +403,7 @@ static void emitIndexDataForSourceFile(SourceFile &PrimarySourceFile,
                                IndexOpts.IncludeSystemModules,
                                IndexOpts.IgnoreStdlib,
                                IndexOpts.IncludeLocals,
+                               IndexOpts.Compress,
                                isDebugCompilation,
                                IndexOpts.DisableImplicitModules,
                                Invocation.getTargetTriple(),

tools/SourceKit/tools/sourcekitd/lib/Service/Requests.cpp

@@ -1511,6 +1511,10 @@ getIndexStoreOpts(const RequestDict &Req, ResponseReceiver Rec) {
   if (auto IncludeLocals = Req.getOptionalInt64(KeyIncludeLocals)) {
     Opts.IncludeLocals = IncludeLocals.value() > 0;
   }
+
+  if (auto Compress = Req.getOptionalInt64(KeyCompress)) {
+    Opts.Compress = Compress.value() > 0;
+  }
 
   if (auto IgnoreClangModules = Req.getOptionalInt64(KeyIgnoreClangModules)) {
     Opts.IgnoreClangModules = IgnoreClangModules.value() > 0;

utils/gyb_sourcekit_support/UIDs.py

@@ -219,6 +219,7 @@ def __init__(self, internal_name, external_name):
     KEY('IndexStorePath', 'key.index_store_path'),
     KEY('IndexUnitOutputPath', 'key.index_unit_output_path'),
     KEY('IncludeLocals', 'key.include_locals'),
+    KEY('Compress', 'key.compress'),
     KEY('IgnoreClangModules', 'key.ignore_clang_modules'),
     KEY('IncludeSystemModules', 'key.include_system_modules'),
     KEY('IgnoreStdlib', 'key.ignore_stdlib'),