Merge pull request #737 from tweag/fetch-and-build-all-grammars

feat: add prefetch subcommand

CHANGELOG.md

@@ -46,6 +46,7 @@ This name should be decided amongst the team before the release.
 
 ### Added
 - [#705](https://github.com/tweag/topiary/pull/705) Added support for Nickel 1.7 extended pattern formatting
+- [#737](https://github.com/tweag/topiary/pull/737) Added the `prefetch` command, that prefetches and caches all grammars in the current configuration
 
 ### Fixed
 - [#720](https://github.com/tweag/topiary/pull/720) [#722](https://github.com/tweag/topiary/pull/722) [#723](https://github.com/tweag/topiary/pull/723) [#724](https://github.com/tweag/topiary/pull/724) [#735](https://github.com/tweag/topiary/pull/735)

Cargo.toml

@@ -66,9 +66,10 @@ nickel-lang-core = { version = "0.8.0", default-features = false }
 predicates = "3.0"
 pretty_assertions = "1.3"
 prettydiff = { version = "0.6.4", default-features = false }
+rayon = "1.10.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-tempfile = "3.5"
+tempfile = "3.12"
 test-log = "0.2"
 tokio = "1.32"
 tokio-test = "0.4"

README.md

@@ -184,6 +184,7 @@ Commands:
   format      Format inputs
   visualise   Visualise the input's Tree-sitter parse tree
   config      Print the current configuration
+  prefetch    Prefetch all languages in the configuration
   completion  Generate shell completion script
   help        Print this message or the help of the given subcommand(s)
 
@@ -345,6 +346,26 @@ For example, in Bash:
 source <(topiary completion)
 ```
 
+#### Prefetching
+
+Topiary dynamically downloads, builds, and loads the tree-sitter grammars. In
+order to ensure offline availability or speed up startup time, the grammars can
+be prefetched and compiled.
+
+<!-- DO NOT REMOVE THE "usage" COMMENTS -->
+<!-- usage:start:prefetch-->
+```
+Prefetch all languages in the configuration
+
+Usage: topiary prefetch [OPTIONS]
+
+Options:
+  -C, --configuration <CONFIGURATION>  Configuration file [env: TOPIARY_CONFIG_FILE]
+  -v, --verbose...                     Logging verbosity (increased per occurrence)
+  -h, --help                           Print help
+```
+<!-- usage:end:prefetch -->
+
 #### Logging
 
 By default, the Topiary CLI will only output error messages. You can

bin/verify-documented-usage.sh

@@ -40,7 +40,7 @@ diff-usage() {
 }
 
 main() {
-  local -a subcommands=(ROOT format visualise config completion)
+  local -a subcommands=(ROOT format visualise config completion prefetch)
 
   local _diff
   local _subcommand

topiary-cli/src/cli.rs

@@ -140,6 +140,10 @@ pub enum Commands {
     #[command(alias = "cfg", display_order = 3)]
     Config,
 
+    /// Prefetch all languages in the configuration
+    #[command(display_order = 4)]
+    Prefetch,
+
     /// Generate shell completion script
     #[command(display_order = 100)]
     Completion {
@@ -170,6 +174,11 @@ fn traverse_fs(files: &mut Vec<PathBuf>) -> CLIResult<()> {
 pub fn get_args() -> CLIResult<Cli> {
     let mut args = Cli::parse();
 
+    // When doing prefetching, we should always output at at least verbosity level two
+    if matches!(args.command, Commands::Prefetch) && args.global.verbose < 2 {
+        args.global.verbose = 2;
+    }
+
     // This is the earliest point that we can initialise the logger, from the --verbose flags,
     // before any fallible operations have started
     env_logger::Builder::new()

topiary-cli/src/error.rs

@@ -1,5 +1,5 @@
 use std::{error, fmt, io, path::PathBuf, process::ExitCode, result};
-use topiary_config::error::TopiaryConfigError;
+use topiary_config::error::{TopiaryConfigError, TopiaryConfigFetchingError};
 use topiary_core::FormatterError;
 
 /// A convenience wrapper around `std::result::Result<T, TopiaryError>`.
@@ -112,6 +112,12 @@ impl From<TopiaryConfigError> for TopiaryError {
     }
 }
 
+impl From<TopiaryConfigFetchingError> for TopiaryError {
+    fn from(e: TopiaryConfigFetchingError) -> Self {
+        Self::Config(TopiaryConfigError::Fetching(e))
+    }
+}
+
 impl From<io::Error> for TopiaryError {
     fn from(e: io::Error) -> Self {
         match e.kind() {

topiary-cli/src/main.rs

@@ -143,6 +143,10 @@ async fn run() -> CLIResult<()> {
             print!("{:#?}", config);
         }
 
+        Commands::Prefetch => {
+            config.prefetch_languages()?;
+        }
+
         Commands::Completion { shell } => {
             // The CLI parser fails if no shell is provided/detected, so it's safe to unwrap here
             cli::completion(shell.unwrap());

topiary-config/Cargo.toml

@@ -17,22 +17,14 @@ directories.workspace = true
 itertools.workspace = true
 log.workspace = true
 nickel-lang-core.workspace = true
+rayon = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
 tempfile.workspace = true
 toml.workspace = true
 tree-sitter.workspace = true
 
 topiary-tree-sitter-facade.workspace = true
 topiary-web-tree-sitter-sys.workspace = true
-# tree-sitter-json = { workspace = true, optional = true }
-# tree-sitter-rust = { workspace = true, optional = true }
-# tree-sitter-toml = { workspace = true, optional = true }
-# tree-sitter-bash = { workspace = true, optional = true }
-# tree-sitter-css = { workspace = true, optional = true }
-# tree-sitter-nickel = { workspace = true, optional = true }
-# tree-sitter-query = { workspace = true, optional = true }
-# tree-sitter-ocaml = { workspace = true, optional = true }
-# tree-sitter-ocamllex = { workspace = true, optional = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 clap = { workspace = true, features = ["derive"] }
@@ -41,6 +33,12 @@ git2.workspace = true
 libloading.workspace = true
 
 [features]
+default = [ "parallel" ]
+
+# Enabling the `parallel` feature enables parallel computation where possible.
+# At the moment, this is only in grammar prefetching
+parallel = [ "dep:rayon" ]
+
 bash = []
 css = []
 json = []

topiary-config/src/error.rs

@@ -10,17 +10,25 @@ pub enum TopiaryConfigError {
     NoExtension(path::PathBuf),
     #[cfg(not(target_arch = "wasm32"))]
     QueryFileNotFound(path::PathBuf),
-    IoError(io::Error),
+    Io(io::Error),
     Missing,
     TreeSitterFacade(topiary_tree_sitter_facade::LanguageError),
     Nickel(nickel_lang_core::error::Error),
     NickelDeserialization(nickel_lang_core::deserialize::RustDeserializationError),
     #[cfg(not(target_arch = "wasm32"))]
-    LibLoading(libloading::Error),
-    #[cfg(not(target_arch = "wasm32"))]
+    Fetching(TopiaryConfigFetchingError),
+}
+
+#[derive(Debug)]
+/// Topiary can fetch an compile grammars, doing so may create errors.
+/// Usually, this error would be part of the `TopiaryConfigError`, however, that enum also includes `nickel_lang_core::error::Error`, which does not implement Sync/Send.
+/// Since fetching an compilation is something that can easily be parallelized, we create a special error that DOES implement Sync/Send.
+#[cfg(not(target_arch = "wasm32"))]
+pub enum TopiaryConfigFetchingError {
     Git(git2::Error),
-    #[cfg(not(target_arch = "wasm32"))]
-    Compilation(String),
+    Subprocess(String),
+    Io(io::Error),
+    LibLoading(libloading::Error),
 }
 
 impl fmt::Display for TopiaryConfigError {
@@ -32,17 +40,29 @@ impl fmt::Display for TopiaryConfigError {
             TopiaryConfigError::NoExtension(path) => write!(f, "You tried to format {} without specifying a language, but we cannot automatically detect the language because we can't find the filetype extension.", path.to_string_lossy()),
             #[cfg(not(target_arch = "wasm32"))]
             TopiaryConfigError::QueryFileNotFound(path) => write!(f, "We could not find the query file: \"{}\" anywhere. If you use the TOPIARY_LANGUAGE_DIR environment variable, make sure it set set correctly.", path.to_string_lossy()),
-            TopiaryConfigError::IoError(error) => write!(f, "We encountered an io error: {error}"),
+            TopiaryConfigError::Io(error) => write!(f, "We encountered an io error: {error}"),
             TopiaryConfigError::Missing => write!(f, "A configuration file is missing. If you passed a configuration file, make sure it exists."),
             TopiaryConfigError::TreeSitterFacade(_) => write!(f, "We could not load the grammar for the given language"),
             TopiaryConfigError::Nickel(e) => write!(f, "Nickel error: {:?}", e),
             TopiaryConfigError::NickelDeserialization(e) => write!(f, "Nickel error: {:?}", e),
             #[cfg(not(target_arch = "wasm32"))]
-            TopiaryConfigError::LibLoading(e) => write!(f, "Libloading error: {:?}", e),
-            #[cfg(not(target_arch = "wasm32"))]
-            TopiaryConfigError::Git(e) => write!(f, "Git error: {:?}", e),
-            #[cfg(not(target_arch = "wasm32"))]
-            TopiaryConfigError::Compilation(e) => write!(f, "Compilation error: {:?},", e),
+            TopiaryConfigError::Fetching(e) => write!(f, "Error Fetching Language: {:?}", e),
+        }
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl fmt::Display for TopiaryConfigFetchingError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TopiaryConfigFetchingError::Git(e) => write!(f, "Git error: {:?}", e),
+            TopiaryConfigFetchingError::Subprocess(e) => {
+                write!(f, "Compilation error: {e},")
+            }
+            TopiaryConfigFetchingError::Io(error) => {
+                write!(f, "We encountered an io error: {error}")
+            }
+            TopiaryConfigFetchingError::LibLoading(e) => write!(f, "Libloading error: {:?}", e),
         }
     }
 }
@@ -59,9 +79,23 @@ impl From<nickel_lang_core::error::Error> for TopiaryConfigError {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+impl From<TopiaryConfigFetchingError> for TopiaryConfigError {
+    fn from(e: TopiaryConfigFetchingError) -> Self {
+        Self::Fetching(e)
+    }
+}
+
 impl From<io::Error> for TopiaryConfigError {
     fn from(e: io::Error) -> Self {
-        Self::IoError(e)
+        Self::Io(e)
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl From<io::Error> for TopiaryConfigFetchingError {
+    fn from(e: io::Error) -> Self {
+        Self::Io(e)
     }
 }
 
@@ -72,14 +106,14 @@ impl From<topiary_tree_sitter_facade::LanguageError> for TopiaryConfigError {
 }
 
 #[cfg(not(target_arch = "wasm32"))]
-impl From<libloading::Error> for TopiaryConfigError {
+impl From<libloading::Error> for TopiaryConfigFetchingError {
     fn from(e: libloading::Error) -> Self {
         Self::LibLoading(e)
     }
 }
 
 #[cfg(not(target_arch = "wasm32"))]
-impl From<git2::Error> for TopiaryConfigError {
+impl From<git2::Error> for TopiaryConfigFetchingError {
     fn from(e: git2::Error) -> Self {
         Self::Git(e)
     }
@@ -89,7 +123,7 @@ impl error::Error for TopiaryConfigError {
     fn source(&self) -> Option<&(dyn error::Error + 'static)> {
         match self {
             #[cfg(not(target_arch = "wasm32"))]
-            TopiaryConfigError::IoError(e) => e.source(),
+            TopiaryConfigError::Io(e) => e.source(),
             _ => None,
         }
     }