Merge pull request #776 from ehuss/parallel-crates

Process crates in parallel in prepare-local

Author: Mark-Simulacrum

Cargo.lock

@@ -24,6 +24,7 @@ clap = { version = "4", features = ["derive"] }
 crates-index = { version = "2.2.0", default-features = false, features = [
     "git-performance",
     "git-https",
+    "parallel",
 ] }
 crossbeam-channel = "0.5"
 csv = "1.0.2"
@@ -45,6 +46,7 @@ percent-encoding = "2.1.0"
 prometheus = "0.13.3"
 r2d2 = "0.8.2"
 rand = "0.8"
+rayon = "1.10"
 regex = "1.0"
 remove_dir_all = "0.7"
 reqwest = { version = "0.11", features = ["blocking", "json"] }

Cargo.toml

@@ -2,48 +2,59 @@ use crate::crates::{lists::List, Crate};
 use crate::dirs::WORK_DIR;
 use crate::prelude::*;
 use crates_index::GitIndex;
+use rayon::iter::ParallelIterator;
 use std::collections::HashMap;
 use std::fs::{self};
+use std::sync::Mutex;
 
 pub(crate) struct RegistryList;
 
 impl List for RegistryList {
     const NAME: &'static str = "registry";
 
     fn fetch(&self) -> Fallible<Vec<Crate>> {
-        let mut list = Vec::new();
-        let mut counts = HashMap::new();
+        let counts = Mutex::new(HashMap::new());
 
+        debug!("updating git index");
         fs::create_dir_all(&*WORK_DIR)?;
         let mut index = GitIndex::with_path(
             WORK_DIR.join("crates.io-index"),
             "https://github.com/rust-lang/crates.io-index",
         )?;
         index.update()?;
+        debug!("collecting crate information");
 
-        for krate in index.crates() {
-            // The versions() method returns the list of published versions starting from the
-            // first one, so its output is reversed to check the latest first
-            for version in krate.versions().iter().rev() {
-                // Try every version until we find a non-yanked one. If all the versions are
-                // yanked the crate is automatically skipped
-                if !version.is_yanked() {
-                    // Increment the counters of this crate's dependencies
-                    for dependency in version.dependencies() {
-                        let count = counts.entry(dependency.name().to_string()).or_insert(0);
-                        *count += 1;
-                    }
-
-                    list.push(Crate::Registry(RegistryCrate {
-                        name: krate.name().to_string(),
-                        version: version.version().to_string(),
-                    }));
-                    break;
-                }
-            }
-        }
+        let mut list: Vec<_> = index
+            .crates_parallel()
+            .filter_map(|krate| {
+                let krate = krate.as_ref().unwrap();
+                // The versions() method returns the list of published versions starting from the
+                // first one, so its output is reversed to check the latest first
+                krate
+                    .versions()
+                    .iter()
+                    .rev()
+                    // Don't include yanked versions. If all versions are
+                    // yanked, then the crate is skipped.
+                    .filter(|version| !version.is_yanked())
+                    .map(|version| {
+                        // Increment the counters of this crate's dependencies
+                        let mut counts = counts.lock().unwrap();
+                        for dependency in version.dependencies() {
+                            let count = counts.entry(dependency.name().to_string()).or_insert(0);
+                            *count += 1;
+                        }
+                        Crate::Registry(RegistryCrate {
+                            name: krate.name().to_string(),
+                            version: version.version().to_string(),
+                        })
+                    })
+                    .next()
+            })
+            .collect();
 
         // Ensure the list is sorted by popularity
+        let counts = counts.lock().unwrap();
         list.sort_by(|a, b| {
             if let (Crate::Registry(ref a), Crate::Registry(ref b)) = (a, b) {
                 let count_a = counts.get(&a.name).cloned().unwrap_or(0);