From abd521f350b14e68aaf2a69b3461963bae4a2f42 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 11 Sep 2024 16:06:59 +0400 Subject: [PATCH 01/70] wip --- crates/compilers/src/cache.rs | 41 +++++++++---- crates/compilers/src/lib.rs | 2 + crates/compilers/src/preprocessor.rs | 90 ++++++++++++++++++++++++++++ crates/core/src/utils.rs | 3 + 4 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 crates/compilers/src/preprocessor.rs diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 0d5d1613..3c9fc9bd 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,6 +4,7 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, + preprocessor::interface_representation, resolver::GraphEdges, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, @@ -173,7 +174,10 @@ impl CompilerCache { pub fn join_entries(&mut self, root: &Path) -> &mut Self { self.files = std::mem::take(&mut self.files) .into_iter() - .map(|(path, entry)| (root.join(path), entry)) + .map(|(path, mut entry)| { + entry.join_imports(root); + (root.join(path), entry) + }) .collect(); self } @@ -182,7 +186,11 @@ impl CompilerCache { pub fn strip_entries_prefix(&mut self, base: &Path) -> &mut Self { self.files = std::mem::take(&mut self.files) .into_iter() - .map(|(path, entry)| (path.strip_prefix(base).map(Into::into).unwrap_or(path), entry)) + .map(|(path, mut entry)| { + let path = path.strip_prefix(base).map(Into::into).unwrap_or(path); + entry.strip_imports_prefixes(base); + (path, entry) + }) .collect(); self } @@ -405,6 +413,8 @@ pub struct CacheEntry { pub last_modification_date: u64, /// hash to identify whether the content of the file changed pub content_hash: String, + /// hash of the interface representation of the file, if it's a source file + pub interface_repr_hash: Option, /// identifier name see [`foundry_compilers_core::utils::source_name()`] pub source_name: PathBuf, /// what config was set when compiling this file @@ -550,6 +560,17 @@ impl CacheEntry { self.artifacts().all(|a| a.path.exists()) } + /// Joins all import paths with `base` + pub fn join_imports(&mut self, base: &Path) { + self.imports = self.imports.iter().map(|i| base.join(i)).collect(); + } + + /// Strips `base` from all import paths + pub fn strip_imports_prefixes(&mut self, base: &Path) { + self.imports = + self.imports.iter().map(|i| i.strip_prefix(base).unwrap_or(i).to_path_buf()).collect(); + } + /// Sets the artifact's paths to `base` adjoined to the artifact's `path`. pub fn join_artifacts_files(&mut self, base: &Path) { self.artifacts_mut().for_each(|a| a.path = base.join(&a.path)) @@ -625,20 +646,18 @@ pub(crate) struct ArtifactsCacheInner<'a, T: ArtifactOutput, C: Compiler> { impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { /// Creates a new cache entry for the file fn create_cache_entry(&mut self, file: PathBuf, source: &Source) { - let imports = self - .edges - .imports(&file) - .into_iter() - .map(|import| strip_prefix(import, self.project.root()).into()) - .collect(); - + let content_hash = source.content_hash(); + let interface_repr_hash = file.starts_with(&self.project.paths.sources).then(|| { + interface_representation(&source.content).unwrap_or_else(|_| content_hash.clone()) + }); let entry = CacheEntry { last_modification_date: CacheEntry::::read_last_modification_date(&file) .unwrap_or_default(), - content_hash: source.content_hash(), + content_hash, + interface_repr_hash, source_name: strip_prefix(&file, self.project.root()).into(), compiler_settings: self.project.settings.clone(), - imports, + imports: self.edges.imports(&file).into_iter().map(|i| i.into()).collect(), version_requirement: self.edges.version_requirement(&file).map(|v| v.to_string()), // artifacts remain empty until we received the compiler output artifacts: Default::default(), diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index fc2f8c3a..74d38cb9 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -24,6 +24,8 @@ pub use resolver::Graph; pub mod compilers; pub use compilers::*; +mod preprocessor; + mod compile; pub use compile::{ output::{AggregatedCompilerOutput, ProjectCompileOutput}, diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs new file mode 100644 index 00000000..763ab35c --- /dev/null +++ b/crates/compilers/src/preprocessor.rs @@ -0,0 +1,90 @@ +use foundry_compilers_core::utils; +use solang_parser::{ + diagnostics::Diagnostic, + helpers::CodeLocation, + pt::{ContractPart, ContractTy, FunctionAttribute, FunctionTy, SourceUnitPart, Visibility}, +}; + +pub(crate) fn interface_representation(content: &str) -> Result> { + let (source_unit, _) = solang_parser::parse(&content, 0)?; + let mut locs_to_remove = Vec::new(); + + for part in source_unit.0 { + if let SourceUnitPart::ContractDefinition(contract) = part { + if matches!(contract.ty, ContractTy::Interface(_) | ContractTy::Library(_)) { + continue; + } + for part in contract.parts { + if let ContractPart::FunctionDefinition(func) = part { + let is_exposed = func.ty == FunctionTy::Function + && func.attributes.iter().any(|attr| { + matches!( + attr, + FunctionAttribute::Visibility( + Visibility::External(_) | Visibility::Public(_) + ) + ) + }) + || matches!( + func.ty, + FunctionTy::Constructor | FunctionTy::Fallback | FunctionTy::Receive + ); + + if !is_exposed { + locs_to_remove.push(func.loc); + } + + if let Some(ref body) = func.body { + locs_to_remove.push(body.loc()); + } + } + } + } + } + + let mut content = content.to_string(); + let mut offset = 0; + + for loc in locs_to_remove { + let start = loc.start() - offset; + let end = loc.end() - offset; + + content.replace_range(start..end, ""); + offset += end - start; + } + + let content = content.replace("\n", ""); + Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_interface_representation() { + let content = r#" +library Lib { + function libFn() internal { + // logic to keep + } +} +contract A { + function a() external {} + function b() public {} + function c() internal { + // logic logic logic + } + function d() private {} + function e() external { + // logic logic logic + } +}"#; + + let result = interface_representation(content).unwrap(); + assert_eq!( + result, + r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# + ); + } +} diff --git a/crates/core/src/utils.rs b/crates/core/src/utils.rs index 877d2d9f..17e80775 100644 --- a/crates/core/src/utils.rs +++ b/crates/core/src/utils.rs @@ -42,6 +42,9 @@ pub static RE_SOL_SDPX_LICENSE_IDENTIFIER: Lazy = /// A regex used to remove extra lines in flatenned files pub static RE_THREE_OR_MORE_NEWLINES: Lazy = Lazy::new(|| Regex::new("\n{3,}").unwrap()); +/// A regex used to remove extra lines in flatenned files +pub static RE_TWO_OR_MORE_SPACES: Lazy = Lazy::new(|| Regex::new(" {2,}").unwrap()); + /// A regex that matches version pragma in a Vyper pub static RE_VYPER_VERSION: Lazy = Lazy::new(|| Regex::new(r"#(?:pragma version|@version)\s+(?P.+)").unwrap()); From fb64ca754b80fa502b8ae3caeb7991f2d9ba57d3 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 11 Sep 2024 17:23:38 +0400 Subject: [PATCH 02/70] wip --- crates/compilers/src/cache.rs | 176 +++++++++++++++++---------- crates/compilers/src/preprocessor.rs | 11 ++ 2 files changed, 120 insertions(+), 67 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 3c9fc9bd..07222b28 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,7 +4,7 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, - preprocessor::interface_representation, + preprocessor::{interface_representation_hash}, resolver::GraphEdges, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, @@ -641,15 +641,24 @@ pub(crate) struct ArtifactsCacheInner<'a, T: ArtifactOutput, C: Compiler> { /// The file hashes. pub content_hashes: HashMap, + + /// The interface representations for source files. + pub interface_repr_hashes: HashMap, } impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { /// Creates a new cache entry for the file - fn create_cache_entry(&mut self, file: PathBuf, source: &Source) { + fn create_cache_entry( + &mut self, + file: PathBuf, + source: &Source, + edges: Option<&GraphEdges>, + ) { + let edges = edges.unwrap_or(&self.edges); let content_hash = source.content_hash(); - let interface_repr_hash = file.starts_with(&self.project.paths.sources).then(|| { - interface_representation(&source.content).unwrap_or_else(|_| content_hash.clone()) - }); + let interface_repr_hash = file + .starts_with(&self.project.paths.sources) + .then(|| interface_representation_hash(&source)); let entry = CacheEntry { last_modification_date: CacheEntry::::read_last_modification_date(&file) .unwrap_or_default(), @@ -657,8 +666,8 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { interface_repr_hash, source_name: strip_prefix(&file, self.project.root()).into(), compiler_settings: self.project.settings.clone(), - imports: self.edges.imports(&file).into_iter().map(|i| i.into()).collect(), - version_requirement: self.edges.version_requirement(&file).map(|v| v.to_string()), + imports: edges.imports(&file).into_iter().map(|i| i.into()).collect(), + version_requirement: edges.version_requirement(&file).map(|v| v.to_string()), // artifacts remain empty until we received the compiler output artifacts: Default::default(), seen_by_compiler: false, @@ -692,7 +701,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { // Ensure that we have a cache entry for all sources. if !self.cache.files.contains_key(file) { - self.create_cache_entry(file.clone(), source); + self.create_cache_entry(file.clone(), source, None); } } @@ -749,62 +758,66 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { return true; } - false - } - - // Walks over all cache entires, detects dirty files and removes them from cache. - fn find_and_remove_dirty(&mut self) { - fn populate_dirty_files( - file: &Path, - dirty_files: &mut HashSet, - edges: &GraphEdges, - ) { - for file in edges.importers(file) { - // If file is marked as dirty we either have already visited it or it was marked as - // dirty initially and will be visited at some point later. - if !dirty_files.contains(file) { - dirty_files.insert(file.to_path_buf()); - populate_dirty_files(file, dirty_files, edges); + // If any requested extra files are missing for any artifact, mark source as dirty to + // generate them + for artifacts in self.cached_artifacts.values() { + for artifacts in artifacts.values() { + for artifact_file in artifacts { + if self.project.artifacts_handler().is_dirty(artifact_file).unwrap_or(true) { + return true; + } } } } - // Iterate over existing cache entries. - let files = self.cache.files.keys().cloned().collect::>(); + false + } + // Walks over all cache entires, detects dirty files and removes them from cache. + fn find_and_remove_dirty(&mut self) { let mut sources = Sources::new(); - // Read all sources, marking entries as dirty on I/O errors. - for file in &files { - let Ok(source) = Source::read(file) else { - self.dirty_sources.insert(file.clone()); + // Read all sources, removing entries on I/O errors. + for file in self.cache.files.keys().cloned().collect::>() { + let Ok(source) = Source::read(&file) else { + self.cache.files.remove(&file); continue; }; sources.insert(file.clone(), source); } - // Build a temporary graph for walking imports. We need this because `self.edges` - // only contains graph data for in-scope sources but we are operating on cache entries. - if let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) { - let (sources, edges) = graph.into_sources(); + let src_files = sources + .keys() + .filter(|f| f.starts_with(&self.project.paths.sources)) + .collect::>(); - // Calculate content hashes for later comparison. - self.fill_hashes(&sources); + // Calculate content hashes for later comparison. + self.fill_hashes(&sources); - // Pre-add all sources that are guaranteed to be dirty - for file in sources.keys() { - if self.is_dirty_impl(file) { - self.dirty_sources.insert(file.clone()); - } + // Pre-add all sources that are guaranteed to be dirty + for file in self.cache.files.keys() { + if self.is_dirty_impl(file, false) { + self.dirty_sources.insert(file.clone()); } + } - // Perform DFS to find direct/indirect importers of dirty files. - for file in self.dirty_sources.clone().iter() { - populate_dirty_files(file, &mut self.dirty_sources, &edges); + // Mark sources as dirty based on their imports + for (file, entry) in &self.cache.files { + if self.dirty_sources.contains(file) { + continue; + } + let is_src = src_files.contains(file); + for import in &entry.imports { + if is_src && self.dirty_sources.contains(import) { + self.dirty_sources.insert(file.clone()); + break; + } else if !is_src + && self.dirty_sources.contains(import) + && (!src_files.contains(import) || self.is_dirty_impl(import, true)) + { + self.dirty_sources.insert(file.clone()); + } } - } else { - // Purge all sources on graph resolution error. - self.dirty_sources.extend(files); } // Remove all dirty files from cache. @@ -812,22 +825,55 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { debug!("removing dirty file from cache: {}", file.display()); self.cache.remove(file); } - } - fn is_dirty_impl(&self, file: &Path) -> bool { - let Some(hash) = self.content_hashes.get(file) else { - trace!("missing content hash"); - return true; + // Build a temporary graph for populating cache. We want to ensure that we preserve all just + // removed entries with updated data. We need separate graph for this because + // `self.edges` only contains graph data for in-scope sources but we are operating on cache + // entries. + let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) + else { + // Purge all sources on graph resolution error. + self.cache.files.clear(); + return; }; + let (sources, edges) = graph.into_sources(); + + for (file, source) in sources { + if self.cache.files.contains_key(&file) { + continue; + } + + self.create_cache_entry(file.clone(), &source, Some(&edges)); + } + } + + fn is_dirty_impl(&self, file: &Path, use_interface_repr: bool) -> bool { let Some(entry) = self.cache.entry(file) else { trace!("missing cache entry"); return true; }; - if entry.content_hash != *hash { - trace!("content hash changed"); - return true; + if use_interface_repr { + let Some(interface_hash) = self.interface_repr_hashes.get(file) else { + trace!("missing interface hash"); + return true; + }; + + if entry.interface_repr_hash.as_ref().map_or(true, |h| h != interface_hash) { + trace!("interface hash changed"); + return true; + }; + } else { + let Some(content_hash) = self.content_hashes.get(file) else { + trace!("missing content hash"); + return true; + }; + + if entry.content_hash != *content_hash { + trace!("content hash changed"); + return true; + } } if !self.project.settings.can_use_cached(&entry.compiler_settings) { @@ -835,18 +881,6 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { return true; } - // If any requested extra files are missing for any artifact, mark source as dirty to - // generate them - for artifacts in self.cached_artifacts.values() { - for artifacts in artifacts.values() { - for artifact_file in artifacts { - if self.project.artifacts_handler().is_dirty(artifact_file).unwrap_or(true) { - return true; - } - } - } - } - // all things match, can be reused false } @@ -857,6 +891,13 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { if let hash_map::Entry::Vacant(entry) = self.content_hashes.entry(file.clone()) { entry.insert(source.content_hash()); } + if file.starts_with(&self.project.paths.sources) { + if let hash_map::Entry::Vacant(entry) = + self.interface_repr_hashes.entry(file.clone()) + { + entry.insert(interface_representation_hash(&source)); + } + } } } } @@ -940,6 +981,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCache<'a, T, C> { dirty_sources: Default::default(), content_hashes: Default::default(), sources_in_scope: Default::default(), + interface_repr_hashes: Default::default(), }; ArtifactsCache::Cached(cache) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 763ab35c..c7d4e0b9 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -1,4 +1,7 @@ +use alloy_primitives::hex; +use foundry_compilers_artifacts::Source; use foundry_compilers_core::utils; +use md5::Digest; use solang_parser::{ diagnostics::Diagnostic, helpers::CodeLocation, @@ -57,6 +60,14 @@ pub(crate) fn interface_representation(content: &str) -> Result String { + let Ok(repr) = interface_representation(&source.content) else { return source.content_hash() }; + let mut hasher = md5::Md5::new(); + hasher.update(&repr); + let result = hasher.finalize(); + hex::encode(result) +} + #[cfg(test)] mod tests { use super::*; From 0876b019f80f01edaf0be5932ae4f7ace1aafbd5 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 11 Sep 2024 17:36:54 +0400 Subject: [PATCH 03/70] add preprocessor --- crates/compilers/src/cache.rs | 2 +- crates/compilers/src/compile/project.rs | 47 +++- crates/compilers/src/preprocessor.rs | 289 +++++++++++++++++++++++- 3 files changed, 324 insertions(+), 14 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 07222b28..e11f37a1 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,7 +4,7 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, - preprocessor::{interface_representation_hash}, + preprocessor::interface_representation_hash, resolver::GraphEdges, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index b76c47a5..f4c3a67c 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -114,11 +114,15 @@ use crate::{ use foundry_compilers_core::error::Result; use rayon::prelude::*; use semver::Version; -use std::{collections::HashMap, path::PathBuf, time::Instant}; +use std::{collections::HashMap, fmt::Debug, path::PathBuf, time::Instant}; /// A set of different Solc installations with their version and the sources to be compiled pub(crate) type VersionedSources = HashMap>; +pub trait Preprocessor: Debug { + fn preprocess(&self, compiler: &C, input: C::Input, dirty: &Vec) -> Result; +} + #[derive(Debug)] pub struct ProjectCompiler<'a, T: ArtifactOutput, C: Compiler> { /// Contains the relationship of the source files and their imports @@ -126,6 +130,8 @@ pub struct ProjectCompiler<'a, T: ArtifactOutput, C: Compiler> { project: &'a Project, /// how to compile all the sources sources: CompilerSources, + /// Optional preprocessor + preprocessor: Option>>, } impl<'a, T: ArtifactOutput, C: Compiler> ProjectCompiler<'a, T, C> { @@ -160,7 +166,14 @@ impl<'a, T: ArtifactOutput, C: Compiler> ProjectCompiler<'a, T, C> { sources, }; - Ok(Self { edges, project, sources }) + Ok(Self { edges, project, sources, preprocessor: None }) + } + + pub fn with_preprocessor( + self, + preprocessor: impl Preprocessor + 'static, + ) -> ProjectCompiler<'a, T, C> { + ProjectCompiler { preprocessor: Some(Box::new(preprocessor)), ..self } } /// Compiles all the sources of the `Project` in the appropriate mode @@ -197,7 +210,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ProjectCompiler<'a, T, C> { /// - check cache fn preprocess(self) -> Result> { trace!("preprocessing"); - let Self { edges, project, mut sources } = self; + let Self { edges, project, mut sources, preprocessor } = self; // convert paths on windows to ensure consistency with the `CompilerOutput` `solc` emits, // which is unix style `/` @@ -207,7 +220,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ProjectCompiler<'a, T, C> { // retain and compile only dirty sources and all their imports sources.filter(&mut cache); - Ok(PreprocessedState { sources, cache }) + Ok(PreprocessedState { sources, cache, preprocessor }) } } @@ -221,15 +234,18 @@ struct PreprocessedState<'a, T: ArtifactOutput, C: Compiler> { /// Cache that holds `CacheEntry` objects if caching is enabled and the project is recompiled cache: ArtifactsCache<'a, T, C>, + + /// Optional preprocessor + preprocessor: Option>>, } impl<'a, T: ArtifactOutput, C: Compiler> PreprocessedState<'a, T, C> { /// advance to the next state by compiling all sources fn compile(self) -> Result> { trace!("compiling"); - let PreprocessedState { sources, mut cache } = self; + let PreprocessedState { sources, mut cache, preprocessor } = self; - let mut output = sources.compile(&mut cache)?; + let mut output = sources.compile(&mut cache, preprocessor)?; // source paths get stripped before handing them over to solc, so solc never uses absolute // paths, instead `--base-path ` is set. this way any metadata that's derived from @@ -410,6 +426,7 @@ impl CompilerSources { fn compile, T: ArtifactOutput>( self, cache: &mut ArtifactsCache<'_, T, C>, + preprocessor: Option>>, ) -> Result> { let project = cache.project(); let graph = cache.graph(); @@ -455,6 +472,18 @@ impl CompilerSources { let mut input = C::Input::build(sources, settings, language, version.clone()); input.strip_prefix(project.paths.root.as_path()); + let actually_dirty = actually_dirty + .into_iter() + .map(|path| { + path.strip_prefix(project.paths.root.as_path()) + .unwrap_or(&path) + .to_path_buf() + }) + .collect(); + + if let Some(preprocessor) = preprocessor.as_ref() { + input = preprocessor.preprocess(&project.compiler, input, &actually_dirty)?; + } jobs.push((input, actually_dirty)); } @@ -478,11 +507,7 @@ impl CompilerSources { let build_info = RawBuildInfo::new(&input, &output, project.build_info)?; - output.retain_files( - actually_dirty - .iter() - .map(|f| f.strip_prefix(project.paths.root.as_path()).unwrap_or(f)), - ); + output.retain_files(actually_dirty); output.join_all(project.paths.root.as_path()); aggregated.extend(version.clone(), build_info, output); diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index c7d4e0b9..13489f70 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -1,12 +1,29 @@ +use super::project::Preprocessor; +use crate::{ + flatten::{apply_updates, Updates}, + multi::{MultiCompiler, MultiCompilerInput}, + solc::{SolcCompiler, SolcVersionedInput}, + Compiler, Result, SolcError, +}; use alloy_primitives::hex; -use foundry_compilers_artifacts::Source; +use foundry_compilers_artifacts::{ + ast::SourceLocation, + output_selection::OutputSelection, + visitor::{Visitor, Walk}, + ContractDefinition, ContractKind, Expression, FunctionCall, MemberAccess, NewExpression, + Source, SourceUnit, SourceUnitPart, Sources, TypeName, +}; use foundry_compilers_core::utils; -use md5::Digest; +use md5::{digest::typenum::Exp, Digest}; use solang_parser::{ diagnostics::Diagnostic, helpers::CodeLocation, pt::{ContractPart, ContractTy, FunctionAttribute, FunctionTy, SourceUnitPart, Visibility}, }; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, +}; pub(crate) fn interface_representation(content: &str) -> Result> { let (source_unit, _) = solang_parser::parse(&content, 0)?; @@ -68,6 +85,274 @@ pub(crate) fn interface_representation_hash(source: &Source) -> String { hex::encode(result) } +#[derive(Debug)] +pub struct ItemLocation { + start: usize, + end: usize, +} + +impl ItemLocation { + fn try_from_loc(loc: SourceLocation) -> Option { + Some(ItemLocation { start: loc.start?, end: loc.start? + loc.length? }) + } +} + +#[derive(Debug)] +enum BytecodeDependencyKind { + CreationCode, + New(ItemLocation, String), +} + +#[derive(Debug)] +struct BytecodeDependency { + kind: BytecodeDependencyKind, + loc: ItemLocation, + referenced_contract: usize, +} + +#[derive(Debug)] +struct BytecodeDependencyCollector<'a> { + source: &'a str, + dependencies: Vec, +} + +impl BytecodeDependencyCollector<'_> { + fn new(source: &str) -> BytecodeDependencyCollector<'_> { + BytecodeDependencyCollector { source, dependencies: Vec::new() } + } +} + +impl Visitor for BytecodeDependencyCollector<'_> { + fn visit_function_call(&mut self, call: &FunctionCall) { + let (new_loc, expr) = match &call.expression { + Expression::NewExpression(expr) => (expr.src, expr), + Expression::FunctionCallOptions(expr) => { + if let Expression::NewExpression(new_expr) = &expr.expression { + (expr.src, new_expr) + } else { + return; + } + } + _ => return, + }; + + let TypeName::UserDefinedTypeName(type_name) = &expr.type_name else { return }; + + let Some(loc) = ItemLocation::try_from_loc(call.src) else { return }; + let Some(name_loc) = ItemLocation::try_from_loc(type_name.src) else { return }; + let Some(new_loc) = ItemLocation::try_from_loc(new_loc) else { return }; + let name = &self.source[name_loc.start..name_loc.end]; + + self.dependencies.push(BytecodeDependency { + kind: BytecodeDependencyKind::New(new_loc, name.to_string()), + loc, + referenced_contract: type_name.referenced_declaration as usize, + }); + } + + fn visit_member_access(&mut self, access: &MemberAccess) { + if access.member_name != "creationCode" { + return; + } + + let Expression::FunctionCall(call) = &access.expression else { return }; + + let Expression::Identifier(ident) = &call.expression else { return }; + + if ident.name != "type" { + return; + } + + let Some(Expression::Identifier(ident)) = call.arguments.first() else { return }; + + let Some(referenced) = ident.referenced_declaration else { return }; + + let Some(loc) = ItemLocation::try_from_loc(access.src) else { return }; + + self.dependencies.push(BytecodeDependency { + kind: BytecodeDependencyKind::CreationCode, + loc, + referenced_contract: referenced as usize, + }); + } +} + +struct TestOptimizer<'a> { + asts: BTreeMap, + dirty: &'a Vec, + sources: &'a mut Sources, +} + +impl TestOptimizer<'_> { + fn new<'a>( + asts: BTreeMap, + dirty: &'a Vec, + sources: &'a mut Sources, + ) -> TestOptimizer<'a> { + TestOptimizer { asts, dirty, sources } + } + + fn optimize(self) { + let mut updates = Updates::default(); + let ignored_contracts = self.collect_ignored_contracts(); + self.rename_contracts_to_abstract(&ignored_contracts, &mut updates); + self.remove_bytecode_dependencies(&ignored_contracts, &mut updates); + + apply_updates(self.sources, updates); + } + + fn collect_ignored_contracts(&self) -> BTreeSet { + let mut ignored_sources = BTreeSet::new(); + + for (path, ast) in &self.asts { + if path.to_str().unwrap().contains("test") || path.to_str().unwrap().contains("script") + { + ignored_sources.insert(ast.id); + } else if self.dirty.contains(path) { + ignored_sources.insert(ast.id); + + for node in &ast.nodes { + if let SourceUnitPart::ImportDirective(import) = node { + ignored_sources.insert(import.source_unit); + } + } + } + } + + let mut ignored_contracts = BTreeSet::new(); + + for ast in self.asts.values() { + if ignored_sources.contains(&ast.id) { + for node in &ast.nodes { + if let SourceUnitPart::ContractDefinition(contract) = node { + ignored_contracts.insert(contract.id); + } + } + } + } + + ignored_contracts + } + + fn rename_contracts_to_abstract( + &self, + ignored_contracts: &BTreeSet, + updates: &mut Updates, + ) { + for (path, ast) in &self.asts { + for node in &ast.nodes { + if let SourceUnitPart::ContractDefinition(contract) = node { + if ignored_contracts.contains(&contract.id) { + continue; + } + if matches!(contract.kind, ContractKind::Contract) && !contract.is_abstract { + if let Some(start) = contract.src.start { + updates.entry(path.clone()).or_default().insert(( + start, + start, + "abstract ".to_string(), + )); + } + } + } + } + } + } + + fn remove_bytecode_dependencies( + &self, + ignored_contracts: &BTreeSet, + updates: &mut Updates, + ) { + for (path, ast) in &self.asts { + let src = self.sources.get(path).unwrap().content.as_str(); + let mut collector = BytecodeDependencyCollector::new(src); + ast.walk(&mut collector); + let updates = updates.entry(path.clone()).or_default(); + for dep in collector.dependencies { + match dep.kind { + BytecodeDependencyKind::CreationCode => { + updates.insert((dep.loc.start, dep.loc.end, "bytes(\"\")".to_string())); + } + BytecodeDependencyKind::New(new_loc, name) => { + updates.insert(( + new_loc.start, + new_loc.end, + format!("{name}(payable(address(uint160(uint256(keccak256(abi.encode"), + )); + updates.insert((dep.loc.end, dep.loc.end, format!("))))))"))); + } + }; + } + } + } +} + +#[derive(Debug)] +pub struct TestOptimizerPreprocessor; + +impl Preprocessor for TestOptimizerPreprocessor { + fn preprocess( + &self, + solc: &SolcCompiler, + mut input: SolcVersionedInput, + dirty: &Vec, + ) -> Result { + let prev_output_selection = std::mem::replace( + &mut input.input.settings.output_selection, + OutputSelection::ast_output_selection(), + ); + let output = solc.compile(&input)?; + + input.input.settings.output_selection = prev_output_selection; + + if let Some(e) = output.errors.iter().find(|e| e.severity.is_error()) { + return Err(SolcError::msg(e)); + } + + let asts = output + .sources + .into_iter() + .filter_map(|(path, source)| { + if !input.input.sources.contains_key(&path) { + return None; + } + + Some((|| { + let ast = source.ast.ok_or_else(|| SolcError::msg("missing AST"))?; + let ast: SourceUnit = serde_json::from_str(&serde_json::to_string(&ast)?)?; + Ok((path, ast)) + })()) + }) + .collect::>>()?; + + TestOptimizer::new(asts, dirty, &mut input.input.sources).optimize(); + + Ok(input) + } +} + +impl Preprocessor for TestOptimizerPreprocessor { + fn preprocess( + &self, + compiler: &MultiCompiler, + input: ::Input, + dirty: &Vec, + ) -> Result<::Input> { + match input { + MultiCompilerInput::Solc(input) => { + if let Some(solc) = &compiler.solc { + let input = self.preprocess(solc, input, dirty)?; + Ok(MultiCompilerInput::Solc(input)) + } else { + Ok(MultiCompilerInput::Solc(input)) + } + } + MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), + } + } +} + #[cfg(test)] mod tests { use super::*; From 314f284e3660701a69b20915a849d72e42735bf6 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 11 Sep 2024 22:48:47 +0400 Subject: [PATCH 04/70] wip modified: crates/compilers/src/preprocessor.rs --- crates/artifacts/solc/src/ast/misc.rs | 2 +- crates/compilers/src/cache.rs | 7 +- crates/compilers/src/compile/project.rs | 20 +- crates/compilers/src/flatten.rs | 94 ++++++---- crates/compilers/src/lib.rs | 2 +- crates/compilers/src/preprocessor.rs | 237 +++++++++++++++--------- crates/core/src/error.rs | 2 +- 7 files changed, 224 insertions(+), 140 deletions(-) diff --git a/crates/artifacts/solc/src/ast/misc.rs b/crates/artifacts/solc/src/ast/misc.rs index 6ec3187b..7144ddc5 100644 --- a/crates/artifacts/solc/src/ast/misc.rs +++ b/crates/artifacts/solc/src/ast/misc.rs @@ -4,7 +4,7 @@ use std::{fmt, fmt::Write, str::FromStr}; /// Represents the source location of a node: `::`. /// /// The `start`, `length` and `index` can be -1 which is represented as `None` -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct SourceLocation { pub start: Option, pub length: Option, diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index e11f37a1..1e81c11d 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -658,7 +658,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { let content_hash = source.content_hash(); let interface_repr_hash = file .starts_with(&self.project.paths.sources) - .then(|| interface_representation_hash(&source)); + .then(|| interface_representation_hash(source)); let entry = CacheEntry { last_modification_date: CacheEntry::::read_last_modification_date(&file) .unwrap_or_default(), @@ -788,7 +788,10 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { let src_files = sources .keys() - .filter(|f| f.starts_with(&self.project.paths.sources)) + .filter(|f| { + !f.starts_with(&self.project.paths.tests) + && !f.starts_with(&self.project.paths.scripts) + }) .collect::>(); // Calculate content hashes for later comparison. diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index f4c3a67c..2d875fa4 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -109,7 +109,8 @@ use crate::{ output::{AggregatedCompilerOutput, Builds}, report, resolver::GraphEdges, - ArtifactOutput, CompilerSettings, Graph, Project, ProjectCompileOutput, Sources, + ArtifactOutput, CompilerSettings, Graph, Project, ProjectCompileOutput, ProjectPathsConfig, + Sources, }; use foundry_compilers_core::error::Result; use rayon::prelude::*; @@ -119,8 +120,14 @@ use std::{collections::HashMap, fmt::Debug, path::PathBuf, time::Instant}; /// A set of different Solc installations with their version and the sources to be compiled pub(crate) type VersionedSources = HashMap>; +/// Invoked before the actual compiler invocation and can override the input. pub trait Preprocessor: Debug { - fn preprocess(&self, compiler: &C, input: C::Input, dirty: &Vec) -> Result; + fn preprocess( + &self, + compiler: &C, + input: C::Input, + paths: &ProjectPathsConfig, + ) -> Result; } #[derive(Debug)] @@ -169,11 +176,8 @@ impl<'a, T: ArtifactOutput, C: Compiler> ProjectCompiler<'a, T, C> { Ok(Self { edges, project, sources, preprocessor: None }) } - pub fn with_preprocessor( - self, - preprocessor: impl Preprocessor + 'static, - ) -> ProjectCompiler<'a, T, C> { - ProjectCompiler { preprocessor: Some(Box::new(preprocessor)), ..self } + pub fn with_preprocessor(self, preprocessor: impl Preprocessor + 'static) -> Self { + Self { preprocessor: Some(Box::new(preprocessor)), ..self } } /// Compiles all the sources of the `Project` in the appropriate mode @@ -482,7 +486,7 @@ impl CompilerSources { .collect(); if let Some(preprocessor) = preprocessor.as_ref() { - input = preprocessor.preprocess(&project.compiler, input, &actually_dirty)?; + input = preprocessor.preprocess(&project.compiler, input, &project.paths)?; } jobs.push((input, actually_dirty)); diff --git a/crates/compilers/src/flatten.rs b/crates/compilers/src/flatten.rs index 45da3902..df4f5f87 100644 --- a/crates/compilers/src/flatten.rs +++ b/crates/compilers/src/flatten.rs @@ -17,9 +17,10 @@ use foundry_compilers_core::{ }; use itertools::Itertools; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeSet, HashMap, HashSet}, hash::Hash, path::{Path, PathBuf}, + sync::Arc, }; use visitor::Walk; @@ -95,7 +96,7 @@ impl Visitor for ReferencesCollector { } fn visit_external_assembly_reference(&mut self, reference: &ExternalInlineAssemblyReference) { - let mut src = reference.src.clone(); + let mut src = reference.src; // If suffix is used in assembly reference (e.g. value.slot), it will be included into src. // However, we are only interested in the referenced name, thus we strip . part. @@ -112,47 +113,32 @@ impl Visitor for ReferencesCollector { /// Updates to be applied to the sources. /// source_path -> (start, end, new_value) -type Updates = HashMap>; +pub type Updates = HashMap>; -pub struct FlatteningResult<'a> { +pub struct FlatteningResult { /// Updated source in the order they shoud be written to the output file. sources: Vec, /// Pragmas that should be present in the target file. pragmas: Vec, /// License identifier that should be present in the target file. - license: Option<&'a str>, + license: Option, } -impl<'a> FlatteningResult<'a> { +impl FlatteningResult { fn new( - flattener: &Flattener, - mut updates: Updates, + mut flattener: Flattener, + updates: Updates, pragmas: Vec, - license: Option<&'a str>, + license: Option, ) -> Self { - let mut sources = Vec::new(); - - for path in &flattener.ordered_sources { - let mut content = flattener.sources.get(path).unwrap().content.as_bytes().to_vec(); - let mut offset: isize = 0; - if let Some(updates) = updates.remove(path) { - let mut updates = updates.iter().collect::>(); - updates.sort_by_key(|(start, _, _)| *start); - for (start, end, new_value) in updates { - let start = (*start as isize + offset) as usize; - let end = (*end as isize + offset) as usize; - - content.splice(start..end, new_value.bytes()); - offset += new_value.len() as isize - (end - start) as isize; - } - } - let content = format!( - "// {}\n{}", - path.strip_prefix(&flattener.project_root).unwrap_or(path).display(), - String::from_utf8(content).unwrap() - ); - sources.push(content); - } + apply_updates(&mut flattener.sources, updates); + + let sources = flattener + .ordered_sources + .iter() + .map(|path| flattener.sources.remove(path).unwrap().content) + .map(Arc::unwrap_or_clone) + .collect(); Self { sources, pragmas, license } } @@ -274,9 +260,10 @@ impl Flattener { /// 3. Remove all imports. /// 4. Remove all pragmas except for the ones in the target file. /// 5. Remove all license identifiers except for the one in the target file. - pub fn flatten(&self) -> String { + pub fn flatten(self) -> String { let mut updates = Updates::new(); + self.append_filenames(&mut updates); let top_level_names = self.rename_top_level_definitions(&mut updates); self.rename_contract_level_types_references(&top_level_names, &mut updates); self.remove_qualified_imports(&mut updates); @@ -289,15 +276,26 @@ impl Flattener { self.flatten_result(updates, target_pragmas, target_license).get_flattened_target() } - fn flatten_result<'a>( - &'a self, + fn flatten_result( + self, updates: Updates, target_pragmas: Vec, - target_license: Option<&'a str>, - ) -> FlatteningResult<'_> { + target_license: Option, + ) -> FlatteningResult { FlatteningResult::new(self, updates, target_pragmas, target_license) } + /// Appends a comment with the file name to the beginning of each source. + fn append_filenames(&self, updates: &mut Updates) { + for path in &self.ordered_sources { + updates.entry(path.clone()).or_default().insert(( + 0, + 0, + format!("// {}\n", path.strip_prefix(&self.project_root).unwrap_or(path).display()), + )); + } + } + /// Finds and goes over all references to file-level definitions and updates them to match /// definition name. This is needed for two reasons: /// 1. We want to rename all aliased or qualified imports. @@ -752,14 +750,14 @@ impl Flattener { /// Removes all license identifiers from all sources. Returns licesnse identifier from target /// file, if any. - fn process_licenses(&self, updates: &mut Updates) -> Option<&str> { + fn process_licenses(&self, updates: &mut Updates) -> Option { let mut target_license = None; for loc in &self.collect_licenses() { if loc.path == self.target { let license_line = self.read_location(loc); let license_start = license_line.find("SPDX-License-Identifier:").unwrap(); - target_license = Some(license_line[license_start..].trim()); + target_license = Some(license_line[license_start..].trim().to_string()); } updates.entry(loc.path.clone()).or_default().insert(( loc.start, @@ -887,3 +885,21 @@ pub fn combine_version_pragmas(pragmas: Vec<&str>) -> Option { None } + +pub fn apply_updates(sources: &mut Sources, mut updates: Updates) { + for (path, source) in sources { + if let Some(updates) = updates.remove(path) { + let mut offset = 0; + let mut content = source.content.as_bytes().to_vec(); + for (start, end, new_value) in updates { + let start = (start as isize + offset) as usize; + let end = (end as isize + offset) as usize; + + content.splice(start..end, new_value.bytes()); + offset += new_value.len() as isize - (end - start) as isize; + } + + source.content = Arc::new(String::from_utf8_lossy(&content).to_string()); + } + } +} diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 74d38cb9..1eebbfe9 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -24,7 +24,7 @@ pub use resolver::Graph; pub mod compilers; pub use compilers::*; -mod preprocessor; +pub mod preprocessor; mod compile; pub use compile::{ diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 13489f70..2678b97d 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -1,53 +1,53 @@ use super::project::Preprocessor; use crate::{ flatten::{apply_updates, Updates}, - multi::{MultiCompiler, MultiCompilerInput}, + multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, solc::{SolcCompiler, SolcVersionedInput}, - Compiler, Result, SolcError, + Compiler, ProjectPathsConfig, Result, SolcError, }; -use alloy_primitives::hex; +use alloy_primitives::{hex}; use foundry_compilers_artifacts::{ ast::SourceLocation, output_selection::OutputSelection, visitor::{Visitor, Walk}, - ContractDefinition, ContractKind, Expression, FunctionCall, MemberAccess, NewExpression, - Source, SourceUnit, SourceUnitPart, Sources, TypeName, + ContractDefinitionPart, Expression, FunctionCall, + FunctionKind, MemberAccess, SolcLanguage, Source, SourceUnit, SourceUnitPart, + Sources, TypeName, }; use foundry_compilers_core::utils; -use md5::{digest::typenum::Exp, Digest}; -use solang_parser::{ - diagnostics::Diagnostic, - helpers::CodeLocation, - pt::{ContractPart, ContractTy, FunctionAttribute, FunctionTy, SourceUnitPart, Visibility}, -}; +use md5::{ Digest}; +use solang_parser::{diagnostics::Diagnostic, helpers::CodeLocation, pt}; use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, }, + fmt::Write, path::{Path, PathBuf}, }; pub(crate) fn interface_representation(content: &str) -> Result> { - let (source_unit, _) = solang_parser::parse(&content, 0)?; + let (source_unit, _) = solang_parser::parse(content, 0)?; let mut locs_to_remove = Vec::new(); for part in source_unit.0 { - if let SourceUnitPart::ContractDefinition(contract) = part { - if matches!(contract.ty, ContractTy::Interface(_) | ContractTy::Library(_)) { + if let pt::SourceUnitPart::ContractDefinition(contract) = part { + if matches!(contract.ty, pt::ContractTy::Interface(_) | pt::ContractTy::Library(_)) { continue; } for part in contract.parts { - if let ContractPart::FunctionDefinition(func) = part { - let is_exposed = func.ty == FunctionTy::Function + if let pt::ContractPart::FunctionDefinition(func) = part { + let is_exposed = func.ty == pt::FunctionTy::Function && func.attributes.iter().any(|attr| { matches!( attr, - FunctionAttribute::Visibility( - Visibility::External(_) | Visibility::Public(_) + pt::FunctionAttribute::Visibility( + pt::Visibility::External(_) | pt::Visibility::Public(_) ) ) }) || matches!( func.ty, - FunctionTy::Constructor | FunctionTy::Fallback | FunctionTy::Receive + pt::FunctionTy::Constructor + | pt::FunctionTy::Fallback + | pt::FunctionTy::Receive ); if !is_exposed { @@ -92,11 +92,17 @@ pub struct ItemLocation { } impl ItemLocation { - fn try_from_loc(loc: SourceLocation) -> Option { - Some(ItemLocation { start: loc.start?, end: loc.start? + loc.length? }) + fn try_from_loc(loc: SourceLocation) -> Option { + Some(Self { start: loc.start?, end: loc.start? + loc.length? }) } } +fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { + let test_dir = paths.tests.strip_prefix(&paths.root).unwrap_or(&paths.root); + let script_dir = paths.scripts.strip_prefix(&paths.root).unwrap_or(&paths.root); + path.starts_with(test_dir) || path.starts_with(script_dir) +} + #[derive(Debug)] enum BytecodeDependencyKind { CreationCode, @@ -177,113 +183,161 @@ impl Visitor for BytecodeDependencyCollector<'_> { } } -struct TestOptimizer<'a> { +/// Keeps data about a single contract definition. +struct ContractData { + /// Artifact ID to use in `getCode`/`deployCode` calls. + artifact: String, + /// Whether contract has a non-empty constructor. + has_constructor: bool, +} + +/// Receives a set of source files along with their ASTs and removes bytecode dependencies from +/// contracts by replacing them with cheatcode invocations. +struct BytecodeDependencyOptimizer<'a> { asts: BTreeMap, - dirty: &'a Vec, + paths: &'a ProjectPathsConfig, sources: &'a mut Sources, } -impl TestOptimizer<'_> { +impl BytecodeDependencyOptimizer<'_> { fn new<'a>( asts: BTreeMap, - dirty: &'a Vec, + paths: &'a ProjectPathsConfig, sources: &'a mut Sources, - ) -> TestOptimizer<'a> { - TestOptimizer { asts, dirty, sources } + ) -> BytecodeDependencyOptimizer<'a> { + BytecodeDependencyOptimizer { asts, paths, sources } } - fn optimize(self) { + fn process(self) { let mut updates = Updates::default(); - let ignored_contracts = self.collect_ignored_contracts(); - self.rename_contracts_to_abstract(&ignored_contracts, &mut updates); - self.remove_bytecode_dependencies(&ignored_contracts, &mut updates); + + let contracts = self.collect_contracts(&mut updates); + self.remove_bytecode_dependencies(&contracts, &mut updates); apply_updates(self.sources, updates); } - fn collect_ignored_contracts(&self) -> BTreeSet { - let mut ignored_sources = BTreeSet::new(); + /// Collects a mapping from contract AST id to [ContractData]. + fn collect_contracts(&self, updates: &mut Updates) -> BTreeMap { + let mut contracts = BTreeMap::new(); for (path, ast) in &self.asts { - if path.to_str().unwrap().contains("test") || path.to_str().unwrap().contains("script") - { - ignored_sources.insert(ast.id); - } else if self.dirty.contains(path) { - ignored_sources.insert(ast.id); - - for node in &ast.nodes { - if let SourceUnitPart::ImportDirective(import) = node { - ignored_sources.insert(import.source_unit); - } - } - } - } - - let mut ignored_contracts = BTreeSet::new(); - - for ast in self.asts.values() { - if ignored_sources.contains(&ast.id) { - for node in &ast.nodes { - if let SourceUnitPart::ContractDefinition(contract) = node { - ignored_contracts.insert(contract.id); - } - } - } - } - - ignored_contracts - } + let src = self.sources.get(path).unwrap().content.as_str(); - fn rename_contracts_to_abstract( - &self, - ignored_contracts: &BTreeSet, - updates: &mut Updates, - ) { - for (path, ast) in &self.asts { for node in &ast.nodes { if let SourceUnitPart::ContractDefinition(contract) = node { - if ignored_contracts.contains(&contract.id) { + let artifact = format!("{}:{}", path.display(), contract.name); + let constructor = contract.nodes.iter().find_map(|node| { + let ContractDefinitionPart::FunctionDefinition(func) = node else { + return None; + }; + if *func.kind() != FunctionKind::Constructor { + return None; + } + + Some(func) + }); + + if constructor.map_or(true, |func| func.parameters.parameters.is_empty()) { + contracts + .insert(contract.id, ContractData { artifact, has_constructor: false }); continue; } - if matches!(contract.kind, ContractKind::Contract) && !contract.is_abstract { - if let Some(start) = contract.src.start { - updates.entry(path.clone()).or_default().insert(( - start, - start, - "abstract ".to_string(), - )); + contracts.insert(contract.id, ContractData { artifact, has_constructor: true }); + + let constructor = constructor.unwrap(); + let updates = updates.entry(path.clone()).or_default(); + + let mut constructor_helper = + format!("struct ConstructorHelper{} {{", contract.id); + + for param in &constructor.parameters.parameters { + if let Some(loc) = ItemLocation::try_from_loc(param.src) { + let param = &src[loc.start..loc.end] + .replace(" memory ", " ") + .replace(" calldata ", " "); + write!(constructor_helper, "{param};").unwrap(); } } + + constructor_helper.push('}'); + + if let Some(loc) = ItemLocation::try_from_loc(constructor.src) { + updates.insert((loc.start, loc.start, constructor_helper)); + } } } } + + contracts } + /// Goes over all source files and replaces bytecode dependencies with cheatcode invocations. fn remove_bytecode_dependencies( &self, - ignored_contracts: &BTreeSet, + contracts: &BTreeMap, updates: &mut Updates, ) { + let test_dir = &self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); + let script_dir = + &self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); for (path, ast) in &self.asts { + if !path.starts_with(test_dir) && !path.starts_with(script_dir) { + continue; + } let src = self.sources.get(path).unwrap().content.as_str(); + + if src.is_empty() { + continue; + } let mut collector = BytecodeDependencyCollector::new(src); ast.walk(&mut collector); + let updates = updates.entry(path.clone()).or_default(); + let vm_interface_name = format!("VmContractHelper{}", ast.id); + let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); + for dep in collector.dependencies { + let ContractData { artifact, has_constructor } = + contracts.get(&dep.referenced_contract).unwrap(); match dep.kind { BytecodeDependencyKind::CreationCode => { - updates.insert((dep.loc.start, dep.loc.end, "bytes(\"\")".to_string())); - } - BytecodeDependencyKind::New(new_loc, name) => { updates.insert(( - new_loc.start, - new_loc.end, - format!("{name}(payable(address(uint160(uint256(keccak256(abi.encode"), + dep.loc.start, + dep.loc.end, + format!("{vm}.getCode(\"{artifact}\")"), )); - updates.insert((dep.loc.end, dep.loc.end, format!("))))))"))); + } + BytecodeDependencyKind::New(new_loc, name) => { + if !*has_constructor { + updates.insert(( + dep.loc.start, + dep.loc.end, + format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), + )); + } else { + updates.insert(( + new_loc.start, + new_loc.end, + format!("{name}(payable({vm}.deployCode(\"{artifact}\", abi.encode({name}.ConstructorHelper{}", dep.referenced_contract), + )); + updates.insert((dep.loc.end, dep.loc.end, "))))".to_string())); + } } }; } + updates.insert(( + src.len() - 1, + src.len() - 1, + format!( + r#" +interface {vm_interface_name} {{ + function deployCode(string memory _artifact, bytes memory _data) external returns (address); + function deployCode(string memory _artifact) external returns (address); + function getCode(string memory _artifact) external returns (bytes memory); +}}"# + ), + )); } } } @@ -296,8 +350,14 @@ impl Preprocessor for TestOptimizerPreprocessor { &self, solc: &SolcCompiler, mut input: SolcVersionedInput, - dirty: &Vec, + paths: &ProjectPathsConfig, ) -> Result { + // Skip if we are not compiling any tests or scripts. Avoids unnecessary solc invocation and + // AST parsing. + if input.input.sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { + return Ok(input); + } + let prev_output_selection = std::mem::replace( &mut input.input.settings.output_selection, OutputSelection::ast_output_selection(), @@ -326,7 +386,7 @@ impl Preprocessor for TestOptimizerPreprocessor { }) .collect::>>()?; - TestOptimizer::new(asts, dirty, &mut input.input.sources).optimize(); + BytecodeDependencyOptimizer::new(asts, paths, &mut input.input.sources).process(); Ok(input) } @@ -337,12 +397,13 @@ impl Preprocessor for TestOptimizerPreprocessor { &self, compiler: &MultiCompiler, input: ::Input, - dirty: &Vec, + paths: &ProjectPathsConfig, ) -> Result<::Input> { match input { MultiCompilerInput::Solc(input) => { if let Some(solc) = &compiler.solc { - let input = self.preprocess(solc, input, dirty)?; + let paths = paths.clone().with_language::(); + let input = self.preprocess(solc, input, &paths)?; Ok(MultiCompilerInput::Solc(input)) } else { Ok(MultiCompilerInput::Solc(input)) diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index aa9e5221..d58b0f44 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -5,7 +5,7 @@ use std::{ }; use thiserror::Error; -pub type Result = std::result::Result; +pub type Result = std::result::Result; #[allow(unused_macros)] #[macro_export] From 86f05629eaf710bc2f09c7350641db2879608a6c Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 11 Sep 2024 22:48:47 +0400 Subject: [PATCH 05/70] wip modified: crates/compilers/src/preprocessor.rs --- crates/compilers/src/cache.rs | 16 +++++------ crates/compilers/src/preprocessor.rs | 43 +++++++++++++++++++++------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 1e81c11d..c41c1923 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -786,14 +786,6 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { sources.insert(file.clone(), source); } - let src_files = sources - .keys() - .filter(|f| { - !f.starts_with(&self.project.paths.tests) - && !f.starts_with(&self.project.paths.scripts) - }) - .collect::>(); - // Calculate content hashes for later comparison. self.fill_hashes(&sources); @@ -804,6 +796,14 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { } } + let src_files = sources + .keys() + .filter(|f| { + !f.starts_with(&self.project.paths.tests) + && !f.starts_with(&self.project.paths.scripts) + }) + .collect::>(); + // Mark sources as dirty based on their imports for (file, entry) in &self.cache.files { if self.dirty_sources.contains(file) { diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 2678b97d..c5e1c6d1 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -10,15 +10,14 @@ use foundry_compilers_artifacts::{ ast::SourceLocation, output_selection::OutputSelection, visitor::{Visitor, Walk}, - ContractDefinitionPart, Expression, FunctionCall, - FunctionKind, MemberAccess, SolcLanguage, Source, SourceUnit, SourceUnitPart, - Sources, TypeName, + ContractDefinitionPart, Expression, FunctionCall, FunctionKind, MemberAccess, NewExpression, + SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, }; use foundry_compilers_core::utils; -use md5::{ Digest}; +use md5::Digest; use solang_parser::{diagnostics::Diagnostic, helpers::CodeLocation, pt}; use std::{ - collections::{BTreeMap, }, + collections::BTreeMap, fmt::Write, path::{Path, PathBuf}, }; @@ -120,15 +119,22 @@ struct BytecodeDependency { struct BytecodeDependencyCollector<'a> { source: &'a str, dependencies: Vec, + total_count: usize, } impl BytecodeDependencyCollector<'_> { fn new(source: &str) -> BytecodeDependencyCollector<'_> { - BytecodeDependencyCollector { source, dependencies: Vec::new() } + BytecodeDependencyCollector { source, dependencies: Vec::new(), total_count: 0 } } } impl Visitor for BytecodeDependencyCollector<'_> { + fn visit_new_expression(&mut self, expr: &NewExpression) { + if let TypeName::UserDefinedTypeName(_) = &expr.type_name { + self.total_count += 1; + } + } + fn visit_function_call(&mut self, call: &FunctionCall) { let (new_loc, expr) = match &call.expression { Expression::NewExpression(expr) => (expr.src, expr), @@ -160,6 +166,7 @@ impl Visitor for BytecodeDependencyCollector<'_> { if access.member_name != "creationCode" { return; } + self.total_count += 1; let Expression::FunctionCall(call) = &access.expression else { return }; @@ -208,13 +215,15 @@ impl BytecodeDependencyOptimizer<'_> { BytecodeDependencyOptimizer { asts, paths, sources } } - fn process(self) { + fn process(self) -> Result<()> { let mut updates = Updates::default(); let contracts = self.collect_contracts(&mut updates); - self.remove_bytecode_dependencies(&contracts, &mut updates); + self.remove_bytecode_dependencies(&contracts, &mut updates)?; apply_updates(self.sources, updates); + + Ok(()) } /// Collects a mapping from contract AST id to [ContractData]. @@ -277,7 +286,7 @@ impl BytecodeDependencyOptimizer<'_> { &self, contracts: &BTreeMap, updates: &mut Updates, - ) { + ) -> Result<()> { let test_dir = &self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); let script_dir = &self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); @@ -293,6 +302,18 @@ impl BytecodeDependencyOptimizer<'_> { let mut collector = BytecodeDependencyCollector::new(src); ast.walk(&mut collector); + + // It is possible to write weird expressions which we won't catch. + // e.g. (((new Contract)))() is valid syntax + // + // We need to ensure that we've collected all dependencies that are in the contract. + if collector.dependencies.len() != collector.total_count { + return Err(SolcError::msg(format!( + "failed to collect all bytecode dependencies for {}", + path.display() + ))); + } + let updates = updates.entry(path.clone()).or_default(); let vm_interface_name = format!("VmContractHelper{}", ast.id); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); @@ -339,6 +360,8 @@ interface {vm_interface_name} {{ ), )); } + + Ok(()) } } @@ -386,7 +409,7 @@ impl Preprocessor for TestOptimizerPreprocessor { }) .collect::>>()?; - BytecodeDependencyOptimizer::new(asts, paths, &mut input.input.sources).process(); + BytecodeDependencyOptimizer::new(asts, paths, &mut input.input.sources).process()?; Ok(input) } From af6550ff86bdc9b44721b954aada37cc7989f82b Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 13 Sep 2024 13:18:19 +0400 Subject: [PATCH 06/70] fixes --- crates/compilers/src/cache.rs | 107 ++++++++++-------------- crates/compilers/src/compile/project.rs | 14 ++-- crates/compilers/src/preprocessor.rs | 3 +- 3 files changed, 52 insertions(+), 72 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index c41c1923..8541a8b0 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -174,10 +174,7 @@ impl CompilerCache { pub fn join_entries(&mut self, root: &Path) -> &mut Self { self.files = std::mem::take(&mut self.files) .into_iter() - .map(|(path, mut entry)| { - entry.join_imports(root); - (root.join(path), entry) - }) + .map(|(path, entry)| (root.join(path), entry)) .collect(); self } @@ -186,11 +183,7 @@ impl CompilerCache { pub fn strip_entries_prefix(&mut self, base: &Path) -> &mut Self { self.files = std::mem::take(&mut self.files) .into_iter() - .map(|(path, mut entry)| { - let path = path.strip_prefix(base).map(Into::into).unwrap_or(path); - entry.strip_imports_prefixes(base); - (path, entry) - }) + .map(|(path, entry)| (path.strip_prefix(base).map(Into::into).unwrap_or(path), entry)) .collect(); self } @@ -560,17 +553,6 @@ impl CacheEntry { self.artifacts().all(|a| a.path.exists()) } - /// Joins all import paths with `base` - pub fn join_imports(&mut self, base: &Path) { - self.imports = self.imports.iter().map(|i| base.join(i)).collect(); - } - - /// Strips `base` from all import paths - pub fn strip_imports_prefixes(&mut self, base: &Path) { - self.imports = - self.imports.iter().map(|i| i.strip_prefix(base).unwrap_or(i).to_path_buf()).collect(); - } - /// Sets the artifact's paths to `base` adjoined to the artifact's `path`. pub fn join_artifacts_files(&mut self, base: &Path) { self.artifacts_mut().for_each(|a| a.path = base.join(&a.path)) @@ -647,27 +629,33 @@ pub(crate) struct ArtifactsCacheInner<'a, T: ArtifactOutput, C: Compiler> { } impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { + /// Whther given file is a source file or a test/script file. + fn is_source_file(&self, file: &Path) -> bool { + !file.starts_with(&self.project.paths.tests) + && !file.starts_with(&self.project.paths.scripts) + } + /// Creates a new cache entry for the file - fn create_cache_entry( - &mut self, - file: PathBuf, - source: &Source, - edges: Option<&GraphEdges>, - ) { - let edges = edges.unwrap_or(&self.edges); - let content_hash = source.content_hash(); - let interface_repr_hash = file - .starts_with(&self.project.paths.sources) - .then(|| interface_representation_hash(source)); + fn create_cache_entry(&mut self, file: PathBuf, source: &Source) { + let imports = self + .edges + .imports(&file) + .into_iter() + .map(|import| strip_prefix(import, self.project.root()).into()) + .collect(); + + let interface_repr_hash = + self.is_source_file(&file).then(|| interface_representation_hash(source)); + let entry = CacheEntry { last_modification_date: CacheEntry::::read_last_modification_date(&file) .unwrap_or_default(), - content_hash, + content_hash: source.content_hash(), interface_repr_hash, source_name: strip_prefix(&file, self.project.root()).into(), compiler_settings: self.project.settings.clone(), - imports: edges.imports(&file).into_iter().map(|i| i.into()).collect(), - version_requirement: edges.version_requirement(&file).map(|v| v.to_string()), + imports, + version_requirement: self.edges.version_requirement(&file).map(|v| v.to_string()), // artifacts remain empty until we received the compiler output artifacts: Default::default(), seen_by_compiler: false, @@ -701,7 +689,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { // Ensure that we have a cache entry for all sources. if !self.cache.files.contains_key(file) { - self.create_cache_entry(file.clone(), source, None); + self.create_cache_entry(file.clone(), source); } } @@ -796,27 +784,35 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { } } - let src_files = sources - .keys() - .filter(|f| { - !f.starts_with(&self.project.paths.tests) - && !f.starts_with(&self.project.paths.scripts) - }) - .collect::>(); + // Build a temporary graph for populating cache. We want to ensure that we preserve all just + // removed entries with updated data. We need separate graph for this because + // `self.edges` only contains graph data for in-scope sources but we are operating on cache + // entries. + let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) + else { + // Purge all sources on graph resolution error. + self.cache.files.clear(); + return; + }; + + let (sources, edges) = graph.into_sources(); // Mark sources as dirty based on their imports - for (file, entry) in &self.cache.files { + for file in sources.keys() { if self.dirty_sources.contains(file) { continue; } - let is_src = src_files.contains(file); - for import in &entry.imports { + let is_src = self.is_source_file(file); + for import in edges.imports(file) { + // Any source file importing dirty source file is dirty. if is_src && self.dirty_sources.contains(import) { self.dirty_sources.insert(file.clone()); break; + // For non-src files we mark them as dirty only if they import dirty non-src file + // or src file for which interface representation changed. } else if !is_src && self.dirty_sources.contains(import) - && (!src_files.contains(import) || self.is_dirty_impl(import, true)) + && (!self.is_source_file(import) || self.is_dirty_impl(import, true)) { self.dirty_sources.insert(file.clone()); } @@ -829,25 +825,13 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { self.cache.remove(file); } - // Build a temporary graph for populating cache. We want to ensure that we preserve all just - // removed entries with updated data. We need separate graph for this because - // `self.edges` only contains graph data for in-scope sources but we are operating on cache - // entries. - let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) - else { - // Purge all sources on graph resolution error. - self.cache.files.clear(); - return; - }; - - let (sources, edges) = graph.into_sources(); - + // Create new entries for all source files for (file, source) in sources { if self.cache.files.contains_key(&file) { continue; } - self.create_cache_entry(file.clone(), &source, Some(&edges)); + self.create_cache_entry(file.clone(), &source); } } @@ -894,7 +878,8 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCacheInner<'a, T, C> { if let hash_map::Entry::Vacant(entry) = self.content_hashes.entry(file.clone()) { entry.insert(source.content_hash()); } - if file.starts_with(&self.project.paths.sources) { + // Fill interface representation hashes for source files + if self.is_source_file(&file) { if let hash_map::Entry::Vacant(entry) = self.interface_repr_hashes.entry(file.clone()) { diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 2d875fa4..6ea0738b 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -476,14 +476,6 @@ impl CompilerSources { let mut input = C::Input::build(sources, settings, language, version.clone()); input.strip_prefix(project.paths.root.as_path()); - let actually_dirty = actually_dirty - .into_iter() - .map(|path| { - path.strip_prefix(project.paths.root.as_path()) - .unwrap_or(&path) - .to_path_buf() - }) - .collect(); if let Some(preprocessor) = preprocessor.as_ref() { input = preprocessor.preprocess(&project.compiler, input, &project.paths)?; @@ -511,7 +503,11 @@ impl CompilerSources { let build_info = RawBuildInfo::new(&input, &output, project.build_info)?; - output.retain_files(actually_dirty); + output.retain_files( + actually_dirty + .iter() + .map(|f| f.strip_prefix(project.paths.root.as_path()).unwrap_or(f)), + ); output.join_all(project.paths.root.as_path()); aggregated.extend(version.clone(), build_info, output); diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index c5e1c6d1..18c95548 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -5,7 +5,7 @@ use crate::{ solc::{SolcCompiler, SolcVersionedInput}, Compiler, ProjectPathsConfig, Result, SolcError, }; -use alloy_primitives::{hex}; +use alloy_primitives::hex; use foundry_compilers_artifacts::{ ast::SourceLocation, output_selection::OutputSelection, @@ -302,7 +302,6 @@ impl BytecodeDependencyOptimizer<'_> { let mut collector = BytecodeDependencyCollector::new(src); ast.walk(&mut collector); - // It is possible to write weird expressions which we won't catch. // e.g. (((new Contract)))() is valid syntax // From 276bc9470709143489697788c883fcd11afe030f Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 13 Sep 2024 19:33:04 +0400 Subject: [PATCH 07/70] helper libs --- crates/compilers/src/preprocessor.rs | 192 ++++++++++++++++++++------- crates/core/src/error.rs | 3 + 2 files changed, 149 insertions(+), 46 deletions(-) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 18c95548..623efe26 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -11,9 +11,10 @@ use foundry_compilers_artifacts::{ output_selection::OutputSelection, visitor::{Visitor, Walk}, ContractDefinitionPart, Expression, FunctionCall, FunctionKind, MemberAccess, NewExpression, - SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, + ParameterList, SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, }; use foundry_compilers_core::utils; +use itertools::Itertools; use md5::Digest; use solang_parser::{diagnostics::Diagnostic, helpers::CodeLocation, pt}; use std::{ @@ -22,6 +23,11 @@ use std::{ path::{Path, PathBuf}, }; +/// Removes parts of the contract which do not alter its interface: +/// - Internal functions +/// - External functions bodies +/// +/// Preserves all libraries and interfaces. pub(crate) fn interface_representation(content: &str) -> Result> { let (source_unit, _) = solang_parser::parse(content, 0)?; let mut locs_to_remove = Vec::new(); @@ -76,6 +82,7 @@ pub(crate) fn interface_representation(content: &str) -> Result String { let Ok(repr) = interface_representation(&source.content) else { return source.content_hash() }; let mut hasher = md5::Md5::new(); @@ -102,12 +109,16 @@ fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { path.starts_with(test_dir) || path.starts_with(script_dir) } +/// Kind of the bytecode dependency. #[derive(Debug)] enum BytecodeDependencyKind { + /// `type(Contract).creationCode` CreationCode, + /// `new Contract` New(ItemLocation, String), } +/// Represents a single bytecode dependency. #[derive(Debug)] struct BytecodeDependency { kind: BytecodeDependencyKind, @@ -190,12 +201,73 @@ impl Visitor for BytecodeDependencyCollector<'_> { } } +fn build_constructor_struct<'a>( + parameters: &'a ParameterList, + src: &'a str, +) -> Result<(String, Vec<&'a str>)> { + let mut s = "struct ConstructorArgs {".to_string(); + let mut param_names = Vec::new(); + + for param in ¶meters.parameters { + param_names.push(param.name.as_str()); + if let Some(loc) = ItemLocation::try_from_loc(param.src) { + let param_def = + &src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " "); + write!(s, "{param_def};")?; + } + } + + s.push('}'); + + Ok((s, param_names)) +} + /// Keeps data about a single contract definition. -struct ContractData { - /// Artifact ID to use in `getCode`/`deployCode` calls. +struct ContractData<'a> { + /// AST id of the contract. + ast_id: usize, + /// Path of the source file. + path: &'a Path, + /// Name of the contract + name: &'a str, + /// Constructor parameters. + constructor_params: Option<&'a ParameterList>, + /// Reference to source code. + src: &'a str, + /// Artifact string to pass into cheatcodes. artifact: String, - /// Whether contract has a non-empty constructor. - has_constructor: bool, +} + +impl ContractData<'_> { + pub fn build_helper(&self) -> Result> { + let Self { ast_id, path, name, constructor_params, src, .. } = self; + + let Some(params) = constructor_params else { return Ok(None) }; + let (constructor_struct, param_names) = build_constructor_struct(params, src)?; + let abi_encode = format!( + "abi.encode({})", + param_names.iter().map(|name| format!("args.{name}")).join(", ") + ); + + let helper = format!( + r#" +pragma solidity >=0.4.0; + +import "{path}"; + +abstract contract DeployHelper{ast_id} is {name} {{ + {constructor_struct} +}} + +function encodeArgs{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ + return {abi_encode}; +}} + "#, + path = path.display(), + ); + + Ok(Some(helper)) + } } /// Receives a set of source files along with their ASTs and removes bytecode dependencies from @@ -215,24 +287,38 @@ impl BytecodeDependencyOptimizer<'_> { BytecodeDependencyOptimizer { asts, paths, sources } } + fn is_src_file(&self, file: &Path) -> bool { + let tests = self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); + let scripts = self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); + + !file.starts_with(tests) && !file.starts_with(scripts) + } + fn process(self) -> Result<()> { let mut updates = Updates::default(); - let contracts = self.collect_contracts(&mut updates); + let contracts = self.collect_contracts(); + let additional_sources = self.create_deploy_helpers(&contracts)?; self.remove_bytecode_dependencies(&contracts, &mut updates)?; + self.sources.extend(additional_sources); + apply_updates(self.sources, updates); Ok(()) } /// Collects a mapping from contract AST id to [ContractData]. - fn collect_contracts(&self, updates: &mut Updates) -> BTreeMap { + fn collect_contracts(&self) -> BTreeMap> { let mut contracts = BTreeMap::new(); for (path, ast) in &self.asts { let src = self.sources.get(path).unwrap().content.as_str(); + if !self.is_src_file(path) { + continue; + } + for node in &ast.nodes { if let SourceUnitPart::ContractDefinition(contract) = node { let artifact = format!("{}:{}", path.display(), contract.name); @@ -247,33 +333,19 @@ impl BytecodeDependencyOptimizer<'_> { Some(func) }); - if constructor.map_or(true, |func| func.parameters.parameters.is_empty()) { - contracts - .insert(contract.id, ContractData { artifact, has_constructor: false }); - continue; - } - contracts.insert(contract.id, ContractData { artifact, has_constructor: true }); - - let constructor = constructor.unwrap(); - let updates = updates.entry(path.clone()).or_default(); - - let mut constructor_helper = - format!("struct ConstructorHelper{} {{", contract.id); - - for param in &constructor.parameters.parameters { - if let Some(loc) = ItemLocation::try_from_loc(param.src) { - let param = &src[loc.start..loc.end] - .replace(" memory ", " ") - .replace(" calldata ", " "); - write!(constructor_helper, "{param};").unwrap(); - } - } - - constructor_helper.push('}'); - - if let Some(loc) = ItemLocation::try_from_loc(constructor.src) { - updates.insert((loc.start, loc.start, constructor_helper)); - } + contracts.insert( + contract.id, + ContractData { + artifact, + constructor_params: constructor + .map(|constructor| &constructor.parameters) + .filter(|params| !params.parameters.is_empty()), + src, + ast_id: contract.id, + path, + name: &contract.name, + }, + ); } } } @@ -281,17 +353,30 @@ impl BytecodeDependencyOptimizer<'_> { contracts } + /// Creates a helper library used to generate helpers for contract deployment. + fn create_deploy_helpers( + &self, + contracts: &BTreeMap>, + ) -> Result { + let mut new_sources = Sources::new(); + for (id, contract) in contracts { + if let Some(code) = contract.build_helper()? { + let path = format!("foundry-pp/DeployHelper{}.sol", id); + new_sources.insert(path.into(), Source::new(code)); + } + } + + Ok(new_sources) + } + /// Goes over all source files and replaces bytecode dependencies with cheatcode invocations. fn remove_bytecode_dependencies( &self, - contracts: &BTreeMap, + contracts: &BTreeMap>, updates: &mut Updates, ) -> Result<()> { - let test_dir = &self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); - let script_dir = - &self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); for (path, ast) in &self.asts { - if !path.starts_with(test_dir) && !path.starts_with(script_dir) { + if self.is_src_file(path) { continue; } let src = self.sources.get(path).unwrap().content.as_str(); @@ -299,6 +384,10 @@ impl BytecodeDependencyOptimizer<'_> { if src.is_empty() { continue; } + + let updates = updates.entry(path.clone()).or_default(); + let mut used_helpers = Vec::new(); + let mut collector = BytecodeDependencyCollector::new(src); ast.walk(&mut collector); @@ -313,15 +402,18 @@ impl BytecodeDependencyOptimizer<'_> { ))); } - let updates = updates.entry(path.clone()).or_default(); let vm_interface_name = format!("VmContractHelper{}", ast.id); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); for dep in collector.dependencies { - let ContractData { artifact, has_constructor } = - contracts.get(&dep.referenced_contract).unwrap(); + let Some(ContractData { artifact, constructor_params, .. }) = + contracts.get(&dep.referenced_contract) + else { + continue; + }; match dep.kind { BytecodeDependencyKind::CreationCode => { + // for creation code we need to just call getCode updates.insert(( dep.loc.start, dep.loc.end, @@ -329,28 +421,36 @@ impl BytecodeDependencyOptimizer<'_> { )); } BytecodeDependencyKind::New(new_loc, name) => { - if !*has_constructor { + if constructor_params.is_none() { updates.insert(( dep.loc.start, dep.loc.end, format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), )); } else { + used_helpers.push(dep.referenced_contract); updates.insert(( new_loc.start, new_loc.end, - format!("{name}(payable({vm}.deployCode(\"{artifact}\", abi.encode({name}.ConstructorHelper{}", dep.referenced_contract), + format!("{name}(payable({vm}.deployCode(\"{artifact}\", encodeArgs{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract), )); updates.insert((dep.loc.end, dep.loc.end, "))))".to_string())); } } }; } + let helper_imports = used_helpers.into_iter().map(|id| { + format!( + "import {{DeployHelper{id}, encodeArgs{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", + ) + }).join("\n"); updates.insert(( - src.len() - 1, - src.len() - 1, + src.len(), + src.len(), format!( r#" +{helper_imports} + interface {vm_interface_name} {{ function deployCode(string memory _artifact, bytes memory _data) external returns (address); function deployCode(string memory _artifact) external returns (address); diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index d58b0f44..db78648c 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -70,6 +70,9 @@ pub enum SolcError { #[error("no artifact found for `{}:{}`", .0.display(), .1)] ArtifactNotFound(PathBuf, String), + #[error(transparent)] + Fmt(#[from] std::fmt::Error), + #[cfg(feature = "project-util")] #[error(transparent)] FsExtra(#[from] fs_extra::error::Error), From fa5d7bfbf9d0779c1703fdd03cf0dbabd7a06213 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 13 Sep 2024 19:59:02 +0400 Subject: [PATCH 08/70] some docs --- crates/compilers/src/preprocessor.rs | 109 ++++++++++++++++++--------- 1 file changed, 74 insertions(+), 35 deletions(-) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 623efe26..42c6140d 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -18,8 +18,7 @@ use itertools::Itertools; use md5::Digest; use solang_parser::{diagnostics::Diagnostic, helpers::CodeLocation, pt}; use std::{ - collections::BTreeMap, - fmt::Write, + collections::{BTreeMap, BTreeSet}, path::{Path, PathBuf}, }; @@ -126,6 +125,7 @@ struct BytecodeDependency { referenced_contract: usize, } +/// Walks over AST and collects [`BytecodeDependency`]s. #[derive(Debug)] struct BytecodeDependencyCollector<'a> { source: &'a str, @@ -201,27 +201,6 @@ impl Visitor for BytecodeDependencyCollector<'_> { } } -fn build_constructor_struct<'a>( - parameters: &'a ParameterList, - src: &'a str, -) -> Result<(String, Vec<&'a str>)> { - let mut s = "struct ConstructorArgs {".to_string(); - let mut param_names = Vec::new(); - - for param in ¶meters.parameters { - param_names.push(param.name.as_str()); - if let Some(loc) = ItemLocation::try_from_loc(param.src) { - let param_def = - &src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " "); - write!(s, "{param_def};")?; - } - } - - s.push('}'); - - Ok((s, param_names)) -} - /// Keeps data about a single contract definition. struct ContractData<'a> { /// AST id of the contract. @@ -239,15 +218,65 @@ struct ContractData<'a> { } impl ContractData<'_> { + /// If contract has a non-empty constructor, generates a helper source file for it containing a + /// helper to encode constructor arguments. + /// + /// This is needed because current preprocessing wraps the arguments, leaving them unchanged. + /// This allows us to handle nested new expressions correctly. However, this requires us to have + /// a way to wrap both named and unnamed arguments. i.e you can't do abi.encode({arg: val}). + /// + /// This function produces a helper struct + a helper function to encode the arguments. The + /// struct is defined in scope of an abstract contract inheriting the contract containing the + /// constructor. This is done as a hack to allow us to inherit the same scope of definitions. + /// + /// The resulted helper looks like this: + /// ```solidity + /// import "lib/openzeppelin-contracts/contracts/token/ERC20.sol"; + /// + /// abstract contract DeployHelper335 is ERC20 { + /// struct ConstructorArgs { + /// string name; + /// string symbol; + /// } + /// } + /// + /// function encodeArgs335(DeployHelper335.ConstructorArgs memory args) pure returns (bytes memory) { + /// return abi.encode(args.name, args.symbol); + /// } + /// ``` + /// + /// Example usage: + /// ```solidity + /// new ERC20(name, symbol) + /// ``` + /// becomes + /// ```solidity + /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs(name, symbol))) + /// ``` + /// With named arguments: + /// ```solidity + /// new ERC20({name: name, symbol: symbol}) + /// ``` + /// becomes + /// ```solidity + /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) + /// ``` pub fn build_helper(&self) -> Result> { let Self { ast_id, path, name, constructor_params, src, .. } = self; let Some(params) = constructor_params else { return Ok(None) }; - let (constructor_struct, param_names) = build_constructor_struct(params, src)?; - let abi_encode = format!( - "abi.encode({})", - param_names.iter().map(|name| format!("args.{name}")).join(", ") - ); + + let struct_fields = params + .parameters + .iter() + .filter_map(|param| { + let loc = ItemLocation::try_from_loc(param.src)?; + Some(src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " ")) + }) + .join("; "); + + let abi_encode_args = + params.parameters.iter().map(|param| format!("args.{}", param.name)).join(", "); let helper = format!( r#" @@ -256,11 +285,13 @@ pragma solidity >=0.4.0; import "{path}"; abstract contract DeployHelper{ast_id} is {name} {{ - {constructor_struct} + struct ConstructorArgs {{ + {struct_fields}; + }} }} function encodeArgs{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ - return {abi_encode}; + return abi.encode({abi_encode_args}); }} "#, path = path.display(), @@ -287,6 +318,7 @@ impl BytecodeDependencyOptimizer<'_> { BytecodeDependencyOptimizer { asts, paths, sources } } + /// Returns true if the file is not a test or script file. fn is_src_file(&self, file: &Path) -> bool { let tests = self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); let scripts = self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); @@ -308,7 +340,8 @@ impl BytecodeDependencyOptimizer<'_> { Ok(()) } - /// Collects a mapping from contract AST id to [ContractData]. + /// Collects a mapping from contract AST id to [ContractData] for all contracts defined in src/ + /// files. fn collect_contracts(&self) -> BTreeMap> { let mut contracts = BTreeMap::new(); @@ -353,7 +386,9 @@ impl BytecodeDependencyOptimizer<'_> { contracts } - /// Creates a helper library used to generate helpers for contract deployment. + /// Creates helper libraries for contracts with a non-empty constructor. + /// + /// See [`ContractData::build_helper`] for more details. fn create_deploy_helpers( &self, contracts: &BTreeMap>, @@ -369,7 +404,8 @@ impl BytecodeDependencyOptimizer<'_> { Ok(new_sources) } - /// Goes over all source files and replaces bytecode dependencies with cheatcode invocations. + /// Goes over all test/script files and replaces bytecode dependencies with cheatcode + /// invocations. fn remove_bytecode_dependencies( &self, contracts: &BTreeMap>, @@ -386,7 +422,7 @@ impl BytecodeDependencyOptimizer<'_> { } let updates = updates.entry(path.clone()).or_default(); - let mut used_helpers = Vec::new(); + let mut used_helpers = BTreeSet::new(); let mut collector = BytecodeDependencyCollector::new(src); ast.walk(&mut collector); @@ -422,13 +458,16 @@ impl BytecodeDependencyOptimizer<'_> { } BytecodeDependencyKind::New(new_loc, name) => { if constructor_params.is_none() { + // if there's no constructor, we can just call deployCode with one + // argument updates.insert(( dep.loc.start, dep.loc.end, format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), )); } else { - used_helpers.push(dep.referenced_contract); + // if there's a constructor, we use our helper + used_helpers.insert(dep.referenced_contract); updates.insert(( new_loc.start, new_loc.end, From 15ce120c2e17aa384a43af79a202a36ac78a0c6f Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 13 Sep 2024 20:06:55 +0400 Subject: [PATCH 09/70] clean up --- crates/compilers/src/preprocessor.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 42c6140d..096a3d5f 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -102,6 +102,7 @@ impl ItemLocation { } } +/// Checks if the given path is a test/script file. fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { let test_dir = paths.tests.strip_prefix(&paths.root).unwrap_or(&paths.root); let script_dir = paths.scripts.strip_prefix(&paths.root).unwrap_or(&paths.root); @@ -318,14 +319,6 @@ impl BytecodeDependencyOptimizer<'_> { BytecodeDependencyOptimizer { asts, paths, sources } } - /// Returns true if the file is not a test or script file. - fn is_src_file(&self, file: &Path) -> bool { - let tests = self.paths.tests.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); - let scripts = self.paths.scripts.strip_prefix(&self.paths.root).unwrap_or(&self.paths.root); - - !file.starts_with(tests) && !file.starts_with(scripts) - } - fn process(self) -> Result<()> { let mut updates = Updates::default(); @@ -348,7 +341,7 @@ impl BytecodeDependencyOptimizer<'_> { for (path, ast) in &self.asts { let src = self.sources.get(path).unwrap().content.as_str(); - if !self.is_src_file(path) { + if is_test_or_script(path, &self.paths) { continue; } @@ -412,7 +405,7 @@ impl BytecodeDependencyOptimizer<'_> { updates: &mut Updates, ) -> Result<()> { for (path, ast) in &self.asts { - if self.is_src_file(path) { + if !is_test_or_script(path, &self.paths) { continue; } let src = self.sources.get(path).unwrap().content.as_str(); From a379db3eba7739500c7062381503c82a6b556eee Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Mon, 16 Sep 2024 21:01:57 +0300 Subject: [PATCH 10/70] fixes --- crates/compilers/src/preprocessor.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 096a3d5f..d79f6619 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -263,7 +263,7 @@ impl ContractData<'_> { /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) /// ``` pub fn build_helper(&self) -> Result> { - let Self { ast_id, path, name, constructor_params, src, .. } = self; + let Self { ast_id, path, name, constructor_params, src, artifact } = self; let Some(params) = constructor_params else { return Ok(None) }; @@ -278,6 +278,9 @@ impl ContractData<'_> { let abi_encode_args = params.parameters.iter().map(|param| format!("args.{}", param.name)).join(", "); + + let vm_interface_name = format!("VmContractHelper{}", ast_id); + let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); let helper = format!( r#" @@ -294,6 +297,16 @@ abstract contract DeployHelper{ast_id} is {name} {{ function encodeArgs{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ return abi.encode({abi_encode_args}); }} + +function deployCode{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) returns({name}) {{ + return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{ast_id}(args)))); +}} + +interface {vm_interface_name} {{ + function deployCode(string memory _artifact, bytes memory _data) external returns (address); + function deployCode(string memory _artifact) external returns (address); + function getCode(string memory _artifact) external returns (bytes memory); +}} "#, path = path.display(), ); @@ -464,16 +477,16 @@ impl BytecodeDependencyOptimizer<'_> { updates.insert(( new_loc.start, new_loc.end, - format!("{name}(payable({vm}.deployCode(\"{artifact}\", encodeArgs{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract), + format!("deployCode{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract), )); - updates.insert((dep.loc.end, dep.loc.end, "))))".to_string())); + updates.insert((dep.loc.end, dep.loc.end, ")".to_string())); } } }; } let helper_imports = used_helpers.into_iter().map(|id| { format!( - "import {{DeployHelper{id}, encodeArgs{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", + "import {{DeployHelper{id}, encodeArgs{id}, deployCode{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", ) }).join("\n"); updates.insert(( From 3faaa0280217dd1b3faedddb0b666b183af70e11 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 07:28:02 +0200 Subject: [PATCH 11/70] Port to solar --- crates/compilers/src/cache.rs | 4 +- crates/compilers/src/preprocessor.rs | 139 ++++++++++++++++----------- 2 files changed, 86 insertions(+), 57 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 10431f91..c437538b 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -681,7 +681,7 @@ impl, C: Compiler> .collect(); let interface_repr_hash = - self.is_source_file(&file).then(|| interface_representation_hash(source)); + self.is_source_file(&file).then(|| interface_representation_hash(source, &file)); let entry = CacheEntry { last_modification_date: CacheEntry::read_last_modification_date(&file) @@ -949,7 +949,7 @@ impl, C: Compiler> if let hash_map::Entry::Vacant(entry) = self.interface_repr_hashes.entry(file.clone()) { - entry.insert(interface_representation_hash(&source)); + entry.insert(interface_representation_hash(&source, file)); } } } diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 2bc60313..632870a4 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -16,74 +16,100 @@ use foundry_compilers_artifacts::{ use foundry_compilers_core::utils; use itertools::Itertools; use md5::Digest; +use solar_parse::{ + ast::{Span, Visibility}, + interface::diagnostics::EmittedDiagnostics, +}; use std::{ collections::{BTreeMap, BTreeSet}, path::{Path, PathBuf}, }; -use solar_parse::interface::diagnostics::Diagnostic; /// Removes parts of the contract which do not alter its interface: /// - Internal functions /// - External functions bodies /// /// Preserves all libraries and interfaces. -pub(crate) fn interface_representation(content: &str) -> Result> { - // let (source_unit, _) = solang_parser::parse(content, 0)?; - // let mut locs_to_remove = Vec::new(); - // - // for part in source_unit.0 { - // if let pt::SourceUnitPart::ContractDefinition(contract) = part { - // if matches!(contract.ty, pt::ContractTy::Interface(_) | pt::ContractTy::Library(_)) { - // continue; - // } - // for part in contract.parts { - // if let pt::ContractPart::FunctionDefinition(func) = part { - // let is_exposed = func.ty == pt::FunctionTy::Function - // && func.attributes.iter().any(|attr| { - // matches!( - // attr, - // pt::FunctionAttribute::Visibility( - // pt::Visibility::External(_) | pt::Visibility::Public(_) - // ) - // ) - // }) - // || matches!( - // func.ty, - // pt::FunctionTy::Constructor - // | pt::FunctionTy::Fallback - // | pt::FunctionTy::Receive - // ); - // - // if !is_exposed { - // locs_to_remove.push(func.loc); - // } - // - // if let Some(ref body) = func.body { - // locs_to_remove.push(body.loc()); - // } - // } - // } - // } - // } - // - // let mut content = content.to_string(); - // let mut offset = 0; - // - // for loc in locs_to_remove { - // let start = loc.start() - offset; - // let end = loc.end() - offset; - // - // content.replace_range(start..end, ""); - // offset += end - start; - // } +pub(crate) fn interface_representation( + content: &str, + file: &PathBuf, +) -> Result { + let mut spans_to_remove: Vec = Vec::new(); + let sess = + solar_parse::interface::Session::builder().with_buffer_emitter(Default::default()).build(); + sess.enter(|| { + let arena = solar_parse::ast::Arena::new(); + let filename = solar_parse::interface::source_map::FileName::Real(file.to_path_buf()); + let Ok(mut parser) = + solar_parse::Parser::from_source_code(&sess, &arena, filename, content.to_string()) + else { + return; + }; + let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; + for item in ast.items { + if let solar_parse::ast::ItemKind::Contract(contract) = &item.kind { + if contract.kind.is_interface() | contract.kind.is_library() { + continue; + } + for contract_item in contract.body.iter() { + if let solar_parse::ast::ItemKind::Function(function) = &contract_item.kind { + let is_exposed = match function.kind { + // Function with external or public visibility + solar_parse::ast::FunctionKind::Function => { + function.header.visibility.is_some_and(|visibility| { + visibility == Visibility::External + || visibility == Visibility::Public + }) + } + solar_parse::ast::FunctionKind::Constructor + | solar_parse::ast::FunctionKind::Fallback + | solar_parse::ast::FunctionKind::Receive => true, + // Other (modifiers) + _ => false, + }; + + // If function is not exposed we remove the entire span (signature and + // body). Otherwise we keep function signature and + // remove only the body. + if !is_exposed { + spans_to_remove.push(contract_item.span); + } else { + spans_to_remove.push(function.body_span); + } + } + } + } + } + }); + + // Return original content if errors. + if let Err(err) = sess.emitted_errors().unwrap() { + let e = err.to_string(); + trace!("failed parsing {file:?}: {e}"); + return Err(err); + } + + let mut content = content.to_string(); + let mut offset = 0; + + for span in spans_to_remove { + let range = span.to_range(); + let start = range.start - offset; + let end = range.end - offset; + + content.replace_range(start..end, ""); + offset += end - start; + } let content = content.replace("\n", ""); Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) } /// Computes hash of [`interface_representation`] of the source. -pub(crate) fn interface_representation_hash(source: &Source) -> String { - let Ok(repr) = interface_representation(&source.content) else { return source.content_hash() }; +pub(crate) fn interface_representation_hash(source: &Source, file: &PathBuf) -> String { + let Ok(repr) = interface_representation(&source.content, file) else { + return source.content_hash(); + }; let mut hasher = md5::Md5::new(); hasher.update(&repr); let result = hasher.finalize(); @@ -278,7 +304,7 @@ impl ContractData<'_> { let abi_encode_args = params.parameters.iter().map(|param| format!("args.{}", param.name)).join(", "); - + let vm_interface_name = format!("VmContractHelper{}", ast_id); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); @@ -477,7 +503,10 @@ impl BytecodeDependencyOptimizer<'_> { updates.insert(( new_loc.start, new_loc.end, - format!("deployCode{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract), + format!( + "deployCode{id}(DeployHelper{id}.ConstructorArgs", + id = dep.referenced_contract + ), )); updates.insert((dep.loc.end, dep.loc.end, ")".to_string())); } @@ -605,7 +634,7 @@ contract A { } }"#; - let result = interface_representation(content).unwrap(); + let result = interface_representation(content, &PathBuf::new()).unwrap(); assert_eq!( result, r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# From 0f781974246d5d8a58cbdf9315330e5a400be6f3 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 09:56:34 +0200 Subject: [PATCH 12/70] Clippy --- crates/compilers/src/cache.rs | 6 +++--- crates/compilers/src/preprocessor.rs | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index c437538b..670331ca 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -918,7 +918,7 @@ impl, C: Compiler> return true; }; - if entry.interface_repr_hash.as_ref().map_or(true, |h| h != interface_hash) { + if entry.interface_repr_hash.as_ref() != Some(interface_hash) { trace!("interface hash changed"); return true; }; @@ -945,11 +945,11 @@ impl, C: Compiler> entry.insert(source.content_hash()); } // Fill interface representation hashes for source files - if self.is_source_file(&file) { + if self.is_source_file(file) { if let hash_map::Entry::Vacant(entry) = self.interface_repr_hashes.entry(file.clone()) { - entry.insert(interface_representation_hash(&source, file)); + entry.insert(interface_representation_hash(source, file)); } } } diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 632870a4..a858c9a7 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -82,7 +82,7 @@ pub(crate) fn interface_representation( } }); - // Return original content if errors. + // Return if any diagnostics emitted during content parsing. if let Err(err) = sess.emitted_errors().unwrap() { let e = err.to_string(); trace!("failed parsing {file:?}: {e}"); @@ -167,12 +167,6 @@ impl BytecodeDependencyCollector<'_> { } impl Visitor for BytecodeDependencyCollector<'_> { - fn visit_new_expression(&mut self, expr: &NewExpression) { - if let TypeName::UserDefinedTypeName(_) = &expr.type_name { - self.total_count += 1; - } - } - fn visit_function_call(&mut self, call: &FunctionCall) { let (new_loc, expr) = match &call.expression { Expression::NewExpression(expr) => (expr.src, expr), @@ -200,6 +194,12 @@ impl Visitor for BytecodeDependencyCollector<'_> { }); } + fn visit_new_expression(&mut self, expr: &NewExpression) { + if let TypeName::UserDefinedTypeName(_) = &expr.type_name { + self.total_count += 1; + } + } + fn visit_member_access(&mut self, access: &MemberAccess) { if access.member_name != "creationCode" { return; @@ -305,7 +305,7 @@ impl ContractData<'_> { let abi_encode_args = params.parameters.iter().map(|param| format!("args.{}", param.name)).join(", "); - let vm_interface_name = format!("VmContractHelper{}", ast_id); + let vm_interface_name = format!("VmContractHelper{ast_id}"); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); let helper = format!( @@ -380,7 +380,7 @@ impl BytecodeDependencyOptimizer<'_> { for (path, ast) in &self.asts { let src = self.sources.get(path).unwrap().content.as_str(); - if is_test_or_script(path, &self.paths) { + if is_test_or_script(path, self.paths) { continue; } @@ -428,7 +428,7 @@ impl BytecodeDependencyOptimizer<'_> { let mut new_sources = Sources::new(); for (id, contract) in contracts { if let Some(code) = contract.build_helper()? { - let path = format!("foundry-pp/DeployHelper{}.sol", id); + let path = format!("foundry-pp/DeployHelper{id}.sol"); new_sources.insert(path.into(), Source::new(code)); } } @@ -444,7 +444,7 @@ impl BytecodeDependencyOptimizer<'_> { updates: &mut Updates, ) -> Result<()> { for (path, ast) in &self.asts { - if !is_test_or_script(path, &self.paths) { + if !is_test_or_script(path, self.paths) { continue; } let src = self.sources.get(path).unwrap().content.as_str(); From 594c09b0a111f5af3c43cadbe5dad0bb05545c71 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 13:29:27 +0200 Subject: [PATCH 13/70] Changes after review --- crates/compilers/src/lib.rs | 68 ++++++++++++++++++++++++++++ crates/compilers/src/preprocessor.rs | 54 ++++++++-------------- 2 files changed, 88 insertions(+), 34 deletions(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index ea2d1af4..2bbd1e90 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -64,6 +64,7 @@ use semver::Version; use solc::SolcSettings; use std::{ collections::{BTreeMap, HashMap, HashSet}, + ops::Range, path::{Path, PathBuf}, }; @@ -884,6 +885,23 @@ fn rebase_path(base: &Path, path: &Path) -> PathBuf { new_path.to_slash_lossy().into_owned().into() } +/// Utility function to change source content ranges with provided updates. +fn replace_source_content<'a>( + source: &str, + updates: impl Iterator, &'a str)>, +) -> String { + let mut updated_content = source.to_string(); + let mut offset = 0; + for (range, update) in updates { + let start = range.start - offset; + let end = range.end - offset; + + updated_content.replace_range(start..end, update); + offset += end - start; + } + updated_content +} + #[cfg(test)] #[cfg(feature = "svm-solc")] mod tests { @@ -1033,4 +1051,54 @@ mod tests { .unwrap(); assert!(resolved.exists()); } + + #[test] + fn test_replace_source_content() { + let original_content = r#" +library Lib { + function libFn() internal { + // logic to keep + } +} +contract A { + function a() external {} + function b() public {} + function c() internal { + // logic logic logic + } + function d() private {} + function e() external { + // logic logic logic + } +}"#; + let updates = vec![ + // Replace function libFn() visibility to external + ((36..44), "external"), + // Replace contract A name to contract B + ((88..98), "contract B"), + // Remove function c() + ((167..230), ""), + // Replace function e() logic + ((294..314), "// no logic"), + ] + .into_iter(); + + assert_eq!( + replace_source_content(original_content, updates), + r#" +library Lib { + function libFn() external { + // logic to keep + } +} +contract B { + function a() external {} + function b() public {} + function d() private {} + function e() external { + // no logic + } +}"# + ); + } } diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index a858c9a7..9aa3efab 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -2,6 +2,7 @@ use super::project::Preprocessor; use crate::{ flatten::{apply_updates, Updates}, multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, + replace_source_content, solc::{SolcCompiler, SolcVersionedInput}, Compiler, ProjectPathsConfig, Result, SolcError, }; @@ -10,15 +11,15 @@ use foundry_compilers_artifacts::{ ast::SourceLocation, output_selection::OutputSelection, visitor::{Visitor, Walk}, - ContractDefinitionPart, Expression, FunctionCall, FunctionKind, MemberAccess, NewExpression, - ParameterList, SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, + ContractDefinitionPart, Expression, FunctionCall, MemberAccess, NewExpression, ParameterList, + SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, }; use foundry_compilers_core::utils; use itertools::Itertools; use md5::Digest; use solar_parse::{ - ast::{Span, Visibility}, - interface::diagnostics::EmittedDiagnostics, + ast::{Arena, FunctionKind, ItemKind, Span, Visibility}, + interface::{diagnostics::EmittedDiagnostics, source_map::FileName}, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -38,8 +39,8 @@ pub(crate) fn interface_representation( let sess = solar_parse::interface::Session::builder().with_buffer_emitter(Default::default()).build(); sess.enter(|| { - let arena = solar_parse::ast::Arena::new(); - let filename = solar_parse::interface::source_map::FileName::Real(file.to_path_buf()); + let arena = Arena::new(); + let filename = FileName::Real(file.to_path_buf()); let Ok(mut parser) = solar_parse::Parser::from_source_code(&sess, &arena, filename, content.to_string()) else { @@ -47,25 +48,21 @@ pub(crate) fn interface_representation( }; let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; for item in ast.items { - if let solar_parse::ast::ItemKind::Contract(contract) = &item.kind { + if let ItemKind::Contract(contract) = &item.kind { if contract.kind.is_interface() | contract.kind.is_library() { continue; } for contract_item in contract.body.iter() { - if let solar_parse::ast::ItemKind::Function(function) = &contract_item.kind { + if let ItemKind::Function(function) = &contract_item.kind { let is_exposed = match function.kind { // Function with external or public visibility - solar_parse::ast::FunctionKind::Function => { - function.header.visibility.is_some_and(|visibility| { - visibility == Visibility::External - || visibility == Visibility::Public - }) + FunctionKind::Function => { + function.header.visibility >= Some(Visibility::Public) } - solar_parse::ast::FunctionKind::Constructor - | solar_parse::ast::FunctionKind::Fallback - | solar_parse::ast::FunctionKind::Receive => true, - // Other (modifiers) - _ => false, + FunctionKind::Constructor + | FunctionKind::Fallback + | FunctionKind::Receive => true, + FunctionKind::Modifier => false, }; // If function is not exposed we remove the entire span (signature and @@ -84,24 +81,13 @@ pub(crate) fn interface_representation( // Return if any diagnostics emitted during content parsing. if let Err(err) = sess.emitted_errors().unwrap() { - let e = err.to_string(); - trace!("failed parsing {file:?}: {e}"); + trace!("failed parsing {file:?}: {err}"); return Err(err); } - let mut content = content.to_string(); - let mut offset = 0; - - for span in spans_to_remove { - let range = span.to_range(); - let start = range.start - offset; - let end = range.end - offset; - - content.replace_range(start..end, ""); - offset += end - start; - } - - let content = content.replace("\n", ""); + let content = + replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) + .replace("\n", ""); Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) } @@ -391,7 +377,7 @@ impl BytecodeDependencyOptimizer<'_> { let ContractDefinitionPart::FunctionDefinition(func) = node else { return None; }; - if *func.kind() != FunctionKind::Constructor { + if *func.kind() != foundry_compilers_artifacts::FunctionKind::Constructor { return None; } From 3105272083e83dfca75de7febf8a3627c8482591 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 14:58:37 +0200 Subject: [PATCH 14/70] Patch solar to get HIR visitor impl --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 0cb4c9d3..64864be7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,3 +67,6 @@ futures-util = "0.3" tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" + +[patch.crates-io] +solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "6e8f4a1" } From 7d0edb851c0bf0234be4130e9950537187c38fab Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 16:36:47 +0200 Subject: [PATCH 15/70] Review changes --- crates/compilers/src/preprocessor.rs | 49 +++++++++++++++------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 9aa3efab..6e974dbd 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -48,31 +48,34 @@ pub(crate) fn interface_representation( }; let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; for item in ast.items { - if let ItemKind::Contract(contract) = &item.kind { - if contract.kind.is_interface() | contract.kind.is_library() { - continue; - } - for contract_item in contract.body.iter() { - if let ItemKind::Function(function) = &contract_item.kind { - let is_exposed = match function.kind { - // Function with external or public visibility - FunctionKind::Function => { - function.header.visibility >= Some(Visibility::Public) - } - FunctionKind::Constructor - | FunctionKind::Fallback - | FunctionKind::Receive => true, - FunctionKind::Modifier => false, - }; + let ItemKind::Contract(contract) = &item.kind else { + continue; + }; - // If function is not exposed we remove the entire span (signature and - // body). Otherwise we keep function signature and - // remove only the body. - if !is_exposed { - spans_to_remove.push(contract_item.span); - } else { - spans_to_remove.push(function.body_span); + if contract.kind.is_interface() || contract.kind.is_library() { + continue; + } + + for contract_item in contract.body.iter() { + if let ItemKind::Function(function) = &contract_item.kind { + let is_exposed = match function.kind { + // Function with external or public visibility + FunctionKind::Function => { + function.header.visibility >= Some(Visibility::Public) } + FunctionKind::Constructor + | FunctionKind::Fallback + | FunctionKind::Receive => true, + FunctionKind::Modifier => false, + }; + + // If function is not exposed we remove the entire span (signature and + // body). Otherwise we keep function signature and + // remove only the body. + if !is_exposed { + spans_to_remove.push(contract_item.span); + } else { + spans_to_remove.push(function.body_span); } } } From 4025e5a045d9b23aa14e2c4d38a6a6f79a399884 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 18:45:00 +0200 Subject: [PATCH 16/70] reuse replace source fn --- crates/compilers/src/flatten.rs | 15 ++++---------- crates/compilers/src/lib.rs | 31 ++++++++++++++-------------- crates/compilers/src/preprocessor.rs | 10 ++++++--- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/crates/compilers/src/flatten.rs b/crates/compilers/src/flatten.rs index 1c755a32..bba0e21d 100644 --- a/crates/compilers/src/flatten.rs +++ b/crates/compilers/src/flatten.rs @@ -1,6 +1,7 @@ use crate::{ compilers::{Compiler, ParsedSource}, filter::MaybeSolData, + replace_source_content, resolver::parse::SolData, ArtifactOutput, CompilerSettings, Graph, Project, ProjectPathsConfig, }; @@ -111,6 +112,7 @@ impl Visitor for ReferencesCollector { } } +pub type Update = (usize, usize, String); /// Updates to be applied to the sources. /// source_path -> (start, end, new_value) pub type Updates = HashMap>; @@ -906,17 +908,8 @@ pub fn combine_version_pragmas(pragmas: Vec<&str>) -> Option { pub fn apply_updates(sources: &mut Sources, mut updates: Updates) { for (path, source) in sources { if let Some(updates) = updates.remove(path) { - let mut offset = 0; - let mut content = source.content.as_bytes().to_vec(); - for (start, end, new_value) in updates { - let start = (start as isize + offset) as usize; - let end = (end as isize + offset) as usize; - - content.splice(start..end, new_value.bytes()); - offset += new_value.len() as isize - (end - start) as isize; - } - - source.content = Arc::new(String::from_utf8_lossy(&content).to_string()); + source.content = + Arc::new(replace_source_content(source.content.as_str(), updates.into_iter())); } } } diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 2bbd1e90..7600b284 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -47,6 +47,7 @@ pub mod project_util; pub use foundry_compilers_artifacts as artifacts; pub use foundry_compilers_core::{error, utils}; +use crate::flatten::Update; use cache::CompilerCache; use compile::output::contracts::VersionedContracts; use compilers::multi::MultiCompiler; @@ -64,7 +65,6 @@ use semver::Version; use solc::SolcSettings; use std::{ collections::{BTreeMap, HashMap, HashSet}, - ops::Range, path::{Path, PathBuf}, }; @@ -886,20 +886,18 @@ fn rebase_path(base: &Path, path: &Path) -> PathBuf { } /// Utility function to change source content ranges with provided updates. -fn replace_source_content<'a>( - source: &str, - updates: impl Iterator, &'a str)>, -) -> String { - let mut updated_content = source.to_string(); +fn replace_source_content(source: &str, updates: impl Iterator) -> String { let mut offset = 0; - for (range, update) in updates { - let start = range.start - offset; - let end = range.end - offset; + let mut content = source.as_bytes().to_vec(); + for (start, end, new_value) in updates { + let start = (start as isize + offset) as usize; + let end = (end as isize + offset) as usize; - updated_content.replace_range(start..end, update); - offset += end - start; + content.splice(start..end, new_value.bytes()); + offset += new_value.len() as isize - (end - start) as isize; } - updated_content + + String::from_utf8_lossy(&content).to_string() } #[cfg(test)] @@ -1071,15 +1069,16 @@ contract A { // logic logic logic } }"#; + let updates = vec![ // Replace function libFn() visibility to external - ((36..44), "external"), + (36, 44, "external".to_string()), // Replace contract A name to contract B - ((88..98), "contract B"), + (80, 90, "contract B".to_string()), // Remove function c() - ((167..230), ""), + (159, 222, "".to_string()), // Replace function e() logic - ((294..314), "// no logic"), + (276, 296, "// no logic".to_string()), ] .into_iter(); diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 6e974dbd..dd4464e6 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -88,9 +88,13 @@ pub(crate) fn interface_representation( return Err(err); } - let content = - replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) - .replace("\n", ""); + let content = replace_source_content( + content, + spans_to_remove + .iter() + .map(|span| (span.to_range().start, span.to_range().end, String::new())), + ) + .replace("\n", ""); Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) } From 0c940d1677a5421d659e65d7089fe4df4f33b56b Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 19:04:54 +0200 Subject: [PATCH 17/70] Clippy --- crates/compilers/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 7600b284..9f973935 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -1076,7 +1076,7 @@ contract A { // Replace contract A name to contract B (80, 90, "contract B".to_string()), // Remove function c() - (159, 222, "".to_string()), + (159, 222, String::new()), // Replace function e() logic (276, 296, "// no logic".to_string()), ] From 79d0a0f74ea20d2b5fb5e1b37864d2bbbcf77749 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 19 Feb 2025 19:22:38 +0200 Subject: [PATCH 18/70] Cleanup, move Update type in lib --- crates/compilers/src/flatten.rs | 9 ++------- crates/compilers/src/lib.rs | 8 ++++++-- crates/compilers/src/preprocessor.rs | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/compilers/src/flatten.rs b/crates/compilers/src/flatten.rs index bba0e21d..331399de 100644 --- a/crates/compilers/src/flatten.rs +++ b/crates/compilers/src/flatten.rs @@ -3,7 +3,7 @@ use crate::{ filter::MaybeSolData, replace_source_content, resolver::parse::SolData, - ArtifactOutput, CompilerSettings, Graph, Project, ProjectPathsConfig, + ArtifactOutput, CompilerSettings, Graph, Project, ProjectPathsConfig, Updates, }; use foundry_compilers_artifacts::{ ast::{visitor::Visitor, *}, @@ -18,7 +18,7 @@ use foundry_compilers_core::{ }; use itertools::Itertools; use std::{ - collections::{BTreeSet, HashMap, HashSet}, + collections::{HashMap, HashSet}, hash::Hash, path::{Path, PathBuf}, sync::Arc, @@ -112,11 +112,6 @@ impl Visitor for ReferencesCollector { } } -pub type Update = (usize, usize, String); -/// Updates to be applied to the sources. -/// source_path -> (start, end, new_value) -pub type Updates = HashMap>; - pub struct FlatteningResult { /// Updated source in the order they should be written to the output file. sources: Vec, diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 9f973935..b08d200a 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -40,6 +40,11 @@ pub use filter::{FileFilter, SparseOutputFilter, TestFileFilter}; pub mod report; +/// Updates to be applied to the sources. +/// source_path -> (start, end, new_value) +pub type Update = (usize, usize, String); +pub type Updates = HashMap>; + /// Utilities for creating, mocking and testing of (temporary) projects #[cfg(feature = "project-util")] pub mod project_util; @@ -47,7 +52,6 @@ pub mod project_util; pub use foundry_compilers_artifacts as artifacts; pub use foundry_compilers_core::{error, utils}; -use crate::flatten::Update; use cache::CompilerCache; use compile::output::contracts::VersionedContracts; use compilers::multi::MultiCompiler; @@ -64,7 +68,7 @@ use project::ProjectCompiler; use semver::Version; use solc::SolcSettings; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, path::{Path, PathBuf}, }; diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index dd4464e6..6cd2e55a 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -1,10 +1,10 @@ use super::project::Preprocessor; use crate::{ - flatten::{apply_updates, Updates}, + flatten::apply_updates, multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, replace_source_content, solc::{SolcCompiler, SolcVersionedInput}, - Compiler, ProjectPathsConfig, Result, SolcError, + Compiler, ProjectPathsConfig, Result, SolcError, Updates, }; use alloy_primitives::hex; use foundry_compilers_artifacts::{ From d37cafb8f47521c4e50b126db2acac6dcd4e002a Mon Sep 17 00:00:00 2001 From: grandizzy Date: Thu, 20 Feb 2025 14:47:04 +0200 Subject: [PATCH 19/70] Change replace_source_content sig, move apply_updates --- crates/compilers/src/flatten.rs | 11 +------- crates/compilers/src/lib.rs | 39 +++++++++++++++++++--------- crates/compilers/src/preprocessor.rs | 12 +++------ 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/crates/compilers/src/flatten.rs b/crates/compilers/src/flatten.rs index 331399de..636e5006 100644 --- a/crates/compilers/src/flatten.rs +++ b/crates/compilers/src/flatten.rs @@ -1,7 +1,7 @@ use crate::{ + apply_updates, compilers::{Compiler, ParsedSource}, filter::MaybeSolData, - replace_source_content, resolver::parse::SolData, ArtifactOutput, CompilerSettings, Graph, Project, ProjectPathsConfig, Updates, }; @@ -899,12 +899,3 @@ pub fn combine_version_pragmas(pragmas: Vec<&str>) -> Option { None } - -pub fn apply_updates(sources: &mut Sources, mut updates: Updates) { - for (path, source) in sources { - if let Some(updates) = updates.remove(path) { - source.content = - Arc::new(replace_source_content(source.content.as_str(), updates.into_iter())); - } - } -} diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index b08d200a..81f38d25 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -42,8 +42,7 @@ pub mod report; /// Updates to be applied to the sources. /// source_path -> (start, end, new_value) -pub type Update = (usize, usize, String); -pub type Updates = HashMap>; +pub type Updates = HashMap>; /// Utilities for creating, mocking and testing of (temporary) projects #[cfg(feature = "project-util")] @@ -69,7 +68,9 @@ use semver::Version; use solc::SolcSettings; use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + ops::Range, path::{Path, PathBuf}, + sync::Arc, }; /// Represents a project workspace and handles `solc` compiling of all contracts in that workspace. @@ -889,13 +890,28 @@ fn rebase_path(base: &Path, path: &Path) -> PathBuf { new_path.to_slash_lossy().into_owned().into() } +/// Utility function to apply a set of updates to provided sources. +fn apply_updates(sources: &mut Sources, updates: Updates) { + for (path, source) in sources { + if let Some(updates) = updates.get(path) { + source.content = Arc::new(replace_source_content( + source.content.as_str(), + updates.iter().map(|(start, end, update)| ((*start..*end), update.as_str())), + )); + } + } +} + /// Utility function to change source content ranges with provided updates. -fn replace_source_content(source: &str, updates: impl Iterator) -> String { +fn replace_source_content<'a>( + source: &str, + updates: impl IntoIterator, &'a str)>, +) -> String { let mut offset = 0; let mut content = source.as_bytes().to_vec(); - for (start, end, new_value) in updates { - let start = (start as isize + offset) as usize; - let end = (end as isize + offset) as usize; + for (range, new_value) in updates { + let start = (range.start as isize + offset) as usize; + let end = (range.end as isize + offset) as usize; content.splice(start..end, new_value.bytes()); offset += new_value.len() as isize - (end - start) as isize; @@ -1076,15 +1092,14 @@ contract A { let updates = vec![ // Replace function libFn() visibility to external - (36, 44, "external".to_string()), + (36..44, "external"), // Replace contract A name to contract B - (80, 90, "contract B".to_string()), + (80..90, "contract B"), // Remove function c() - (159, 222, String::new()), + (159..222, ""), // Replace function e() logic - (276, 296, "// no logic".to_string()), - ] - .into_iter(); + (276..296, "// no logic"), + ]; assert_eq!( replace_source_content(original_content, updates), diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs index 6cd2e55a..f6f98f87 100644 --- a/crates/compilers/src/preprocessor.rs +++ b/crates/compilers/src/preprocessor.rs @@ -1,6 +1,6 @@ use super::project::Preprocessor; use crate::{ - flatten::apply_updates, + apply_updates, multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, replace_source_content, solc::{SolcCompiler, SolcVersionedInput}, @@ -88,13 +88,9 @@ pub(crate) fn interface_representation( return Err(err); } - let content = replace_source_content( - content, - spans_to_remove - .iter() - .map(|span| (span.to_range().start, span.to_range().end, String::new())), - ) - .replace("\n", ""); + let content = + replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) + .replace("\n", ""); Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) } From c526df6548e0ba0162235019195f6fd324929e04 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Feb 2025 10:20:24 +0200 Subject: [PATCH 20/70] Port to solar HIR, refactor --- Cargo.toml | 5 +- crates/compilers/Cargo.toml | 1 + crates/compilers/src/preprocessor.rs | 632 ---------------------- crates/compilers/src/preprocessor/data.rs | 215 ++++++++ crates/compilers/src/preprocessor/deps.rs | 244 +++++++++ crates/compilers/src/preprocessor/mod.rs | 257 +++++++++ 6 files changed, 721 insertions(+), 633 deletions(-) delete mode 100644 crates/compilers/src/preprocessor.rs create mode 100644 crates/compilers/src/preprocessor/data.rs create mode 100644 crates/compilers/src/preprocessor/deps.rs create mode 100644 crates/compilers/src/preprocessor/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 64864be7..43d42926 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ serde = { version = "1", features = ["derive", "rc"] } serde_json = "1.0" similar-asserts = "1" solar-parse = { version = "=0.1.1", default-features = false } +solar-sema = { version = "=0.1.1", default-features = false } svm = { package = "svm-rs", version = "0.5", default-features = false } tempfile = "3.9" thiserror = "2" @@ -69,4 +70,6 @@ tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" [patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "6e8f4a1" } +solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "964b054" } +solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "964b054" } + diff --git a/crates/compilers/Cargo.toml b/crates/compilers/Cargo.toml index e0fd8a68..7034d649 100644 --- a/crates/compilers/Cargo.toml +++ b/crates/compilers/Cargo.toml @@ -33,6 +33,7 @@ thiserror.workspace = true path-slash.workspace = true yansi.workspace = true solar-parse.workspace = true +solar-sema.workspace = true futures-util = { workspace = true, optional = true } tokio = { workspace = true, optional = true } diff --git a/crates/compilers/src/preprocessor.rs b/crates/compilers/src/preprocessor.rs deleted file mode 100644 index f6f98f87..00000000 --- a/crates/compilers/src/preprocessor.rs +++ /dev/null @@ -1,632 +0,0 @@ -use super::project::Preprocessor; -use crate::{ - apply_updates, - multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, - replace_source_content, - solc::{SolcCompiler, SolcVersionedInput}, - Compiler, ProjectPathsConfig, Result, SolcError, Updates, -}; -use alloy_primitives::hex; -use foundry_compilers_artifacts::{ - ast::SourceLocation, - output_selection::OutputSelection, - visitor::{Visitor, Walk}, - ContractDefinitionPart, Expression, FunctionCall, MemberAccess, NewExpression, ParameterList, - SolcLanguage, Source, SourceUnit, SourceUnitPart, Sources, TypeName, -}; -use foundry_compilers_core::utils; -use itertools::Itertools; -use md5::Digest; -use solar_parse::{ - ast::{Arena, FunctionKind, ItemKind, Span, Visibility}, - interface::{diagnostics::EmittedDiagnostics, source_map::FileName}, -}; -use std::{ - collections::{BTreeMap, BTreeSet}, - path::{Path, PathBuf}, -}; - -/// Removes parts of the contract which do not alter its interface: -/// - Internal functions -/// - External functions bodies -/// -/// Preserves all libraries and interfaces. -pub(crate) fn interface_representation( - content: &str, - file: &PathBuf, -) -> Result { - let mut spans_to_remove: Vec = Vec::new(); - let sess = - solar_parse::interface::Session::builder().with_buffer_emitter(Default::default()).build(); - sess.enter(|| { - let arena = Arena::new(); - let filename = FileName::Real(file.to_path_buf()); - let Ok(mut parser) = - solar_parse::Parser::from_source_code(&sess, &arena, filename, content.to_string()) - else { - return; - }; - let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; - for item in ast.items { - let ItemKind::Contract(contract) = &item.kind else { - continue; - }; - - if contract.kind.is_interface() || contract.kind.is_library() { - continue; - } - - for contract_item in contract.body.iter() { - if let ItemKind::Function(function) = &contract_item.kind { - let is_exposed = match function.kind { - // Function with external or public visibility - FunctionKind::Function => { - function.header.visibility >= Some(Visibility::Public) - } - FunctionKind::Constructor - | FunctionKind::Fallback - | FunctionKind::Receive => true, - FunctionKind::Modifier => false, - }; - - // If function is not exposed we remove the entire span (signature and - // body). Otherwise we keep function signature and - // remove only the body. - if !is_exposed { - spans_to_remove.push(contract_item.span); - } else { - spans_to_remove.push(function.body_span); - } - } - } - } - }); - - // Return if any diagnostics emitted during content parsing. - if let Err(err) = sess.emitted_errors().unwrap() { - trace!("failed parsing {file:?}: {err}"); - return Err(err); - } - - let content = - replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) - .replace("\n", ""); - Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) -} - -/// Computes hash of [`interface_representation`] of the source. -pub(crate) fn interface_representation_hash(source: &Source, file: &PathBuf) -> String { - let Ok(repr) = interface_representation(&source.content, file) else { - return source.content_hash(); - }; - let mut hasher = md5::Md5::new(); - hasher.update(&repr); - let result = hasher.finalize(); - hex::encode(result) -} - -#[derive(Debug)] -pub struct ItemLocation { - start: usize, - end: usize, -} - -impl ItemLocation { - fn try_from_loc(loc: SourceLocation) -> Option { - Some(Self { start: loc.start?, end: loc.start? + loc.length? }) - } -} - -/// Checks if the given path is a test/script file. -fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { - let test_dir = paths.tests.strip_prefix(&paths.root).unwrap_or(&paths.root); - let script_dir = paths.scripts.strip_prefix(&paths.root).unwrap_or(&paths.root); - path.starts_with(test_dir) || path.starts_with(script_dir) -} - -/// Kind of the bytecode dependency. -#[derive(Debug)] -enum BytecodeDependencyKind { - /// `type(Contract).creationCode` - CreationCode, - /// `new Contract` - New(ItemLocation, String), -} - -/// Represents a single bytecode dependency. -#[derive(Debug)] -struct BytecodeDependency { - kind: BytecodeDependencyKind, - loc: ItemLocation, - referenced_contract: usize, -} - -/// Walks over AST and collects [`BytecodeDependency`]s. -#[derive(Debug)] -struct BytecodeDependencyCollector<'a> { - source: &'a str, - dependencies: Vec, - total_count: usize, -} - -impl BytecodeDependencyCollector<'_> { - fn new(source: &str) -> BytecodeDependencyCollector<'_> { - BytecodeDependencyCollector { source, dependencies: Vec::new(), total_count: 0 } - } -} - -impl Visitor for BytecodeDependencyCollector<'_> { - fn visit_function_call(&mut self, call: &FunctionCall) { - let (new_loc, expr) = match &call.expression { - Expression::NewExpression(expr) => (expr.src, expr), - Expression::FunctionCallOptions(expr) => { - if let Expression::NewExpression(new_expr) = &expr.expression { - (expr.src, new_expr) - } else { - return; - } - } - _ => return, - }; - - let TypeName::UserDefinedTypeName(type_name) = &expr.type_name else { return }; - - let Some(loc) = ItemLocation::try_from_loc(call.src) else { return }; - let Some(name_loc) = ItemLocation::try_from_loc(type_name.src) else { return }; - let Some(new_loc) = ItemLocation::try_from_loc(new_loc) else { return }; - let name = &self.source[name_loc.start..name_loc.end]; - - self.dependencies.push(BytecodeDependency { - kind: BytecodeDependencyKind::New(new_loc, name.to_string()), - loc, - referenced_contract: type_name.referenced_declaration as usize, - }); - } - - fn visit_new_expression(&mut self, expr: &NewExpression) { - if let TypeName::UserDefinedTypeName(_) = &expr.type_name { - self.total_count += 1; - } - } - - fn visit_member_access(&mut self, access: &MemberAccess) { - if access.member_name != "creationCode" { - return; - } - self.total_count += 1; - - let Expression::FunctionCall(call) = &access.expression else { return }; - - let Expression::Identifier(ident) = &call.expression else { return }; - - if ident.name != "type" { - return; - } - - let Some(Expression::Identifier(ident)) = call.arguments.first() else { return }; - - let Some(referenced) = ident.referenced_declaration else { return }; - - let Some(loc) = ItemLocation::try_from_loc(access.src) else { return }; - - self.dependencies.push(BytecodeDependency { - kind: BytecodeDependencyKind::CreationCode, - loc, - referenced_contract: referenced as usize, - }); - } -} - -/// Keeps data about a single contract definition. -struct ContractData<'a> { - /// AST id of the contract. - ast_id: usize, - /// Path of the source file. - path: &'a Path, - /// Name of the contract - name: &'a str, - /// Constructor parameters. - constructor_params: Option<&'a ParameterList>, - /// Reference to source code. - src: &'a str, - /// Artifact string to pass into cheatcodes. - artifact: String, -} - -impl ContractData<'_> { - /// If contract has a non-empty constructor, generates a helper source file for it containing a - /// helper to encode constructor arguments. - /// - /// This is needed because current preprocessing wraps the arguments, leaving them unchanged. - /// This allows us to handle nested new expressions correctly. However, this requires us to have - /// a way to wrap both named and unnamed arguments. i.e you can't do abi.encode({arg: val}). - /// - /// This function produces a helper struct + a helper function to encode the arguments. The - /// struct is defined in scope of an abstract contract inheriting the contract containing the - /// constructor. This is done as a hack to allow us to inherit the same scope of definitions. - /// - /// The resulted helper looks like this: - /// ```solidity - /// import "lib/openzeppelin-contracts/contracts/token/ERC20.sol"; - /// - /// abstract contract DeployHelper335 is ERC20 { - /// struct ConstructorArgs { - /// string name; - /// string symbol; - /// } - /// } - /// - /// function encodeArgs335(DeployHelper335.ConstructorArgs memory args) pure returns (bytes memory) { - /// return abi.encode(args.name, args.symbol); - /// } - /// ``` - /// - /// Example usage: - /// ```solidity - /// new ERC20(name, symbol) - /// ``` - /// becomes - /// ```solidity - /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs(name, symbol))) - /// ``` - /// With named arguments: - /// ```solidity - /// new ERC20({name: name, symbol: symbol}) - /// ``` - /// becomes - /// ```solidity - /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) - /// ``` - pub fn build_helper(&self) -> Result> { - let Self { ast_id, path, name, constructor_params, src, artifact } = self; - - let Some(params) = constructor_params else { return Ok(None) }; - - let struct_fields = params - .parameters - .iter() - .filter_map(|param| { - let loc = ItemLocation::try_from_loc(param.src)?; - Some(src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " ")) - }) - .join("; "); - - let abi_encode_args = - params.parameters.iter().map(|param| format!("args.{}", param.name)).join(", "); - - let vm_interface_name = format!("VmContractHelper{ast_id}"); - let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); - - let helper = format!( - r#" -pragma solidity >=0.4.0; - -import "{path}"; - -abstract contract DeployHelper{ast_id} is {name} {{ - struct ConstructorArgs {{ - {struct_fields}; - }} -}} - -function encodeArgs{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ - return abi.encode({abi_encode_args}); -}} - -function deployCode{ast_id}(DeployHelper{ast_id}.ConstructorArgs memory args) returns({name}) {{ - return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{ast_id}(args)))); -}} - -interface {vm_interface_name} {{ - function deployCode(string memory _artifact, bytes memory _data) external returns (address); - function deployCode(string memory _artifact) external returns (address); - function getCode(string memory _artifact) external returns (bytes memory); -}} - "#, - path = path.display(), - ); - - Ok(Some(helper)) - } -} - -/// Receives a set of source files along with their ASTs and removes bytecode dependencies from -/// contracts by replacing them with cheatcode invocations. -struct BytecodeDependencyOptimizer<'a> { - asts: BTreeMap, - paths: &'a ProjectPathsConfig, - sources: &'a mut Sources, -} - -impl BytecodeDependencyOptimizer<'_> { - fn new<'a>( - asts: BTreeMap, - paths: &'a ProjectPathsConfig, - sources: &'a mut Sources, - ) -> BytecodeDependencyOptimizer<'a> { - BytecodeDependencyOptimizer { asts, paths, sources } - } - - fn process(self) -> Result<()> { - let mut updates = Updates::default(); - - let contracts = self.collect_contracts(); - let additional_sources = self.create_deploy_helpers(&contracts)?; - self.remove_bytecode_dependencies(&contracts, &mut updates)?; - - self.sources.extend(additional_sources); - - apply_updates(self.sources, updates); - - Ok(()) - } - - /// Collects a mapping from contract AST id to [ContractData] for all contracts defined in src/ - /// files. - fn collect_contracts(&self) -> BTreeMap> { - let mut contracts = BTreeMap::new(); - - for (path, ast) in &self.asts { - let src = self.sources.get(path).unwrap().content.as_str(); - - if is_test_or_script(path, self.paths) { - continue; - } - - for node in &ast.nodes { - if let SourceUnitPart::ContractDefinition(contract) = node { - let artifact = format!("{}:{}", path.display(), contract.name); - let constructor = contract.nodes.iter().find_map(|node| { - let ContractDefinitionPart::FunctionDefinition(func) = node else { - return None; - }; - if *func.kind() != foundry_compilers_artifacts::FunctionKind::Constructor { - return None; - } - - Some(func) - }); - - contracts.insert( - contract.id, - ContractData { - artifact, - constructor_params: constructor - .map(|constructor| &constructor.parameters) - .filter(|params| !params.parameters.is_empty()), - src, - ast_id: contract.id, - path, - name: &contract.name, - }, - ); - } - } - } - - contracts - } - - /// Creates helper libraries for contracts with a non-empty constructor. - /// - /// See [`ContractData::build_helper`] for more details. - fn create_deploy_helpers( - &self, - contracts: &BTreeMap>, - ) -> Result { - let mut new_sources = Sources::new(); - for (id, contract) in contracts { - if let Some(code) = contract.build_helper()? { - let path = format!("foundry-pp/DeployHelper{id}.sol"); - new_sources.insert(path.into(), Source::new(code)); - } - } - - Ok(new_sources) - } - - /// Goes over all test/script files and replaces bytecode dependencies with cheatcode - /// invocations. - fn remove_bytecode_dependencies( - &self, - contracts: &BTreeMap>, - updates: &mut Updates, - ) -> Result<()> { - for (path, ast) in &self.asts { - if !is_test_or_script(path, self.paths) { - continue; - } - let src = self.sources.get(path).unwrap().content.as_str(); - - if src.is_empty() { - continue; - } - - let updates = updates.entry(path.clone()).or_default(); - let mut used_helpers = BTreeSet::new(); - - let mut collector = BytecodeDependencyCollector::new(src); - ast.walk(&mut collector); - - // It is possible to write weird expressions which we won't catch. - // e.g. (((new Contract)))() is valid syntax - // - // We need to ensure that we've collected all dependencies that are in the contract. - if collector.dependencies.len() != collector.total_count { - return Err(SolcError::msg(format!( - "failed to collect all bytecode dependencies for {}", - path.display() - ))); - } - - let vm_interface_name = format!("VmContractHelper{}", ast.id); - let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); - - for dep in collector.dependencies { - let Some(ContractData { artifact, constructor_params, .. }) = - contracts.get(&dep.referenced_contract) - else { - continue; - }; - match dep.kind { - BytecodeDependencyKind::CreationCode => { - // for creation code we need to just call getCode - updates.insert(( - dep.loc.start, - dep.loc.end, - format!("{vm}.getCode(\"{artifact}\")"), - )); - } - BytecodeDependencyKind::New(new_loc, name) => { - if constructor_params.is_none() { - // if there's no constructor, we can just call deployCode with one - // argument - updates.insert(( - dep.loc.start, - dep.loc.end, - format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), - )); - } else { - // if there's a constructor, we use our helper - used_helpers.insert(dep.referenced_contract); - updates.insert(( - new_loc.start, - new_loc.end, - format!( - "deployCode{id}(DeployHelper{id}.ConstructorArgs", - id = dep.referenced_contract - ), - )); - updates.insert((dep.loc.end, dep.loc.end, ")".to_string())); - } - } - }; - } - let helper_imports = used_helpers.into_iter().map(|id| { - format!( - "import {{DeployHelper{id}, encodeArgs{id}, deployCode{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", - ) - }).join("\n"); - updates.insert(( - src.len(), - src.len(), - format!( - r#" -{helper_imports} - -interface {vm_interface_name} {{ - function deployCode(string memory _artifact, bytes memory _data) external returns (address); - function deployCode(string memory _artifact) external returns (address); - function getCode(string memory _artifact) external returns (bytes memory); -}}"# - ), - )); - } - - Ok(()) - } -} - -#[derive(Debug)] -pub struct TestOptimizerPreprocessor; - -impl Preprocessor for TestOptimizerPreprocessor { - fn preprocess( - &self, - solc: &SolcCompiler, - mut input: SolcVersionedInput, - paths: &ProjectPathsConfig, - ) -> Result { - // Skip if we are not compiling any tests or scripts. Avoids unnecessary solc invocation and - // AST parsing. - if input.input.sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { - return Ok(input); - } - - let prev_output_selection = std::mem::replace( - &mut input.input.settings.output_selection, - OutputSelection::ast_output_selection(), - ); - let output = solc.compile(&input)?; - - input.input.settings.output_selection = prev_output_selection; - - if let Some(e) = output.errors.iter().find(|e| e.severity.is_error()) { - return Err(SolcError::msg(e)); - } - - let asts = output - .sources - .into_iter() - .filter_map(|(path, source)| { - if !input.input.sources.contains_key(&path) { - return None; - } - - Some((|| { - let ast = source.ast.ok_or_else(|| SolcError::msg("missing AST"))?; - let ast: SourceUnit = serde_json::from_str(&serde_json::to_string(&ast)?)?; - Ok((path, ast)) - })()) - }) - .collect::>>()?; - - BytecodeDependencyOptimizer::new(asts, paths, &mut input.input.sources).process()?; - - Ok(input) - } -} - -impl Preprocessor for TestOptimizerPreprocessor { - fn preprocess( - &self, - compiler: &MultiCompiler, - input: ::Input, - paths: &ProjectPathsConfig, - ) -> Result<::Input> { - match input { - MultiCompilerInput::Solc(input) => { - if let Some(solc) = &compiler.solc { - let paths = paths.clone().with_language::(); - let input = self.preprocess(solc, input, &paths)?; - Ok(MultiCompilerInput::Solc(input)) - } else { - Ok(MultiCompilerInput::Solc(input)) - } - } - MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_interface_representation() { - let content = r#" -library Lib { - function libFn() internal { - // logic to keep - } -} -contract A { - function a() external {} - function b() public {} - function c() internal { - // logic logic logic - } - function d() private {} - function e() external { - // logic logic logic - } -}"#; - - let result = interface_representation(content, &PathBuf::new()).unwrap(); - assert_eq!( - result, - r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# - ); - } -} diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs new file mode 100644 index 00000000..325662af --- /dev/null +++ b/crates/compilers/src/preprocessor/data.rs @@ -0,0 +1,215 @@ +use crate::preprocessor::SourceMapLocation; +use foundry_compilers_artifacts::{Source, Sources}; +use itertools::Itertools; +use solar_parse::interface::{Session, SourceMap}; +use solar_sema::{ + hir::{Contract, ContractId, Hir}, + interface::source_map::FileName, +}; +use std::{ + collections::{BTreeMap, HashSet}, + path::{Path, PathBuf}, +}; + +/// Keeps data about project contracts definitions referenced from tests and scripts. +/// HIR id -> Contract data definition mapping. +pub type PreprocessorData = BTreeMap; + +/// Collects preprocessor data from referenced contracts. +pub fn collect_preprocessor_data( + sess: &Session, + hir: &Hir<'_>, + libs: &[PathBuf], + referenced_contracts: HashSet, +) -> PreprocessorData { + let mut data = PreprocessorData::default(); + for contract_id in referenced_contracts { + let contract = Hir::contract(hir, ContractId::new(contract_id)); + let source = Hir::source(hir, contract.source); + + let FileName::Real(path) = &source.file.name else { + continue; + }; + + // Do not include external dependencies / libs. + // TODO: better to include only files from project src in order to avoid processing mocks + // within test dir. + if libs.iter().any(|lib_paths| path.starts_with(lib_paths)) { + continue; + } + + let contract_data = ContractData::new(hir, contract, path, source, sess.source_map()); + data.insert(contract_data.hir_id, contract_data); + } + data +} + +/// Creates helper libraries for contracts with a non-empty constructor. +/// +/// See [`ContractData::build_helper`] for more details. +pub fn create_deploy_helpers(data: &BTreeMap) -> Sources { + let mut deploy_helpers = Sources::new(); + for (hir_id, contract) in data { + if let Some(code) = contract.build_helper() { + let path = format!("foundry-pp/DeployHelper{hir_id}.sol"); + deploy_helpers.insert(path.into(), Source::new(code)); + } + } + deploy_helpers +} + +/// Keeps data about a contract constructor. +#[derive(Debug)] +pub struct ContractConstructorData { + /// ABI encoded args. + pub abi_encode_args: String, + /// Constructor struct fields. + pub struct_fields: String, +} + +/// Keeps data about a single contract definition. +#[derive(Debug)] +pub struct ContractData { + /// HIR Id of the contract. + hir_id: u32, + /// Path of the source file. + path: PathBuf, + /// Name of the contract + name: String, + /// Constructor parameters, if any. + pub constructor_data: Option, + /// Artifact string to pass into cheatcodes. + pub artifact: String, +} + +impl ContractData { + fn new( + hir: &Hir<'_>, + contract: &Contract<'_>, + path: &Path, + source: &solar_sema::hir::Source<'_>, + source_map: &SourceMap, + ) -> Self { + let artifact = format!("{}:{}", path.display(), contract.name); + + // Process data for contracts with constructor and parameters. + let constructor_data = contract + .ctor + .map(|ctor_id| Hir::function(hir, ctor_id)) + .filter(|ctor| !ctor.parameters.is_empty()) + .map(|ctor| { + let abi_encode_args = ctor + .parameters + .iter() + .map(|param_id| { + format!("args.{}", Hir::variable(hir, *param_id).name.unwrap().name) + }) + .join(", "); + let struct_fields = ctor + .parameters + .iter() + .map(|param_id| { + let src = source.file.src.as_str(); + let loc = SourceMapLocation::from_span( + source_map, + Hir::variable(hir, *param_id).span, + ); + src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " ") + }) + .join("; "); + ContractConstructorData { abi_encode_args, struct_fields } + }); + + Self { + hir_id: contract.linearized_bases[0].get(), + path: path.to_path_buf(), + name: contract.name.to_string(), + constructor_data, + artifact, + } + } + + /// If contract has a non-empty constructor, generates a helper source file for it containing a + /// helper to encode constructor arguments. + /// + /// This is needed because current preprocessing wraps the arguments, leaving them unchanged. + /// This allows us to handle nested new expressions correctly. However, this requires us to have + /// a way to wrap both named and unnamed arguments. i.e you can't do abi.encode({arg: val}). + /// + /// This function produces a helper struct + a helper function to encode the arguments. The + /// struct is defined in scope of an abstract contract inheriting the contract containing the + /// constructor. This is done as a hack to allow us to inherit the same scope of definitions. + /// + /// The resulted helper looks like this: + /// ```solidity + /// import "lib/openzeppelin-contracts/contracts/token/ERC20.sol"; + /// + /// abstract contract DeployHelper335 is ERC20 { + /// struct ConstructorArgs { + /// string name; + /// string symbol; + /// } + /// } + /// + /// function encodeArgs335(DeployHelper335.ConstructorArgs memory args) pure returns (bytes memory) { + /// return abi.encode(args.name, args.symbol); + /// } + /// ``` + /// + /// Example usage: + /// ```solidity + /// new ERC20(name, symbol) + /// ``` + /// becomes + /// ```solidity + /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs(name, symbol))) + /// ``` + /// With named arguments: + /// ```solidity + /// new ERC20({name: name, symbol: symbol}) + /// ``` + /// becomes + /// ```solidity + /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) + /// ``` + pub fn build_helper(&self) -> Option { + let Self { hir_id, path, name, constructor_data, artifact } = self; + + let Some(constructor_details) = constructor_data else { return None }; + let struct_fields = &constructor_details.struct_fields; + let abi_encode_args = &constructor_details.abi_encode_args; + let vm_interface_name = format!("VmContractHelper{hir_id}"); + let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); + + let helper = format!( + r#" +pragma solidity >=0.4.0; + +import "{path}"; + +abstract contract DeployHelper{hir_id} is {name} {{ + struct ConstructorArgs {{ + {struct_fields}; + }} +}} + +function encodeArgs{hir_id}(DeployHelper{hir_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ + return abi.encode({abi_encode_args}); +}} + +function deployCode{hir_id}(DeployHelper{hir_id}.ConstructorArgs memory args) returns({name}) {{ + return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{hir_id}(args)))); +}} + +interface {vm_interface_name} {{ + function deployCode(string memory _artifact, bytes memory _data) external returns (address); + function deployCode(string memory _artifact) external returns (address); + function getCode(string memory _artifact) external returns (bytes memory); +}} + "#, + path = path.display(), + ); + + Some(helper) + } +} diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs new file mode 100644 index 00000000..cabcf628 --- /dev/null +++ b/crates/compilers/src/preprocessor/deps.rs @@ -0,0 +1,244 @@ +use crate::{ + preprocessor::{ + data::{ContractData, PreprocessorData}, + SourceMapLocation, + }, + Updates, +}; +use itertools::Itertools; +use solar_parse::interface::Session; +use solar_sema::{ + ast::Span, + hir::{ContractId, Expr, ExprKind, Hir, TypeKind, Visit}, + interface::{data_structures::Never, source_map::FileName, SourceMap}, +}; +use std::{ + collections::{BTreeMap, BTreeSet, HashSet}, + ops::ControlFlow, + path::PathBuf, +}; + +/// Holds data about referenced source contracts and bytecode dependencies. +pub struct PreprocessorDependencies { + // Mapping test contract id -> test contract bytecode dependencies. + pub bytecode_deps: BTreeMap>, + // Referenced contract ids. + pub referenced_contracts: HashSet, +} + +impl PreprocessorDependencies { + pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf]) -> Self { + let mut inner = BTreeMap::new(); + let mut references = HashSet::default(); + for contract in Hir::contracts(hir) { + let source = Hir::source(hir, contract.source); + + let FileName::Real(path) = &source.file.name else { + continue; + }; + + // Collect dependencies only for tests and scripts. + if !paths.contains(path) { + continue; + } + + let mut deps_collector = + BytecodeDependencyCollector::new(sess.source_map(), hir, source.file.src.as_str()); + // Analyze current contract. + deps_collector.walk_contract(contract); + // Ignore empty test contracts declared in source files with other contracts. + if !deps_collector.dependencies.is_empty() { + inner.insert(contract.linearized_bases[0].get(), deps_collector.dependencies); + } + // Record collected referenced contract ids. + references.extend(deps_collector.referenced_contracts); + } + Self { bytecode_deps: inner, referenced_contracts: references } + } +} + +/// Represents a bytecode dependency kind. +#[derive(Debug)] +enum BytecodeDependencyKind { + /// `type(Contract).creationCode` + CreationCode, + /// `new Contract`. Holds the name of the contract and args length. + New(String, usize), +} + +/// Represents a single bytecode dependency. +#[derive(Debug)] +pub struct BytecodeDependency { + /// Dependency kind. + kind: BytecodeDependencyKind, + /// Source map location of this dependency. + loc: SourceMapLocation, + /// HIR id of referenced contract. + referenced_contract: u32, +} + +/// Walks over contract HIR and collects [`BytecodeDependency`]s and referenced contracts. +struct BytecodeDependencyCollector<'hir> { + /// Source map, used for determining contract item locations. + source_map: &'hir SourceMap, + /// Parsed HIR. + hir: &'hir Hir<'hir>, + /// Source content of current contract. + src: &'hir str, + /// Dependencies collected for current contract. + dependencies: Vec, + /// HIR ids of contracts referenced from current contract. + referenced_contracts: HashSet, +} + +impl<'hir> BytecodeDependencyCollector<'hir> { + fn new(source_map: &'hir SourceMap, hir: &'hir Hir<'hir>, src: &'hir str) -> Self { + Self { + source_map, + hir, + src, + dependencies: vec![], + referenced_contracts: HashSet::default(), + } + } +} + +impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { + type BreakValue = Never; + + fn hir(&self) -> &'hir Hir<'hir> { + self.hir + } + + fn visit_expr(&mut self, expr: &'hir Expr<'hir>) -> ControlFlow { + match &expr.kind { + ExprKind::New(ty) => { + if let TypeKind::Custom(item_id) = ty.kind { + if let Some(contract_id) = item_id.as_contract() { + let name_loc = SourceMapLocation::from_span(self.source_map, ty.span); + let name = &self.src[name_loc.start..name_loc.end]; + // TODO: check if there's a better way to determine where constructor call + // ends. + let args_len = self.src[name_loc.end..].split_once(';').unwrap().0.len(); + self.dependencies.push(BytecodeDependency { + kind: BytecodeDependencyKind::New(name.to_string(), args_len), + loc: SourceMapLocation::from_span( + self.source_map, + Span::new(expr.span.lo(), expr.span.hi()), + ), + referenced_contract: contract_id.get(), + }); + self.referenced_contracts.insert(contract_id.get()); + } + } + } + ExprKind::Member(member_expr, ident) => { + if ident.name.to_string() == "creationCode" { + if let ExprKind::TypeCall(ty) = &member_expr.kind { + if let TypeKind::Custom(contract_id) = &ty.kind { + if let Some(contract_id) = contract_id.as_contract() { + self.dependencies.push(BytecodeDependency { + kind: BytecodeDependencyKind::CreationCode, + loc: SourceMapLocation::from_span(self.source_map, expr.span), + referenced_contract: contract_id.get(), + }); + self.referenced_contracts.insert(contract_id.get()); + } + } + } + } + } + _ => {} + } + self.walk_expr(expr) + } +} + +/// Goes over all test/script files and replaces bytecode dependencies with cheatcode +/// invocations. +pub fn remove_bytecode_dependencies( + hir: &Hir<'_>, + deps: &PreprocessorDependencies, + data: &PreprocessorData, +) -> Updates { + let mut updates = Updates::default(); + for (contract_id, deps) in &deps.bytecode_deps { + let contract = Hir::contract(hir, ContractId::new(*contract_id)); + let source = Hir::source(hir, contract.source); + let FileName::Real(path) = &source.file.name else { + continue; + }; + + let updates = updates.entry(path.clone()).or_default(); + let mut used_helpers = BTreeSet::new(); + + let vm_interface_name = format!("VmContractHelper{contract_id}"); + let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); + + for dep in deps { + let Some(ContractData { artifact, constructor_data, .. }) = + data.get(&dep.referenced_contract) + else { + continue; + }; + + match &dep.kind { + BytecodeDependencyKind::CreationCode => { + // for creation code we need to just call getCode + updates.insert(( + dep.loc.start, + dep.loc.end, + format!("{vm}.getCode(\"{artifact}\")"), + )); + } + BytecodeDependencyKind::New(name, args_length) => { + if constructor_data.is_none() { + // if there's no constructor, we can just call deployCode with one + // argument + updates.insert(( + dep.loc.start, + dep.loc.end + args_length, + format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), + )); + } else { + // if there's a constructor, we use our helper + used_helpers.insert(dep.referenced_contract); + updates.insert(( + dep.loc.start, + dep.loc.end, + format!( + "deployCode{id}(DeployHelper{id}.ConstructorArgs", + id = dep.referenced_contract + ), + )); + updates.insert(( + dep.loc.end + args_length, + dep.loc.end + args_length, + ")".to_string(), + )); + } + } + }; + } + let helper_imports = used_helpers.into_iter().map(|id| { + format!( + "import {{DeployHelper{id}, encodeArgs{id}, deployCode{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", + ) + }).join("\n"); + updates.insert(( + source.file.src.len(), + source.file.src.len(), + format!( + r#" +{helper_imports} + +interface {vm_interface_name} {{ + function deployCode(string memory _artifact, bytes memory _data) external returns (address); + function deployCode(string memory _artifact) external returns (address); + function getCode(string memory _artifact) external returns (bytes memory); +}}"# + ), + )); + } + updates +} diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs new file mode 100644 index 00000000..70554af6 --- /dev/null +++ b/crates/compilers/src/preprocessor/mod.rs @@ -0,0 +1,257 @@ +use crate::{ + apply_updates, + multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, + preprocessor::{ + data::{collect_preprocessor_data, create_deploy_helpers}, + deps::{remove_bytecode_dependencies, PreprocessorDependencies}, + }, + project::Preprocessor, + replace_source_content, + solc::{SolcCompiler, SolcVersionedInput}, + Compiler, ProjectPathsConfig, Result, +}; +use alloy_primitives::hex; +use foundry_compilers_artifacts::{SolcLanguage, Source}; +use foundry_compilers_core::{error::SolcError, utils}; +use itertools::Itertools; +use md5::Digest; +use solar_parse::{ + ast::{FunctionKind, ItemKind, Span, Visibility}, + interface::{ + diagnostics::EmittedDiagnostics, source_map::FileName, BytePos, Session, SourceMap, + }, + Parser, +}; +use solar_sema::{hir::Arena, ParsingContext}; +use std::path::{Path, PathBuf}; + +mod data; +mod deps; + +/// Represents location of an item in the source map. +/// Used to generate source code updates. +#[derive(Debug)] +pub struct SourceMapLocation { + /// Source map location start. + start: usize, + /// Source map location end. + end: usize, +} + +impl SourceMapLocation { + /// Creates source map location from an item location within a source file. + fn from_span(source_map: &SourceMap, span: Span) -> Self { + let range = span.to_range(); + let start_pos = BytePos::from_usize(range.start); + let end_pos = BytePos::from_usize(range.end); + Self { + start: source_map.lookup_byte_offset(start_pos).pos.to_usize(), + end: source_map.lookup_byte_offset(end_pos).pos.to_usize(), + } + } +} + +#[derive(Debug)] +pub struct TestOptimizerPreprocessor; + +impl Preprocessor for TestOptimizerPreprocessor { + fn preprocess( + &self, + _solc: &SolcCompiler, + mut input: SolcVersionedInput, + paths: &ProjectPathsConfig, + ) -> Result { + let sources = &mut input.input.sources; + // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. + if sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { + trace!("no tests or sources to preprocess"); + return Ok(input); + } + + let sess = Session::builder().with_buffer_emitter(Default::default()).build(); + let _ = sess.enter_parallel(|| -> solar_parse::interface::Result<()> { + let hir_arena = Arena::new(); + let mut parsing_context = ParsingContext::new(&sess); + // Set remappings into HIR parsing context. + for remapping in &paths.remappings { + parsing_context + .file_resolver + .add_import_map(PathBuf::from(&remapping.name), PathBuf::from(&remapping.path)); + } + // Load and parse test and script contracts only (dependencies are automatically + // resolved). + let preprocessed_paths = sources + .into_iter() + .filter(|(path, source)| { + is_test_or_script(path, paths) && !source.content.is_empty() + }) + .map(|(path, _)| path.clone()) + .collect_vec(); + parsing_context.load_files(&preprocessed_paths)?; + + let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; + // Collect tests and scripts dependencies. + let deps = PreprocessorDependencies::new(&sess, hir, &preprocessed_paths); + // Collect data of source contracts referenced in tests and scripts. + let data = collect_preprocessor_data( + &sess, + hir, + &paths.libraries, + deps.referenced_contracts.clone(), + ); + + // Extend existing sources with preprocessor deploy helper sources. + sources.extend(create_deploy_helpers(&data)); + + // Generate and apply preprocessor source updates. + apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); + + Ok(()) + }); + + // Return if any diagnostics emitted during content parsing. + if let Err(err) = sess.emitted_errors().unwrap() { + trace!("failed preprocessing {err}"); + return Err(SolcError::Message(err.to_string())); + } + + Ok(input) + } +} + +impl Preprocessor for TestOptimizerPreprocessor { + fn preprocess( + &self, + compiler: &MultiCompiler, + input: ::Input, + paths: &ProjectPathsConfig, + ) -> Result<::Input> { + match input { + MultiCompilerInput::Solc(input) => { + if let Some(solc) = &compiler.solc { + let paths = paths.clone().with_language::(); + let input = self.preprocess(solc, input, &paths)?; + Ok(MultiCompilerInput::Solc(input)) + } else { + Ok(MultiCompilerInput::Solc(input)) + } + } + MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), + } + } +} + +/// Helper function to compute hash of [`interface_representation`] of the source. +pub(crate) fn interface_representation_hash(source: &Source, file: &PathBuf) -> String { + let Ok(repr) = interface_representation(&source.content, file) else { + return source.content_hash(); + }; + let mut hasher = md5::Md5::new(); + hasher.update(&repr); + let result = hasher.finalize(); + hex::encode(result) +} + +/// Helper function to remove parts of the contract which do not alter its interface: +/// - Internal functions +/// - External functions bodies +/// +/// Preserves all libraries and interfaces. +fn interface_representation(content: &str, file: &PathBuf) -> Result { + let mut spans_to_remove: Vec = Vec::new(); + let sess = + solar_parse::interface::Session::builder().with_buffer_emitter(Default::default()).build(); + sess.enter(|| { + let arena = solar_parse::ast::Arena::new(); + let filename = FileName::Real(file.to_path_buf()); + let Ok(mut parser) = Parser::from_source_code(&sess, &arena, filename, content.to_string()) + else { + return; + }; + let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; + for item in ast.items { + let ItemKind::Contract(contract) = &item.kind else { + continue; + }; + + if contract.kind.is_interface() || contract.kind.is_library() { + continue; + } + + for contract_item in contract.body.iter() { + if let ItemKind::Function(function) = &contract_item.kind { + let is_exposed = match function.kind { + // Function with external or public visibility + FunctionKind::Function => { + function.header.visibility >= Some(Visibility::Public) + } + FunctionKind::Constructor + | FunctionKind::Fallback + | FunctionKind::Receive => true, + FunctionKind::Modifier => false, + }; + + // If function is not exposed we remove the entire span (signature and + // body). Otherwise we keep function signature and + // remove only the body. + if !is_exposed { + spans_to_remove.push(contract_item.span); + } else { + spans_to_remove.push(function.body_span); + } + } + } + } + }); + + // Return if any diagnostics emitted during content parsing. + if let Err(err) = sess.emitted_errors().unwrap() { + trace!("failed parsing {file:?}: {err}"); + return Err(err); + } + + let content = + replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) + .replace("\n", ""); + Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) +} + +/// Checks if the given path is a test/script file. +fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { + let test_dir = paths.tests.strip_prefix(&paths.root).unwrap_or(&paths.root); + let script_dir = paths.scripts.strip_prefix(&paths.root).unwrap_or(&paths.root); + path.starts_with(test_dir) || path.starts_with(script_dir) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_interface_representation() { + let content = r#" +library Lib { + function libFn() internal { + // logic to keep + } +} +contract A { + function a() external {} + function b() public {} + function c() internal { + // logic logic logic + } + function d() private {} + function e() external { + // logic logic logic + } +}"#; + + let result = interface_representation(content, &PathBuf::new()).unwrap(); + assert_eq!( + result, + r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# + ); + } +} From e691a96b71825479b19a043e01d4fa8804527f40 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Feb 2025 15:52:58 +0200 Subject: [PATCH 21/70] add preprocessing parse constructors --- crates/compilers/tests/preprocessor.rs | 40 +++++++++++++++++++++ test-data/preprocessor/src/Counter.sol | 12 +++++++ test-data/preprocessor/src/CounterB.sol | 8 +++++ test-data/preprocessor/src/CounterC.sol | 10 ++++++ test-data/preprocessor/src/v1/Counter.sol | 12 +++++++ test-data/preprocessor/test/CounterTest.sol | 23 ++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 crates/compilers/tests/preprocessor.rs create mode 100644 test-data/preprocessor/src/Counter.sol create mode 100644 test-data/preprocessor/src/CounterB.sol create mode 100644 test-data/preprocessor/src/CounterC.sol create mode 100644 test-data/preprocessor/src/v1/Counter.sol create mode 100644 test-data/preprocessor/test/CounterTest.sol diff --git a/crates/compilers/tests/preprocessor.rs b/crates/compilers/tests/preprocessor.rs new file mode 100644 index 00000000..554c8ff8 --- /dev/null +++ b/crates/compilers/tests/preprocessor.rs @@ -0,0 +1,40 @@ +//! preprocessor tests + +use foundry_compilers::{ + preprocessor::TestOptimizerPreprocessor, + project::ProjectCompiler, + solc::{SolcCompiler, SolcLanguage}, + ProjectBuilder, ProjectPathsConfig, +}; +use foundry_compilers_core::utils::canonicalize; +use std::{env, path::Path}; + +#[test] +fn can_handle_constructors_and_creation_code() { + let root = + canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) + .unwrap(); + + let paths = ProjectPathsConfig::builder() + .sources(root.join("src")) + .tests(root.join("test")) + .root(&root) + .build::() + .unwrap(); + + let project = ProjectBuilder::::new(Default::default()) + .paths(paths) + .build(SolcCompiler::default()) + .unwrap(); + + // TODO: figure out how to set root to parsing context. + let cur_dir = env::current_dir().unwrap(); + env::set_current_dir(root).unwrap(); + let compiled = ProjectCompiler::new(&project) + .unwrap() + .with_preprocessor(TestOptimizerPreprocessor) + .compile() + .expect("failed to compile"); + compiled.assert_success(); + env::set_current_dir(cur_dir).unwrap(); +} diff --git a/test-data/preprocessor/src/Counter.sol b/test-data/preprocessor/src/Counter.sol new file mode 100644 index 00000000..d7d69d25 --- /dev/null +++ b/test-data/preprocessor/src/Counter.sol @@ -0,0 +1,12 @@ +// Contract without constructor +contract Counter { + uint256 public number; + + function setNumber(uint256 newNumber) public { + number = newNumber; + } + + function increment() public { + number++; + } +} diff --git a/test-data/preprocessor/src/CounterB.sol b/test-data/preprocessor/src/CounterB.sol new file mode 100644 index 00000000..20f72922 --- /dev/null +++ b/test-data/preprocessor/src/CounterB.sol @@ -0,0 +1,8 @@ +// Contract without constructor +contract CounterB { + uint256 public number; + + constructor(address a, uint256 b, + bool c, + address d) {} +} diff --git a/test-data/preprocessor/src/CounterC.sol b/test-data/preprocessor/src/CounterC.sol new file mode 100644 index 00000000..2d5f5158 --- /dev/null +++ b/test-data/preprocessor/src/CounterC.sol @@ -0,0 +1,10 @@ +// Contract without constructor +contract CounterC { + struct CounterCStruct { + address a; + bool b; + } + uint256 public number; + + constructor(string memory _name, uint _age, address _wallet) {} +} diff --git a/test-data/preprocessor/src/v1/Counter.sol b/test-data/preprocessor/src/v1/Counter.sol new file mode 100644 index 00000000..e302c8b3 --- /dev/null +++ b/test-data/preprocessor/src/v1/Counter.sol @@ -0,0 +1,12 @@ +// Same as Counter but different version. Test preprocessor aliased imports. +contract Counter { + uint256 public number; + + function setNumber(uint256 newNumber) public { + number = newNumber; + } + + function increment() public { + number++; + } +} diff --git a/test-data/preprocessor/test/CounterTest.sol b/test-data/preprocessor/test/CounterTest.sol new file mode 100644 index 00000000..3ad55df6 --- /dev/null +++ b/test-data/preprocessor/test/CounterTest.sol @@ -0,0 +1,23 @@ +import {Counter} from "src/Counter.sol"; +import {Counter as CounterV1} from "src/v1/Counter.sol"; +import "src/CounterB.sol"; +import "src/CounterC.sol"; + +contract CounterTest { + Counter public counter; + Counter public counter2 = new Counter(); + CounterB public counter3 = new CounterB(address(this), 44, true, address(this)); + CounterV1 public counterv1; + + function setUp() public { + counter = new Counter(); + counterv1 = new CounterV1( ); + type(CounterV1).creationCode; + CounterB counterB = new CounterB(address(this), 15, false, address(counter)); + CounterC counterC = new CounterC( + "something", + 35, + address(this) + ); + } +} \ No newline at end of file From 4241d2bd5f12978acf22ef6dbed38da6d2218ce2 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Feb 2025 19:26:21 +0200 Subject: [PATCH 22/70] Contract id cleanup --- crates/compilers/src/preprocessor/data.rs | 26 ++++++++++++----------- crates/compilers/src/preprocessor/deps.rs | 5 +++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 325662af..274e44b7 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -38,8 +38,9 @@ pub fn collect_preprocessor_data( continue; } - let contract_data = ContractData::new(hir, contract, path, source, sess.source_map()); - data.insert(contract_data.hir_id, contract_data); + let contract_data = + ContractData::new(hir, contract_id, contract, path, source, sess.source_map()); + data.insert(contract_id, contract_data); } data } @@ -49,9 +50,9 @@ pub fn collect_preprocessor_data( /// See [`ContractData::build_helper`] for more details. pub fn create_deploy_helpers(data: &BTreeMap) -> Sources { let mut deploy_helpers = Sources::new(); - for (hir_id, contract) in data { + for (contract_id, contract) in data { if let Some(code) = contract.build_helper() { - let path = format!("foundry-pp/DeployHelper{hir_id}.sol"); + let path = format!("foundry-pp/DeployHelper{contract_id}.sol"); deploy_helpers.insert(path.into(), Source::new(code)); } } @@ -71,7 +72,7 @@ pub struct ContractConstructorData { #[derive(Debug)] pub struct ContractData { /// HIR Id of the contract. - hir_id: u32, + contract_id: u32, /// Path of the source file. path: PathBuf, /// Name of the contract @@ -85,6 +86,7 @@ pub struct ContractData { impl ContractData { fn new( hir: &Hir<'_>, + contract_id: u32, contract: &Contract<'_>, path: &Path, source: &solar_sema::hir::Source<'_>, @@ -121,7 +123,7 @@ impl ContractData { }); Self { - hir_id: contract.linearized_bases[0].get(), + contract_id, path: path.to_path_buf(), name: contract.name.to_string(), constructor_data, @@ -173,12 +175,12 @@ impl ContractData { /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) /// ``` pub fn build_helper(&self) -> Option { - let Self { hir_id, path, name, constructor_data, artifact } = self; + let Self { contract_id, path, name, constructor_data, artifact } = self; let Some(constructor_details) = constructor_data else { return None }; let struct_fields = &constructor_details.struct_fields; let abi_encode_args = &constructor_details.abi_encode_args; - let vm_interface_name = format!("VmContractHelper{hir_id}"); + let vm_interface_name = format!("VmContractHelper{contract_id}"); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); let helper = format!( @@ -187,18 +189,18 @@ pragma solidity >=0.4.0; import "{path}"; -abstract contract DeployHelper{hir_id} is {name} {{ +abstract contract DeployHelper{contract_id} is {name} {{ struct ConstructorArgs {{ {struct_fields}; }} }} -function encodeArgs{hir_id}(DeployHelper{hir_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ +function encodeArgs{contract_id}(DeployHelper{contract_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ return abi.encode({abi_encode_args}); }} -function deployCode{hir_id}(DeployHelper{hir_id}.ConstructorArgs memory args) returns({name}) {{ - return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{hir_id}(args)))); +function deployCode{contract_id}(DeployHelper{contract_id}.ConstructorArgs memory args) returns({name}) {{ + return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{contract_id}(args)))); }} interface {vm_interface_name} {{ diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index cabcf628..5fb68164 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -30,7 +30,8 @@ impl PreprocessorDependencies { pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf]) -> Self { let mut inner = BTreeMap::new(); let mut references = HashSet::default(); - for contract in Hir::contracts(hir) { + for contract_id in Hir::contract_ids(hir) { + let contract = Hir::contract(hir, contract_id); let source = Hir::source(hir, contract.source); let FileName::Real(path) = &source.file.name else { @@ -48,7 +49,7 @@ impl PreprocessorDependencies { deps_collector.walk_contract(contract); // Ignore empty test contracts declared in source files with other contracts. if !deps_collector.dependencies.is_empty() { - inner.insert(contract.linearized_bases[0].get(), deps_collector.dependencies); + inner.insert(contract_id.get(), deps_collector.dependencies); } // Record collected referenced contract ids. references.extend(deps_collector.referenced_contracts); From c86c4aabd6edb9df366662214401f4c945102d55 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Thu, 27 Feb 2025 12:59:38 +0200 Subject: [PATCH 23/70] Cleanup, filter collected dependencies to be source contracts --- crates/compilers/src/preprocessor/data.rs | 8 --- crates/compilers/src/preprocessor/deps.rs | 64 +++++++++++++++++------ crates/compilers/src/preprocessor/mod.rs | 10 ++-- 3 files changed, 52 insertions(+), 30 deletions(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 274e44b7..b52edf5f 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -19,7 +19,6 @@ pub type PreprocessorData = BTreeMap; pub fn collect_preprocessor_data( sess: &Session, hir: &Hir<'_>, - libs: &[PathBuf], referenced_contracts: HashSet, ) -> PreprocessorData { let mut data = PreprocessorData::default(); @@ -31,13 +30,6 @@ pub fn collect_preprocessor_data( continue; }; - // Do not include external dependencies / libs. - // TODO: better to include only files from project src in order to avoid processing mocks - // within test dir. - if libs.iter().any(|lib_paths| path.starts_with(lib_paths)) { - continue; - } - let contract_data = ContractData::new(hir, contract_id, contract, path, source, sess.source_map()); data.insert(contract_id, contract_data); diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 5fb68164..701dac82 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -20,16 +20,16 @@ use std::{ /// Holds data about referenced source contracts and bytecode dependencies. pub struct PreprocessorDependencies { - // Mapping test contract id -> test contract bytecode dependencies. - pub bytecode_deps: BTreeMap>, + // Mapping contract id to preprocess -> contract bytecode dependencies. + pub preprocessed_contracts: BTreeMap>, // Referenced contract ids. pub referenced_contracts: HashSet, } impl PreprocessorDependencies { - pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf]) -> Self { - let mut inner = BTreeMap::new(); - let mut references = HashSet::default(); + pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf], src_dir: &PathBuf) -> Self { + let mut preprocessed_contracts = BTreeMap::new(); + let mut referenced_contracts = HashSet::new(); for contract_id in Hir::contract_ids(hir) { let contract = Hir::contract(hir, contract_id); let source = Hir::source(hir, contract.source); @@ -43,18 +43,22 @@ impl PreprocessorDependencies { continue; } - let mut deps_collector = - BytecodeDependencyCollector::new(sess.source_map(), hir, source.file.src.as_str()); + let mut deps_collector = BytecodeDependencyCollector::new( + sess.source_map(), + hir, + source.file.src.as_str(), + src_dir, + ); // Analyze current contract. deps_collector.walk_contract(contract); // Ignore empty test contracts declared in source files with other contracts. if !deps_collector.dependencies.is_empty() { - inner.insert(contract_id.get(), deps_collector.dependencies); + preprocessed_contracts.insert(contract_id.get(), deps_collector.dependencies); } // Record collected referenced contract ids. - references.extend(deps_collector.referenced_contracts); + referenced_contracts.extend(deps_collector.referenced_contracts); } - Self { bytecode_deps: inner, referenced_contracts: references } + Self { preprocessed_contracts, referenced_contracts } } } @@ -86,22 +90,50 @@ struct BytecodeDependencyCollector<'hir> { hir: &'hir Hir<'hir>, /// Source content of current contract. src: &'hir str, + /// Project source dir, used to determine if referenced contract is a source contract. + src_dir: &'hir PathBuf, /// Dependencies collected for current contract. dependencies: Vec, - /// HIR ids of contracts referenced from current contract. + /// Unique HIR ids of contracts referenced from current contract. referenced_contracts: HashSet, } impl<'hir> BytecodeDependencyCollector<'hir> { - fn new(source_map: &'hir SourceMap, hir: &'hir Hir<'hir>, src: &'hir str) -> Self { + fn new( + source_map: &'hir SourceMap, + hir: &'hir Hir<'hir>, + src: &'hir str, + src_dir: &'hir PathBuf, + ) -> Self { Self { source_map, hir, src, + src_dir, dependencies: vec![], referenced_contracts: HashSet::default(), } } + + /// Collects reference identified as bytecode dependency of analyzed contract. + /// Discards any reference that is not in project src directory (e.g. external + /// libraries or mock contracts that extend source contracts). + fn collect_dependency(&mut self, dependency: BytecodeDependency) { + let contract = Hir::contract(self.hir, ContractId::new(dependency.referenced_contract)); + let source = Hir::source(self.hir, contract.source); + let FileName::Real(path) = &source.file.name else { + return; + }; + + if !path.starts_with(self.src_dir) { + let path = path.display(); + trace!("ignore dependency {path}"); + return; + } + + self.referenced_contracts.insert(dependency.referenced_contract); + self.dependencies.push(dependency); + } } impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { @@ -121,7 +153,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { // TODO: check if there's a better way to determine where constructor call // ends. let args_len = self.src[name_loc.end..].split_once(';').unwrap().0.len(); - self.dependencies.push(BytecodeDependency { + self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::New(name.to_string(), args_len), loc: SourceMapLocation::from_span( self.source_map, @@ -129,7 +161,6 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { ), referenced_contract: contract_id.get(), }); - self.referenced_contracts.insert(contract_id.get()); } } } @@ -138,12 +169,11 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { if let ExprKind::TypeCall(ty) = &member_expr.kind { if let TypeKind::Custom(contract_id) = &ty.kind { if let Some(contract_id) = contract_id.as_contract() { - self.dependencies.push(BytecodeDependency { + self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::CreationCode, loc: SourceMapLocation::from_span(self.source_map, expr.span), referenced_contract: contract_id.get(), }); - self.referenced_contracts.insert(contract_id.get()); } } } @@ -163,7 +193,7 @@ pub fn remove_bytecode_dependencies( data: &PreprocessorData, ) -> Updates { let mut updates = Updates::default(); - for (contract_id, deps) in &deps.bytecode_deps { + for (contract_id, deps) in &deps.preprocessed_contracts { let contract = Hir::contract(hir, ContractId::new(*contract_id)); let source = Hir::source(hir, contract.source); let FileName::Real(path) = &source.file.name else { diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 70554af6..34230438 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -91,14 +91,14 @@ impl Preprocessor for TestOptimizerPreprocessor { let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; // Collect tests and scripts dependencies. - let deps = PreprocessorDependencies::new(&sess, hir, &preprocessed_paths); - // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data( + let deps = PreprocessorDependencies::new( &sess, hir, - &paths.libraries, - deps.referenced_contracts.clone(), + &preprocessed_paths, + &paths.paths_relative().sources, ); + // Collect data of source contracts referenced in tests and scripts. + let data = collect_preprocessor_data(&sess, hir, deps.referenced_contracts.clone()); // Extend existing sources with preprocessor deploy helper sources. sources.extend(create_deploy_helpers(&data)); From f4a109d53bfe97bba287335404c2f5cc20e7dbf6 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Fri, 28 Feb 2025 21:00:03 +0200 Subject: [PATCH 24/70] Cleanup, remove Hir:: usage, use ContractIds --- crates/compilers/src/lib.rs | 1 + crates/compilers/src/preprocessor/data.rs | 39 +++++++++----------- crates/compilers/src/preprocessor/deps.rs | 45 ++++++++++++----------- crates/compilers/src/preprocessor/mod.rs | 2 +- 4 files changed, 43 insertions(+), 44 deletions(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 81f38d25..99b6e0dc 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -903,6 +903,7 @@ fn apply_updates(sources: &mut Sources, updates: Updates) { } /// Utility function to change source content ranges with provided updates. +/// Assumes that the updates are sorted. fn replace_source_content<'a>( source: &str, updates: impl IntoIterator, &'a str)>, diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index b52edf5f..d88edea0 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -12,27 +12,27 @@ use std::{ }; /// Keeps data about project contracts definitions referenced from tests and scripts. -/// HIR id -> Contract data definition mapping. -pub type PreprocessorData = BTreeMap; +/// Contract id -> Contract data definition mapping. +pub type PreprocessorData = BTreeMap; /// Collects preprocessor data from referenced contracts. -pub fn collect_preprocessor_data( +pub(crate) fn collect_preprocessor_data( sess: &Session, hir: &Hir<'_>, - referenced_contracts: HashSet, + referenced_contracts: &HashSet, ) -> PreprocessorData { let mut data = PreprocessorData::default(); for contract_id in referenced_contracts { - let contract = Hir::contract(hir, ContractId::new(contract_id)); - let source = Hir::source(hir, contract.source); + let contract = hir.contract(*contract_id); + let source = hir.source(contract.source); let FileName::Real(path) = &source.file.name else { continue; }; let contract_data = - ContractData::new(hir, contract_id, contract, path, source, sess.source_map()); - data.insert(contract_id, contract_data); + ContractData::new(hir, *contract_id, contract, path, source, sess.source_map()); + data.insert(*contract_id, contract_data); } data } @@ -40,11 +40,11 @@ pub fn collect_preprocessor_data( /// Creates helper libraries for contracts with a non-empty constructor. /// /// See [`ContractData::build_helper`] for more details. -pub fn create_deploy_helpers(data: &BTreeMap) -> Sources { +pub(crate) fn create_deploy_helpers(data: &BTreeMap) -> Sources { let mut deploy_helpers = Sources::new(); for (contract_id, contract) in data { if let Some(code) = contract.build_helper() { - let path = format!("foundry-pp/DeployHelper{contract_id}.sol"); + let path = format!("foundry-pp/DeployHelper{}.sol", contract_id.get()); deploy_helpers.insert(path.into(), Source::new(code)); } } @@ -62,9 +62,9 @@ pub struct ContractConstructorData { /// Keeps data about a single contract definition. #[derive(Debug)] -pub struct ContractData { +pub(crate) struct ContractData { /// HIR Id of the contract. - contract_id: u32, + contract_id: ContractId, /// Path of the source file. path: PathBuf, /// Name of the contract @@ -78,7 +78,7 @@ pub struct ContractData { impl ContractData { fn new( hir: &Hir<'_>, - contract_id: u32, + contract_id: ContractId, contract: &Contract<'_>, path: &Path, source: &solar_sema::hir::Source<'_>, @@ -89,25 +89,21 @@ impl ContractData { // Process data for contracts with constructor and parameters. let constructor_data = contract .ctor - .map(|ctor_id| Hir::function(hir, ctor_id)) + .map(|ctor_id| hir.function(ctor_id)) .filter(|ctor| !ctor.parameters.is_empty()) .map(|ctor| { let abi_encode_args = ctor .parameters .iter() - .map(|param_id| { - format!("args.{}", Hir::variable(hir, *param_id).name.unwrap().name) - }) + .map(|param_id| format!("args.{}", hir.variable(*param_id).name.unwrap().name)) .join(", "); let struct_fields = ctor .parameters .iter() .map(|param_id| { let src = source.file.src.as_str(); - let loc = SourceMapLocation::from_span( - source_map, - Hir::variable(hir, *param_id).span, - ); + let loc = + SourceMapLocation::from_span(source_map, hir.variable(*param_id).span); src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " ") }) .join("; "); @@ -170,6 +166,7 @@ impl ContractData { let Self { contract_id, path, name, constructor_data, artifact } = self; let Some(constructor_details) = constructor_data else { return None }; + let contract_id = contract_id.get(); let struct_fields = &constructor_details.struct_fields; let abi_encode_args = &constructor_details.abi_encode_args; let vm_interface_name = format!("VmContractHelper{contract_id}"); diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 701dac82..3ca88c4e 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -19,20 +19,20 @@ use std::{ }; /// Holds data about referenced source contracts and bytecode dependencies. -pub struct PreprocessorDependencies { +pub(crate) struct PreprocessorDependencies { // Mapping contract id to preprocess -> contract bytecode dependencies. - pub preprocessed_contracts: BTreeMap>, + pub preprocessed_contracts: BTreeMap>, // Referenced contract ids. - pub referenced_contracts: HashSet, + pub referenced_contracts: HashSet, } impl PreprocessorDependencies { pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf], src_dir: &PathBuf) -> Self { let mut preprocessed_contracts = BTreeMap::new(); let mut referenced_contracts = HashSet::new(); - for contract_id in Hir::contract_ids(hir) { - let contract = Hir::contract(hir, contract_id); - let source = Hir::source(hir, contract.source); + for contract_id in hir.contract_ids() { + let contract = hir.contract(contract_id); + let source = hir.source(contract.source); let FileName::Real(path) = &source.file.name else { continue; @@ -53,7 +53,7 @@ impl PreprocessorDependencies { deps_collector.walk_contract(contract); // Ignore empty test contracts declared in source files with other contracts. if !deps_collector.dependencies.is_empty() { - preprocessed_contracts.insert(contract_id.get(), deps_collector.dependencies); + preprocessed_contracts.insert(contract_id, deps_collector.dependencies); } // Record collected referenced contract ids. referenced_contracts.extend(deps_collector.referenced_contracts); @@ -73,13 +73,13 @@ enum BytecodeDependencyKind { /// Represents a single bytecode dependency. #[derive(Debug)] -pub struct BytecodeDependency { +pub(crate) struct BytecodeDependency { /// Dependency kind. kind: BytecodeDependencyKind, /// Source map location of this dependency. loc: SourceMapLocation, /// HIR id of referenced contract. - referenced_contract: u32, + referenced_contract: ContractId, } /// Walks over contract HIR and collects [`BytecodeDependency`]s and referenced contracts. @@ -95,7 +95,7 @@ struct BytecodeDependencyCollector<'hir> { /// Dependencies collected for current contract. dependencies: Vec, /// Unique HIR ids of contracts referenced from current contract. - referenced_contracts: HashSet, + referenced_contracts: HashSet, } impl<'hir> BytecodeDependencyCollector<'hir> { @@ -119,8 +119,8 @@ impl<'hir> BytecodeDependencyCollector<'hir> { /// Discards any reference that is not in project src directory (e.g. external /// libraries or mock contracts that extend source contracts). fn collect_dependency(&mut self, dependency: BytecodeDependency) { - let contract = Hir::contract(self.hir, ContractId::new(dependency.referenced_contract)); - let source = Hir::source(self.hir, contract.source); + let contract = self.hir.contract(dependency.referenced_contract); + let source = self.hir.source(contract.source); let FileName::Real(path) = &source.file.name else { return; }; @@ -159,20 +159,20 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { self.source_map, Span::new(expr.span.lo(), expr.span.hi()), ), - referenced_contract: contract_id.get(), + referenced_contract: contract_id, }); } } } ExprKind::Member(member_expr, ident) => { - if ident.name.to_string() == "creationCode" { - if let ExprKind::TypeCall(ty) = &member_expr.kind { - if let TypeKind::Custom(contract_id) = &ty.kind { + if let ExprKind::TypeCall(ty) = &member_expr.kind { + if let TypeKind::Custom(contract_id) = &ty.kind { + if ident.name.as_str() == "creationCode" { if let Some(contract_id) = contract_id.as_contract() { self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::CreationCode, loc: SourceMapLocation::from_span(self.source_map, expr.span), - referenced_contract: contract_id.get(), + referenced_contract: contract_id, }); } } @@ -187,15 +187,15 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { /// Goes over all test/script files and replaces bytecode dependencies with cheatcode /// invocations. -pub fn remove_bytecode_dependencies( +pub(crate) fn remove_bytecode_dependencies( hir: &Hir<'_>, deps: &PreprocessorDependencies, data: &PreprocessorData, ) -> Updates { let mut updates = Updates::default(); for (contract_id, deps) in &deps.preprocessed_contracts { - let contract = Hir::contract(hir, ContractId::new(*contract_id)); - let source = Hir::source(hir, contract.source); + let contract = hir.contract(*contract_id); + let source = hir.source(contract.source); let FileName::Real(path) = &source.file.name else { continue; }; @@ -203,7 +203,7 @@ pub fn remove_bytecode_dependencies( let updates = updates.entry(path.clone()).or_default(); let mut used_helpers = BTreeSet::new(); - let vm_interface_name = format!("VmContractHelper{contract_id}"); + let vm_interface_name = format!("VmContractHelper{}", contract_id.get()); let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); for dep in deps { @@ -239,7 +239,7 @@ pub fn remove_bytecode_dependencies( dep.loc.end, format!( "deployCode{id}(DeployHelper{id}.ConstructorArgs", - id = dep.referenced_contract + id = dep.referenced_contract.get() ), )); updates.insert(( @@ -252,6 +252,7 @@ pub fn remove_bytecode_dependencies( }; } let helper_imports = used_helpers.into_iter().map(|id| { + let id = id.get(); format!( "import {{DeployHelper{id}, encodeArgs{id}, deployCode{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", ) diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 34230438..264b9ae5 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -98,7 +98,7 @@ impl Preprocessor for TestOptimizerPreprocessor { &paths.paths_relative().sources, ); // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data(&sess, hir, deps.referenced_contracts.clone()); + let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); // Extend existing sources with preprocessor deploy helper sources. sources.extend(create_deploy_helpers(&data)); From f0818ea3de2fcf8cc6b314df52140506b2c9a5b3 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 3 Mar 2025 16:10:23 +0200 Subject: [PATCH 25/70] Optional preprocessed cache --- crates/compilers/src/cache.rs | 127 ++++++++++++++++++++---- crates/compilers/src/compile/project.rs | 2 +- 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 670331ca..327422e9 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -46,16 +46,18 @@ pub struct CompilerCache { pub files: BTreeMap, pub builds: BTreeSet, pub profiles: BTreeMap, + pub preprocessed: bool, } impl CompilerCache { - pub fn new(format: String, paths: ProjectPaths) -> Self { + pub fn new(format: String, paths: ProjectPaths, preprocessed: bool) -> Self { Self { format, paths, files: Default::default(), builds: Default::default(), profiles: Default::default(), + preprocessed, } } } @@ -378,6 +380,7 @@ impl Default for CompilerCache { files: Default::default(), paths: Default::default(), profiles: Default::default(), + preprocessed: false, } } } @@ -385,7 +388,7 @@ impl Default for CompilerCache { impl<'a, S: CompilerSettings> From<&'a ProjectPathsConfig> for CompilerCache { fn from(config: &'a ProjectPathsConfig) -> Self { let paths = config.paths_relative(); - Self::new(Default::default(), paths) + Self::new(Default::default(), paths, false) } } @@ -680,8 +683,11 @@ impl, C: Compiler> .map(|import| strip_prefix(import, self.project.root()).into()) .collect(); - let interface_repr_hash = - self.is_source_file(&file).then(|| interface_representation_hash(source, &file)); + let interface_repr_hash = if self.cache.preprocessed { + self.is_source_file(&file).then(|| interface_representation_hash(source, &file)) + } else { + None + }; let entry = CacheEntry { last_modification_date: CacheEntry::read_last_modification_date(&file) @@ -783,21 +789,30 @@ impl, C: Compiler> // If any requested extra files are missing for any artifact, mark source as dirty to // generate them - for artifacts in self.cached_artifacts.values() { - for artifacts in artifacts.values() { - for artifact_file in artifacts { - if self.project.artifacts_handler().is_dirty(artifact_file).unwrap_or(true) { - return true; - } - } - } + if self.cache.preprocessed { + return self.missing_extra_files(); } false } - // Walks over all cache entires, detects dirty files and removes them from cache. + // Walks over all cache entries, detects dirty files and removes them from cache. fn find_and_remove_dirty(&mut self) { + fn populate_dirty_files( + file: &Path, + dirty_files: &mut HashSet, + edges: &GraphEdges, + ) { + for file in edges.importers(file) { + // If file is marked as dirty we either have already visited it or it was marked as + // dirty initially and will be visited at some point later. + if !dirty_files.contains(file) { + dirty_files.insert(file.to_path_buf()); + populate_dirty_files(file, dirty_files, edges); + } + } + } + let existing_profiles = self.project.settings_profiles().collect::>(); let mut dirty_profiles = HashSet::new(); @@ -834,6 +849,59 @@ impl, C: Compiler> } } + if !self.cache.preprocessed { + // Iterate over existing cache entries. + let files = self.cache.files.keys().cloned().collect::>(); + + let mut sources = Sources::new(); + + // Read all sources, marking entries as dirty on I/O errors. + for file in &files { + let Ok(source) = Source::read(file) else { + self.dirty_sources.insert(file.clone()); + continue; + }; + sources.insert(file.clone(), source); + } + + // Build a temporary graph for walking imports. We need this because `self.edges` + // only contains graph data for in-scope sources but we are operating on cache entries. + if let Ok(graph) = + Graph::::resolve_sources(&self.project.paths, sources) + { + let (sources, edges) = graph.into_sources(); + + // Calculate content hashes for later comparison. + self.fill_hashes(&sources); + + // Pre-add all sources that are guaranteed to be dirty + for file in sources.keys() { + if self.is_dirty_impl(file, false) { + self.dirty_sources.insert(file.clone()); + } + } + + // Perform DFS to find direct/indirect importers of dirty files. + for file in self.dirty_sources.clone().iter() { + populate_dirty_files(file, &mut self.dirty_sources, &edges); + } + } else { + // Purge all sources on graph resolution error. + self.dirty_sources.extend(files); + } + + // Remove all dirty files from cache. + for file in &self.dirty_sources { + debug!("removing dirty file from cache: {}", file.display()); + self.cache.remove(file); + } + } else { + self.find_and_remove_dirty_preprocessed() + } + } + + // Walks over all cache entries, detects dirty files and removes them from preprocessed cache. + fn find_and_remove_dirty_preprocessed(&mut self) { let mut sources = Sources::new(); // Read all sources, removing entries on I/O errors. @@ -912,7 +980,7 @@ impl, C: Compiler> return true; }; - if use_interface_repr { + if use_interface_repr && self.cache.preprocessed { let Some(interface_hash) = self.interface_repr_hashes.get(file) else { trace!("missing interface hash"); return true; @@ -932,6 +1000,12 @@ impl, C: Compiler> trace!("content hash changed"); return true; } + + if !self.cache.preprocessed { + // If any requested extra files are missing for any artifact, mark source as dirty + // to generate them + return self.missing_extra_files(); + } } // all things match, can be reused @@ -954,6 +1028,20 @@ impl, C: Compiler> } } } + + /// Helper function to check if any requested extra files are missing for any artifact. + fn missing_extra_files(&self) -> bool { + for artifacts in self.cached_artifacts.values() { + for artifacts in artifacts.values() { + for artifact_file in artifacts { + if self.project.artifacts_handler().is_dirty(artifact_file).unwrap_or(true) { + return true; + } + } + } + } + false + } } /// Abstraction over configured caching which can be either non-existent or an already loaded cache @@ -974,13 +1062,18 @@ impl<'a, T: ArtifactOutput, C: Compiler> ArtifactsCache<'a, T, C> { /// Create a new cache instance with the given files - pub fn new(project: &'a Project, edges: GraphEdges) -> Result { + pub fn new( + project: &'a Project, + edges: GraphEdges, + preprocessed: bool, + ) -> Result { /// Returns the [CompilerCache] to use /// /// Returns a new empty cache if the cache does not exist or `invalidate_cache` is set. fn get_cache, C: Compiler>( project: &Project, invalidate_cache: bool, + preprocessed: bool, ) -> CompilerCache { // the currently configured paths let paths = project.paths.paths_relative(); @@ -995,7 +1088,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> } // new empty cache - CompilerCache::new(Default::default(), paths) + CompilerCache::new(Default::default(), paths, preprocessed) } let cache = if project.cached { @@ -1005,7 +1098,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> let invalidate_cache = !edges.unresolved_imports().is_empty(); // read the cache file if it already exists - let mut cache = get_cache(project, invalidate_cache); + let mut cache = get_cache(project, invalidate_cache, preprocessed); cache.remove_missing_files(); diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 899c2bc9..e6a94543 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -225,7 +225,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> // which is unix style `/` sources.slash_paths(); - let mut cache = ArtifactsCache::new(project, edges)?; + let mut cache = ArtifactsCache::new(project, edges, preprocessor.is_some())?; // retain and compile only dirty sources and all their imports sources.filter(&mut cache); From 2d1e59fa5aa8f18b04242d09d077d838e95412e4 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 3 Mar 2025 17:28:57 +0200 Subject: [PATCH 26/70] Review cleanup --- crates/compilers/src/cache.rs | 196 +++++++++++++++------------------- 1 file changed, 87 insertions(+), 109 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 327422e9..59ee6294 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -789,11 +789,7 @@ impl, C: Compiler> // If any requested extra files are missing for any artifact, mark source as dirty to // generate them - if self.cache.preprocessed { - return self.missing_extra_files(); - } - - false + self.missing_extra_files() } // Walks over all cache entries, detects dirty files and removes them from cache. @@ -849,114 +845,100 @@ impl, C: Compiler> } } - if !self.cache.preprocessed { - // Iterate over existing cache entries. - let files = self.cache.files.keys().cloned().collect::>(); - - let mut sources = Sources::new(); - - // Read all sources, marking entries as dirty on I/O errors. - for file in &files { - let Ok(source) = Source::read(file) else { - self.dirty_sources.insert(file.clone()); - continue; - }; - sources.insert(file.clone(), source); - } - - // Build a temporary graph for walking imports. We need this because `self.edges` - // only contains graph data for in-scope sources but we are operating on cache entries. - if let Ok(graph) = - Graph::::resolve_sources(&self.project.paths, sources) - { - let (sources, edges) = graph.into_sources(); - - // Calculate content hashes for later comparison. - self.fill_hashes(&sources); - - // Pre-add all sources that are guaranteed to be dirty - for file in sources.keys() { - if self.is_dirty_impl(file, false) { - self.dirty_sources.insert(file.clone()); - } - } - - // Perform DFS to find direct/indirect importers of dirty files. - for file in self.dirty_sources.clone().iter() { - populate_dirty_files(file, &mut self.dirty_sources, &edges); - } - } else { - // Purge all sources on graph resolution error. - self.dirty_sources.extend(files); - } - - // Remove all dirty files from cache. - for file in &self.dirty_sources { - debug!("removing dirty file from cache: {}", file.display()); - self.cache.remove(file); - } - } else { - self.find_and_remove_dirty_preprocessed() - } - } - - // Walks over all cache entries, detects dirty files and removes them from preprocessed cache. - fn find_and_remove_dirty_preprocessed(&mut self) { let mut sources = Sources::new(); - - // Read all sources, removing entries on I/O errors. + // Iterate over existing cache entries. + let files = self.cache.files.keys().cloned().collect::>(); + // Read all sources, marking entries as dirty on I/O errors. for file in self.cache.files.keys().cloned().collect::>() { let Ok(source) = Source::read(&file) else { - self.cache.files.remove(&file); + if !self.cache.preprocessed { + self.dirty_sources.insert(file.clone()); + } else { + self.cache.files.remove(&file); + } continue; }; sources.insert(file.clone(), source); } - // Calculate content hashes for later comparison. - self.fill_hashes(&sources); + if self.cache.preprocessed { + // Calculate content hashes for later comparison. + self.fill_hashes(&sources); - // Pre-add all sources that are guaranteed to be dirty - for file in self.cache.files.keys() { - if self.is_dirty_impl(file, false) { - self.dirty_sources.insert(file.clone()); + // Pre-add all sources that are guaranteed to be dirty + for file in self.cache.files.keys() { + if self.is_dirty_impl(file, false) { + self.dirty_sources.insert(file.clone()); + } } } - // Build a temporary graph for populating cache. We want to ensure that we preserve all just - // removed entries with updated data. We need separate graph for this because - // `self.edges` only contains graph data for in-scope sources but we are operating on cache - // entries. - let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) - else { - // Purge all sources on graph resolution error. - self.cache.files.clear(); - return; - }; - - let (sources, edges) = graph.into_sources(); + // Build a temporary graph for walking imports or populate cache (if preprocessed). + // For non preprocessed caches we need this because `self.edges` only contains graph data + // for in-scope sources. + // For preprocessed caches we want to ensure that we preserve all just removed entries + // with updated data. + // We need separate graph for this because `self.edges` only contains graph data for + // in-scope sources but we are operating on cache entries. + let sources = match Graph::::resolve_sources(&self.project.paths, sources) + { + Ok(graph) => { + let (sources, edges) = graph.into_sources(); + if !self.cache.preprocessed { + // Calculate content hashes for later comparison. + self.fill_hashes(&sources); + + // Pre-add all sources that are guaranteed to be dirty + for file in sources.keys() { + if self.is_dirty_impl(file, false) { + self.dirty_sources.insert(file.clone()); + } + } - // Mark sources as dirty based on their imports - for file in sources.keys() { - if self.dirty_sources.contains(file) { - continue; + // Perform DFS to find direct/indirect importers of dirty files. + for file in self.dirty_sources.clone().iter() { + populate_dirty_files(file, &mut self.dirty_sources, &edges); + } + None + } else { + // Mark sources as dirty based on their imports + for file in sources.keys() { + if self.dirty_sources.contains(file) { + continue; + } + let is_src = self.is_source_file(file); + for import in edges.imports(file) { + // Any source file importing dirty source file is dirty. + if is_src && self.dirty_sources.contains(import) { + self.dirty_sources.insert(file.clone()); + break; + // For non-src files we mark them as dirty only if they import dirty + // non-src file or src file for which + // interface representation changed. + } else if !is_src + && self.dirty_sources.contains(import) + && (!self.is_source_file(import) + || self.is_dirty_impl(import, true)) + { + self.dirty_sources.insert(file.clone()); + } + } + } + Some(sources) + } } - let is_src = self.is_source_file(file); - for import in edges.imports(file) { - // Any source file importing dirty source file is dirty. - if is_src && self.dirty_sources.contains(import) { - self.dirty_sources.insert(file.clone()); - break; - // For non-src files we mark them as dirty only if they import dirty non-src file - // or src file for which interface representation changed. - } else if !is_src - && self.dirty_sources.contains(import) - && (!self.is_source_file(import) || self.is_dirty_impl(import, true)) - { - self.dirty_sources.insert(file.clone()); + Err(_) => { + if !self.cache.preprocessed { + // Purge all sources on graph resolution error. + self.dirty_sources.extend(files); + } else { + // Purge all sources on graph resolution error and return. + self.cache.files.clear(); + return; } + None } - } + }; // Remove all dirty files from cache. for file in &self.dirty_sources { @@ -964,13 +946,15 @@ impl, C: Compiler> self.cache.remove(file); } - // Create new entries for all source files - for (file, source) in sources { - if self.cache.files.contains_key(&file) { - continue; - } + if let Some(sources) = sources { + // Create new entries for all source files + for (file, source) in sources { + if self.cache.files.contains_key(&file) { + continue; + } - self.create_cache_entry(file.clone(), &source); + self.create_cache_entry(file.clone(), &source); + } } } @@ -1000,12 +984,6 @@ impl, C: Compiler> trace!("content hash changed"); return true; } - - if !self.cache.preprocessed { - // If any requested extra files are missing for any artifact, mark source as dirty - // to generate them - return self.missing_extra_files(); - } } // all things match, can be reused From c354a7c8dac82c3b44b75ce6afd2aca42d7ec198 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 3 Mar 2025 20:38:47 +0200 Subject: [PATCH 27/70] Simplify find and remove branches --- crates/compilers/src/cache.rs | 127 +++++++++++----------------------- 1 file changed, 42 insertions(+), 85 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 59ee6294..026dca06 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -792,7 +792,7 @@ impl, C: Compiler> self.missing_extra_files() } - // Walks over all cache entries, detects dirty files and removes them from cache. + // Walks over all cache entires, detects dirty files and removes them from cache. fn find_and_remove_dirty(&mut self) { fn populate_dirty_files( file: &Path, @@ -845,117 +845,74 @@ impl, C: Compiler> } } - let mut sources = Sources::new(); // Iterate over existing cache entries. let files = self.cache.files.keys().cloned().collect::>(); + + let mut sources = Sources::new(); + // Read all sources, marking entries as dirty on I/O errors. - for file in self.cache.files.keys().cloned().collect::>() { - let Ok(source) = Source::read(&file) else { - if !self.cache.preprocessed { - self.dirty_sources.insert(file.clone()); - } else { - self.cache.files.remove(&file); - } + for file in &files { + let Ok(source) = Source::read(file) else { + self.dirty_sources.insert(file.clone()); continue; }; sources.insert(file.clone(), source); } - if self.cache.preprocessed { + // Build a temporary graph for walking imports. We need this because `self.edges` + // only contains graph data for in-scope sources but we are operating on cache entries. + if let Ok(graph) = Graph::::resolve_sources(&self.project.paths, sources) { + let (sources, edges) = graph.into_sources(); + // Calculate content hashes for later comparison. self.fill_hashes(&sources); // Pre-add all sources that are guaranteed to be dirty - for file in self.cache.files.keys() { + for file in sources.keys() { if self.is_dirty_impl(file, false) { self.dirty_sources.insert(file.clone()); } } - } - // Build a temporary graph for walking imports or populate cache (if preprocessed). - // For non preprocessed caches we need this because `self.edges` only contains graph data - // for in-scope sources. - // For preprocessed caches we want to ensure that we preserve all just removed entries - // with updated data. - // We need separate graph for this because `self.edges` only contains graph data for - // in-scope sources but we are operating on cache entries. - let sources = match Graph::::resolve_sources(&self.project.paths, sources) - { - Ok(graph) => { - let (sources, edges) = graph.into_sources(); - if !self.cache.preprocessed { - // Calculate content hashes for later comparison. - self.fill_hashes(&sources); - - // Pre-add all sources that are guaranteed to be dirty - for file in sources.keys() { - if self.is_dirty_impl(file, false) { - self.dirty_sources.insert(file.clone()); - } - } - - // Perform DFS to find direct/indirect importers of dirty files. - for file in self.dirty_sources.clone().iter() { - populate_dirty_files(file, &mut self.dirty_sources, &edges); + if !self.cache.preprocessed { + // Perform DFS to find direct/indirect importers of dirty files. + for file in self.dirty_sources.clone().iter() { + populate_dirty_files(file, &mut self.dirty_sources, &edges); + } + } else { + // Mark sources as dirty based on their imports + for file in sources.keys() { + if self.dirty_sources.contains(file) { + continue; } - None - } else { - // Mark sources as dirty based on their imports - for file in sources.keys() { - if self.dirty_sources.contains(file) { - continue; - } - let is_src = self.is_source_file(file); - for import in edges.imports(file) { - // Any source file importing dirty source file is dirty. - if is_src && self.dirty_sources.contains(import) { - self.dirty_sources.insert(file.clone()); - break; - // For non-src files we mark them as dirty only if they import dirty - // non-src file or src file for which - // interface representation changed. - } else if !is_src - && self.dirty_sources.contains(import) - && (!self.is_source_file(import) - || self.is_dirty_impl(import, true)) - { - self.dirty_sources.insert(file.clone()); - } + let is_src = self.is_source_file(file); + for import in edges.imports(file) { + // Any source file importing dirty source file is dirty. + if is_src && self.dirty_sources.contains(import) { + self.dirty_sources.insert(file.clone()); + break; + // For non-src files we mark them as dirty only if they import dirty + // non-src file or src file for which + // interface representation changed. + } else if !is_src + && self.dirty_sources.contains(import) + && (!self.is_source_file(import) || self.is_dirty_impl(import, true)) + { + self.dirty_sources.insert(file.clone()); } } - Some(sources) } } - Err(_) => { - if !self.cache.preprocessed { - // Purge all sources on graph resolution error. - self.dirty_sources.extend(files); - } else { - // Purge all sources on graph resolution error and return. - self.cache.files.clear(); - return; - } - None - } - }; + } else { + // Purge all sources on graph resolution error. + self.dirty_sources.extend(files); + } // Remove all dirty files from cache. for file in &self.dirty_sources { debug!("removing dirty file from cache: {}", file.display()); self.cache.remove(file); } - - if let Some(sources) = sources { - // Create new entries for all source files - for (file, source) in sources { - if self.cache.files.contains_key(&file) { - continue; - } - - self.create_cache_entry(file.clone(), &source); - } - } } fn is_dirty_impl(&self, file: &Path, use_interface_repr: bool) -> bool { @@ -997,7 +954,7 @@ impl, C: Compiler> entry.insert(source.content_hash()); } // Fill interface representation hashes for source files - if self.is_source_file(file) { + if self.cache.preprocessed && self.is_source_file(file) { if let hash_map::Entry::Vacant(entry) = self.interface_repr_hashes.entry(file.clone()) { From 719d4e8dc56e5fc37d8402d52ca627313b5e04f4 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 4 Mar 2025 09:24:34 +0200 Subject: [PATCH 28/70] Autodetect and recompile mocks --- crates/compilers/src/cache.rs | 31 ++++++-- crates/compilers/src/compile/project.rs | 16 +++- crates/compilers/src/preprocessor/deps.rs | 33 +++++++- crates/compilers/src/preprocessor/mod.rs | 95 ++++++++++++----------- 4 files changed, 120 insertions(+), 55 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 026dca06..5a563c42 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -47,6 +47,7 @@ pub struct CompilerCache { pub builds: BTreeSet, pub profiles: BTreeMap, pub preprocessed: bool, + pub mocks: HashSet, } impl CompilerCache { @@ -58,6 +59,7 @@ impl CompilerCache { builds: Default::default(), profiles: Default::default(), preprocessed, + mocks: Default::default(), } } } @@ -381,6 +383,7 @@ impl Default for CompilerCache { paths: Default::default(), profiles: Default::default(), preprocessed: false, + mocks: Default::default(), } } } @@ -792,7 +795,7 @@ impl, C: Compiler> self.missing_extra_files() } - // Walks over all cache entires, detects dirty files and removes them from cache. + // Walks over all cache entries, detects dirty files and removes them from cache. fn find_and_remove_dirty(&mut self) { fn populate_dirty_files( file: &Path, @@ -892,13 +895,21 @@ impl, C: Compiler> self.dirty_sources.insert(file.clone()); break; // For non-src files we mark them as dirty only if they import dirty - // non-src file or src file for which - // interface representation changed. + // non-src file or src file for which interface representation changed. + // For identified mock contracts (non-src contracts that extends contracts + // from src file) we mark edges as dirty. } else if !is_src && self.dirty_sources.contains(import) - && (!self.is_source_file(import) || self.is_dirty_impl(import, true)) + && (!self.is_source_file(import) + || self.is_dirty_impl(import, true) + || self.cache.mocks.contains(file)) { - self.dirty_sources.insert(file.clone()); + if self.cache.mocks.contains(file) { + // Mark all mock edges as dirty. + populate_dirty_files(file, &mut self.dirty_sources, &edges); + } else { + self.dirty_sources.insert(file.clone()); + } } } } @@ -1122,6 +1133,16 @@ impl<'a, T: ArtifactOutput, C: Compiler> } } + /// Adds the file's hashes to the set if not set yet + pub fn add_mocks(&mut self, mocks: Option>) { + if let Some(mocks) = mocks { + match self { + ArtifactsCache::Ephemeral(..) => {} + ArtifactsCache::Cached(cache) => cache.cache.mocks.extend(mocks), + } + } + } + /// Filters out those sources that don't need to be compiled pub fn filter(&mut self, sources: &mut Sources, version: &Version, profile: &str) { match self { diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index e6a94543..eb29250f 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -115,19 +115,25 @@ use crate::{ use foundry_compilers_core::error::Result; use rayon::prelude::*; use semver::Version; -use std::{collections::HashMap, fmt::Debug, path::PathBuf, time::Instant}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + path::PathBuf, + time::Instant, +}; /// A set of different Solc installations with their version and the sources to be compiled pub(crate) type VersionedSources<'a, L, S> = HashMap>; /// Invoked before the actual compiler invocation and can override the input. +/// Returns preprocessed compiler input and identified mocks (if any) to be stored in cache. pub trait Preprocessor: Debug { fn preprocess( &self, compiler: &C, input: C::Input, paths: &ProjectPathsConfig, - ) -> Result; + ) -> Result<(C::Input, Option>)>; } #[derive(Debug)] @@ -469,6 +475,7 @@ impl CompilerSources<'_, L, S> { include_paths.extend(graph.include_paths().clone()); let mut jobs = Vec::new(); + let mut mocks = None; for (language, versioned_sources) in self.sources { for (version, sources, (profile, opt_settings)) in versioned_sources { let mut opt_settings = opt_settings.clone(); @@ -503,13 +510,16 @@ impl CompilerSources<'_, L, S> { input.strip_prefix(project.paths.root.as_path()); if let Some(preprocessor) = preprocessor.as_ref() { - input = preprocessor.preprocess(&project.compiler, input, &project.paths)?; + (input, mocks) = + preprocessor.preprocess(&project.compiler, input, &project.paths)?; } jobs.push((input, profile, actually_dirty)); } } + cache.add_mocks(mocks); + let results = if let Some(num_jobs) = jobs_cnt { compile_parallel(&project.compiler, jobs, num_jobs) } else { diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 3ca88c4e..ad410ed7 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -15,7 +15,7 @@ use solar_sema::{ use std::{ collections::{BTreeMap, BTreeSet, HashSet}, ops::ControlFlow, - path::PathBuf, + path::{Path, PathBuf}, }; /// Holds data about referenced source contracts and bytecode dependencies. @@ -24,12 +24,21 @@ pub(crate) struct PreprocessorDependencies { pub preprocessed_contracts: BTreeMap>, // Referenced contract ids. pub referenced_contracts: HashSet, + // Mock contract paths (with a base contract from src dir). + pub mocks: HashSet, } impl PreprocessorDependencies { - pub fn new(sess: &Session, hir: &Hir<'_>, paths: &[PathBuf], src_dir: &PathBuf) -> Self { + pub fn new( + sess: &Session, + hir: &Hir<'_>, + paths: &[PathBuf], + src_dir: &PathBuf, + root_dir: &Path, + ) -> Self { let mut preprocessed_contracts = BTreeMap::new(); let mut referenced_contracts = HashSet::new(); + let mut mocks = HashSet::new(); for contract_id in hir.contract_ids() { let contract = hir.contract(contract_id); let source = hir.source(contract.source); @@ -40,6 +49,24 @@ impl PreprocessorDependencies { // Collect dependencies only for tests and scripts. if !paths.contains(path) { + let path = path.display(); + trace!("{path} is not test or script"); + continue; + } + + // Do not collect dependencies for mock contracts. Walk through base contracts and + // check if they're from src dir. + if contract.linearized_bases.iter().any(|base_contract_id| { + let base_contract = hir.contract(*base_contract_id); + let FileName::Real(path) = &hir.source(base_contract.source).file.name else { + return false; + }; + path.starts_with(src_dir) + }) { + // Record mock contracts to be evicted from preprocessed cache. + mocks.insert(root_dir.join(path)); + let path = path.display(); + trace!("found mock contract {path}"); continue; } @@ -58,7 +85,7 @@ impl PreprocessorDependencies { // Record collected referenced contract ids. referenced_contracts.extend(deps_collector.referenced_contracts); } - Self { preprocessed_contracts, referenced_contracts } + Self { preprocessed_contracts, referenced_contracts, mocks } } } diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 264b9ae5..fd3dfdc0 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -23,7 +23,10 @@ use solar_parse::{ Parser, }; use solar_sema::{hir::Arena, ParsingContext}; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; mod data; mod deps; @@ -60,54 +63,58 @@ impl Preprocessor for TestOptimizerPreprocessor { _solc: &SolcCompiler, mut input: SolcVersionedInput, paths: &ProjectPathsConfig, - ) -> Result { + ) -> Result<(SolcVersionedInput, Option>)> { let sources = &mut input.input.sources; // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. if sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { trace!("no tests or sources to preprocess"); - return Ok(input); + return Ok((input, None)); } let sess = Session::builder().with_buffer_emitter(Default::default()).build(); - let _ = sess.enter_parallel(|| -> solar_parse::interface::Result<()> { - let hir_arena = Arena::new(); - let mut parsing_context = ParsingContext::new(&sess); - // Set remappings into HIR parsing context. - for remapping in &paths.remappings { - parsing_context - .file_resolver - .add_import_map(PathBuf::from(&remapping.name), PathBuf::from(&remapping.path)); - } - // Load and parse test and script contracts only (dependencies are automatically - // resolved). - let preprocessed_paths = sources - .into_iter() - .filter(|(path, source)| { - is_test_or_script(path, paths) && !source.content.is_empty() - }) - .map(|(path, _)| path.clone()) - .collect_vec(); - parsing_context.load_files(&preprocessed_paths)?; + let mocks = sess + .enter_parallel(|| -> solar_parse::interface::Result>> { + let hir_arena = Arena::new(); + let mut parsing_context = ParsingContext::new(&sess); + // Set remappings into HIR parsing context. + for remapping in &paths.remappings { + parsing_context.file_resolver.add_import_map( + PathBuf::from(&remapping.name), + PathBuf::from(&remapping.path), + ); + } + // Load and parse test and script contracts only (dependencies are automatically + // resolved). + let preprocessed_paths = sources + .into_iter() + .filter(|(path, source)| { + is_test_or_script(path, paths) && !source.content.is_empty() + }) + .map(|(path, _)| path.clone()) + .collect_vec(); + parsing_context.load_files(&preprocessed_paths)?; - let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; - // Collect tests and scripts dependencies. - let deps = PreprocessorDependencies::new( - &sess, - hir, - &preprocessed_paths, - &paths.paths_relative().sources, - ); - // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); + let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; + // Collect tests and scripts dependencies and identify mock contracts. + let deps = PreprocessorDependencies::new( + &sess, + hir, + &preprocessed_paths, + &paths.paths_relative().sources, + &paths.root, + ); + // Collect data of source contracts referenced in tests and scripts. + let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); - // Extend existing sources with preprocessor deploy helper sources. - sources.extend(create_deploy_helpers(&data)); + // Extend existing sources with preprocessor deploy helper sources. + sources.extend(create_deploy_helpers(&data)); - // Generate and apply preprocessor source updates. - apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); + // Generate and apply preprocessor source updates. + apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); - Ok(()) - }); + Ok(Some(deps.mocks)) + }) + .unwrap_or_default(); // Return if any diagnostics emitted during content parsing. if let Err(err) = sess.emitted_errors().unwrap() { @@ -115,7 +122,7 @@ impl Preprocessor for TestOptimizerPreprocessor { return Err(SolcError::Message(err.to_string())); } - Ok(input) + Ok((input, mocks)) } } @@ -125,18 +132,18 @@ impl Preprocessor for TestOptimizerPreprocessor { compiler: &MultiCompiler, input: ::Input, paths: &ProjectPathsConfig, - ) -> Result<::Input> { + ) -> Result<(::Input, Option>)> { match input { MultiCompilerInput::Solc(input) => { if let Some(solc) = &compiler.solc { let paths = paths.clone().with_language::(); - let input = self.preprocess(solc, input, &paths)?; - Ok(MultiCompilerInput::Solc(input)) + let (input, mocks) = self.preprocess(solc, input, &paths)?; + Ok((MultiCompilerInput::Solc(input), mocks)) } else { - Ok(MultiCompilerInput::Solc(input)) + Ok((MultiCompilerInput::Solc(input), None)) } } - MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), + MultiCompilerInput::Vyper(input) => Ok((MultiCompilerInput::Vyper(input), None)), } } } From 4342f82e03c97922e2900ef88beacc17e6fa9922 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 4 Mar 2025 10:55:52 +0200 Subject: [PATCH 29/70] Fix description --- crates/compilers/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 5a563c42..c5f62767 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -1133,7 +1133,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> } } - /// Adds the file's hashes to the set if not set yet + /// Collects files with mock contracts identified in preprocess phase. pub fn add_mocks(&mut self, mocks: Option>) { if let Some(mocks) = mocks { match self { From d8ece14595783bfc58262e9cd377e2b397f0e1ae Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 5 Mar 2025 09:00:51 +0200 Subject: [PATCH 30/70] Cleanup autodetect and update cached mocks: - pass the set of cached mocks to preprocessor - preprocessor updates the set accordingly (add new mocks and cleanup old mocks - handles case when a mock is refactored to non mock impl) - replace set of mocks in cache --- crates/compilers/src/cache.rs | 22 ++++-- crates/compilers/src/compile/project.rs | 18 +++-- crates/compilers/src/preprocessor/deps.rs | 10 ++- crates/compilers/src/preprocessor/mod.rs | 94 +++++++++++------------ 4 files changed, 80 insertions(+), 64 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index c5f62767..0542ac35 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -1133,13 +1133,21 @@ impl<'a, T: ArtifactOutput, C: Compiler> } } - /// Collects files with mock contracts identified in preprocess phase. - pub fn add_mocks(&mut self, mocks: Option>) { - if let Some(mocks) = mocks { - match self { - ArtifactsCache::Ephemeral(..) => {} - ArtifactsCache::Cached(cache) => cache.cache.mocks.extend(mocks), - } + /// Updates files with mock contracts identified in preprocess phase. + pub fn update_mocks(&mut self, mocks: HashSet) { + match self { + ArtifactsCache::Ephemeral(..) => {} + ArtifactsCache::Cached(cache) => cache.cache.mocks = mocks, + } + } + + /// Returns the set of files with mock contracts currently in cache. + /// This set is passed to preprocessors and updated accordingly. + /// Cache is then updated by using `update_mocks` call. + pub fn mocks(&self) -> HashSet { + match self { + ArtifactsCache::Ephemeral(..) => HashSet::default(), + ArtifactsCache::Cached(cache) => cache.cache.mocks.clone(), } } diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index eb29250f..9119b38e 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -126,14 +126,16 @@ use std::{ pub(crate) type VersionedSources<'a, L, S> = HashMap>; /// Invoked before the actual compiler invocation and can override the input. -/// Returns preprocessed compiler input and identified mocks (if any) to be stored in cache. +/// Updates the list of identified cached mocks (if any) to be stored in cache and returns +/// preprocessed compiler input. pub trait Preprocessor: Debug { fn preprocess( &self, compiler: &C, input: C::Input, paths: &ProjectPathsConfig, - ) -> Result<(C::Input, Option>)>; + mocks: &mut HashSet, + ) -> Result; } #[derive(Debug)] @@ -474,8 +476,11 @@ impl CompilerSources<'_, L, S> { let mut include_paths = project.paths.include_paths.clone(); include_paths.extend(graph.include_paths().clone()); + // Get current list of mocks from cache. This will be passed to preprocessors and updated + // accordingly, then set back in cache. + let mocks = &mut cache.mocks(); + let mut jobs = Vec::new(); - let mut mocks = None; for (language, versioned_sources) in self.sources { for (version, sources, (profile, opt_settings)) in versioned_sources { let mut opt_settings = opt_settings.clone(); @@ -510,15 +515,16 @@ impl CompilerSources<'_, L, S> { input.strip_prefix(project.paths.root.as_path()); if let Some(preprocessor) = preprocessor.as_ref() { - (input, mocks) = - preprocessor.preprocess(&project.compiler, input, &project.paths)?; + input = + preprocessor.preprocess(&project.compiler, input, &project.paths, mocks)?; } jobs.push((input, profile, actually_dirty)); } } - cache.add_mocks(mocks); + // Update cache with mocks updated by preprocessors. + cache.update_mocks(mocks.clone()); let results = if let Some(num_jobs) = jobs_cnt { compile_parallel(&project.compiler, jobs, num_jobs) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index ad410ed7..7ea0400b 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -24,8 +24,6 @@ pub(crate) struct PreprocessorDependencies { pub preprocessed_contracts: BTreeMap>, // Referenced contract ids. pub referenced_contracts: HashSet, - // Mock contract paths (with a base contract from src dir). - pub mocks: HashSet, } impl PreprocessorDependencies { @@ -35,10 +33,10 @@ impl PreprocessorDependencies { paths: &[PathBuf], src_dir: &PathBuf, root_dir: &Path, + mocks: &mut HashSet, ) -> Self { let mut preprocessed_contracts = BTreeMap::new(); let mut referenced_contracts = HashSet::new(); - let mut mocks = HashSet::new(); for contract_id in hir.contract_ids() { let contract = hir.contract(contract_id); let source = hir.source(contract.source); @@ -68,6 +66,10 @@ impl PreprocessorDependencies { let path = path.display(); trace!("found mock contract {path}"); continue; + } else { + // Make sure current contract is not in list of mocks (could happen when a contract + // which used to be a mock is refactored to a non-mock implementation). + mocks.remove(&root_dir.join(path)); } let mut deps_collector = BytecodeDependencyCollector::new( @@ -85,7 +87,7 @@ impl PreprocessorDependencies { // Record collected referenced contract ids. referenced_contracts.extend(deps_collector.referenced_contracts); } - Self { preprocessed_contracts, referenced_contracts, mocks } + Self { preprocessed_contracts, referenced_contracts } } } diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index fd3dfdc0..b18a56f9 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -63,58 +63,57 @@ impl Preprocessor for TestOptimizerPreprocessor { _solc: &SolcCompiler, mut input: SolcVersionedInput, paths: &ProjectPathsConfig, - ) -> Result<(SolcVersionedInput, Option>)> { + mocks: &mut HashSet, + ) -> Result { let sources = &mut input.input.sources; // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. if sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { trace!("no tests or sources to preprocess"); - return Ok((input, None)); + return Ok(input); } let sess = Session::builder().with_buffer_emitter(Default::default()).build(); - let mocks = sess - .enter_parallel(|| -> solar_parse::interface::Result>> { - let hir_arena = Arena::new(); - let mut parsing_context = ParsingContext::new(&sess); - // Set remappings into HIR parsing context. - for remapping in &paths.remappings { - parsing_context.file_resolver.add_import_map( - PathBuf::from(&remapping.name), - PathBuf::from(&remapping.path), - ); - } - // Load and parse test and script contracts only (dependencies are automatically - // resolved). - let preprocessed_paths = sources - .into_iter() - .filter(|(path, source)| { - is_test_or_script(path, paths) && !source.content.is_empty() - }) - .map(|(path, _)| path.clone()) - .collect_vec(); - parsing_context.load_files(&preprocessed_paths)?; + let _ = sess.enter_parallel(|| -> solar_parse::interface::Result { + let hir_arena = Arena::new(); + let mut parsing_context = ParsingContext::new(&sess); + // Set remappings into HIR parsing context. + for remapping in &paths.remappings { + parsing_context + .file_resolver + .add_import_map(PathBuf::from(&remapping.name), PathBuf::from(&remapping.path)); + } + // Load and parse test and script contracts only (dependencies are automatically + // resolved). + let preprocessed_paths = sources + .into_iter() + .filter(|(path, source)| { + is_test_or_script(path, paths) && !source.content.is_empty() + }) + .map(|(path, _)| path.clone()) + .collect_vec(); + parsing_context.load_files(&preprocessed_paths)?; - let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; - // Collect tests and scripts dependencies and identify mock contracts. - let deps = PreprocessorDependencies::new( - &sess, - hir, - &preprocessed_paths, - &paths.paths_relative().sources, - &paths.root, - ); - // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); + let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; + // Collect tests and scripts dependencies and identify mock contracts. + let deps = PreprocessorDependencies::new( + &sess, + hir, + &preprocessed_paths, + &paths.paths_relative().sources, + &paths.root, + mocks, + ); + // Collect data of source contracts referenced in tests and scripts. + let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); - // Extend existing sources with preprocessor deploy helper sources. - sources.extend(create_deploy_helpers(&data)); + // Extend existing sources with preprocessor deploy helper sources. + sources.extend(create_deploy_helpers(&data)); - // Generate and apply preprocessor source updates. - apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); + // Generate and apply preprocessor source updates. + apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); - Ok(Some(deps.mocks)) - }) - .unwrap_or_default(); + Ok(()) + }); // Return if any diagnostics emitted during content parsing. if let Err(err) = sess.emitted_errors().unwrap() { @@ -122,7 +121,7 @@ impl Preprocessor for TestOptimizerPreprocessor { return Err(SolcError::Message(err.to_string())); } - Ok((input, mocks)) + Ok(input) } } @@ -132,18 +131,19 @@ impl Preprocessor for TestOptimizerPreprocessor { compiler: &MultiCompiler, input: ::Input, paths: &ProjectPathsConfig, - ) -> Result<(::Input, Option>)> { + mocks: &mut HashSet, + ) -> Result<::Input> { match input { MultiCompilerInput::Solc(input) => { if let Some(solc) = &compiler.solc { let paths = paths.clone().with_language::(); - let (input, mocks) = self.preprocess(solc, input, &paths)?; - Ok((MultiCompilerInput::Solc(input), mocks)) + let input = self.preprocess(solc, input, &paths, mocks)?; + Ok(MultiCompilerInput::Solc(input)) } else { - Ok((MultiCompilerInput::Solc(input), None)) + Ok(MultiCompilerInput::Solc(input)) } } - MultiCompilerInput::Vyper(input) => Ok((MultiCompilerInput::Vyper(input), None)), + MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), } } } From c4ec52cfe34192b2b50f90909f3bedac008de58d Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 5 Mar 2025 09:45:00 +0200 Subject: [PATCH 31/70] Invalidate cache on preprocess option toggle --- crates/compilers/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 0542ac35..90481dc7 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -1026,8 +1026,8 @@ impl<'a, T: ArtifactOutput, C: Compiler> if !invalidate_cache && project.cache_path().exists() { if let Ok(cache) = CompilerCache::read_joined(&project.paths) { - if cache.paths == paths { - // unchanged project paths + if cache.paths == paths && preprocessed == cache.preprocessed { + // unchanged project paths and same preprocess cache option return cache; } } From 15ec56fcf720b88dc7151d679aef1a494eb95ca6 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Fri, 14 Mar 2025 10:11:31 +0200 Subject: [PATCH 32/70] Bump solar rev --- Cargo.toml | 4 +-- crates/compilers/src/preprocessor/mod.rs | 39 +++++++++++++----------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 43d42926..64e43868 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,6 @@ tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" [patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "964b054" } -solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "964b054" } +solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "77fb82b" } +solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "77fb82b" } diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index b18a56f9..98b2a09d 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -22,11 +22,12 @@ use solar_parse::{ }, Parser, }; -use solar_sema::{hir::Arena, ParsingContext}; +use solar_sema::ParsingContext; use std::{ collections::HashSet, path::{Path, PathBuf}, }; +use solar_sema::thread_local::ThreadLocal; mod data; mod deps; @@ -74,7 +75,6 @@ impl Preprocessor for TestOptimizerPreprocessor { let sess = Session::builder().with_buffer_emitter(Default::default()).build(); let _ = sess.enter_parallel(|| -> solar_parse::interface::Result { - let hir_arena = Arena::new(); let mut parsing_context = ParsingContext::new(&sess); // Set remappings into HIR parsing context. for remapping in &paths.remappings { @@ -93,24 +93,27 @@ impl Preprocessor for TestOptimizerPreprocessor { .collect_vec(); parsing_context.load_files(&preprocessed_paths)?; - let hir = &parsing_context.parse_and_lower_to_hir(&hir_arena)?; - // Collect tests and scripts dependencies and identify mock contracts. - let deps = PreprocessorDependencies::new( - &sess, - hir, - &preprocessed_paths, - &paths.paths_relative().sources, - &paths.root, - mocks, - ); - // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); + let hir_arena = ThreadLocal::new(); + if let Some(gcx) = parsing_context.parse_and_lower(&hir_arena)? { + let hir = &gcx.get().hir; + // Collect tests and scripts dependencies and identify mock contracts. + let deps = PreprocessorDependencies::new( + &sess, + hir, + &preprocessed_paths, + &paths.paths_relative().sources, + &paths.root, + mocks, + ); + // Collect data of source contracts referenced in tests and scripts. + let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); - // Extend existing sources with preprocessor deploy helper sources. - sources.extend(create_deploy_helpers(&data)); + // Extend existing sources with preprocessor deploy helper sources. + sources.extend(create_deploy_helpers(&data)); - // Generate and apply preprocessor source updates. - apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); + // Generate and apply preprocessor source updates. + apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); + } Ok(()) }); From 2f803668faf12f69cb1d656f21fa626ee4573d63 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Fri, 14 Mar 2025 10:37:06 +0200 Subject: [PATCH 33/70] Move preproc tests --- crates/compilers/src/preprocessor/mod.rs | 3 +- crates/compilers/tests/preprocessor.rs | 40 ------------------------ crates/compilers/tests/project.rs | 33 +++++++++++++++++++ 3 files changed, 34 insertions(+), 42 deletions(-) delete mode 100644 crates/compilers/tests/preprocessor.rs diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 98b2a09d..b4601eba 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -22,12 +22,11 @@ use solar_parse::{ }, Parser, }; -use solar_sema::ParsingContext; +use solar_sema::{thread_local::ThreadLocal, ParsingContext}; use std::{ collections::HashSet, path::{Path, PathBuf}, }; -use solar_sema::thread_local::ThreadLocal; mod data; mod deps; diff --git a/crates/compilers/tests/preprocessor.rs b/crates/compilers/tests/preprocessor.rs deleted file mode 100644 index 554c8ff8..00000000 --- a/crates/compilers/tests/preprocessor.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! preprocessor tests - -use foundry_compilers::{ - preprocessor::TestOptimizerPreprocessor, - project::ProjectCompiler, - solc::{SolcCompiler, SolcLanguage}, - ProjectBuilder, ProjectPathsConfig, -}; -use foundry_compilers_core::utils::canonicalize; -use std::{env, path::Path}; - -#[test] -fn can_handle_constructors_and_creation_code() { - let root = - canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) - .unwrap(); - - let paths = ProjectPathsConfig::builder() - .sources(root.join("src")) - .tests(root.join("test")) - .root(&root) - .build::() - .unwrap(); - - let project = ProjectBuilder::::new(Default::default()) - .paths(paths) - .build(SolcCompiler::default()) - .unwrap(); - - // TODO: figure out how to set root to parsing context. - let cur_dir = env::current_dir().unwrap(); - env::set_current_dir(root).unwrap(); - let compiled = ProjectCompiler::new(&project) - .unwrap() - .with_preprocessor(TestOptimizerPreprocessor) - .compile() - .expect("failed to compile"); - compiled.assert_success(); - env::set_current_dir(cur_dir).unwrap(); -} diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index ef84d65a..e260706b 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -15,6 +15,8 @@ use foundry_compilers::{ flatten::Flattener, info::ContractInfo, multi::MultiCompilerRestrictions, + preprocessor::TestOptimizerPreprocessor, + project::ProjectCompiler, project_util::*, solc::{Restriction, SolcRestrictions, SolcSettings}, take_solc_installer_lock, Artifact, ConfigurableArtifacts, ExtraOutputValues, Graph, Project, @@ -34,6 +36,7 @@ use semver::Version; use similar_asserts::assert_eq; use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + env, fs::{self}, io, path::{Path, PathBuf, MAIN_SEPARATOR}, @@ -4154,3 +4157,33 @@ contract A { } ); }); } + +#[test] +fn can_preprocess_constructors_and_creation_code() { + let root = + canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) + .unwrap(); + + let paths = ProjectPathsConfig::builder() + .sources(root.join("src")) + .tests(root.join("test")) + .root(&root) + .build::() + .unwrap(); + + let project = ProjectBuilder::::new(Default::default()) + .paths(paths) + .build(SolcCompiler::default()) + .unwrap(); + + // TODO: figure out how to set root to parsing context. + let cur_dir = env::current_dir().unwrap(); + env::set_current_dir(root).unwrap(); + let compiled = ProjectCompiler::new(&project) + .unwrap() + .with_preprocessor(TestOptimizerPreprocessor) + .compile() + .expect("failed to compile"); + compiled.assert_success(); + env::set_current_dir(cur_dir).unwrap(); +} From 1a3de9f9740cbdbd8853639468d6e06d5f3ce39c Mon Sep 17 00:00:00 2001 From: grandizzy Date: Fri, 14 Mar 2025 11:00:39 +0200 Subject: [PATCH 34/70] Preprocess by input reference --- crates/compilers/src/compile/project.rs | 12 ++++++--- crates/compilers/src/preprocessor/mod.rs | 31 ++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 9119b38e..f5cd26e0 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -132,10 +132,10 @@ pub trait Preprocessor: Debug { fn preprocess( &self, compiler: &C, - input: C::Input, + input: &mut C::Input, paths: &ProjectPathsConfig, mocks: &mut HashSet, - ) -> Result; + ) -> Result<()>; } #[derive(Debug)] @@ -515,8 +515,12 @@ impl CompilerSources<'_, L, S> { input.strip_prefix(project.paths.root.as_path()); if let Some(preprocessor) = preprocessor.as_ref() { - input = - preprocessor.preprocess(&project.compiler, input, &project.paths, mocks)?; + preprocessor.preprocess( + &project.compiler, + &mut input, + &project.paths, + mocks, + )?; } jobs.push((input, profile, actually_dirty)); diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index b4601eba..04242d74 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -61,15 +61,15 @@ impl Preprocessor for TestOptimizerPreprocessor { fn preprocess( &self, _solc: &SolcCompiler, - mut input: SolcVersionedInput, + input: &mut SolcVersionedInput, paths: &ProjectPathsConfig, mocks: &mut HashSet, - ) -> Result { + ) -> Result<()> { let sources = &mut input.input.sources; // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. if sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { trace!("no tests or sources to preprocess"); - return Ok(input); + return Ok(()); } let sess = Session::builder().with_buffer_emitter(Default::default()).build(); @@ -123,7 +123,7 @@ impl Preprocessor for TestOptimizerPreprocessor { return Err(SolcError::Message(err.to_string())); } - Ok(input) + Ok(()) } } @@ -131,22 +131,17 @@ impl Preprocessor for TestOptimizerPreprocessor { fn preprocess( &self, compiler: &MultiCompiler, - input: ::Input, + input: &mut ::Input, paths: &ProjectPathsConfig, mocks: &mut HashSet, - ) -> Result<::Input> { - match input { - MultiCompilerInput::Solc(input) => { - if let Some(solc) = &compiler.solc { - let paths = paths.clone().with_language::(); - let input = self.preprocess(solc, input, &paths, mocks)?; - Ok(MultiCompilerInput::Solc(input)) - } else { - Ok(MultiCompilerInput::Solc(input)) - } - } - MultiCompilerInput::Vyper(input) => Ok(MultiCompilerInput::Vyper(input)), - } + ) -> Result<()> { + // Preprocess only Solc compilers. + let MultiCompilerInput::Solc(input) = input else { return Ok(()) }; + + let Some(solc) = &compiler.solc else { return Ok(()) }; + + let paths = paths.clone().with_language::(); + self.preprocess(solc, input, &paths, mocks) } } From 0fd4caa076b00223674783df560bb552a7f803e4 Mon Sep 17 00:00:00 2001 From: grandizzy <38490174+grandizzy@users.noreply.github.com> Date: Thu, 20 Mar 2025 12:08:58 +0200 Subject: [PATCH 35/70] Update crates/compilers/src/preprocessor/deps.rs Co-authored-by: zerosnacks <95942363+zerosnacks@users.noreply.github.com> --- crates/compilers/src/preprocessor/deps.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 7ea0400b..d1460357 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -233,6 +233,7 @@ pub(crate) fn remove_bytecode_dependencies( let mut used_helpers = BTreeSet::new(); let vm_interface_name = format!("VmContractHelper{}", contract_id.get()); + // `address(uint160(uint256(keccak256("hevm cheat code"))))` let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); for dep in deps { From c41692db641787a396c97c9aa9ba1c7e44a2e995 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 24 Mar 2025 16:47:48 +0200 Subject: [PATCH 36/70] Bump solar --- Cargo.toml | 4 ++-- crates/compilers/src/preprocessor/deps.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82f2ebc2..94ebeec2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,6 @@ tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" [patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "77fb82b" } -solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "77fb82b" } +solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "63bfcb9" } +solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "63bfcb9" } diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index d1460357..fb083809 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -79,7 +79,7 @@ impl PreprocessorDependencies { src_dir, ); // Analyze current contract. - deps_collector.walk_contract(contract); + let _ = deps_collector.walk_contract(contract); // Ignore empty test contracts declared in source files with other contracts. if !deps_collector.dependencies.is_empty() { preprocessed_contracts.insert(contract_id, deps_collector.dependencies); From 8138172acea5dc5e14ea67d6385155c1f45454d5 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 24 Mar 2025 18:05:53 +0200 Subject: [PATCH 37/70] Rust backtrace to debug win failure --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 965c17b0..81a1a0c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: env: CARGO_TERM_COLOR: always + RUST_BACKTRACE: full concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From 2947ec641dbbd523cbbe87f8c752fbec443c6a0e Mon Sep 17 00:00:00 2001 From: grandizzy <38490174+grandizzy@users.noreply.github.com> Date: Mon, 24 Mar 2025 19:21:02 +0200 Subject: [PATCH 38/70] Update crates/compilers/src/lib.rs Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/compilers/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 99b6e0dc..4720a932 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -918,7 +918,7 @@ fn replace_source_content<'a>( offset += new_value.len() as isize - (end - start) as isize; } - String::from_utf8_lossy(&content).to_string() + String::from_utf8(content).unwrap() } #[cfg(test)] From af7cc598576c0f41002ede34e9257c8b1e19efc6 Mon Sep 17 00:00:00 2001 From: grandizzy <38490174+grandizzy@users.noreply.github.com> Date: Mon, 24 Mar 2025 19:21:43 +0200 Subject: [PATCH 39/70] Update crates/compilers/src/preprocessor/deps.rs Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/compilers/src/preprocessor/deps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index fb083809..0414c2a3 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -186,7 +186,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { kind: BytecodeDependencyKind::New(name.to_string(), args_len), loc: SourceMapLocation::from_span( self.source_map, - Span::new(expr.span.lo(), expr.span.hi()), + expr.span, ), referenced_contract: contract_id, }); From cb0f1d8bdc8ef0253cadd7a0e3c7c5819838da1b Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 24 Mar 2025 20:31:55 +0200 Subject: [PATCH 40/70] Handle vars without name in ctor --- crates/compilers/src/preprocessor/data.rs | 41 ++++++++++++--------- crates/compilers/src/preprocessor/deps.rs | 6 +-- test-data/preprocessor/src/CounterD.sol | 4 ++ test-data/preprocessor/test/CounterTest.sol | 2 + 4 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 test-data/preprocessor/src/CounterD.sol diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index d88edea0..0e8e7c1e 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -1,6 +1,5 @@ use crate::preprocessor::SourceMapLocation; use foundry_compilers_artifacts::{Source, Sources}; -use itertools::Itertools; use solar_parse::interface::{Session, SourceMap}; use solar_sema::{ hir::{Contract, ContractId, Hir}, @@ -92,22 +91,30 @@ impl ContractData { .map(|ctor_id| hir.function(ctor_id)) .filter(|ctor| !ctor.parameters.is_empty()) .map(|ctor| { - let abi_encode_args = ctor - .parameters - .iter() - .map(|param_id| format!("args.{}", hir.variable(*param_id).name.unwrap().name)) - .join(", "); - let struct_fields = ctor - .parameters - .iter() - .map(|param_id| { - let src = source.file.src.as_str(); - let loc = - SourceMapLocation::from_span(source_map, hir.variable(*param_id).span); - src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " ") - }) - .join("; "); - ContractConstructorData { abi_encode_args, struct_fields } + let mut abi_encode_args = vec![]; + let mut struct_fields = vec![]; + let mut arg_index = 0; + for param_id in ctor.parameters { + let src = source.file.src.as_str(); + let loc = + SourceMapLocation::from_span(source_map, hir.variable(*param_id).span); + let mut new_src = + src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " "); + if let Some(ident) = hir.variable(*param_id).name { + abi_encode_args.push(format!("args.{}", ident.name)); + } else { + // Generate an unique name if constructor arg doesn't have one. + arg_index += 1; + abi_encode_args.push(format!("args.foundry_pp_ctor_arg{arg_index}")); + new_src.push_str(&format!(" foundry_pp_ctor_arg{arg_index}")); + } + struct_fields.push(new_src); + } + + ContractConstructorData { + abi_encode_args: abi_encode_args.join(", "), + struct_fields: struct_fields.join("; "), + } }); Self { diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 0414c2a3..3a1bd0b2 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -8,7 +8,6 @@ use crate::{ use itertools::Itertools; use solar_parse::interface::Session; use solar_sema::{ - ast::Span, hir::{ContractId, Expr, ExprKind, Hir, TypeKind, Visit}, interface::{data_structures::Never, source_map::FileName, SourceMap}, }; @@ -184,10 +183,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { let args_len = self.src[name_loc.end..].split_once(';').unwrap().0.len(); self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::New(name.to_string(), args_len), - loc: SourceMapLocation::from_span( - self.source_map, - expr.span, - ), + loc: SourceMapLocation::from_span(self.source_map, expr.span), referenced_contract: contract_id, }); } diff --git a/test-data/preprocessor/src/CounterD.sol b/test-data/preprocessor/src/CounterD.sol new file mode 100644 index 00000000..0b11bf30 --- /dev/null +++ b/test-data/preprocessor/src/CounterD.sol @@ -0,0 +1,4 @@ +// Contract with constructor args without name +contract CounterD { + constructor(address, uint256 x, uint256) {} +} diff --git a/test-data/preprocessor/test/CounterTest.sol b/test-data/preprocessor/test/CounterTest.sol index 3ad55df6..097a9d42 100644 --- a/test-data/preprocessor/test/CounterTest.sol +++ b/test-data/preprocessor/test/CounterTest.sol @@ -2,6 +2,7 @@ import {Counter} from "src/Counter.sol"; import {Counter as CounterV1} from "src/v1/Counter.sol"; import "src/CounterB.sol"; import "src/CounterC.sol"; +import "src/CounterD.sol"; contract CounterTest { Counter public counter; @@ -19,5 +20,6 @@ contract CounterTest { 35, address(this) ); + CounterD counterD = new CounterD(address(this), 15, 15); } } \ No newline at end of file From ebda056fdfd28afba65733b44575e43bcd63293a Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 24 Mar 2025 21:56:37 +0200 Subject: [PATCH 41/70] Better way to determine constructor call --- crates/compilers/src/preprocessor/deps.rs | 30 +++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 3a1bd0b2..f98c7b0f 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -173,19 +173,23 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { fn visit_expr(&mut self, expr: &'hir Expr<'hir>) -> ControlFlow { match &expr.kind { - ExprKind::New(ty) => { - if let TypeKind::Custom(item_id) = ty.kind { - if let Some(contract_id) = item_id.as_contract() { - let name_loc = SourceMapLocation::from_span(self.source_map, ty.span); - let name = &self.src[name_loc.start..name_loc.end]; - // TODO: check if there's a better way to determine where constructor call - // ends. - let args_len = self.src[name_loc.end..].split_once(';').unwrap().0.len(); - self.collect_dependency(BytecodeDependency { - kind: BytecodeDependencyKind::New(name.to_string(), args_len), - loc: SourceMapLocation::from_span(self.source_map, expr.span), - referenced_contract: contract_id, - }); + ExprKind::Call(ty, _, _) => { + if let ExprKind::New(ty_new) = &ty.kind { + if let TypeKind::Custom(item_id) = ty_new.kind { + if let Some(contract_id) = item_id.as_contract() { + let name_loc = + SourceMapLocation::from_span(self.source_map, ty_new.span); + let name = &self.src[name_loc.start..name_loc.end]; + let args_len = expr.span.hi() - ty_new.span.hi(); + self.collect_dependency(BytecodeDependency { + kind: BytecodeDependencyKind::New( + name.to_string(), + args_len.to_usize(), + ), + loc: SourceMapLocation::from_span(self.source_map, ty.span), + referenced_contract: contract_id, + }); + } } } } From 64c5ecdc8572b6eef410e3385d29dbb0f61c7558 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 25 Mar 2025 13:02:03 +0200 Subject: [PATCH 42/70] Use Path, add named args test --- crates/compilers/src/preprocessor/deps.rs | 16 ++++++++++++---- crates/compilers/src/preprocessor/mod.rs | 7 +++---- test-data/preprocessor/test/CounterTest.sol | 3 +++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index f98c7b0f..9748e6ee 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -96,7 +96,7 @@ enum BytecodeDependencyKind { /// `type(Contract).creationCode` CreationCode, /// `new Contract`. Holds the name of the contract and args length. - New(String, usize), + New(String, usize, usize), } /// Represents a single bytecode dependency. @@ -173,18 +173,26 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { fn visit_expr(&mut self, expr: &'hir Expr<'hir>) -> ControlFlow { match &expr.kind { - ExprKind::Call(ty, _, _) => { + ExprKind::Call(ty, call_args, named_args) => { if let ExprKind::New(ty_new) = &ty.kind { if let TypeKind::Custom(item_id) = ty_new.kind { if let Some(contract_id) = item_id.as_contract() { let name_loc = SourceMapLocation::from_span(self.source_map, ty_new.span); let name = &self.src[name_loc.start..name_loc.end]; + + let offset = if named_args.is_some() && !call_args.is_empty() { + (call_args.span().lo() - ty_new.span.hi()).to_usize() - 1 + } else { + 0 + }; + let args_len = expr.span.hi() - ty_new.span.hi(); self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::New( name.to_string(), args_len.to_usize(), + offset, ), loc: SourceMapLocation::from_span(self.source_map, ty.span), referenced_contract: contract_id, @@ -252,7 +260,7 @@ pub(crate) fn remove_bytecode_dependencies( format!("{vm}.getCode(\"{artifact}\")"), )); } - BytecodeDependencyKind::New(name, args_length) => { + BytecodeDependencyKind::New(name, args_length, offset) => { if constructor_data.is_none() { // if there's no constructor, we can just call deployCode with one // argument @@ -266,7 +274,7 @@ pub(crate) fn remove_bytecode_dependencies( used_helpers.insert(dep.referenced_contract); updates.insert(( dep.loc.start, - dep.loc.end, + dep.loc.end + offset, format!( "deployCode{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract.get() diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 04242d74..3d79e433 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -146,7 +146,7 @@ impl Preprocessor for TestOptimizerPreprocessor { } /// Helper function to compute hash of [`interface_representation`] of the source. -pub(crate) fn interface_representation_hash(source: &Source, file: &PathBuf) -> String { +pub(crate) fn interface_representation_hash(source: &Source, file: &Path) -> String { let Ok(repr) = interface_representation(&source.content, file) else { return source.content_hash(); }; @@ -161,10 +161,9 @@ pub(crate) fn interface_representation_hash(source: &Source, file: &PathBuf) -> /// - External functions bodies /// /// Preserves all libraries and interfaces. -fn interface_representation(content: &str, file: &PathBuf) -> Result { +fn interface_representation(content: &str, file: &Path) -> Result { let mut spans_to_remove: Vec = Vec::new(); - let sess = - solar_parse::interface::Session::builder().with_buffer_emitter(Default::default()).build(); + let sess = Session::builder().with_buffer_emitter(Default::default()).build(); sess.enter(|| { let arena = solar_parse::ast::Arena::new(); let filename = FileName::Real(file.to_path_buf()); diff --git a/test-data/preprocessor/test/CounterTest.sol b/test-data/preprocessor/test/CounterTest.sol index 097a9d42..2b09c631 100644 --- a/test-data/preprocessor/test/CounterTest.sol +++ b/test-data/preprocessor/test/CounterTest.sol @@ -8,7 +8,10 @@ contract CounterTest { Counter public counter; Counter public counter2 = new Counter(); CounterB public counter3 = new CounterB(address(this), 44, true, address(this)); + CounterB public counter4 = new CounterB({a:address(this), b: 44, c: true, d: address(this)}); CounterV1 public counterv1; + Counter public counter5 = new Counter{salt: bytes32("123")}(); + CounterB public counter6 = new CounterB {salt: bytes32("123")} (address(this), 44, true, address(this)); function setUp() public { counter = new Counter(); From 67ac483d536dad7e6de298cb2a98219fe4d14fe7 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 25 Mar 2025 15:54:12 +0200 Subject: [PATCH 43/70] Ensure / for win --- crates/compilers/src/preprocessor/data.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 0e8e7c1e..96a76915 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -1,5 +1,6 @@ use crate::preprocessor::SourceMapLocation; use foundry_compilers_artifacts::{Source, Sources}; +use path_slash::PathExt; use solar_parse::interface::{Session, SourceMap}; use solar_sema::{ hir::{Contract, ContractId, Hir}, @@ -83,7 +84,7 @@ impl ContractData { source: &solar_sema::hir::Source<'_>, source_map: &SourceMap, ) -> Self { - let artifact = format!("{}:{}", path.display(), contract.name); + let artifact = format!("{}:{}", path.to_slash_lossy(), contract.name); // Process data for contracts with constructor and parameters. let constructor_data = contract From 6b3e887ddbe6509226f9c3d6dcf1e812b4f292f3 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:13:13 +0100 Subject: [PATCH 44/70] clean --- crates/compilers/src/preprocessor/mod.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 3d79e433..1b0fa522 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -17,9 +17,7 @@ use itertools::Itertools; use md5::Digest; use solar_parse::{ ast::{FunctionKind, ItemKind, Span, Visibility}, - interface::{ - diagnostics::EmittedDiagnostics, source_map::FileName, BytePos, Session, SourceMap, - }, + interface::{diagnostics::EmittedDiagnostics, source_map::FileName, Session, SourceMap}, Parser, }; use solar_sema::{thread_local::ThreadLocal, ParsingContext}; @@ -44,12 +42,9 @@ pub struct SourceMapLocation { impl SourceMapLocation { /// Creates source map location from an item location within a source file. fn from_span(source_map: &SourceMap, span: Span) -> Self { - let range = span.to_range(); - let start_pos = BytePos::from_usize(range.start); - let end_pos = BytePos::from_usize(range.end); Self { - start: source_map.lookup_byte_offset(start_pos).pos.to_usize(), - end: source_map.lookup_byte_offset(end_pos).pos.to_usize(), + start: source_map.lookup_byte_offset(span.lo()).pos.to_usize(), + end: source_map.lookup_byte_offset(span.hi()).pos.to_usize(), } } } From c04627d34e5151ac4c5feb5a657a94e41017a088 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 25 Mar 2025 20:38:56 +0200 Subject: [PATCH 45/70] Use solar main --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 94ebeec2..9018e6b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,6 @@ tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" [patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", rev = "63bfcb9" } -solar-sema = { git = "https://github.com/paradigmxyz/solar", rev = "63bfcb9" } +solar-parse = { git = "https://github.com/paradigmxyz/solar", branch = "main" } +solar-sema = { git = "https://github.com/paradigmxyz/solar", branch = "main" } From 0d0f5d9093ee008f70c02a1527b8fc81983fa053 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Tue, 25 Mar 2025 21:04:45 +0200 Subject: [PATCH 46/70] Ensure / for win in import --- crates/compilers/src/preprocessor/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 96a76915..48627bd4 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -206,7 +206,7 @@ interface {vm_interface_name} {{ function getCode(string memory _artifact) external returns (bytes memory); }} "#, - path = path.display(), + path = path.to_slash_lossy(), ); Some(helper) From 6a8114749d040997390cef5e5483f2a40b791bc8 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Mar 2025 08:55:13 +0200 Subject: [PATCH 47/70] Support value and salt in constructors --- crates/compilers/src/preprocessor/data.rs | 14 +--- crates/compilers/src/preprocessor/deps.rs | 79 ++++++++++++++------- test-data/preprocessor/src/CounterE.sol | 13 ++++ test-data/preprocessor/test/CounterTest.sol | 5 ++ 4 files changed, 74 insertions(+), 37 deletions(-) create mode 100644 test-data/preprocessor/src/CounterE.sol diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 48627bd4..bcc88ced 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -171,14 +171,12 @@ impl ContractData { /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) /// ``` pub fn build_helper(&self) -> Option { - let Self { contract_id, path, name, constructor_data, artifact } = self; + let Self { contract_id, path, name, constructor_data, artifact: _ } = self; let Some(constructor_details) = constructor_data else { return None }; let contract_id = contract_id.get(); let struct_fields = &constructor_details.struct_fields; let abi_encode_args = &constructor_details.abi_encode_args; - let vm_interface_name = format!("VmContractHelper{contract_id}"); - let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); let helper = format!( r#" @@ -195,16 +193,6 @@ abstract contract DeployHelper{contract_id} is {name} {{ function encodeArgs{contract_id}(DeployHelper{contract_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ return abi.encode({abi_encode_args}); }} - -function deployCode{contract_id}(DeployHelper{contract_id}.ConstructorArgs memory args) returns({name}) {{ - return {name}(payable({vm}.deployCode("{artifact}", encodeArgs{contract_id}(args)))); -}} - -interface {vm_interface_name} {{ - function deployCode(string memory _artifact, bytes memory _data) external returns (address); - function deployCode(string memory _artifact) external returns (address); - function getCode(string memory _artifact) external returns (bytes memory); -}} "#, path = path.to_slash_lossy(), ); diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 9748e6ee..20b445ae 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -8,7 +8,7 @@ use crate::{ use itertools::Itertools; use solar_parse::interface::Session; use solar_sema::{ - hir::{ContractId, Expr, ExprKind, Hir, TypeKind, Visit}, + hir::{ContractId, Expr, ExprKind, Hir, NamedArg, TypeKind, Visit}, interface::{data_structures::Never, source_map::FileName, SourceMap}, }; use std::{ @@ -95,8 +95,9 @@ impl PreprocessorDependencies { enum BytecodeDependencyKind { /// `type(Contract).creationCode` CreationCode, - /// `new Contract`. Holds the name of the contract and args length. - New(String, usize, usize), + /// `new Contract`. Holds the name of the contract, args length and offset, `msg.value` (if + /// any) and salt (if any). + New(String, usize, usize, Option, Option), } /// Represents a single bytecode dependency. @@ -193,6 +194,8 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { name.to_string(), args_len.to_usize(), offset, + named_arg(self.src, named_args, "value", self.source_map), + named_arg(self.src, named_args, "salt", self.source_map), ), loc: SourceMapLocation::from_span(self.source_map, ty.span), referenced_contract: contract_id, @@ -222,6 +225,21 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { } } +/// Helper function to extract value of a given named arg. +fn named_arg( + src: &str, + named_args: &Option<&[NamedArg<'_>]>, + arg: &str, + source_map: &SourceMap, +) -> Option { + named_args.unwrap_or_default().iter().find(|named_arg| named_arg.name.as_str() == arg).map( + |named_arg| { + let named_arg_loc = SourceMapLocation::from_span(source_map, named_arg.value.span); + src[named_arg_loc.start..named_arg_loc.end].to_string() + }, + ) +} + /// Goes over all test/script files and replaces bytecode dependencies with cheatcode /// invocations. pub(crate) fn remove_bytecode_dependencies( @@ -260,31 +278,38 @@ pub(crate) fn remove_bytecode_dependencies( format!("{vm}.getCode(\"{artifact}\")"), )); } - BytecodeDependencyKind::New(name, args_length, offset) => { - if constructor_data.is_none() { - // if there's no constructor, we can just call deployCode with one - // argument - updates.insert(( - dep.loc.start, - dep.loc.end + args_length, - format!("{name}(payable({vm}.deployCode(\"{artifact}\")))"), - )); - } else { - // if there's a constructor, we use our helper + BytecodeDependencyKind::New(name, args_length, offset, value, salt) => { + let mut update = format!("{name}(payable({vm}.deployCode({{"); + update.push_str(&format!("_artifact: \"{artifact}\"")); + + if let Some(value) = value { + update.push_str(", "); + update.push_str(&format!("_value: {value}")); + } + + if let Some(salt) = salt { + update.push_str(", "); + update.push_str(&format!("_salt: {salt}")); + } + + if constructor_data.is_some() { + // Insert our helper used_helpers.insert(dep.referenced_contract); - updates.insert(( - dep.loc.start, - dep.loc.end + offset, - format!( - "deployCode{id}(DeployHelper{id}.ConstructorArgs", - id = dep.referenced_contract.get() - ), + + update.push_str(", "); + update.push_str(&format!( + "_args: encodeArgs{id}(DeployHelper{id}.ConstructorArgs", + id = dep.referenced_contract.get() )); + updates.insert((dep.loc.start, dep.loc.end + offset, update)); updates.insert(( dep.loc.end + args_length, dep.loc.end + args_length, - ")".to_string(), + ")})))".to_string(), )); + } else { + update.push_str("})))"); + updates.insert((dep.loc.start, dep.loc.end + args_length, update)); } } }; @@ -292,7 +317,7 @@ pub(crate) fn remove_bytecode_dependencies( let helper_imports = used_helpers.into_iter().map(|id| { let id = id.get(); format!( - "import {{DeployHelper{id}, encodeArgs{id}, deployCode{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", + "import {{DeployHelper{id}, encodeArgs{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", ) }).join("\n"); updates.insert(( @@ -303,8 +328,14 @@ pub(crate) fn remove_bytecode_dependencies( {helper_imports} interface {vm_interface_name} {{ - function deployCode(string memory _artifact, bytes memory _data) external returns (address); function deployCode(string memory _artifact) external returns (address); + function deployCode(string memory _artifact, bytes32 _salt) external returns (address); + function deployCode(string memory _artifact, bytes memory _args) external returns (address); + function deployCode(string memory _artifact, bytes memory _args, bytes32 _salt) external returns (address); + function deployCode(string memory _artifact, uint256 _value) external returns (address); + function deployCode(string memory _artifact, uint256 _value, bytes32 _salt) external returns (address); + function deployCode(string memory _artifact, bytes memory _args, uint256 _value) external returns (address); + function deployCode(string memory _artifact, bytes memory _args, uint256 _value, bytes32 _salt) external returns (address); function getCode(string memory _artifact) external returns (bytes memory); }}"# ), diff --git a/test-data/preprocessor/src/CounterE.sol b/test-data/preprocessor/src/CounterE.sol new file mode 100644 index 00000000..a7f5e5a4 --- /dev/null +++ b/test-data/preprocessor/src/CounterE.sol @@ -0,0 +1,13 @@ +// Contracts with payable constructor +contract CounterE { + constructor() payable {} +} + +contract CounterF { + constructor(uint256 x) payable {} +} + +contract CounterG { + constructor(address) payable {} +} + diff --git a/test-data/preprocessor/test/CounterTest.sol b/test-data/preprocessor/test/CounterTest.sol index 2b09c631..2594b4ab 100644 --- a/test-data/preprocessor/test/CounterTest.sol +++ b/test-data/preprocessor/test/CounterTest.sol @@ -3,6 +3,7 @@ import {Counter as CounterV1} from "src/v1/Counter.sol"; import "src/CounterB.sol"; import "src/CounterC.sol"; import "src/CounterD.sol"; +import "src/CounterE.sol"; contract CounterTest { Counter public counter; @@ -12,6 +13,10 @@ contract CounterTest { CounterV1 public counterv1; Counter public counter5 = new Counter{salt: bytes32("123")}(); CounterB public counter6 = new CounterB {salt: bytes32("123")} (address(this), 44, true, address(this)); + CounterE public counter7 = new CounterE{ value: 111, salt: bytes32("123")}(); + CounterF public counter8 = new CounterF{value: 222, salt: bytes32("123")}(11); + CounterG public counter9 = new CounterG { value: 333, salt: bytes32("123") } (address(this)); + CounterG public counter10 = new CounterG{ value: 333 }(address(this)); function setUp() public { counter = new Counter(); From 322e80ac30b02e3a05cde1201dcc4d0b9acc10e7 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Mar 2025 12:09:17 +0200 Subject: [PATCH 48/70] Handle named args with call args and offset --- crates/compilers/src/preprocessor/deps.rs | 21 ++++++++++++++------- test-data/preprocessor/test/CounterTest.sol | 4 ++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 20b445ae..963638ca 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -95,8 +95,8 @@ impl PreprocessorDependencies { enum BytecodeDependencyKind { /// `type(Contract).creationCode` CreationCode, - /// `new Contract`. Holds the name of the contract, args length and offset, `msg.value` (if - /// any) and salt (if any). + /// `new Contract`. Holds the name of the contract, args length and call args offset, + /// `msg.value` (if any) and salt (if any). New(String, usize, usize, Option, Option), } @@ -182,8 +182,12 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { SourceMapLocation::from_span(self.source_map, ty_new.span); let name = &self.src[name_loc.start..name_loc.end]; - let offset = if named_args.is_some() && !call_args.is_empty() { - (call_args.span().lo() - ty_new.span.hi()).to_usize() - 1 + // Calculate offset to remove named args, e.g. for an expression like + // `new Counter {value: 333} ( address(this))` + // the offset will be used to replace `{value: 333} ( ` with `(` + let call_args_offset = if named_args.is_some() && !call_args.is_empty() + { + (call_args.span().lo() - ty_new.span.hi()).to_usize() } else { 0 }; @@ -193,7 +197,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { kind: BytecodeDependencyKind::New( name.to_string(), args_len.to_usize(), - offset, + call_args_offset, named_arg(self.src, named_args, "value", self.source_map), named_arg(self.src, named_args, "salt", self.source_map), ), @@ -278,7 +282,7 @@ pub(crate) fn remove_bytecode_dependencies( format!("{vm}.getCode(\"{artifact}\")"), )); } - BytecodeDependencyKind::New(name, args_length, offset, value, salt) => { + BytecodeDependencyKind::New(name, args_length, call_args_offset, value, salt) => { let mut update = format!("{name}(payable({vm}.deployCode({{"); update.push_str(&format!("_artifact: \"{artifact}\"")); @@ -301,7 +305,10 @@ pub(crate) fn remove_bytecode_dependencies( "_args: encodeArgs{id}(DeployHelper{id}.ConstructorArgs", id = dep.referenced_contract.get() )); - updates.insert((dep.loc.start, dep.loc.end + offset, update)); + if *call_args_offset > 0 { + update.push('('); + } + updates.insert((dep.loc.start, dep.loc.end + call_args_offset, update)); updates.insert(( dep.loc.end + args_length, dep.loc.end + args_length, diff --git a/test-data/preprocessor/test/CounterTest.sol b/test-data/preprocessor/test/CounterTest.sol index 2594b4ab..7eb79bce 100644 --- a/test-data/preprocessor/test/CounterTest.sol +++ b/test-data/preprocessor/test/CounterTest.sol @@ -12,10 +12,10 @@ contract CounterTest { CounterB public counter4 = new CounterB({a:address(this), b: 44, c: true, d: address(this)}); CounterV1 public counterv1; Counter public counter5 = new Counter{salt: bytes32("123")}(); - CounterB public counter6 = new CounterB {salt: bytes32("123")} (address(this), 44, true, address(this)); + CounterB public counter6 = new CounterB {salt: bytes32("123")} ( address(this), 44, true, address(this)); CounterE public counter7 = new CounterE{ value: 111, salt: bytes32("123")}(); CounterF public counter8 = new CounterF{value: 222, salt: bytes32("123")}(11); - CounterG public counter9 = new CounterG { value: 333, salt: bytes32("123") } (address(this)); + CounterG public counter9 = new CounterG { value: 333, salt: bytes32("123") } ( address(this)); CounterG public counter10 = new CounterG{ value: 333 }(address(this)); function setUp() public { From 5d0255532acd46166ace8d7388ff9af17defc512 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:47:20 +0100 Subject: [PATCH 49/70] cleaning --- crates/compilers/src/lib.rs | 2 +- crates/compilers/src/preprocessor/data.rs | 2 +- crates/compilers/src/preprocessor/deps.rs | 10 ++++------ crates/compilers/src/preprocessor/mod.rs | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 4720a932..d5f057ac 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -42,7 +42,7 @@ pub mod report; /// Updates to be applied to the sources. /// source_path -> (start, end, new_value) -pub type Updates = HashMap>; +pub(crate) type Updates = HashMap>; /// Utilities for creating, mocking and testing of (temporary) projects #[cfg(feature = "project-util")] diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index bcc88ced..11468e4c 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -1,4 +1,4 @@ -use crate::preprocessor::SourceMapLocation; +use super::SourceMapLocation; use foundry_compilers_artifacts::{Source, Sources}; use path_slash::PathExt; use solar_parse::interface::{Session, SourceMap}; diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 963638ca..a6547965 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -1,10 +1,8 @@ -use crate::{ - preprocessor::{ - data::{ContractData, PreprocessorData}, - SourceMapLocation, - }, - Updates, +use super::{ + data::{ContractData, PreprocessorData}, + SourceMapLocation, }; +use crate::Updates; use itertools::Itertools; use solar_parse::interface::Session; use solar_sema::{ diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 1b0fa522..a07e2583 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -32,7 +32,7 @@ mod deps; /// Represents location of an item in the source map. /// Used to generate source code updates. #[derive(Debug)] -pub struct SourceMapLocation { +struct SourceMapLocation { /// Source map location start. start: usize, /// Source map location end. From 8dd35c34edbb78b846c82fc6556f79d457ba9d0c Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:54:08 +0100 Subject: [PATCH 50/70] chore: replace SourceMapLocation with Range --- crates/compilers/src/preprocessor/data.rs | 9 ++++----- crates/compilers/src/preprocessor/deps.rs | 19 +++++++++---------- crates/compilers/src/preprocessor/mod.rs | 23 +++++------------------ 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index 11468e4c..cbc82c00 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -1,4 +1,3 @@ -use super::SourceMapLocation; use foundry_compilers_artifacts::{Source, Sources}; use path_slash::PathExt; use solar_parse::interface::{Session, SourceMap}; @@ -11,6 +10,8 @@ use std::{ path::{Path, PathBuf}, }; +use super::span_to_range; + /// Keeps data about project contracts definitions referenced from tests and scripts. /// Contract id -> Contract data definition mapping. pub type PreprocessorData = BTreeMap; @@ -97,10 +98,8 @@ impl ContractData { let mut arg_index = 0; for param_id in ctor.parameters { let src = source.file.src.as_str(); - let loc = - SourceMapLocation::from_span(source_map, hir.variable(*param_id).span); - let mut new_src = - src[loc.start..loc.end].replace(" memory ", " ").replace(" calldata ", " "); + let loc = span_to_range(source_map, hir.variable(*param_id).span); + let mut new_src = src[loc].replace(" memory ", " ").replace(" calldata ", " "); if let Some(ident) = hir.variable(*param_id).name { abi_encode_args.push(format!("args.{}", ident.name)); } else { diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index a6547965..f6e4373d 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -1,6 +1,6 @@ use super::{ data::{ContractData, PreprocessorData}, - SourceMapLocation, + span_to_range, }; use crate::Updates; use itertools::Itertools; @@ -11,7 +11,7 @@ use solar_sema::{ }; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, - ops::ControlFlow, + ops::{ControlFlow, Range}, path::{Path, PathBuf}, }; @@ -104,7 +104,7 @@ pub(crate) struct BytecodeDependency { /// Dependency kind. kind: BytecodeDependencyKind, /// Source map location of this dependency. - loc: SourceMapLocation, + loc: Range, /// HIR id of referenced contract. referenced_contract: ContractId, } @@ -176,9 +176,8 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { if let ExprKind::New(ty_new) = &ty.kind { if let TypeKind::Custom(item_id) = ty_new.kind { if let Some(contract_id) = item_id.as_contract() { - let name_loc = - SourceMapLocation::from_span(self.source_map, ty_new.span); - let name = &self.src[name_loc.start..name_loc.end]; + let name_loc = span_to_range(self.source_map, ty_new.span); + let name = &self.src[name_loc]; // Calculate offset to remove named args, e.g. for an expression like // `new Counter {value: 333} ( address(this))` @@ -199,7 +198,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { named_arg(self.src, named_args, "value", self.source_map), named_arg(self.src, named_args, "salt", self.source_map), ), - loc: SourceMapLocation::from_span(self.source_map, ty.span), + loc: span_to_range(self.source_map, ty.span), referenced_contract: contract_id, }); } @@ -213,7 +212,7 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { if let Some(contract_id) = contract_id.as_contract() { self.collect_dependency(BytecodeDependency { kind: BytecodeDependencyKind::CreationCode, - loc: SourceMapLocation::from_span(self.source_map, expr.span), + loc: span_to_range(self.source_map, expr.span), referenced_contract: contract_id, }); } @@ -236,8 +235,8 @@ fn named_arg( ) -> Option { named_args.unwrap_or_default().iter().find(|named_arg| named_arg.name.as_str() == arg).map( |named_arg| { - let named_arg_loc = SourceMapLocation::from_span(source_map, named_arg.value.span); - src[named_arg_loc.start..named_arg_loc.end].to_string() + let named_arg_loc = span_to_range(source_map, named_arg.value.span); + src[named_arg_loc].to_string() }, ) } diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index a07e2583..968909e2 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -23,30 +23,17 @@ use solar_parse::{ use solar_sema::{thread_local::ThreadLocal, ParsingContext}; use std::{ collections::HashSet, + ops::Range, path::{Path, PathBuf}, }; mod data; mod deps; -/// Represents location of an item in the source map. -/// Used to generate source code updates. -#[derive(Debug)] -struct SourceMapLocation { - /// Source map location start. - start: usize, - /// Source map location end. - end: usize, -} - -impl SourceMapLocation { - /// Creates source map location from an item location within a source file. - fn from_span(source_map: &SourceMap, span: Span) -> Self { - Self { - start: source_map.lookup_byte_offset(span.lo()).pos.to_usize(), - end: source_map.lookup_byte_offset(span.hi()).pos.to_usize(), - } - } +/// Returns the range of the given span in the source map. +#[track_caller] +fn span_to_range(source_map: &SourceMap, span: Span) -> Range { + source_map.span_to_source(span).unwrap().1 } #[derive(Debug)] From a7bc520c0ecc4202f32cd58c4fa0b446eab7748b Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 26 Mar 2025 14:54:50 +0100 Subject: [PATCH 51/70] fmt --- crates/compilers/src/preprocessor/data.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs index cbc82c00..5b51dd0a 100644 --- a/crates/compilers/src/preprocessor/data.rs +++ b/crates/compilers/src/preprocessor/data.rs @@ -1,3 +1,4 @@ +use super::span_to_range; use foundry_compilers_artifacts::{Source, Sources}; use path_slash::PathExt; use solar_parse::interface::{Session, SourceMap}; @@ -10,8 +11,6 @@ use std::{ path::{Path, PathBuf}, }; -use super::span_to_range; - /// Keeps data about project contracts definitions referenced from tests and scripts. /// Contract id -> Contract data definition mapping. pub type PreprocessorData = BTreeMap; From 6b907d1abae6446996db900103ec155c7d7f5dd8 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Mar 2025 16:01:50 +0200 Subject: [PATCH 52/70] Cleanup BytecodeDependencyKind::New --- crates/compilers/src/preprocessor/deps.rs | 41 +++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index f6e4373d..8a0554e5 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -93,9 +93,19 @@ impl PreprocessorDependencies { enum BytecodeDependencyKind { /// `type(Contract).creationCode` CreationCode, - /// `new Contract`. Holds the name of the contract, args length and call args offset, - /// `msg.value` (if any) and salt (if any). - New(String, usize, usize, Option, Option), + /// `new Contract`. + New { + /// Contract name. + name: String, + /// Constructor args length. + args_length: usize, + /// Constructor call args offset. + call_args_offset: usize, + /// `msg.value` (if any) used when creating contract. + value: Option, + /// `salt` (if any) used when creating contract. + salt: Option, + }, } /// Represents a single bytecode dependency. @@ -191,13 +201,18 @@ impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { let args_len = expr.span.hi() - ty_new.span.hi(); self.collect_dependency(BytecodeDependency { - kind: BytecodeDependencyKind::New( - name.to_string(), - args_len.to_usize(), + kind: BytecodeDependencyKind::New { + name: name.to_string(), + args_length: args_len.to_usize(), call_args_offset, - named_arg(self.src, named_args, "value", self.source_map), - named_arg(self.src, named_args, "salt", self.source_map), - ), + value: named_arg( + self.src, + named_args, + "value", + self.source_map, + ), + salt: named_arg(self.src, named_args, "salt", self.source_map), + }, loc: span_to_range(self.source_map, ty.span), referenced_contract: contract_id, }); @@ -279,7 +294,13 @@ pub(crate) fn remove_bytecode_dependencies( format!("{vm}.getCode(\"{artifact}\")"), )); } - BytecodeDependencyKind::New(name, args_length, call_args_offset, value, salt) => { + BytecodeDependencyKind::New { + name, + args_length, + call_args_offset, + value, + salt, + } => { let mut update = format!("{name}(payable({vm}.deployCode({{"); update.push_str(&format!("_artifact: \"{artifact}\"")); From 446a5d817c83f8b2ff27d06cafe4ffdfb5db686a Mon Sep 17 00:00:00 2001 From: grandizzy Date: Wed, 26 Mar 2025 22:06:12 +0200 Subject: [PATCH 53/70] Use sources that are already read --- crates/compilers/src/lib.rs | 7 +++---- crates/compilers/src/preprocessor/mod.rs | 22 +++++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index d5f057ac..4233de8c 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -911,11 +911,10 @@ fn replace_source_content<'a>( let mut offset = 0; let mut content = source.as_bytes().to_vec(); for (range, new_value) in updates { - let start = (range.start as isize + offset) as usize; - let end = (range.end as isize + offset) as usize; + let update_range = utils::range_by_offset(&range, offset); - content.splice(start..end, new_value.bytes()); - offset += new_value.len() as isize - (end - start) as isize; + content.splice(update_range.start..update_range.end, new_value.bytes()); + offset += new_value.len() as isize - (update_range.end - update_range.start) as isize; } String::from_utf8(content).unwrap() diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 968909e2..f36999f5 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -13,7 +13,6 @@ use crate::{ use alloy_primitives::hex; use foundry_compilers_artifacts::{SolcLanguage, Source}; use foundry_compilers_core::{error::SolcError, utils}; -use itertools::Itertools; use md5::Digest; use solar_parse::{ ast::{FunctionKind, ItemKind, Span, Visibility}, @@ -65,14 +64,19 @@ impl Preprocessor for TestOptimizerPreprocessor { } // Load and parse test and script contracts only (dependencies are automatically // resolved). - let preprocessed_paths = sources - .into_iter() - .filter(|(path, source)| { - is_test_or_script(path, paths) && !source.content.is_empty() - }) - .map(|(path, _)| path.clone()) - .collect_vec(); - parsing_context.load_files(&preprocessed_paths)?; + + let mut preprocessed_paths = vec![]; + for (path, source) in sources.iter() { + if is_test_or_script(path, paths) && !source.content.is_empty() { + if let Ok(src_file) = sess + .source_map() + .new_dummy_source_file(path.clone(), source.content.to_string()) + { + parsing_context.add_file(src_file); + preprocessed_paths.push(path.clone()); + } + } + } let hir_arena = ThreadLocal::new(); if let Some(gcx) = parsing_context.parse_and_lower(&hir_arena)? { From 4eefc936fe684d7c54d655c402350835f0892147 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 27 Mar 2025 21:51:35 +0100 Subject: [PATCH 54/70] fix: correctly set paths in ParsingContext --- Cargo.toml | 5 ++-- crates/compilers/src/preprocessor/mod.rs | 17 ++++++++------ crates/compilers/src/project_util/mod.rs | 17 ++++++++++++-- crates/compilers/tests/project.rs | 29 ++++++++---------------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9018e6b5..fcc437ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,5 @@ tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" [patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", branch = "main" } -solar-sema = { git = "https://github.com/paradigmxyz/solar", branch = "main" } - +solar-parse = { git = "https://github.com/paradigmxyz/solar", branch = "main" } +solar-sema = { git = "https://github.com/paradigmxyz/solar", branch = "main" } diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index f36999f5..42c45c3a 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -55,22 +55,24 @@ impl Preprocessor for TestOptimizerPreprocessor { let sess = Session::builder().with_buffer_emitter(Default::default()).build(); let _ = sess.enter_parallel(|| -> solar_parse::interface::Result { + // Set up the parsing context with the project paths. let mut parsing_context = ParsingContext::new(&sess); - // Set remappings into HIR parsing context. + parsing_context.file_resolver.set_current_dir(&paths.root); for remapping in &paths.remappings { parsing_context .file_resolver .add_import_map(PathBuf::from(&remapping.name), PathBuf::from(&remapping.path)); } - // Load and parse test and script contracts only (dependencies are automatically - // resolved). + for include_path in &paths.include_paths { + let _ = parsing_context.file_resolver.add_import_path(include_path.clone()); + } + // Add the sources into the context. let mut preprocessed_paths = vec![]; for (path, source) in sources.iter() { - if is_test_or_script(path, paths) && !source.content.is_empty() { - if let Ok(src_file) = sess - .source_map() - .new_dummy_source_file(path.clone(), source.content.to_string()) + if is_test_or_script(path, paths) { + if let Ok(src_file) = + sess.source_map().new_source_file(path.clone(), source.content.as_str()) { parsing_context.add_file(src_file); preprocessed_paths.push(path.clone()); @@ -78,6 +80,7 @@ impl Preprocessor for TestOptimizerPreprocessor { } } + // Parse and preprocess. let hir_arena = ThreadLocal::new(); if let Some(gcx) = parsing_context.parse_and_lower(&hir_arena)? { let hir = &gcx.get().hir; diff --git a/crates/compilers/src/project_util/mod.rs b/crates/compilers/src/project_util/mod.rs index f8844aeb..a79b9866 100644 --- a/crates/compilers/src/project_util/mod.rs +++ b/crates/compilers/src/project_util/mod.rs @@ -124,7 +124,7 @@ impl TempProject { .artifacts(HardhatArtifacts::default()) .paths(paths) .build(Default::default())?; - Ok(Self::create_new(tmp_dir, inner)?) + Ok(TempProject::create_new(tmp_dir, inner)?) } } @@ -133,7 +133,12 @@ impl< T: ArtifactOutput + Default, > TempProject { - /// Makes sure all resources are created + /// Wraps an existing project in a temp dir. + pub fn from_project(inner: Project) -> std::result::Result { + Self::create_new(tempdir("tmp_project")?, inner) + } + + /// Makes sure all resources are created. pub fn create_new( root: TempDir, inner: Project, @@ -215,6 +220,14 @@ impl< &mut self.project_mut().paths } + /// Deletes the current project and copies it from `source`. + pub fn copy_project_from(&self, source: &Path) -> Result<()> { + let root = self.root(); + std::fs::remove_dir_all(root).map_err(|e| SolcIoError::new(e, root))?; + std::fs::create_dir_all(root).map_err(|e| SolcIoError::new(e, root))?; + copy_dir(source, root) + } + /// Copies a single file into the projects source pub fn copy_source(&self, source: &Path) -> Result<()> { copy_file(source, &self.paths().sources) diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index e260706b..0571b489 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -4164,26 +4164,17 @@ fn can_preprocess_constructors_and_creation_code() { canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) .unwrap(); - let paths = ProjectPathsConfig::builder() - .sources(root.join("src")) - .tests(root.join("test")) - .root(&root) - .build::() - .unwrap(); - - let project = ProjectBuilder::::new(Default::default()) - .paths(paths) - .build(SolcCompiler::default()) - .unwrap(); - - // TODO: figure out how to set root to parsing context. - let cur_dir = env::current_dir().unwrap(); - env::set_current_dir(root).unwrap(); - let compiled = ProjectCompiler::new(&project) + let project = TempProject::hardhat().unwrap(); + project.copy_project_from(&root).unwrap(); + let r = ProjectCompiler::new(&project.project()) .unwrap() .with_preprocessor(TestOptimizerPreprocessor) - .compile() - .expect("failed to compile"); + .compile(); + + let compiled = match r { + Ok(compiled) => compiled, + Err(e) => panic!("failed to compile: {e}"), + }; compiled.assert_success(); - env::set_current_dir(cur_dir).unwrap(); + assert!(!compiled.is_unchanged()); } From 9b15b7a0b3af37d0d7e0eff2e9df6771e1d92f5d Mon Sep 17 00:00:00 2001 From: grandizzy Date: Fri, 28 Mar 2025 08:45:09 +0200 Subject: [PATCH 55/70] clippy --- crates/compilers/src/project_util/mod.rs | 2 +- crates/compilers/tests/project.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/compilers/src/project_util/mod.rs b/crates/compilers/src/project_util/mod.rs index a79b9866..6473dc62 100644 --- a/crates/compilers/src/project_util/mod.rs +++ b/crates/compilers/src/project_util/mod.rs @@ -124,7 +124,7 @@ impl TempProject { .artifacts(HardhatArtifacts::default()) .paths(paths) .build(Default::default())?; - Ok(TempProject::create_new(tmp_dir, inner)?) + Ok(Self::create_new(tmp_dir, inner)?) } } diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index 0571b489..75f902ce 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -4166,7 +4166,7 @@ fn can_preprocess_constructors_and_creation_code() { let project = TempProject::hardhat().unwrap(); project.copy_project_from(&root).unwrap(); - let r = ProjectCompiler::new(&project.project()) + let r = ProjectCompiler::new(project.project()) .unwrap() .with_preprocessor(TestOptimizerPreprocessor) .compile(); From fb5ebc251f58727d364e5c58d330be14797b73f6 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Fri, 28 Mar 2025 16:04:24 +0000 Subject: [PATCH 56/70] bump --- crates/compilers/src/preprocessor/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 42c45c3a..6266bc00 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -59,13 +59,15 @@ impl Preprocessor for TestOptimizerPreprocessor { let mut parsing_context = ParsingContext::new(&sess); parsing_context.file_resolver.set_current_dir(&paths.root); for remapping in &paths.remappings { - parsing_context - .file_resolver - .add_import_map(PathBuf::from(&remapping.name), PathBuf::from(&remapping.path)); - } - for include_path in &paths.include_paths { - let _ = parsing_context.file_resolver.add_import_path(include_path.clone()); + parsing_context.file_resolver.add_import_remapping( + solar_sema::interface::config::ImportRemapping { + context: remapping.context.clone().unwrap_or_default(), + prefix: remapping.name.clone(), + path: remapping.path.clone(), + }, + ); } + parsing_context.file_resolver.add_include_paths(paths.include_paths.iter().cloned()); // Add the sources into the context. let mut preprocessed_paths = vec![]; From 59ebda6a2111b27f352a414849934715151d2e8c Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:00:40 +0200 Subject: [PATCH 57/70] feat: avoid reparsing for interface_repr_hash --- crates/artifacts/solc/src/sources.rs | 8 +- crates/compilers/src/cache.rs | 36 +++--- crates/compilers/src/compilers/mod.rs | 5 + crates/compilers/src/compilers/multi.rs | 7 ++ crates/compilers/src/compilers/solc/mod.rs | 4 + crates/compilers/src/config.rs | 10 ++ crates/compilers/src/preprocessor/mod.rs | 129 ++++++++++----------- crates/compilers/src/resolver/parse.rs | 25 ++-- 8 files changed, 121 insertions(+), 103 deletions(-) diff --git a/crates/artifacts/solc/src/sources.rs b/crates/artifacts/solc/src/sources.rs index 01c52086..c9ef1fe6 100644 --- a/crates/artifacts/solc/src/sources.rs +++ b/crates/artifacts/solc/src/sources.rs @@ -198,7 +198,13 @@ impl Source { /// Generate a non-cryptographically secure checksum of the file's content. #[cfg(feature = "checksum")] pub fn content_hash(&self) -> String { - alloy_primitives::hex::encode(::digest(self.content.as_bytes())) + Self::content_hash_of(&self.content) + } + + /// Generate a non-cryptographically secure checksum of the given source. + #[cfg(feature = "checksum")] + pub fn content_hash_of(src: &str) -> String { + alloy_primitives::hex::encode(::digest(src)) } } diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 90481dc7..bbfb82b8 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,10 +4,9 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, - preprocessor::interface_representation_hash, resolver::GraphEdges, - ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, - ProjectPaths, ProjectPathsConfig, SourceCompilationKind, + ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, ParsedSource, + Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, }; use foundry_compilers_artifacts::{ sources::{Source, Sources}, @@ -366,10 +365,7 @@ impl CompilerCache { { match tokio::task::spawn_blocking(f).await { Ok(res) => res, - Err(_) => Err(SolcError::io( - std::io::Error::new(std::io::ErrorKind::Other, "background task failed"), - "", - )), + Err(_) => Err(SolcError::io(std::io::Error::other("background task failed"), "")), } } } @@ -673,8 +669,7 @@ impl, C: Compiler> { /// Whether given file is a source file or a test/script file. fn is_source_file(&self, file: &Path) -> bool { - !file.starts_with(&self.project.paths.tests) - && !file.starts_with(&self.project.paths.scripts) + !self.project.paths.is_test_or_script(file) } /// Creates a new cache entry for the file @@ -686,8 +681,8 @@ impl, C: Compiler> .map(|import| strip_prefix(import, self.project.root()).into()) .collect(); - let interface_repr_hash = if self.cache.preprocessed { - self.is_source_file(&file).then(|| interface_representation_hash(source, &file)) + let interface_repr_hash = if self.cache.preprocessed && self.is_source_file(&file) { + self.edges.get_parsed_source(&file).and_then(ParsedSource::interface_repr_hash) } else { None }; @@ -961,16 +956,17 @@ impl, C: Compiler> /// Adds the file's hashes to the set if not set yet fn fill_hashes(&mut self, sources: &Sources) { for (file, source) in sources { - if let hash_map::Entry::Vacant(entry) = self.content_hashes.entry(file.clone()) { - entry.insert(source.content_hash()); - } + let content_hash = + self.content_hashes.entry(file.clone()).or_insert_with(|| source.content_hash()); + // Fill interface representation hashes for source files - if self.cache.preprocessed && self.is_source_file(file) { - if let hash_map::Entry::Vacant(entry) = - self.interface_repr_hashes.entry(file.clone()) - { - entry.insert(interface_representation_hash(source, file)); - } + if self.cache.preprocessed && self.project.paths.is_source_file(file) { + self.interface_repr_hashes.entry(file.clone()).or_insert_with(|| { + self.edges + .get_parsed_source(file) + .and_then(ParsedSource::interface_repr_hash) + .unwrap_or_else(|| content_hash.clone()) + }); } } } diff --git a/crates/compilers/src/compilers/mod.rs b/crates/compilers/src/compilers/mod.rs index 5abb74b7..2e923c6d 100644 --- a/crates/compilers/src/compilers/mod.rs +++ b/crates/compilers/src/compilers/mod.rs @@ -186,6 +186,11 @@ pub trait ParsedSource: Debug + Sized + Send + Clone { { vec![].into_iter() } + + /// Returns the hash of the interface of the source. + fn interface_repr_hash(&self) -> Option { + None + } } /// Error returned by compiler. Might also represent a warning or informational message. diff --git a/crates/compilers/src/compilers/multi.rs b/crates/compilers/src/compilers/multi.rs index d206076b..ec2cc94e 100644 --- a/crates/compilers/src/compilers/multi.rs +++ b/crates/compilers/src/compilers/multi.rs @@ -387,6 +387,13 @@ impl ParsedSource for MultiCompilerParsedSource { } .into_iter() } + + fn interface_repr_hash(&self) -> Option { + match self { + Self::Solc(parsed) => parsed.interface_repr_hash(), + Self::Vyper(parsed) => parsed.interface_repr_hash(), + } + } } impl CompilationError for MultiCompilerError { diff --git a/crates/compilers/src/compilers/solc/mod.rs b/crates/compilers/src/compilers/solc/mod.rs index 3c1d6526..0a612ed4 100644 --- a/crates/compilers/src/compilers/solc/mod.rs +++ b/crates/compilers/src/compilers/solc/mod.rs @@ -397,6 +397,10 @@ impl ParsedSource for SolData { { imported_nodes.filter_map(|(path, node)| (!node.libraries.is_empty()).then_some(path)) } + + fn interface_repr_hash(&self) -> Option { + self.interface_repr_hash.clone() + } } impl CompilationError for Error { diff --git a/crates/compilers/src/config.rs b/crates/compilers/src/config.rs index 920ecdaa..901108a7 100644 --- a/crates/compilers/src/config.rs +++ b/crates/compilers/src/config.rs @@ -250,6 +250,16 @@ impl ProjectPathsConfig { Self::dapptools(&std::env::current_dir().map_err(|err| SolcError::io(err, "."))?) } + pub(crate) fn is_test_or_script(&self, path: &Path) -> bool { + let test_dir = self.tests.strip_prefix(&self.root).unwrap_or(&self.tests); + let script_dir = self.scripts.strip_prefix(&self.root).unwrap_or(&self.scripts); + path.starts_with(test_dir) || path.starts_with(script_dir) + } + + pub(crate) fn is_source_file(&self, path: &Path) -> bool { + !self.is_test_or_script(path) + } + /// Returns a new [ProjectPaths] instance that contains all directories configured for this /// project pub fn paths(&self) -> ProjectPaths { diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 6266bc00..75979912 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -10,10 +10,8 @@ use crate::{ solc::{SolcCompiler, SolcVersionedInput}, Compiler, ProjectPathsConfig, Result, }; -use alloy_primitives::hex; -use foundry_compilers_artifacts::{SolcLanguage, Source}; +use foundry_compilers_artifacts::SolcLanguage; use foundry_compilers_core::{error::SolcError, utils}; -use md5::Digest; use solar_parse::{ ast::{FunctionKind, ItemKind, Span, Visibility}, interface::{diagnostics::EmittedDiagnostics, source_map::FileName, Session, SourceMap}, @@ -48,7 +46,7 @@ impl Preprocessor for TestOptimizerPreprocessor { ) -> Result<()> { let sources = &mut input.input.sources; // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. - if sources.iter().all(|(path, _)| !is_test_or_script(path, paths)) { + if sources.iter().all(|(path, _)| !paths.is_test_or_script(path)) { trace!("no tests or sources to preprocess"); return Ok(()); } @@ -72,7 +70,7 @@ impl Preprocessor for TestOptimizerPreprocessor { // Add the sources into the context. let mut preprocessed_paths = vec![]; for (path, source) in sources.iter() { - if is_test_or_script(path, paths) { + if paths.is_test_or_script(path) { if let Ok(src_file) = sess.source_map().new_source_file(path.clone(), source.content.as_str()) { @@ -136,15 +134,27 @@ impl Preprocessor for TestOptimizerPreprocessor { } } -/// Helper function to compute hash of [`interface_representation`] of the source. -pub(crate) fn interface_representation_hash(source: &Source, file: &Path) -> String { - let Ok(repr) = interface_representation(&source.content, file) else { - return source.content_hash(); - }; - let mut hasher = md5::Md5::new(); - hasher.update(&repr); - let result = hasher.finalize(); - hex::encode(result) +pub(crate) fn parse_one_source( + content: &str, + path: &Path, + f: impl FnOnce(solar_sema::ast::SourceUnit<'_>) -> R, +) -> Result { + let sess = Session::builder().with_buffer_emitter(Default::default()).build(); + let res = sess.enter(|| -> solar_parse::interface::Result<_> { + let arena = solar_parse::ast::Arena::new(); + let filename = FileName::Real(path.to_path_buf()); + let mut parser = Parser::from_source_code(&sess, &arena, filename, content.to_string())?; + let ast = parser.parse_file().map_err(|e| e.emit())?; + Ok(f(ast)) + }); + + // Return if any diagnostics emitted during content parsing. + if let Err(err) = sess.emitted_errors().unwrap() { + trace!("failed parsing {path:?}:\n{err}"); + return Err(err); + } + + Ok(res.unwrap()) } /// Helper function to remove parts of the contract which do not alter its interface: @@ -152,75 +162,58 @@ pub(crate) fn interface_representation_hash(source: &Source, file: &Path) -> Str /// - External functions bodies /// /// Preserves all libraries and interfaces. -fn interface_representation(content: &str, file: &Path) -> Result { +pub(crate) fn interface_representation_ast( + content: &str, + ast: &solar_parse::ast::SourceUnit<'_>, +) -> String { let mut spans_to_remove: Vec = Vec::new(); - let sess = Session::builder().with_buffer_emitter(Default::default()).build(); - sess.enter(|| { - let arena = solar_parse::ast::Arena::new(); - let filename = FileName::Real(file.to_path_buf()); - let Ok(mut parser) = Parser::from_source_code(&sess, &arena, filename, content.to_string()) - else { - return; + for item in ast.items.iter() { + let ItemKind::Contract(contract) = &item.kind else { + continue; }; - let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; - for item in ast.items { - let ItemKind::Contract(contract) = &item.kind else { - continue; - }; - - if contract.kind.is_interface() || contract.kind.is_library() { - continue; - } - for contract_item in contract.body.iter() { - if let ItemKind::Function(function) = &contract_item.kind { - let is_exposed = match function.kind { - // Function with external or public visibility - FunctionKind::Function => { - function.header.visibility >= Some(Visibility::Public) - } - FunctionKind::Constructor - | FunctionKind::Fallback - | FunctionKind::Receive => true, - FunctionKind::Modifier => false, - }; - - // If function is not exposed we remove the entire span (signature and - // body). Otherwise we keep function signature and - // remove only the body. - if !is_exposed { - spans_to_remove.push(contract_item.span); - } else { - spans_to_remove.push(function.body_span); + if contract.kind.is_interface() || contract.kind.is_library() { + continue; + } + + for contract_item in contract.body.iter() { + if let ItemKind::Function(function) = &contract_item.kind { + let is_exposed = match function.kind { + // Function with external or public visibility + FunctionKind::Function => { + function.header.visibility >= Some(Visibility::Public) + } + FunctionKind::Constructor | FunctionKind::Fallback | FunctionKind::Receive => { + true } + FunctionKind::Modifier => false, + }; + + // If function is not exposed we remove the entire span (signature and + // body). Otherwise we keep function signature and + // remove only the body. + if !is_exposed { + spans_to_remove.push(contract_item.span); + } else { + spans_to_remove.push(function.body_span); } } } - }); - - // Return if any diagnostics emitted during content parsing. - if let Err(err) = sess.emitted_errors().unwrap() { - trace!("failed parsing {file:?}: {err}"); - return Err(err); } - let content = replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) .replace("\n", ""); - Ok(utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").to_string()) -} - -/// Checks if the given path is a test/script file. -fn is_test_or_script(path: &Path, paths: &ProjectPathsConfig) -> bool { - let test_dir = paths.tests.strip_prefix(&paths.root).unwrap_or(&paths.root); - let script_dir = paths.scripts.strip_prefix(&paths.root).unwrap_or(&paths.root); - path.starts_with(test_dir) || path.starts_with(script_dir) + utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").into_owned() } #[cfg(test)] mod tests { use super::*; - use std::path::PathBuf; + + fn interface_representation(content: &str) -> String { + parse_one_source(content, Path::new(""), |ast| interface_representation_ast(content, &ast)) + .unwrap() + } #[test] fn test_interface_representation() { @@ -242,7 +235,7 @@ contract A { } }"#; - let result = interface_representation(content, &PathBuf::new()).unwrap(); + let result = interface_representation(content); assert_eq!( result, r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# diff --git a/crates/compilers/src/resolver/parse.rs b/crates/compilers/src/resolver/parse.rs index 07bca60f..9153f4a6 100644 --- a/crates/compilers/src/resolver/parse.rs +++ b/crates/compilers/src/resolver/parse.rs @@ -8,6 +8,7 @@ use std::{ /// Represents various information about a Solidity file. #[derive(Clone, Debug)] +#[non_exhaustive] pub struct SolData { pub license: Option>, pub version: Option>, @@ -18,6 +19,7 @@ pub struct SolData { pub contract_names: Vec, pub is_yul: bool, pub parse_result: Result<(), String>, + pub interface_repr_hash: Option, } impl SolData { @@ -49,20 +51,10 @@ impl SolData { let mut libraries = Vec::new(); let mut contract_names = Vec::new(); let mut parse_result = Ok(()); + let mut interface_repr_hash = None; - let sess = solar_parse::interface::Session::builder() - .with_buffer_emitter(Default::default()) - .build(); - sess.enter(|| { - let arena = ast::Arena::new(); - let filename = solar_parse::interface::source_map::FileName::Real(file.to_path_buf()); - let Ok(mut parser) = - solar_parse::Parser::from_source_code(&sess, &arena, filename, content.to_string()) - else { - return; - }; - let Ok(ast) = parser.parse_file().map_err(|e| e.emit()) else { return }; - for item in ast.items { + let result = crate::preprocessor::parse_one_source(content, file, |ast| { + for item in ast.items.iter() { let loc = item.span.lo().to_usize()..item.span.hi().to_usize(); match &item.kind { ast::ItemKind::Pragma(pragma) => match &pragma.tokens { @@ -111,9 +103,13 @@ impl SolData { _ => {} } + + interface_repr_hash = Some(foundry_compilers_artifacts::Source::content_hash_of( + &crate::preprocessor::interface_representation_ast(content, &ast), + )); } }); - if let Err(e) = sess.emitted_errors().unwrap() { + if let Err(e) = result { let e = e.to_string(); trace!("failed parsing {file:?}: {e}"); parse_result = Err(e); @@ -157,6 +153,7 @@ impl SolData { contract_names, is_yul, parse_result, + interface_repr_hash, } } From 4451c71fbf2fc552002c4a4b90ecc5e352fb431b Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:17:48 +0200 Subject: [PATCH 58/70] perf: cache interface_hash in ArtifactsCacheInner --- crates/compilers/src/cache.rs | 43 ++++++++++++++-------- crates/compilers/src/compilers/mod.rs | 5 --- crates/compilers/src/compilers/multi.rs | 7 ---- crates/compilers/src/compilers/solc/mod.rs | 4 -- crates/compilers/src/preprocessor/mod.rs | 16 +++++--- crates/compilers/src/resolver/parse.rs | 7 ---- 6 files changed, 38 insertions(+), 44 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index bbfb82b8..6eb72388 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,9 +4,10 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, + preprocessor::interface_repr_hash, resolver::GraphEdges, - ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, ParsedSource, - Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, + ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, + ProjectPaths, ProjectPathsConfig, SourceCompilationKind, }; use foundry_compilers_artifacts::{ sources::{Source, Sources}, @@ -681,11 +682,8 @@ impl, C: Compiler> .map(|import| strip_prefix(import, self.project.root()).into()) .collect(); - let interface_repr_hash = if self.cache.preprocessed && self.is_source_file(&file) { - self.edges.get_parsed_source(&file).and_then(ParsedSource::interface_repr_hash) - } else { - None - }; + let interface_repr_hash = (self.cache.preprocessed && self.is_source_file(&file)) + .then(|| self.interface_repr_hash(source, &file).to_string()); let entry = CacheEntry { last_modification_date: CacheEntry::read_last_modification_date(&file) @@ -703,6 +701,27 @@ impl, C: Compiler> self.cache.files.insert(file, entry); } + /// Gets or calculates the content hash for the given source file. + fn content_hash(&mut self, source: &Source, file: &Path) -> &str { + self.content_hashes.entry(file.to_path_buf()).or_insert_with(|| source.content_hash()) + } + + /// Gets or calculates the interface representation hash for the given source file. + fn interface_repr_hash(&mut self, source: &Source, file: &Path) -> &str { + self.interface_repr_hashes + .entry(file.to_path_buf()) + .or_insert_with(|| { + if let Some(r) = interface_repr_hash(&source.content, file) { + return r; + } + self.content_hashes + .entry(file.to_path_buf()) + .or_insert_with(|| source.content_hash()) + .clone() + }) + .as_str() + } + /// Returns the set of [Source]s that need to be compiled to produce artifacts for requested /// input. /// @@ -956,17 +975,11 @@ impl, C: Compiler> /// Adds the file's hashes to the set if not set yet fn fill_hashes(&mut self, sources: &Sources) { for (file, source) in sources { - let content_hash = - self.content_hashes.entry(file.clone()).or_insert_with(|| source.content_hash()); + let _ = self.content_hash(source, file); // Fill interface representation hashes for source files if self.cache.preprocessed && self.project.paths.is_source_file(file) { - self.interface_repr_hashes.entry(file.clone()).or_insert_with(|| { - self.edges - .get_parsed_source(file) - .and_then(ParsedSource::interface_repr_hash) - .unwrap_or_else(|| content_hash.clone()) - }); + let _ = self.interface_repr_hash(source, file); } } } diff --git a/crates/compilers/src/compilers/mod.rs b/crates/compilers/src/compilers/mod.rs index 2e923c6d..5abb74b7 100644 --- a/crates/compilers/src/compilers/mod.rs +++ b/crates/compilers/src/compilers/mod.rs @@ -186,11 +186,6 @@ pub trait ParsedSource: Debug + Sized + Send + Clone { { vec![].into_iter() } - - /// Returns the hash of the interface of the source. - fn interface_repr_hash(&self) -> Option { - None - } } /// Error returned by compiler. Might also represent a warning or informational message. diff --git a/crates/compilers/src/compilers/multi.rs b/crates/compilers/src/compilers/multi.rs index ec2cc94e..d206076b 100644 --- a/crates/compilers/src/compilers/multi.rs +++ b/crates/compilers/src/compilers/multi.rs @@ -387,13 +387,6 @@ impl ParsedSource for MultiCompilerParsedSource { } .into_iter() } - - fn interface_repr_hash(&self) -> Option { - match self { - Self::Solc(parsed) => parsed.interface_repr_hash(), - Self::Vyper(parsed) => parsed.interface_repr_hash(), - } - } } impl CompilationError for MultiCompilerError { diff --git a/crates/compilers/src/compilers/solc/mod.rs b/crates/compilers/src/compilers/solc/mod.rs index 0a612ed4..3c1d6526 100644 --- a/crates/compilers/src/compilers/solc/mod.rs +++ b/crates/compilers/src/compilers/solc/mod.rs @@ -397,10 +397,6 @@ impl ParsedSource for SolData { { imported_nodes.filter_map(|(path, node)| (!node.libraries.is_empty()).then_some(path)) } - - fn interface_repr_hash(&self) -> Option { - self.interface_repr_hash.clone() - } } impl CompilationError for Error { diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index 75979912..b5da5237 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -134,6 +134,15 @@ impl Preprocessor for TestOptimizerPreprocessor { } } +pub(crate) fn interface_repr_hash(content: &str, path: &Path) -> Option { + let src = interface_repr(content, path).ok()?; + Some(foundry_compilers_artifacts::Source::content_hash_of(&src)) +} + +pub(crate) fn interface_repr(content: &str, path: &Path) -> Result { + parse_one_source(content, path, |ast| interface_representation_ast(content, &ast)) +} + pub(crate) fn parse_one_source( content: &str, path: &Path, @@ -210,11 +219,6 @@ pub(crate) fn interface_representation_ast( mod tests { use super::*; - fn interface_representation(content: &str) -> String { - parse_one_source(content, Path::new(""), |ast| interface_representation_ast(content, &ast)) - .unwrap() - } - #[test] fn test_interface_representation() { let content = r#" @@ -235,7 +239,7 @@ contract A { } }"#; - let result = interface_representation(content); + let result = interface_repr(content, Path::new("")).unwrap(); assert_eq!( result, r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# diff --git a/crates/compilers/src/resolver/parse.rs b/crates/compilers/src/resolver/parse.rs index 9153f4a6..5493205f 100644 --- a/crates/compilers/src/resolver/parse.rs +++ b/crates/compilers/src/resolver/parse.rs @@ -19,7 +19,6 @@ pub struct SolData { pub contract_names: Vec, pub is_yul: bool, pub parse_result: Result<(), String>, - pub interface_repr_hash: Option, } impl SolData { @@ -51,7 +50,6 @@ impl SolData { let mut libraries = Vec::new(); let mut contract_names = Vec::new(); let mut parse_result = Ok(()); - let mut interface_repr_hash = None; let result = crate::preprocessor::parse_one_source(content, file, |ast| { for item in ast.items.iter() { @@ -103,10 +101,6 @@ impl SolData { _ => {} } - - interface_repr_hash = Some(foundry_compilers_artifacts::Source::content_hash_of( - &crate::preprocessor::interface_representation_ast(content, &ast), - )); } }); if let Err(e) = result { @@ -153,7 +147,6 @@ impl SolData { contract_names, is_yul, parse_result, - interface_repr_hash, } } From 73ae218471a8cd71db51964be477c12d21f9cce3 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:46:48 +0200 Subject: [PATCH 59/70] chore: change &PathBuf to &Path --- crates/compilers/src/cache.rs | 30 ++++++++++++-------------- crates/compilers/src/filter.rs | 2 +- crates/compilers/src/lib.rs | 4 ++-- crates/compilers/src/resolver/mod.rs | 18 ++++++++-------- crates/compilers/src/resolver/parse.rs | 4 ++-- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 6eb72388..a4b52996 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -708,18 +708,16 @@ impl, C: Compiler> /// Gets or calculates the interface representation hash for the given source file. fn interface_repr_hash(&mut self, source: &Source, file: &Path) -> &str { - self.interface_repr_hashes - .entry(file.to_path_buf()) - .or_insert_with(|| { - if let Some(r) = interface_repr_hash(&source.content, file) { - return r; - } - self.content_hashes - .entry(file.to_path_buf()) - .or_insert_with(|| source.content_hash()) - .clone() - }) - .as_str() + self.interface_repr_hashes.entry(file.to_path_buf()).or_insert_with(|| { + if let Some(r) = interface_repr_hash(&source.content, file) { + return r; + } + // self.content_hash(source, file).into() + self.content_hashes + .entry(file.to_path_buf()) + .or_insert_with(|| source.content_hash()) + .clone() + }) } /// Returns the set of [Source]s that need to be compiled to produce artifacts for requested @@ -742,7 +740,7 @@ impl, C: Compiler> // If we are missing artifact for file, compile it. if self.is_missing_artifacts(file, version, profile) { - compile_complete.insert(file.clone()); + compile_complete.insert(file.to_path_buf()); } // Ensure that we have a cache entry for all sources. @@ -756,15 +754,15 @@ impl, C: Compiler> for source in &compile_complete { for import in self.edges.imports(source) { if !compile_complete.contains(import) { - compile_optimized.insert(import.clone()); + compile_optimized.insert(import); } } } sources.retain(|file, source| { - source.kind = if compile_complete.contains(file) { + source.kind = if compile_complete.contains(file.as_path()) { SourceCompilationKind::Complete - } else if compile_optimized.contains(file) { + } else if compile_optimized.contains(file.as_path()) { SourceCompilationKind::Optimized } else { return false; diff --git a/crates/compilers/src/filter.rs b/crates/compilers/src/filter.rs index 119d9674..979b1e5b 100644 --- a/crates/compilers/src/filter.rs +++ b/crates/compilers/src/filter.rs @@ -121,7 +121,7 @@ impl<'a> SparseOutputFilter<'a> { let mut required_sources = vec![file.clone()]; if let Some(data) = graph.get_parsed_source(file) { let imports = graph.imports(file).into_iter().filter_map(|import| { - graph.get_parsed_source(import).map(|data| (import.as_path(), data)) + graph.get_parsed_source(import).map(|data| (import, data)) }); for import in data.compilation_dependencies(imports) { let import = import.to_path_buf(); diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 4233de8c..0bcefa15 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -180,13 +180,13 @@ where let mut sources = Vec::new(); let mut unique_paths = HashSet::new(); let (path, source) = graph.node(*target_index).unpack(); - unique_paths.insert(path.clone()); + unique_paths.insert(path); sources.push((path, source)); sources.extend( graph .all_imported_nodes(*target_index) .map(|index| graph.node(index).unpack()) - .filter(|(p, _)| unique_paths.insert(p.to_path_buf())), + .filter(|(p, _)| unique_paths.insert(*p)), ); let root = self.root(); diff --git a/crates/compilers/src/resolver/mod.rs b/crates/compilers/src/resolver/mod.rs index ddd36dfe..4b0c722c 100644 --- a/crates/compilers/src/resolver/mod.rs +++ b/crates/compilers/src/resolver/mod.rs @@ -169,18 +169,18 @@ impl GraphEdges { } /// Returns all files imported by the given file - pub fn imports(&self, file: &Path) -> HashSet<&PathBuf> { + pub fn imports(&self, file: &Path) -> HashSet<&Path> { if let Some(start) = self.indices.get(file).copied() { - NodesIter::new(start, self).skip(1).map(move |idx| &self.rev_indices[&idx]).collect() + NodesIter::new(start, self).skip(1).map(move |idx| &*self.rev_indices[&idx]).collect() } else { HashSet::new() } } /// Returns all files that import the given file - pub fn importers(&self, file: &Path) -> HashSet<&PathBuf> { + pub fn importers(&self, file: &Path) -> HashSet<&Path> { if let Some(start) = self.indices.get(file).copied() { - self.rev_edges[start].iter().map(move |idx| &self.rev_indices[idx]).collect() + self.rev_edges[start].iter().map(move |idx| &*self.rev_indices[idx]).collect() } else { HashSet::new() } @@ -192,7 +192,7 @@ impl GraphEdges { } /// Returns the path of the given node - pub fn node_path(&self, id: usize) -> &PathBuf { + pub fn node_path(&self, id: usize) -> &Path { &self.rev_indices[&id] } @@ -327,7 +327,7 @@ impl> Graph { } /// Returns all files imported by the given file - pub fn imports(&self, path: &Path) -> HashSet<&PathBuf> { + pub fn imports(&self, path: &Path) -> HashSet<&Path> { self.edges.imports(path) } @@ -1121,7 +1121,7 @@ impl Node { &self.source.content } - pub fn unpack(&self) -> (&PathBuf, &Source) { + pub fn unpack(&self) -> (&Path, &Source) { (&self.path, &self.source) } } @@ -1199,8 +1199,8 @@ mod tests { let dapp_test = graph.node(1); assert_eq!(dapp_test.path, paths.sources.join("Dapp.t.sol")); assert_eq!( - dapp_test.data.imports.iter().map(|i| i.data().path()).collect::>(), - vec![&PathBuf::from("ds-test/test.sol"), &PathBuf::from("./Dapp.sol")] + dapp_test.data.imports.iter().map(|i| i.data().path()).collect::>(), + vec![Path::new("ds-test/test.sol"), Path::new("./Dapp.sol")] ); assert_eq!(graph.imported_nodes(1).to_vec(), vec![2, 0]); } diff --git a/crates/compilers/src/resolver/parse.rs b/crates/compilers/src/resolver/parse.rs index 5493205f..9de8ab6f 100644 --- a/crates/compilers/src/resolver/parse.rs +++ b/crates/compilers/src/resolver/parse.rs @@ -187,11 +187,11 @@ impl SolImport { Self { path, aliases: vec![] } } - pub fn path(&self) -> &PathBuf { + pub fn path(&self) -> &Path { &self.path } - pub fn aliases(&self) -> &Vec { + pub fn aliases(&self) -> &[SolImportAlias] { &self.aliases } From a9341af983376ce64beb9ec94e5d4fb3e9b8001b Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:00:36 +0200 Subject: [PATCH 60/70] com --- crates/compilers/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index a4b52996..a0cd3ab1 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -712,7 +712,7 @@ impl, C: Compiler> if let Some(r) = interface_repr_hash(&source.content, file) { return r; } - // self.content_hash(source, file).into() + // Equivalent to: self.content_hash(source, file).into() self.content_hashes .entry(file.to_path_buf()) .or_insert_with(|| source.content_hash()) From a48b275a404767708ebca58ddb5f4820974feb68 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 2 Apr 2025 16:41:01 +0200 Subject: [PATCH 61/70] fix: correctly create Session and ParsingContext from solc input --- crates/compilers/src/compilers/solc/mod.rs | 2 +- crates/compilers/src/preprocessor/mod.rs | 68 ++++++++++++++++------ crates/compilers/src/resolver/parse.rs | 4 +- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/crates/compilers/src/compilers/solc/mod.rs b/crates/compilers/src/compilers/solc/mod.rs index 3c1d6526..e4381aa8 100644 --- a/crates/compilers/src/compilers/solc/mod.rs +++ b/crates/compilers/src/compilers/solc/mod.rs @@ -113,7 +113,7 @@ pub struct SolcVersionedInput { #[serde(flatten)] pub input: SolcInput, #[serde(flatten)] - cli_settings: CliSettings, + pub cli_settings: CliSettings, } impl CompilerInput for SolcVersionedInput { diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index b5da5237..e9dbaed0 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -44,36 +44,27 @@ impl Preprocessor for TestOptimizerPreprocessor { paths: &ProjectPathsConfig, mocks: &mut HashSet, ) -> Result<()> { - let sources = &mut input.input.sources; // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. - if sources.iter().all(|(path, _)| !paths.is_test_or_script(path)) { + if !input.input.sources.iter().any(|(path, _)| paths.is_test_or_script(path)) { trace!("no tests or sources to preprocess"); return Ok(()); } - let sess = Session::builder().with_buffer_emitter(Default::default()).build(); + let sess = solar_session_from_solc(input); let _ = sess.enter_parallel(|| -> solar_parse::interface::Result { // Set up the parsing context with the project paths. - let mut parsing_context = ParsingContext::new(&sess); - parsing_context.file_resolver.set_current_dir(&paths.root); - for remapping in &paths.remappings { - parsing_context.file_resolver.add_import_remapping( - solar_sema::interface::config::ImportRemapping { - context: remapping.context.clone().unwrap_or_default(), - prefix: remapping.name.clone(), - path: remapping.path.clone(), - }, - ); - } - parsing_context.file_resolver.add_include_paths(paths.include_paths.iter().cloned()); + let mut parsing_context = solar_pcx_from_solc_no_sources(&sess, input, paths); // Add the sources into the context. + // Include all sources in the source map so as to not re-load them from disk, but only + // parse and preprocess tests and scripts. let mut preprocessed_paths = vec![]; + let sources = &mut input.input.sources; for (path, source) in sources.iter() { - if paths.is_test_or_script(path) { - if let Ok(src_file) = - sess.source_map().new_source_file(path.clone(), source.content.as_str()) - { + if let Ok(src_file) = + sess.source_map().new_source_file(path.clone(), source.content.as_str()) + { + if paths.is_test_or_script(path) { parsing_context.add_file(src_file); preprocessed_paths.push(path.clone()); } @@ -134,6 +125,45 @@ impl Preprocessor for TestOptimizerPreprocessor { } } +fn solar_session_from_solc(solc: &SolcVersionedInput) -> Session { + use solar_parse::interface::config; + + Session::builder() + .with_buffer_emitter(Default::default()) + .opts(config::Opts { + language: match solc.input.language { + SolcLanguage::Solidity => config::Language::Solidity, + SolcLanguage::Yul => config::Language::Yul, + _ => unimplemented!(), + }, + + // TODO: ... + /* + evm_version: solc.input.settings.evm_version, + */ + ..Default::default() + }) + .build() +} + +fn solar_pcx_from_solc_no_sources<'sess>( + sess: &'sess Session, + solc: &SolcVersionedInput, + paths: &ProjectPathsConfig, +) -> ParsingContext<'sess> { + let mut pcx = ParsingContext::new(sess); + pcx.file_resolver.set_current_dir(solc.cli_settings.base_path.as_ref().unwrap_or(&paths.root)); + for remapping in &paths.remappings { + pcx.file_resolver.add_import_remapping(solar_sema::interface::config::ImportRemapping { + context: remapping.context.clone().unwrap_or_default(), + prefix: remapping.name.clone(), + path: remapping.path.clone(), + }); + } + pcx.file_resolver.add_include_paths(solc.cli_settings.include_paths.iter().cloned()); + pcx +} + pub(crate) fn interface_repr_hash(content: &str, path: &Path) -> Option { let src = interface_repr(content, path).ok()?; Some(foundry_compilers_artifacts::Source::content_hash_of(&src)) diff --git a/crates/compilers/src/resolver/parse.rs b/crates/compilers/src/resolver/parse.rs index 9de8ab6f..bd3398b9 100644 --- a/crates/compilers/src/resolver/parse.rs +++ b/crates/compilers/src/resolver/parse.rs @@ -72,9 +72,9 @@ impl SolData { ast::ItemKind::Import(import) => { let path = import.path.value.to_string(); let aliases = match &import.items { - ast::ImportItems::Plain(None) | ast::ImportItems::Glob(None) => &[][..], + ast::ImportItems::Plain(None) => &[][..], ast::ImportItems::Plain(Some(alias)) - | ast::ImportItems::Glob(Some(alias)) => &[(*alias, None)][..], + | ast::ImportItems::Glob(alias) => &[(*alias, None)][..], ast::ImportItems::Aliases(aliases) => aliases, }; let sol_import = SolImport::new(PathBuf::from(path)).set_aliases( From 49b65bfff24e24ffb746ae3160198d2069ed6eea Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:36:46 +0200 Subject: [PATCH 62/70] fix: better check for 'is_source_file' --- crates/compilers/src/cache.rs | 2 +- crates/compilers/src/config.rs | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index a0cd3ab1..38508877 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -670,7 +670,7 @@ impl, C: Compiler> { /// Whether given file is a source file or a test/script file. fn is_source_file(&self, file: &Path) -> bool { - !self.project.paths.is_test_or_script(file) + self.project.paths.is_source_file(file) } /// Creates a new cache entry for the file diff --git a/crates/compilers/src/config.rs b/crates/compilers/src/config.rs index 901108a7..bcf1c255 100644 --- a/crates/compilers/src/config.rs +++ b/crates/compilers/src/config.rs @@ -251,9 +251,8 @@ impl ProjectPathsConfig { } pub(crate) fn is_test_or_script(&self, path: &Path) -> bool { - let test_dir = self.tests.strip_prefix(&self.root).unwrap_or(&self.tests); - let script_dir = self.scripts.strip_prefix(&self.root).unwrap_or(&self.scripts); - path.starts_with(test_dir) || path.starts_with(script_dir) + path_starts_with_rooted(path, &self.tests, &self.root) + || path_starts_with_rooted(path, &self.scripts, &self.root) } pub(crate) fn is_source_file(&self, path: &Path) -> bool { @@ -984,6 +983,17 @@ impl SolcConfigBuilder { } } +/// Return true if `a` starts with `b` or `b - root`. +fn path_starts_with_rooted(a: &Path, b: &Path, root: &Path) -> bool { + if a.starts_with(b) { + return true; + } + if let Ok(b) = b.strip_prefix(root) { + return a.starts_with(b); + } + false +} + #[cfg(test)] mod tests { use super::*; From de0bebce1b8ea109f668bbb4f7bf8fbecd549df0 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 12:02:51 +0200 Subject: [PATCH 63/70] chore: rm &PathBuf --- Cargo.toml | 5 +++-- crates/compilers/src/flatten.rs | 12 ++++++------ crates/compilers/src/lib.rs | 16 ++++++++-------- crates/compilers/src/preprocessor/deps.rs | 6 +++--- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fcc437ea..5510f78d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,8 +22,9 @@ manual-string-new = "warn" uninlined-format-args = "warn" use-self = "warn" redundant-clone = "warn" -# until is fixed -literal-string-with-formatting-args = "allow" + +result-large-err = "allow" +large-enum-variant = "allow" [workspace.lints.rust] rust-2018-idioms = "warn" diff --git a/crates/compilers/src/flatten.rs b/crates/compilers/src/flatten.rs index 636e5006..f5440000 100644 --- a/crates/compilers/src/flatten.rs +++ b/crates/compilers/src/flatten.rs @@ -212,7 +212,7 @@ impl Flattener { let sources = Source::read_all_files(vec![target.to_path_buf()])?; let graph = Graph::::resolve_sources(&project.paths, sources)?; - let ordered_sources = collect_ordered_deps(&target.to_path_buf(), &project.paths, &graph)?; + let ordered_sources = collect_ordered_deps(target, &project.paths, &graph)?; #[cfg(windows)] let ordered_sources = { @@ -244,7 +244,7 @@ impl Flattener { sources, asts, ordered_sources, - project_root: project.root().clone(), + project_root: project.root().to_path_buf(), }) } @@ -795,12 +795,12 @@ impl Flattener { /// Performs DFS to collect all dependencies of a target fn collect_deps( - path: &PathBuf, + path: &Path, paths: &ProjectPathsConfig, graph: &Graph, deps: &mut HashSet, ) -> Result<()> { - if deps.insert(path.clone()) { + if deps.insert(path.to_path_buf()) { let target_dir = path.parent().ok_or_else(|| { SolcError::msg(format!("failed to get parent directory for \"{}\"", path.display())) })?; @@ -831,7 +831,7 @@ fn collect_deps( /// order. If files have the same number of dependencies, we sort them alphabetically. /// Target file is always placed last. pub fn collect_ordered_deps( - path: &PathBuf, + path: &Path, paths: &ProjectPathsConfig, graph: &Graph, ) -> Result> { @@ -871,7 +871,7 @@ pub fn collect_ordered_deps( let mut ordered_deps = paths_with_deps_count.into_iter().map(|(_, path)| path).collect::>(); - ordered_deps.push(path.clone()); + ordered_deps.push(path.to_path_buf()); Ok(ordered_deps) } diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 0bcefa15..2536309a 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -213,27 +213,27 @@ where impl, C: Compiler> Project { /// Returns the path to the artifacts directory - pub fn artifacts_path(&self) -> &PathBuf { + pub fn artifacts_path(&self) -> &Path { &self.paths.artifacts } /// Returns the path to the sources directory - pub fn sources_path(&self) -> &PathBuf { + pub fn sources_path(&self) -> &Path { &self.paths.sources } /// Returns the path to the cache file - pub fn cache_path(&self) -> &PathBuf { + pub fn cache_path(&self) -> &Path { &self.paths.cache } /// Returns the path to the `build-info` directory nested in the artifacts dir - pub fn build_info_path(&self) -> &PathBuf { + pub fn build_info_path(&self) -> &Path { &self.paths.build_infos } /// Returns the root directory of the project - pub fn root(&self) -> &PathBuf { + pub fn root(&self) -> &Path { &self.paths.root } @@ -349,7 +349,7 @@ impl, C: Compiler> Pro std::fs::remove_file(self.cache_path()) .map_err(|err| SolcIoError::new(err, self.cache_path()))?; if let Some(cache_folder) = - self.cache_path().parent().filter(|cache_folder| self.root() != cache_folder) + self.cache_path().parent().filter(|cache_folder| self.root() != *cache_folder) { // remove the cache folder if the cache file was the only file if cache_folder @@ -368,14 +368,14 @@ impl, C: Compiler> Pro // clean the artifacts dir if self.artifacts_path().exists() && self.root() != self.artifacts_path() { std::fs::remove_dir_all(self.artifacts_path()) - .map_err(|err| SolcIoError::new(err, self.artifacts_path().clone()))?; + .map_err(|err| SolcIoError::new(err, self.artifacts_path()))?; trace!("removed artifacts dir \"{}\"", self.artifacts_path().display()); } // also clean the build-info dir, in case it's not nested in the artifacts dir if self.build_info_path().exists() && self.root() != self.build_info_path() { std::fs::remove_dir_all(self.build_info_path()) - .map_err(|err| SolcIoError::new(err, self.build_info_path().clone()))?; + .map_err(|err| SolcIoError::new(err, self.build_info_path()))?; tracing::trace!("removed build-info dir \"{}\"", self.build_info_path().display()); } diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs index 8a0554e5..58b1a9c8 100644 --- a/crates/compilers/src/preprocessor/deps.rs +++ b/crates/compilers/src/preprocessor/deps.rs @@ -28,7 +28,7 @@ impl PreprocessorDependencies { sess: &Session, hir: &Hir<'_>, paths: &[PathBuf], - src_dir: &PathBuf, + src_dir: &Path, root_dir: &Path, mocks: &mut HashSet, ) -> Self { @@ -128,7 +128,7 @@ struct BytecodeDependencyCollector<'hir> { /// Source content of current contract. src: &'hir str, /// Project source dir, used to determine if referenced contract is a source contract. - src_dir: &'hir PathBuf, + src_dir: &'hir Path, /// Dependencies collected for current contract. dependencies: Vec, /// Unique HIR ids of contracts referenced from current contract. @@ -140,7 +140,7 @@ impl<'hir> BytecodeDependencyCollector<'hir> { source_map: &'hir SourceMap, hir: &'hir Hir<'hir>, src: &'hir str, - src_dir: &'hir PathBuf, + src_dir: &'hir Path, ) -> Self { Self { source_map, From 14cabe33fb92617433a0a0af20fcd0b71618f5d1 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 12:03:01 +0200 Subject: [PATCH 64/70] test: dapptools instead of hardhat paths --- crates/compilers/tests/project.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index 75f902ce..97b6b820 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -4164,7 +4164,7 @@ fn can_preprocess_constructors_and_creation_code() { canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) .unwrap(); - let project = TempProject::hardhat().unwrap(); + let project = TempProject::::dapptools().unwrap(); project.copy_project_from(&root).unwrap(); let r = ProjectCompiler::new(project.project()) .unwrap() From 93b3c3f5b79e8fca45be3e786a2a9e8a25d6fb59 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Thu, 3 Apr 2025 13:54:04 +0300 Subject: [PATCH 65/70] Continue compiling if preprocessor fails parsing --- crates/compilers/src/preprocessor/mod.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs index e9dbaed0..3d9367a0 100644 --- a/crates/compilers/src/preprocessor/mod.rs +++ b/crates/compilers/src/preprocessor/mod.rs @@ -11,7 +11,7 @@ use crate::{ Compiler, ProjectPathsConfig, Result, }; use foundry_compilers_artifacts::SolcLanguage; -use foundry_compilers_core::{error::SolcError, utils}; +use foundry_compilers_core::utils; use solar_parse::{ ast::{FunctionKind, ItemKind, Span, Visibility}, interface::{diagnostics::EmittedDiagnostics, source_map::FileName, Session, SourceMap}, @@ -97,10 +97,9 @@ impl Preprocessor for TestOptimizerPreprocessor { Ok(()) }); - // Return if any diagnostics emitted during content parsing. + // Warn if any diagnostics emitted during content parsing. if let Err(err) = sess.emitted_errors().unwrap() { - trace!("failed preprocessing {err}"); - return Err(SolcError::Message(err.to_string())); + warn!("failed preprocessing {err}"); } Ok(()) From 53775896584b58e58dcbd14af2128675b81a933b Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:13:21 +0200 Subject: [PATCH 66/70] chore: move preprocessor to foundry --- crates/compilers/src/cache.rs | 4 +- crates/compilers/src/compile/project.rs | 5 +- crates/compilers/src/config.rs | 37 ++- crates/compilers/src/lib.rs | 47 ++- crates/compilers/src/preprocessor/data.rs | 200 ------------ crates/compilers/src/preprocessor/deps.rs | 370 ---------------------- crates/compilers/src/preprocessor/mod.rs | 277 ---------------- crates/compilers/src/resolver/parse.rs | 2 +- crates/compilers/tests/project.rs | 31 +- 9 files changed, 102 insertions(+), 871 deletions(-) delete mode 100644 crates/compilers/src/preprocessor/data.rs delete mode 100644 crates/compilers/src/preprocessor/deps.rs delete mode 100644 crates/compilers/src/preprocessor/mod.rs diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 38508877..e22c5386 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -4,7 +4,6 @@ use crate::{ buildinfo::RawBuildInfo, compilers::{Compiler, CompilerSettings, Language}, output::Builds, - preprocessor::interface_repr_hash, resolver::GraphEdges, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Graph, OutputContext, Project, ProjectPaths, ProjectPathsConfig, SourceCompilationKind, @@ -26,6 +25,9 @@ use std::{ time::{Duration, UNIX_EPOCH}, }; +mod iface; +use iface::interface_repr_hash; + /// ethers-rs format version /// /// `ethers-solc` uses a different format version id, but the actual format is consistent with diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index f5cd26e0..45106d83 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -126,8 +126,9 @@ use std::{ pub(crate) type VersionedSources<'a, L, S> = HashMap>; /// Invoked before the actual compiler invocation and can override the input. -/// Updates the list of identified cached mocks (if any) to be stored in cache and returns -/// preprocessed compiler input. +/// +/// Updates the list of identified cached mocks (if any) to be stored in cache and updates the +/// compiler input. pub trait Preprocessor: Debug { fn preprocess( &self, diff --git a/crates/compilers/src/config.rs b/crates/compilers/src/config.rs index bcf1c255..becd845c 100644 --- a/crates/compilers/src/config.rs +++ b/crates/compilers/src/config.rs @@ -250,12 +250,23 @@ impl ProjectPathsConfig { Self::dapptools(&std::env::current_dir().map_err(|err| SolcError::io(err, "."))?) } - pub(crate) fn is_test_or_script(&self, path: &Path) -> bool { + /// Returns true if the given path is a test or script file. + pub fn is_test_or_script(&self, path: &Path) -> bool { + self.is_test(path) || self.is_script(path) + } + + /// Returns true if the given path is a test file. + pub fn is_test(&self, path: &Path) -> bool { path_starts_with_rooted(path, &self.tests, &self.root) - || path_starts_with_rooted(path, &self.scripts, &self.root) } - pub(crate) fn is_source_file(&self, path: &Path) -> bool { + /// Returns true if the given path is a script file. + pub fn is_script(&self, path: &Path) -> bool { + path_starts_with_rooted(path, &self.scripts, &self.root) + } + + /// Returns true if the given path is a test or script file. + pub fn is_source_file(&self, path: &Path) -> bool { !self.is_test_or_script(path) } @@ -692,6 +703,26 @@ impl ProjectPaths { .collect(); self } + + /// Returns true if the given path is a test or script file. + pub fn is_test_or_script(&self, path: &Path) -> bool { + self.is_test(path) || self.is_script(path) + } + + /// Returns true if the given path is a test file. + pub fn is_test(&self, path: &Path) -> bool { + path.starts_with(&self.tests) + } + + /// Returns true if the given path is a script file. + pub fn is_script(&self, path: &Path) -> bool { + path.starts_with(&self.scripts) + } + + /// Returns true if the given path is a test or script file. + pub fn is_source_file(&self, path: &Path) -> bool { + !self.is_test_or_script(path) + } } impl Default for ProjectPaths { diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 2536309a..16ac2c8b 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -24,8 +24,6 @@ pub use resolver::Graph; pub mod compilers; pub use compilers::*; -pub mod preprocessor; - mod compile; pub use compile::{ output::{AggregatedCompilerOutput, ProjectCompileOutput}, @@ -41,8 +39,9 @@ pub use filter::{FileFilter, SparseOutputFilter, TestFileFilter}; pub mod report; /// Updates to be applied to the sources. -/// source_path -> (start, end, new_value) -pub(crate) type Updates = HashMap>; +/// +/// `source_path -> (start, end, new_value)` +pub type Updates = HashMap>; /// Utilities for creating, mocking and testing of (temporary) projects #[cfg(feature = "project-util")] @@ -65,6 +64,8 @@ use foundry_compilers_core::error::{Result, SolcError, SolcIoError}; use output::sources::{VersionedSourceFile, VersionedSourceFiles}; use project::ProjectCompiler; use semver::Version; +use solar_parse::Parser; +use solar_sema::interface::{diagnostics::EmittedDiagnostics, source_map::FileName, Session}; use solc::SolcSettings; use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, @@ -891,7 +892,7 @@ fn rebase_path(base: &Path, path: &Path) -> PathBuf { } /// Utility function to apply a set of updates to provided sources. -fn apply_updates(sources: &mut Sources, updates: Updates) { +pub fn apply_updates(sources: &mut Sources, updates: Updates) { for (path, source) in sources { if let Some(updates) = updates.get(path) { source.content = Arc::new(replace_source_content( @@ -904,20 +905,42 @@ fn apply_updates(sources: &mut Sources, updates: Updates) { /// Utility function to change source content ranges with provided updates. /// Assumes that the updates are sorted. -fn replace_source_content<'a>( - source: &str, - updates: impl IntoIterator, &'a str)>, +pub fn replace_source_content<'a>( + source: impl Into, + updates: impl IntoIterator, impl AsRef)>, ) -> String { let mut offset = 0; - let mut content = source.as_bytes().to_vec(); + let mut content = source.into(); for (range, new_value) in updates { let update_range = utils::range_by_offset(&range, offset); - - content.splice(update_range.start..update_range.end, new_value.bytes()); + let new_value = new_value.as_ref(); + content.replace_range(update_range.clone(), new_value); offset += new_value.len() as isize - (update_range.end - update_range.start) as isize; } + content +} - String::from_utf8(content).unwrap() +pub(crate) fn parse_one_source( + content: &str, + path: &Path, + f: impl FnOnce(solar_sema::ast::SourceUnit<'_>) -> R, +) -> Result { + let sess = Session::builder().with_buffer_emitter(Default::default()).build(); + let res = sess.enter(|| -> solar_parse::interface::Result<_> { + let arena = solar_parse::ast::Arena::new(); + let filename = FileName::Real(path.to_path_buf()); + let mut parser = Parser::from_source_code(&sess, &arena, filename, content.to_string())?; + let ast = parser.parse_file().map_err(|e| e.emit())?; + Ok(f(ast)) + }); + + // Return if any diagnostics emitted during content parsing. + if let Err(err) = sess.emitted_errors().unwrap() { + trace!("failed parsing {path:?}:\n{err}"); + return Err(err); + } + + Ok(res.unwrap()) } #[cfg(test)] diff --git a/crates/compilers/src/preprocessor/data.rs b/crates/compilers/src/preprocessor/data.rs deleted file mode 100644 index 5b51dd0a..00000000 --- a/crates/compilers/src/preprocessor/data.rs +++ /dev/null @@ -1,200 +0,0 @@ -use super::span_to_range; -use foundry_compilers_artifacts::{Source, Sources}; -use path_slash::PathExt; -use solar_parse::interface::{Session, SourceMap}; -use solar_sema::{ - hir::{Contract, ContractId, Hir}, - interface::source_map::FileName, -}; -use std::{ - collections::{BTreeMap, HashSet}, - path::{Path, PathBuf}, -}; - -/// Keeps data about project contracts definitions referenced from tests and scripts. -/// Contract id -> Contract data definition mapping. -pub type PreprocessorData = BTreeMap; - -/// Collects preprocessor data from referenced contracts. -pub(crate) fn collect_preprocessor_data( - sess: &Session, - hir: &Hir<'_>, - referenced_contracts: &HashSet, -) -> PreprocessorData { - let mut data = PreprocessorData::default(); - for contract_id in referenced_contracts { - let contract = hir.contract(*contract_id); - let source = hir.source(contract.source); - - let FileName::Real(path) = &source.file.name else { - continue; - }; - - let contract_data = - ContractData::new(hir, *contract_id, contract, path, source, sess.source_map()); - data.insert(*contract_id, contract_data); - } - data -} - -/// Creates helper libraries for contracts with a non-empty constructor. -/// -/// See [`ContractData::build_helper`] for more details. -pub(crate) fn create_deploy_helpers(data: &BTreeMap) -> Sources { - let mut deploy_helpers = Sources::new(); - for (contract_id, contract) in data { - if let Some(code) = contract.build_helper() { - let path = format!("foundry-pp/DeployHelper{}.sol", contract_id.get()); - deploy_helpers.insert(path.into(), Source::new(code)); - } - } - deploy_helpers -} - -/// Keeps data about a contract constructor. -#[derive(Debug)] -pub struct ContractConstructorData { - /// ABI encoded args. - pub abi_encode_args: String, - /// Constructor struct fields. - pub struct_fields: String, -} - -/// Keeps data about a single contract definition. -#[derive(Debug)] -pub(crate) struct ContractData { - /// HIR Id of the contract. - contract_id: ContractId, - /// Path of the source file. - path: PathBuf, - /// Name of the contract - name: String, - /// Constructor parameters, if any. - pub constructor_data: Option, - /// Artifact string to pass into cheatcodes. - pub artifact: String, -} - -impl ContractData { - fn new( - hir: &Hir<'_>, - contract_id: ContractId, - contract: &Contract<'_>, - path: &Path, - source: &solar_sema::hir::Source<'_>, - source_map: &SourceMap, - ) -> Self { - let artifact = format!("{}:{}", path.to_slash_lossy(), contract.name); - - // Process data for contracts with constructor and parameters. - let constructor_data = contract - .ctor - .map(|ctor_id| hir.function(ctor_id)) - .filter(|ctor| !ctor.parameters.is_empty()) - .map(|ctor| { - let mut abi_encode_args = vec![]; - let mut struct_fields = vec![]; - let mut arg_index = 0; - for param_id in ctor.parameters { - let src = source.file.src.as_str(); - let loc = span_to_range(source_map, hir.variable(*param_id).span); - let mut new_src = src[loc].replace(" memory ", " ").replace(" calldata ", " "); - if let Some(ident) = hir.variable(*param_id).name { - abi_encode_args.push(format!("args.{}", ident.name)); - } else { - // Generate an unique name if constructor arg doesn't have one. - arg_index += 1; - abi_encode_args.push(format!("args.foundry_pp_ctor_arg{arg_index}")); - new_src.push_str(&format!(" foundry_pp_ctor_arg{arg_index}")); - } - struct_fields.push(new_src); - } - - ContractConstructorData { - abi_encode_args: abi_encode_args.join(", "), - struct_fields: struct_fields.join("; "), - } - }); - - Self { - contract_id, - path: path.to_path_buf(), - name: contract.name.to_string(), - constructor_data, - artifact, - } - } - - /// If contract has a non-empty constructor, generates a helper source file for it containing a - /// helper to encode constructor arguments. - /// - /// This is needed because current preprocessing wraps the arguments, leaving them unchanged. - /// This allows us to handle nested new expressions correctly. However, this requires us to have - /// a way to wrap both named and unnamed arguments. i.e you can't do abi.encode({arg: val}). - /// - /// This function produces a helper struct + a helper function to encode the arguments. The - /// struct is defined in scope of an abstract contract inheriting the contract containing the - /// constructor. This is done as a hack to allow us to inherit the same scope of definitions. - /// - /// The resulted helper looks like this: - /// ```solidity - /// import "lib/openzeppelin-contracts/contracts/token/ERC20.sol"; - /// - /// abstract contract DeployHelper335 is ERC20 { - /// struct ConstructorArgs { - /// string name; - /// string symbol; - /// } - /// } - /// - /// function encodeArgs335(DeployHelper335.ConstructorArgs memory args) pure returns (bytes memory) { - /// return abi.encode(args.name, args.symbol); - /// } - /// ``` - /// - /// Example usage: - /// ```solidity - /// new ERC20(name, symbol) - /// ``` - /// becomes - /// ```solidity - /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs(name, symbol))) - /// ``` - /// With named arguments: - /// ```solidity - /// new ERC20({name: name, symbol: symbol}) - /// ``` - /// becomes - /// ```solidity - /// vm.deployCode("artifact path", encodeArgs335(DeployHelper335.ConstructorArgs({name: name, symbol: symbol}))) - /// ``` - pub fn build_helper(&self) -> Option { - let Self { contract_id, path, name, constructor_data, artifact: _ } = self; - - let Some(constructor_details) = constructor_data else { return None }; - let contract_id = contract_id.get(); - let struct_fields = &constructor_details.struct_fields; - let abi_encode_args = &constructor_details.abi_encode_args; - - let helper = format!( - r#" -pragma solidity >=0.4.0; - -import "{path}"; - -abstract contract DeployHelper{contract_id} is {name} {{ - struct ConstructorArgs {{ - {struct_fields}; - }} -}} - -function encodeArgs{contract_id}(DeployHelper{contract_id}.ConstructorArgs memory args) pure returns (bytes memory) {{ - return abi.encode({abi_encode_args}); -}} - "#, - path = path.to_slash_lossy(), - ); - - Some(helper) - } -} diff --git a/crates/compilers/src/preprocessor/deps.rs b/crates/compilers/src/preprocessor/deps.rs deleted file mode 100644 index 58b1a9c8..00000000 --- a/crates/compilers/src/preprocessor/deps.rs +++ /dev/null @@ -1,370 +0,0 @@ -use super::{ - data::{ContractData, PreprocessorData}, - span_to_range, -}; -use crate::Updates; -use itertools::Itertools; -use solar_parse::interface::Session; -use solar_sema::{ - hir::{ContractId, Expr, ExprKind, Hir, NamedArg, TypeKind, Visit}, - interface::{data_structures::Never, source_map::FileName, SourceMap}, -}; -use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, - ops::{ControlFlow, Range}, - path::{Path, PathBuf}, -}; - -/// Holds data about referenced source contracts and bytecode dependencies. -pub(crate) struct PreprocessorDependencies { - // Mapping contract id to preprocess -> contract bytecode dependencies. - pub preprocessed_contracts: BTreeMap>, - // Referenced contract ids. - pub referenced_contracts: HashSet, -} - -impl PreprocessorDependencies { - pub fn new( - sess: &Session, - hir: &Hir<'_>, - paths: &[PathBuf], - src_dir: &Path, - root_dir: &Path, - mocks: &mut HashSet, - ) -> Self { - let mut preprocessed_contracts = BTreeMap::new(); - let mut referenced_contracts = HashSet::new(); - for contract_id in hir.contract_ids() { - let contract = hir.contract(contract_id); - let source = hir.source(contract.source); - - let FileName::Real(path) = &source.file.name else { - continue; - }; - - // Collect dependencies only for tests and scripts. - if !paths.contains(path) { - let path = path.display(); - trace!("{path} is not test or script"); - continue; - } - - // Do not collect dependencies for mock contracts. Walk through base contracts and - // check if they're from src dir. - if contract.linearized_bases.iter().any(|base_contract_id| { - let base_contract = hir.contract(*base_contract_id); - let FileName::Real(path) = &hir.source(base_contract.source).file.name else { - return false; - }; - path.starts_with(src_dir) - }) { - // Record mock contracts to be evicted from preprocessed cache. - mocks.insert(root_dir.join(path)); - let path = path.display(); - trace!("found mock contract {path}"); - continue; - } else { - // Make sure current contract is not in list of mocks (could happen when a contract - // which used to be a mock is refactored to a non-mock implementation). - mocks.remove(&root_dir.join(path)); - } - - let mut deps_collector = BytecodeDependencyCollector::new( - sess.source_map(), - hir, - source.file.src.as_str(), - src_dir, - ); - // Analyze current contract. - let _ = deps_collector.walk_contract(contract); - // Ignore empty test contracts declared in source files with other contracts. - if !deps_collector.dependencies.is_empty() { - preprocessed_contracts.insert(contract_id, deps_collector.dependencies); - } - // Record collected referenced contract ids. - referenced_contracts.extend(deps_collector.referenced_contracts); - } - Self { preprocessed_contracts, referenced_contracts } - } -} - -/// Represents a bytecode dependency kind. -#[derive(Debug)] -enum BytecodeDependencyKind { - /// `type(Contract).creationCode` - CreationCode, - /// `new Contract`. - New { - /// Contract name. - name: String, - /// Constructor args length. - args_length: usize, - /// Constructor call args offset. - call_args_offset: usize, - /// `msg.value` (if any) used when creating contract. - value: Option, - /// `salt` (if any) used when creating contract. - salt: Option, - }, -} - -/// Represents a single bytecode dependency. -#[derive(Debug)] -pub(crate) struct BytecodeDependency { - /// Dependency kind. - kind: BytecodeDependencyKind, - /// Source map location of this dependency. - loc: Range, - /// HIR id of referenced contract. - referenced_contract: ContractId, -} - -/// Walks over contract HIR and collects [`BytecodeDependency`]s and referenced contracts. -struct BytecodeDependencyCollector<'hir> { - /// Source map, used for determining contract item locations. - source_map: &'hir SourceMap, - /// Parsed HIR. - hir: &'hir Hir<'hir>, - /// Source content of current contract. - src: &'hir str, - /// Project source dir, used to determine if referenced contract is a source contract. - src_dir: &'hir Path, - /// Dependencies collected for current contract. - dependencies: Vec, - /// Unique HIR ids of contracts referenced from current contract. - referenced_contracts: HashSet, -} - -impl<'hir> BytecodeDependencyCollector<'hir> { - fn new( - source_map: &'hir SourceMap, - hir: &'hir Hir<'hir>, - src: &'hir str, - src_dir: &'hir Path, - ) -> Self { - Self { - source_map, - hir, - src, - src_dir, - dependencies: vec![], - referenced_contracts: HashSet::default(), - } - } - - /// Collects reference identified as bytecode dependency of analyzed contract. - /// Discards any reference that is not in project src directory (e.g. external - /// libraries or mock contracts that extend source contracts). - fn collect_dependency(&mut self, dependency: BytecodeDependency) { - let contract = self.hir.contract(dependency.referenced_contract); - let source = self.hir.source(contract.source); - let FileName::Real(path) = &source.file.name else { - return; - }; - - if !path.starts_with(self.src_dir) { - let path = path.display(); - trace!("ignore dependency {path}"); - return; - } - - self.referenced_contracts.insert(dependency.referenced_contract); - self.dependencies.push(dependency); - } -} - -impl<'hir> Visit<'hir> for BytecodeDependencyCollector<'hir> { - type BreakValue = Never; - - fn hir(&self) -> &'hir Hir<'hir> { - self.hir - } - - fn visit_expr(&mut self, expr: &'hir Expr<'hir>) -> ControlFlow { - match &expr.kind { - ExprKind::Call(ty, call_args, named_args) => { - if let ExprKind::New(ty_new) = &ty.kind { - if let TypeKind::Custom(item_id) = ty_new.kind { - if let Some(contract_id) = item_id.as_contract() { - let name_loc = span_to_range(self.source_map, ty_new.span); - let name = &self.src[name_loc]; - - // Calculate offset to remove named args, e.g. for an expression like - // `new Counter {value: 333} ( address(this))` - // the offset will be used to replace `{value: 333} ( ` with `(` - let call_args_offset = if named_args.is_some() && !call_args.is_empty() - { - (call_args.span().lo() - ty_new.span.hi()).to_usize() - } else { - 0 - }; - - let args_len = expr.span.hi() - ty_new.span.hi(); - self.collect_dependency(BytecodeDependency { - kind: BytecodeDependencyKind::New { - name: name.to_string(), - args_length: args_len.to_usize(), - call_args_offset, - value: named_arg( - self.src, - named_args, - "value", - self.source_map, - ), - salt: named_arg(self.src, named_args, "salt", self.source_map), - }, - loc: span_to_range(self.source_map, ty.span), - referenced_contract: contract_id, - }); - } - } - } - } - ExprKind::Member(member_expr, ident) => { - if let ExprKind::TypeCall(ty) = &member_expr.kind { - if let TypeKind::Custom(contract_id) = &ty.kind { - if ident.name.as_str() == "creationCode" { - if let Some(contract_id) = contract_id.as_contract() { - self.collect_dependency(BytecodeDependency { - kind: BytecodeDependencyKind::CreationCode, - loc: span_to_range(self.source_map, expr.span), - referenced_contract: contract_id, - }); - } - } - } - } - } - _ => {} - } - self.walk_expr(expr) - } -} - -/// Helper function to extract value of a given named arg. -fn named_arg( - src: &str, - named_args: &Option<&[NamedArg<'_>]>, - arg: &str, - source_map: &SourceMap, -) -> Option { - named_args.unwrap_or_default().iter().find(|named_arg| named_arg.name.as_str() == arg).map( - |named_arg| { - let named_arg_loc = span_to_range(source_map, named_arg.value.span); - src[named_arg_loc].to_string() - }, - ) -} - -/// Goes over all test/script files and replaces bytecode dependencies with cheatcode -/// invocations. -pub(crate) fn remove_bytecode_dependencies( - hir: &Hir<'_>, - deps: &PreprocessorDependencies, - data: &PreprocessorData, -) -> Updates { - let mut updates = Updates::default(); - for (contract_id, deps) in &deps.preprocessed_contracts { - let contract = hir.contract(*contract_id); - let source = hir.source(contract.source); - let FileName::Real(path) = &source.file.name else { - continue; - }; - - let updates = updates.entry(path.clone()).or_default(); - let mut used_helpers = BTreeSet::new(); - - let vm_interface_name = format!("VmContractHelper{}", contract_id.get()); - // `address(uint160(uint256(keccak256("hevm cheat code"))))` - let vm = format!("{vm_interface_name}(0x7109709ECfa91a80626fF3989D68f67F5b1DD12D)"); - - for dep in deps { - let Some(ContractData { artifact, constructor_data, .. }) = - data.get(&dep.referenced_contract) - else { - continue; - }; - - match &dep.kind { - BytecodeDependencyKind::CreationCode => { - // for creation code we need to just call getCode - updates.insert(( - dep.loc.start, - dep.loc.end, - format!("{vm}.getCode(\"{artifact}\")"), - )); - } - BytecodeDependencyKind::New { - name, - args_length, - call_args_offset, - value, - salt, - } => { - let mut update = format!("{name}(payable({vm}.deployCode({{"); - update.push_str(&format!("_artifact: \"{artifact}\"")); - - if let Some(value) = value { - update.push_str(", "); - update.push_str(&format!("_value: {value}")); - } - - if let Some(salt) = salt { - update.push_str(", "); - update.push_str(&format!("_salt: {salt}")); - } - - if constructor_data.is_some() { - // Insert our helper - used_helpers.insert(dep.referenced_contract); - - update.push_str(", "); - update.push_str(&format!( - "_args: encodeArgs{id}(DeployHelper{id}.ConstructorArgs", - id = dep.referenced_contract.get() - )); - if *call_args_offset > 0 { - update.push('('); - } - updates.insert((dep.loc.start, dep.loc.end + call_args_offset, update)); - updates.insert(( - dep.loc.end + args_length, - dep.loc.end + args_length, - ")})))".to_string(), - )); - } else { - update.push_str("})))"); - updates.insert((dep.loc.start, dep.loc.end + args_length, update)); - } - } - }; - } - let helper_imports = used_helpers.into_iter().map(|id| { - let id = id.get(); - format!( - "import {{DeployHelper{id}, encodeArgs{id}}} from \"foundry-pp/DeployHelper{id}.sol\";", - ) - }).join("\n"); - updates.insert(( - source.file.src.len(), - source.file.src.len(), - format!( - r#" -{helper_imports} - -interface {vm_interface_name} {{ - function deployCode(string memory _artifact) external returns (address); - function deployCode(string memory _artifact, bytes32 _salt) external returns (address); - function deployCode(string memory _artifact, bytes memory _args) external returns (address); - function deployCode(string memory _artifact, bytes memory _args, bytes32 _salt) external returns (address); - function deployCode(string memory _artifact, uint256 _value) external returns (address); - function deployCode(string memory _artifact, uint256 _value, bytes32 _salt) external returns (address); - function deployCode(string memory _artifact, bytes memory _args, uint256 _value) external returns (address); - function deployCode(string memory _artifact, bytes memory _args, uint256 _value, bytes32 _salt) external returns (address); - function getCode(string memory _artifact) external returns (bytes memory); -}}"# - ), - )); - } - updates -} diff --git a/crates/compilers/src/preprocessor/mod.rs b/crates/compilers/src/preprocessor/mod.rs deleted file mode 100644 index 3d9367a0..00000000 --- a/crates/compilers/src/preprocessor/mod.rs +++ /dev/null @@ -1,277 +0,0 @@ -use crate::{ - apply_updates, - multi::{MultiCompiler, MultiCompilerInput, MultiCompilerLanguage}, - preprocessor::{ - data::{collect_preprocessor_data, create_deploy_helpers}, - deps::{remove_bytecode_dependencies, PreprocessorDependencies}, - }, - project::Preprocessor, - replace_source_content, - solc::{SolcCompiler, SolcVersionedInput}, - Compiler, ProjectPathsConfig, Result, -}; -use foundry_compilers_artifacts::SolcLanguage; -use foundry_compilers_core::utils; -use solar_parse::{ - ast::{FunctionKind, ItemKind, Span, Visibility}, - interface::{diagnostics::EmittedDiagnostics, source_map::FileName, Session, SourceMap}, - Parser, -}; -use solar_sema::{thread_local::ThreadLocal, ParsingContext}; -use std::{ - collections::HashSet, - ops::Range, - path::{Path, PathBuf}, -}; - -mod data; -mod deps; - -/// Returns the range of the given span in the source map. -#[track_caller] -fn span_to_range(source_map: &SourceMap, span: Span) -> Range { - source_map.span_to_source(span).unwrap().1 -} - -#[derive(Debug)] -pub struct TestOptimizerPreprocessor; - -impl Preprocessor for TestOptimizerPreprocessor { - fn preprocess( - &self, - _solc: &SolcCompiler, - input: &mut SolcVersionedInput, - paths: &ProjectPathsConfig, - mocks: &mut HashSet, - ) -> Result<()> { - // Skip if we are not preprocessing any tests or scripts. Avoids unnecessary AST parsing. - if !input.input.sources.iter().any(|(path, _)| paths.is_test_or_script(path)) { - trace!("no tests or sources to preprocess"); - return Ok(()); - } - - let sess = solar_session_from_solc(input); - let _ = sess.enter_parallel(|| -> solar_parse::interface::Result { - // Set up the parsing context with the project paths. - let mut parsing_context = solar_pcx_from_solc_no_sources(&sess, input, paths); - - // Add the sources into the context. - // Include all sources in the source map so as to not re-load them from disk, but only - // parse and preprocess tests and scripts. - let mut preprocessed_paths = vec![]; - let sources = &mut input.input.sources; - for (path, source) in sources.iter() { - if let Ok(src_file) = - sess.source_map().new_source_file(path.clone(), source.content.as_str()) - { - if paths.is_test_or_script(path) { - parsing_context.add_file(src_file); - preprocessed_paths.push(path.clone()); - } - } - } - - // Parse and preprocess. - let hir_arena = ThreadLocal::new(); - if let Some(gcx) = parsing_context.parse_and_lower(&hir_arena)? { - let hir = &gcx.get().hir; - // Collect tests and scripts dependencies and identify mock contracts. - let deps = PreprocessorDependencies::new( - &sess, - hir, - &preprocessed_paths, - &paths.paths_relative().sources, - &paths.root, - mocks, - ); - // Collect data of source contracts referenced in tests and scripts. - let data = collect_preprocessor_data(&sess, hir, &deps.referenced_contracts); - - // Extend existing sources with preprocessor deploy helper sources. - sources.extend(create_deploy_helpers(&data)); - - // Generate and apply preprocessor source updates. - apply_updates(sources, remove_bytecode_dependencies(hir, &deps, &data)); - } - - Ok(()) - }); - - // Warn if any diagnostics emitted during content parsing. - if let Err(err) = sess.emitted_errors().unwrap() { - warn!("failed preprocessing {err}"); - } - - Ok(()) - } -} - -impl Preprocessor for TestOptimizerPreprocessor { - fn preprocess( - &self, - compiler: &MultiCompiler, - input: &mut ::Input, - paths: &ProjectPathsConfig, - mocks: &mut HashSet, - ) -> Result<()> { - // Preprocess only Solc compilers. - let MultiCompilerInput::Solc(input) = input else { return Ok(()) }; - - let Some(solc) = &compiler.solc else { return Ok(()) }; - - let paths = paths.clone().with_language::(); - self.preprocess(solc, input, &paths, mocks) - } -} - -fn solar_session_from_solc(solc: &SolcVersionedInput) -> Session { - use solar_parse::interface::config; - - Session::builder() - .with_buffer_emitter(Default::default()) - .opts(config::Opts { - language: match solc.input.language { - SolcLanguage::Solidity => config::Language::Solidity, - SolcLanguage::Yul => config::Language::Yul, - _ => unimplemented!(), - }, - - // TODO: ... - /* - evm_version: solc.input.settings.evm_version, - */ - ..Default::default() - }) - .build() -} - -fn solar_pcx_from_solc_no_sources<'sess>( - sess: &'sess Session, - solc: &SolcVersionedInput, - paths: &ProjectPathsConfig, -) -> ParsingContext<'sess> { - let mut pcx = ParsingContext::new(sess); - pcx.file_resolver.set_current_dir(solc.cli_settings.base_path.as_ref().unwrap_or(&paths.root)); - for remapping in &paths.remappings { - pcx.file_resolver.add_import_remapping(solar_sema::interface::config::ImportRemapping { - context: remapping.context.clone().unwrap_or_default(), - prefix: remapping.name.clone(), - path: remapping.path.clone(), - }); - } - pcx.file_resolver.add_include_paths(solc.cli_settings.include_paths.iter().cloned()); - pcx -} - -pub(crate) fn interface_repr_hash(content: &str, path: &Path) -> Option { - let src = interface_repr(content, path).ok()?; - Some(foundry_compilers_artifacts::Source::content_hash_of(&src)) -} - -pub(crate) fn interface_repr(content: &str, path: &Path) -> Result { - parse_one_source(content, path, |ast| interface_representation_ast(content, &ast)) -} - -pub(crate) fn parse_one_source( - content: &str, - path: &Path, - f: impl FnOnce(solar_sema::ast::SourceUnit<'_>) -> R, -) -> Result { - let sess = Session::builder().with_buffer_emitter(Default::default()).build(); - let res = sess.enter(|| -> solar_parse::interface::Result<_> { - let arena = solar_parse::ast::Arena::new(); - let filename = FileName::Real(path.to_path_buf()); - let mut parser = Parser::from_source_code(&sess, &arena, filename, content.to_string())?; - let ast = parser.parse_file().map_err(|e| e.emit())?; - Ok(f(ast)) - }); - - // Return if any diagnostics emitted during content parsing. - if let Err(err) = sess.emitted_errors().unwrap() { - trace!("failed parsing {path:?}:\n{err}"); - return Err(err); - } - - Ok(res.unwrap()) -} - -/// Helper function to remove parts of the contract which do not alter its interface: -/// - Internal functions -/// - External functions bodies -/// -/// Preserves all libraries and interfaces. -pub(crate) fn interface_representation_ast( - content: &str, - ast: &solar_parse::ast::SourceUnit<'_>, -) -> String { - let mut spans_to_remove: Vec = Vec::new(); - for item in ast.items.iter() { - let ItemKind::Contract(contract) = &item.kind else { - continue; - }; - - if contract.kind.is_interface() || contract.kind.is_library() { - continue; - } - - for contract_item in contract.body.iter() { - if let ItemKind::Function(function) = &contract_item.kind { - let is_exposed = match function.kind { - // Function with external or public visibility - FunctionKind::Function => { - function.header.visibility >= Some(Visibility::Public) - } - FunctionKind::Constructor | FunctionKind::Fallback | FunctionKind::Receive => { - true - } - FunctionKind::Modifier => false, - }; - - // If function is not exposed we remove the entire span (signature and - // body). Otherwise we keep function signature and - // remove only the body. - if !is_exposed { - spans_to_remove.push(contract_item.span); - } else { - spans_to_remove.push(function.body_span); - } - } - } - } - let content = - replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) - .replace("\n", ""); - utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").into_owned() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_interface_representation() { - let content = r#" -library Lib { - function libFn() internal { - // logic to keep - } -} -contract A { - function a() external {} - function b() public {} - function c() internal { - // logic logic logic - } - function d() private {} - function e() external { - // logic logic logic - } -}"#; - - let result = interface_repr(content, Path::new("")).unwrap(); - assert_eq!( - result, - r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# - ); - } -} diff --git a/crates/compilers/src/resolver/parse.rs b/crates/compilers/src/resolver/parse.rs index bd3398b9..61c145d5 100644 --- a/crates/compilers/src/resolver/parse.rs +++ b/crates/compilers/src/resolver/parse.rs @@ -51,7 +51,7 @@ impl SolData { let mut contract_names = Vec::new(); let mut parse_result = Ok(()); - let result = crate::preprocessor::parse_one_source(content, file, |ast| { + let result = crate::parse_one_source(content, file, |ast| { for item in ast.items.iter() { let loc = item.span.lo().to_usize()..item.span.hi().to_usize(); match &item.kind { diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index 97b6b820..d8c08529 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -14,9 +14,8 @@ use foundry_compilers::{ }, flatten::Flattener, info::ContractInfo, - multi::MultiCompilerRestrictions, - preprocessor::TestOptimizerPreprocessor, - project::ProjectCompiler, + multi::{MultiCompilerInput, MultiCompilerRestrictions}, + project::{Preprocessor, ProjectCompiler}, project_util::*, solc::{Restriction, SolcRestrictions, SolcSettings}, take_solc_installer_lock, Artifact, ConfigurableArtifacts, ExtraOutputValues, Graph, Project, @@ -4159,7 +4158,29 @@ contract A { } } #[test] -fn can_preprocess_constructors_and_creation_code() { +fn can_preprocess() { + #[derive(Debug)] + struct SimplePreprocessor(tempfile::NamedTempFile); + + impl Preprocessor for SimplePreprocessor { + fn preprocess( + &self, + _compiler: &MultiCompiler, + input: &mut MultiCompilerInput, + _paths: &ProjectPathsConfig, + mocks: &mut HashSet, + ) -> foundry_compilers::error::Result<()> { + let MultiCompilerInput::Solc(input) = input else { + return Ok(()); + }; + for src in input.input.sources.values_mut() { + src.content = src.content.replace("++", "--").into(); + } + mocks.insert(self.0.path().to_path_buf()); + Ok(()) + } + } + let root = canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/preprocessor")) .unwrap(); @@ -4168,7 +4189,7 @@ fn can_preprocess_constructors_and_creation_code() { project.copy_project_from(&root).unwrap(); let r = ProjectCompiler::new(project.project()) .unwrap() - .with_preprocessor(TestOptimizerPreprocessor) + .with_preprocessor(SimplePreprocessor(tempfile::NamedTempFile::new().unwrap())) .compile(); let compiled = match r { From 48df36df7801842688f893e3d2b78f57c0a9f779 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:19:31 +0200 Subject: [PATCH 67/70] check --- .gitignore | 5 +- crates/compilers/src/cache/iface.rs | 96 +++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 crates/compilers/src/cache/iface.rs diff --git a/.gitignore b/.gitignore index 62eb994d..0f5e46a2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ /target /Cargo.lock -cache/ +/cache +test-data/**/cache/ .vscode /.envrc @@ -12,4 +13,4 @@ cache/ devenv.local.nix .direnv .pre-commit-config.yaml -.lock \ No newline at end of file +.lock diff --git a/crates/compilers/src/cache/iface.rs b/crates/compilers/src/cache/iface.rs new file mode 100644 index 00000000..8f006ef4 --- /dev/null +++ b/crates/compilers/src/cache/iface.rs @@ -0,0 +1,96 @@ +use crate::{parse_one_source, replace_source_content}; +use solar_sema::{ + ast::{self, Span}, + interface::diagnostics::EmittedDiagnostics, +}; +use std::path::Path; + +pub(crate) fn interface_repr_hash(content: &str, path: &Path) -> Option { + let src = interface_repr(content, path).ok()?; + Some(foundry_compilers_artifacts::Source::content_hash_of(&src)) +} + +pub(crate) fn interface_repr(content: &str, path: &Path) -> Result { + parse_one_source(content, path, |ast| interface_representation_ast(content, &ast)) +} + +/// Helper function to remove parts of the contract which do not alter its interface: +/// - Internal functions +/// - External functions bodies +/// +/// Preserves all libraries and interfaces. +pub(crate) fn interface_representation_ast( + content: &str, + ast: &solar_parse::ast::SourceUnit<'_>, +) -> String { + let mut spans_to_remove: Vec = Vec::new(); + for item in ast.items.iter() { + let ast::ItemKind::Contract(contract) = &item.kind else { + continue; + }; + + if contract.kind.is_interface() || contract.kind.is_library() { + continue; + } + + for contract_item in contract.body.iter() { + if let ast::ItemKind::Function(function) = &contract_item.kind { + let is_exposed = match function.kind { + // Function with external or public visibility + ast::FunctionKind::Function => { + function.header.visibility >= Some(ast::Visibility::Public) + } + ast::FunctionKind::Constructor + | ast::FunctionKind::Fallback + | ast::FunctionKind::Receive => true, + ast::FunctionKind::Modifier => false, + }; + + // If function is not exposed we remove the entire span (signature and + // body). Otherwise we keep function signature and + // remove only the body. + if !is_exposed { + spans_to_remove.push(contract_item.span); + } else { + spans_to_remove.push(function.body_span); + } + } + } + } + let content = + replace_source_content(content, spans_to_remove.iter().map(|span| (span.to_range(), ""))) + .replace("\n", ""); + crate::utils::RE_TWO_OR_MORE_SPACES.replace_all(&content, "").into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_interface_representation() { + let content = r#" +library Lib { + function libFn() internal { + // logic to keep + } +} +contract A { + function a() external {} + function b() public {} + function c() internal { + // logic logic logic + } + function d() private {} + function e() external { + // logic logic logic + } +}"#; + + let result = interface_repr(content, Path::new("")).unwrap(); + assert_eq!( + result, + r#"library Lib {function libFn() internal {// logic to keep}}contract A {function a() externalfunction b() publicfunction e() external }"# + ); + } +} From 743ff47ab7157bddbfe58cad937c9acc1465463b Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:21:13 +0200 Subject: [PATCH 68/70] clippy --- crates/compilers/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compilers/src/lib.rs b/crates/compilers/src/lib.rs index 16ac2c8b..bd4bc98a 100644 --- a/crates/compilers/src/lib.rs +++ b/crates/compilers/src/lib.rs @@ -905,7 +905,7 @@ pub fn apply_updates(sources: &mut Sources, updates: Updates) { /// Utility function to change source content ranges with provided updates. /// Assumes that the updates are sorted. -pub fn replace_source_content<'a>( +pub fn replace_source_content( source: impl Into, updates: impl IntoIterator, impl AsRef)>, ) -> String { From a3f4c4db117f20197f2442411ec8b993dba6951d Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:47:06 +0200 Subject: [PATCH 69/70] nit --- crates/compilers/src/compile/project.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 45106d83..6ffc26f4 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -479,7 +479,7 @@ impl CompilerSources<'_, L, S> { // Get current list of mocks from cache. This will be passed to preprocessors and updated // accordingly, then set back in cache. - let mocks = &mut cache.mocks(); + let mut mocks = cache.mocks(); let mut jobs = Vec::new(); for (language, versioned_sources) in self.sources { @@ -520,7 +520,7 @@ impl CompilerSources<'_, L, S> { &project.compiler, &mut input, &project.paths, - mocks, + &mut mocks, )?; } @@ -529,7 +529,7 @@ impl CompilerSources<'_, L, S> { } // Update cache with mocks updated by preprocessors. - cache.update_mocks(mocks.clone()); + cache.update_mocks(mocks); let results = if let Some(num_jobs) = jobs_cnt { compile_parallel(&project.compiler, jobs, num_jobs) From 87b9346ac9d882025de51674592a7a90efb511af Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 7 Apr 2025 19:41:04 +0300 Subject: [PATCH 70/70] Bump solar version --- Cargo.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5510f78d..681b0308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,8 +55,8 @@ semver = { version = "1.0", features = ["serde"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1.0" similar-asserts = "1" -solar-parse = { version = "=0.1.1", default-features = false } -solar-sema = { version = "=0.1.1", default-features = false } +solar-parse = { version = "=0.1.2", default-features = false } +solar-sema = { version = "=0.1.2", default-features = false } svm = { package = "svm-rs", version = "0.5", default-features = false } tempfile = "3.9" thiserror = "2" @@ -69,7 +69,3 @@ futures-util = "0.3" tokio = { version = "1.35", features = ["rt-multi-thread"] } snapbox = "0.6.9" - -[patch.crates-io] -solar-parse = { git = "https://github.com/paradigmxyz/solar", branch = "main" } -solar-sema = { git = "https://github.com/paradigmxyz/solar", branch = "main" }