op-chain-ops: source maps fixes + FS (ethereum-optimism#11574)

* op-chain-ops: source maps fixes + FS

* op-chain-ops/srcmap: add doc-comment, remove replaced test

* op-chain-ops: address review comments

* op-chain-ops: fix missing .sol extension

* op-chain-ops: fix artifacts traversal; check extension again, just don't trim the extension

Author: protolambda

.gitignore

@@ -15,6 +15,9 @@ dist
 artifacts
 cache
 
+!op-chain-ops/foundry/testdata/srcmaps/cache
+!op-chain-ops/foundry/testdata/srcmaps/artifacts
+
 packages/contracts-bedrock/deployments/devnetL1
 packages/contracts-bedrock/deployments/anvil
 

op-chain-ops/foundry/artifactsfs.go

@@ -32,24 +32,29 @@ type ArtifactsFS struct {
 	FS statDirFs
 }
 
+// ListArtifacts lists the artifacts. Each artifact matches a source-file name.
+// This name includes the extension, e.g. ".sol"
+// (no other artifact-types are supported at this time).
 func (af *ArtifactsFS) ListArtifacts() ([]string, error) {
 	entries, err := af.FS.ReadDir(".")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list artifacts: %w", err)
 	}
 	out := make([]string, 0, len(entries))
 	for _, d := range entries {
+		// Some artifacts may be nested in directories not suffixed with ".sol"
+		// Nested artifacts, and non-solidity artifacts, are not supported.
 		if name := d.Name(); strings.HasSuffix(name, ".sol") {
-			out = append(out, strings.TrimSuffix(name, ".sol"))
+			out = append(out, d.Name())
 		}
 	}
 	return out, nil
 }
 
-// ListContracts lists the contracts of the named artifact.
-// E.g. "Owned" might list "Owned.0.8.15", "Owned.0.8.25", and "Owned".
+// ListContracts lists the contracts of the named artifact, including the file extension.
+// E.g. "Owned.sol" might list "Owned.0.8.15", "Owned.0.8.25", and "Owned".
 func (af *ArtifactsFS) ListContracts(name string) ([]string, error) {
-	f, err := af.FS.Open(name + ".sol")
+	f, err := af.FS.Open(name)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open artifact %q: %w", name, err)
 	}
@@ -73,8 +78,10 @@ func (af *ArtifactsFS) ListContracts(name string) ([]string, error) {
 
 // ReadArtifact reads a specific JSON contract artifact from the FS.
 // The contract name may be suffixed by a solidity compiler version, e.g. "Owned.0.8.25".
+// The contract name does not include ".json", this is a detail internal to the artifacts.
+// The name of the artifact is the source-file name, this must include the suffix such as ".sol".
 func (af *ArtifactsFS) ReadArtifact(name string, contract string) (*Artifact, error) {
-	artifactPath := path.Join(name+".sol", contract+".json")
+	artifactPath := path.Join(name, contract+".json")
 	f, err := af.FS.Open(artifactPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open artifact %q: %w", artifactPath, err)

op-chain-ops/foundry/sourcefs.go

@@ -0,0 +1,147 @@
+package foundry
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/exp/maps"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/srcmap"
+)
+
+// SourceMapFS wraps an FS to provide source-maps.
+// This FS relies on the following file path assumptions:
+// - `/artifacts/build-info/X.json` (build-info path is read from the below file): build files, of foundry incremental builds.
+// - `/cache/solidity-files-cache.json`: a JSON file enumerating all files, and when the build last changed.
+// - `/` a root dir, relative to where the source files are located (as per the compilationTarget metadata in an artifact).
+type SourceMapFS struct {
+	fs fs.FS
+}
+
+// NewSourceMapFS creates a new SourceMapFS.
+// The source-map FS loads identifiers for srcmap.ParseSourceMap
+// and provides a util to retrieve a source-map for an Artifact.
+// The solidity source-files are lazy-loaded when using the produced sourcemap.
+func NewSourceMapFS(fs fs.FS) *SourceMapFS {
+	return &SourceMapFS{fs: fs}
+}
+
+// ForgeBuild represents the JSON content of a forge-build entry in the `artifacts/build-info` output.
+type ForgeBuild struct {
+	ID             string                     `json:"id"`                // ID of the build itself
+	SourceIDToPath map[srcmap.SourceID]string `json:"source_id_to_path"` // srcmap ID to source filepath
+}
+
+func (s *SourceMapFS) readBuild(buildInfoPath string, id string) (*ForgeBuild, error) {
+	buildPath := path.Join(buildInfoPath, id+".json")
+	f, err := s.fs.Open(buildPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open build: %w", err)
+	}
+	defer f.Close()
+	var build ForgeBuild
+	if err := json.NewDecoder(f).Decode(&build); err != nil {
+		return nil, fmt.Errorf("failed to read build: %w", err)
+	}
+	return &build, nil
+}
+
+// ForgeBuildEntry represents a JSON entry that links the build job of a contract source file.
+type ForgeBuildEntry struct {
+	Path    string `json:"path"`
+	BuildID string `json:"build_id"`
+}
+
+// ForgeBuildInfo represents a JSON entry that enumerates the latest builds per contract per compiler version.
+type ForgeBuildInfo struct {
+	// contract name -> solidity version -> build entry
+	Artifacts map[string]map[string]ForgeBuildEntry `json:"artifacts"`
+}
+
+// ForgeBuildCache rep
+type ForgeBuildCache struct {
+	Paths struct {
+		BuildInfos string `json:"build_infos"`
+	} `json:"paths"`
+	Files map[string]ForgeBuildInfo `json:"files"`
+}
+
+func (s *SourceMapFS) readBuildCache() (*ForgeBuildCache, error) {
+	cachePath := path.Join("cache", "solidity-files-cache.json")
+	f, err := s.fs.Open(cachePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open build cache: %w", err)
+	}
+	defer f.Close()
+	var buildCache ForgeBuildCache
+	if err := json.NewDecoder(f).Decode(&buildCache); err != nil {
+		return nil, fmt.Errorf("failed to read build cache: %w", err)
+	}
+	return &buildCache, nil
+}
+
+// ReadSourceIDs reads the source-identifier to source file-path mapping that is needed to translate a source-map
+// of the given contract, the given compiler version, and within the given source file path.
+func (s *SourceMapFS) ReadSourceIDs(path string, contract string, compilerVersion string) (map[srcmap.SourceID]string, error) {
+	buildCache, err := s.readBuildCache()
+	if err != nil {
+		return nil, err
+	}
+	artifactBuilds, ok := buildCache.Files[path]
+	if !ok {
+		return nil, fmt.Errorf("no known builds for path %q", path)
+	}
+	byCompilerVersion, ok := artifactBuilds.Artifacts[contract]
+	if !ok {
+		return nil, fmt.Errorf("contract not found in artifact: %q", contract)
+	}
+	var buildEntry ForgeBuildEntry
+	if compilerVersion != "" {
+		entry, ok := byCompilerVersion[compilerVersion]
+		if !ok {
+			return nil, fmt.Errorf("no known build for compiler version: %q", compilerVersion)
+		}
+		buildEntry = entry
+	} else {
+		if len(byCompilerVersion) == 0 {
+			return nil, errors.New("no known build, unspecified compiler version")
+		}
+		if len(byCompilerVersion) > 1 {
+			return nil, fmt.Errorf("no compiler version specified, and more than one option: %s", strings.Join(maps.Keys(byCompilerVersion), ", "))
+		}
+		for _, entry := range byCompilerVersion {
+			buildEntry = entry
+		}
+	}
+	build, err := s.readBuild(filepath.ToSlash(buildCache.Paths.BuildInfos), buildEntry.BuildID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read build %q of contract %q: %w", buildEntry.BuildID, contract, err)
+	}
+	return build.SourceIDToPath, nil
+}
+
+// SourceMap retrieves a source-map for a given contract of a foundry Artifact.
+func (s *SourceMapFS) SourceMap(artifact *Artifact, contract string) (*srcmap.SourceMap, error) {
+	srcPath := ""
+	for path, name := range artifact.Metadata.Settings.CompilationTarget {
+		if name == contract {
+			srcPath = path
+			break
+		}
+	}
+	if srcPath == "" {
+		return nil, fmt.Errorf("no known source path for contract %s in artifact", contract)
+	}
+	// The commit suffix is ignored, the core semver part is what is used in the resolution of builds.
+	basicCompilerVersion := strings.SplitN(artifact.Metadata.Compiler.Version, "+", 2)[0]
+	ids, err := s.ReadSourceIDs(srcPath, contract, basicCompilerVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read source IDs of %q: %w", srcPath, err)
+	}
+	return srcmap.ParseSourceMap(s.fs, ids, artifact.DeployedBytecode.Object, artifact.DeployedBytecode.SourceMap)
+}

op-chain-ops/foundry/sourcefs_test.go

@@ -0,0 +1,25 @@
+package foundry
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+//go:generate ./testdata/srcmaps/generate.sh
+
+func TestSourceMapFS(t *testing.T) {
+	artifactFS := OpenArtifactsDir("./testdata/srcmaps/test-artifacts")
+	exampleArtifact, err := artifactFS.ReadArtifact("SimpleStorage.sol", "SimpleStorage")
+	require.NoError(t, err)
+	srcFS := NewSourceMapFS(os.DirFS("./testdata/srcmaps"))
+	srcMap, err := srcFS.SourceMap(exampleArtifact, "SimpleStorage")
+	require.NoError(t, err)
+	seenInfo := make(map[string]struct{})
+	for i := range exampleArtifact.DeployedBytecode.Object {
+		seenInfo[srcMap.FormattedInfo(uint64(i))] = struct{}{}
+	}
+	require.Contains(t, seenInfo, "src/SimpleStorage.sol:11:5")
+	require.Contains(t, seenInfo, "src/StorageLibrary.sol:8:9")
+}

op-chain-ops/foundry/testdata/README.md

@@ -1,3 +1,3 @@
-# artifacts test data
+# source-map test data
 
-This is a small selection of `forge-artifacts` specifically for testing of Artifact decoding and the Artifacts-FS.
+Simple small multi-contract forge setup, to test Go forge map functionality against.

op-chain-ops/foundry/testdata/srcmaps/artifacts/build-info/c79aa2c3b4578aee2dd8f02d20b1aeb6.json

@@ -0,0 +1 @@
+{"id":"c79aa2c3b4578aee2dd8f02d20b1aeb6","source_id_to_path":{"0":"src/SimpleStorage.sol","1":"src/StorageLibrary.sol"},"language":"Solidity"}

op-chain-ops/foundry/testdata/srcmaps/cache/solidity-files-cache.json

@@ -0,0 +1 @@
+{"_format":"","paths":{"artifacts":"test-artifacts","build_infos":"artifacts/build-info","sources":"src","tests":"test","scripts":"scripts","libraries":["lib","node_modules"]},"files":{"src/SimpleStorage.sol":{"lastModificationDate":1724351550959,"contentHash":"25499c2e202ada22ebd26f8e886cc2e1","sourceName":"src/SimpleStorage.sol","compilerSettings":{"solc":{"optimizer":{"enabled":true,"runs":999999},"metadata":{"useLiteralContent":false,"bytecodeHash":"none","appendCBOR":true},"outputSelection":{"*":{"":["ast"],"*":["abi","evm.bytecode","evm.deployedBytecode","evm.methodIdentifiers","metadata","storageLayout","devdoc","userdoc"]}},"evmVersion":"cancun","viaIR":false,"libraries":{}},"vyper":{"evmVersion":"cancun","outputSelection":{"*":{"*":["abi","evm.bytecode","evm.deployedBytecode"]}}}},"imports":["src/StorageLibrary.sol"],"versionRequirement":"=0.8.15","artifacts":{"SimpleStorage":{"0.8.15":{"path":"SimpleStorage.sol/SimpleStorage.json","build_id":"c79aa2c3b4578aee2dd8f02d20b1aeb6"}}},"seenByCompiler":true},"src/StorageLibrary.sol":{"lastModificationDate":1724351550967,"contentHash":"61545ea51326b6aa0e3bafaf3116b0a8","sourceName":"src/StorageLibrary.sol","compilerSettings":{"solc":{"optimizer":{"enabled":true,"runs":999999},"metadata":{"useLiteralContent":false,"bytecodeHash":"none","appendCBOR":true},"outputSelection":{"*":{"":["ast"],"*":["abi","evm.bytecode","evm.deployedBytecode","evm.methodIdentifiers","metadata","storageLayout","devdoc","userdoc"]}},"evmVersion":"cancun","viaIR":false,"libraries":{}},"vyper":{"evmVersion":"cancun","outputSelection":{"*":{"*":["abi","evm.bytecode","evm.deployedBytecode"]}}}},"imports":[],"versionRequirement":"=0.8.15","artifacts":{"StorageLibrary":{"0.8.15":{"path":"StorageLibrary.sol/StorageLibrary.json","build_id":"c79aa2c3b4578aee2dd8f02d20b1aeb6"}}},"seenByCompiler":true}},"builds":["c79aa2c3b4578aee2dd8f02d20b1aeb6"]}

op-chain-ops/foundry/testdata/srcmaps/foundry.toml

@@ -0,0 +1,31 @@
+################################################################
+#                   PROFILE: DEFAULT (Local)                   #
+################################################################
+
+[profile.default]
+
+# Compilation settings
+src = 'src'
+out = 'test-artifacts'
+script = 'scripts'
+optimizer = true
+optimizer_runs = 999999
+remappings = []
+extra_output = ['devdoc', 'userdoc', 'metadata', 'storageLayout']
+bytecode_hash = 'none'
+build_info_path = 'artifacts/build-info'
+ast = true
+evm_version = "cancun"
+# 5159 error code is selfdestruct error code
+ignored_error_codes = ["transient-storage", "code-size", "init-code-size", 5159]
+
+# We set the gas limit to max int64 to avoid running out of gas during testing, since the default
+# gas limit is 1B and some of our tests require more gas than that, such as `test_callWithMinGas_noLeakageLow_succeeds`.
+# We use this gas limit since it was the default gas limit prior to https://github.com/foundry-rs/foundry/pull/8274.
+# Due to toml-rs limitations, if you increase the gas limit above this value it must be a string.
+gas_limit = 9223372036854775807
+
+# Test / Script Runner Settings
+ffi = false
+fs_permissions = []
+libs = ["node_modules", "lib"]

op-chain-ops/foundry/testdata/srcmaps/generate.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -euo
+
+# Don't include previous build outputs
+forge clean
+
+forge build

op-chain-ops/foundry/testdata/srcmaps/src/SimpleStorage.sol

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: MIT
+pragma solidity 0.8.15;
+
+import {StorageLibrary} from "./StorageLibrary.sol";
+
+// @notice SimpleStorage is a contract to test Go <> foundry integration.
+// @dev uses a dependency, to test source-mapping with multiple sources.
+contract SimpleStorage {
+
+    // @dev example getter
+    function getExampleData() public pure returns (uint256) {
+        return StorageLibrary.addData(42);
+    }
+}