test: refactor testdata/ tests to be run in forge test (foundry-rs#12049)

* test: run forge test on testdata/

* chore: refactor to use common Test contract

* chore: disable testGasMeteringExternal, via-ir

* test: rm unused repros

* fix: paths

* upd

* fmt

* fix more tests

* test: turn testNonExistingContractRevert into expectRevert

* fix some more paths

* legacy assertions

* compile paris with paris

* fix: set configs for fs tests

* fix remaining paths in cheats

* restrict fs permissions

* fix: set runtime evm_version too

* fix vyper

* fix: a couple of repros

* fix: we have storage layouts

* fix: 3223, 3674: set sender

* reorder

* feat: move repros expected failures to snapshots

* feat: migrate remaining repros tests

* feat: rm migrated files

* skip testRevertIfGetUnlinked

* move expected core/ failures

* upd

* move logs/

* move all forgetest tests from it/ to cli/

* fix fork test

* move trace/

* tmp: move fuzz/invariant out of fuzz/

* move fuzz/

* forge fmt

* wips

* fix: both vyper and paris; set src/

* canon

* lib log

* logs

* Revert "fix: set runtime evm_version too"

This reverts commit 7ca544b.

Contract-level inline config will set evm version for libraries too,
which means we fail on deploying libraries that are compiled with
newer evm version.

* fix: set evm version where needed, per test function

* test: reduce gas wastage

* chore: clippy

* invariant mod.rs

* test: fix linking tests with new utils

* redact_with

* Revert "wips"

This reverts commit ee2c17a.

* migrate invariant/target{,Abi}

* migrate InvariantAfterInvariant.t.sol

* migrate InvariantAssume.t.sol

* migrate InvariantCalldataDictionary.t.sol, more test utils

* migrate InvariantCustomError.t.sol

* migrate InvariantExcludedSenders.t.sol

* migrate InvariantFixtures.t.sol

* migrate InvariantHandlerFailure.t.sol

* interlude: forgot to use a new file

* migrate InvariantInnerContract.t.sol

* migrate InvariantPreserveState.t.sol

* migrate InvariantReentrancy.t.sol

* migrate InvariantRollFork.t.sol

* migrate InvariantScrapeValues.t.sol

* migrate InvariantSequenceNoReverts.t.sol

* migrate InvariantShrinkBigSequence.t.sol

* migrate InvariantShrinkFailOnRevert.t.sol

* migrate InvariantShrinkWithAssert.t.sol

* migrate InvariantTest1.t.sol

* fix InvariantInnerContract.t.sol

* update new Rlp test

* com

* better com

* nuke tests/it

* test: fix testdata paths in script tester

* test: fix relative paths in test_cmd

* test: redact more in issue_2851

* fix: copy testdata correctly

* trace addrs

* manual retry logic with --retry

* fix nondeterministic output

* debug: fs lock error context

* test: fix project root for windows

* test: skip project root test if unset

* normalize both

* typo

* Revert "typo"

This reverts commit 402bea1.

* Revert "debug: fs lock error context"

This reverts commit e5caedd.

* fix

* fix: locked_write_line for windows

* chore: clippy

* fmt

* chore: speed up fuzzed_selected_targets

* other way

* fix nondeterministic output 2

* fix: disable persistence

* test: revert old via-ir

* ci: tweak cache key

* do not run trace test when isolate

---------

Co-authored-by: grandizzy <grandizzy.the.egg@gmail.com>

Author: DaniPopes

.github/workflows/nextest.yml

@@ -98,7 +98,10 @@ jobs:
             ~/.config/.foundry/cache
             testdata/cache
             testdata/out
-          key: ${{ runner.os }}-foundry-${{ matrix.name }}
+          # Use a unique key for each run to always update the cache.
+          key: foundry-${{ matrix.name }}-${{ github.run_id }}
+          restore-keys: |
+            foundry-${{ matrix.name }}-
       - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
       - name: Setup Git config
         run: |

crates/cheatcodes/spec/src/lib.rs

@@ -117,8 +117,7 @@ mod tests {
     #[cfg(feature = "schema")]
     const SCHEMA_PATH: &str =
         concat!(env!("CARGO_MANIFEST_DIR"), "/../assets/cheatcodes.schema.json");
-    const IFACE_PATH: &str =
-        concat!(env!("CARGO_MANIFEST_DIR"), "/../../../testdata/cheats/Vm.sol");
+    const IFACE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../testdata/utils/Vm.sol");
 
     /// Generates the `cheatcodes.json` file contents.
     fn json_cheatcodes() -> String {

crates/chisel/src/source.rs

@@ -26,7 +26,7 @@ use walkdir::WalkDir;
 pub const MIN_VM_VERSION: Version = Version::new(0, 6, 2);
 
 /// Solidity source for the `Vm` interface in [forge-std](https://github.com/foundry-rs/forge-std)
-static VM_SOURCE: &str = include_str!("../../../testdata/cheats/Vm.sol");
+static VM_SOURCE: &str = include_str!("../../../testdata/utils/Vm.sol");
 
 /// [`SessionSource`] build output.
 pub struct GeneratedOutput {

crates/common/src/fs.rs

@@ -5,7 +5,7 @@ use flate2::{Compression, read::GzDecoder, write::GzEncoder};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
     fs::{self, File},
-    io::{BufReader, BufWriter, Read, Write},
+    io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
     path::{Component, Path, PathBuf},
 };
 
@@ -62,25 +62,29 @@ pub fn read_json_gzip_file<T: DeserializeOwned>(path: &Path) -> Result<T> {
 /// Reads the entire contents of a locked shared file into a string.
 pub fn locked_read_to_string(path: impl AsRef<Path>) -> Result<String> {
     let path = path.as_ref();
-    let file =
+    let mut file =
         fs::OpenOptions::new().read(true).open(path).map_err(|err| FsPathError::open(err, path))?;
     file.lock_shared().map_err(|err| FsPathError::lock(err, path))?;
-    let mut contents = String::new();
-    (&file).read_to_string(&mut contents).map_err(|err| FsPathError::read(err, path))?;
+    let contents = read_inner(path, &mut file)?;
     file.unlock().map_err(|err| FsPathError::unlock(err, path))?;
-    Ok(contents)
+    String::from_utf8(contents).map_err(|err| FsPathError::read(std::io::Error::other(err), path))
 }
 
 /// Reads the entire contents of a locked shared file into a bytes vector.
 pub fn locked_read(path: impl AsRef<Path>) -> Result<Vec<u8>> {
     let path = path.as_ref();
-    let file =
+    let mut file =
         fs::OpenOptions::new().read(true).open(path).map_err(|err| FsPathError::open(err, path))?;
     file.lock_shared().map_err(|err| FsPathError::lock(err, path))?;
+    let contents = read_inner(path, &mut file)?;
+    file.unlock().map_err(|err| FsPathError::unlock(err, path))?;
+    Ok(contents)
+}
+
+fn read_inner(path: &Path, file: &mut File) -> Result<Vec<u8>> {
     let file_len = file.metadata().map_err(|err| FsPathError::open(err, path))?.len() as usize;
     let mut buffer = Vec::with_capacity(file_len);
-    (&file).read_to_end(&mut buffer).map_err(|err| FsPathError::read(err, path))?;
-    file.unlock().map_err(|err| FsPathError::unlock(err, path))?;
+    file.read_to_end(&mut buffer).map_err(|err| FsPathError::read(err, path))?;
     Ok(buffer)
 }
 
@@ -136,18 +140,39 @@ pub fn locked_write(path: impl AsRef<Path>, contents: impl AsRef<[u8]>) -> Resul
 }
 
 /// Writes a line in an exclusive locked file.
-pub fn locked_write_line(path: impl AsRef<Path>, line: &String) -> Result<()> {
+pub fn locked_write_line(path: impl AsRef<Path>, line: &str) -> Result<()> {
     let path = path.as_ref();
+    if cfg!(windows) {
+        return locked_write_line_windows(path, line);
+    }
+
     let mut file = std::fs::OpenOptions::new()
         .append(true)
         .create(true)
         .open(path)
         .map_err(|err| FsPathError::open(err, path))?;
+
     file.lock().map_err(|err| FsPathError::lock(err, path))?;
     writeln!(file, "{line}").map_err(|err| FsPathError::write(err, path))?;
     file.unlock().map_err(|err| FsPathError::unlock(err, path))
 }
 
+// Locking fails on Windows if the file is opened in append mode.
+fn locked_write_line_windows(path: &Path, line: &str) -> Result<()> {
+    let mut file = std::fs::OpenOptions::new()
+        .write(true)
+        .truncate(false)
+        .create(true)
+        .open(path)
+        .map_err(|err| FsPathError::open(err, path))?;
+    file.lock().map_err(|err| FsPathError::lock(err, path))?;
+
+    file.seek(SeekFrom::End(0)).map_err(|err| FsPathError::write(err, path))?;
+    writeln!(file, "{line}").map_err(|err| FsPathError::write(err, path))?;
+
+    file.unlock().map_err(|err| FsPathError::unlock(err, path))
+}
+
 /// Wrapper for `std::fs::copy`
 pub fn copy(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<u64> {
     let from = from.as_ref();

crates/forge/tests/cli/test_cmd/core.rs

@@ -0,0 +1,142 @@
+//! Core test functionality tests
+
+use foundry_test_utils::str;
+
+forgetest_init!(failing_test_after_failed_setup, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.add_test(
+        "FailingTestAfterFailedSetup.t.sol",
+        r#"
+import "forge-std/Test.sol";
+
+contract FailingTestAfterFailedSetupTest is Test {
+    function setUp() public {
+        assertTrue(false);
+    }
+
+    function testAssertSuccess() public {
+        assertTrue(true);
+    }
+
+    function testAssertFailure() public {
+        assertTrue(false);
+    }
+}
+"#,
+    );
+
+    cmd.arg("test").assert_failure().stdout_eq(str![[r#"
+...
+Ran 1 test for test/FailingTestAfterFailedSetup.t.sol:FailingTestAfterFailedSetupTest
+[FAIL: assertion failed] setUp() ([GAS])
+Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 0 tests passed, 1 failed, 0 skipped (1 total tests)
+
+Failing tests:
+Encountered 1 failing test in test/FailingTestAfterFailedSetup.t.sol:FailingTestAfterFailedSetupTest
+[FAIL: assertion failed] setUp() ([GAS])
+
+Encountered a total of 1 failing tests, 0 tests succeeded
+
+Tip: Run `forge test --rerun` to retry only the 1 failed test
+
+"#]]);
+});
+
+forgetest_init!(legacy_assertions, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.add_test(
+        "LegacyAssertions.t.sol",
+        r#"
+import "forge-std/Test.sol";
+
+contract NoAssertionsRevertTest is Test {
+    function testMultipleAssertFailures() public {
+        vm.assertEq(uint256(1), uint256(2));
+        vm.assertLt(uint256(5), uint256(4));
+    }
+}
+
+/// forge-config: default.legacy_assertions = true
+contract LegacyAssertionsTest {
+    bool public failed;
+
+    function testFlagNotSetSuccess() public {}
+
+    function testFlagSetFailure() public {
+        failed = true;
+    }
+}
+"#,
+    );
+
+    cmd.args(["test", "-j1"]).assert_failure().stdout_eq(str![[r#"
+...
+Ran 2 tests for test/LegacyAssertions.t.sol:LegacyAssertionsTest
+[PASS] testFlagNotSetSuccess() ([GAS])
+[FAIL] testFlagSetFailure() ([GAS])
+Suite result: FAILED. 1 passed; 1 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test for test/LegacyAssertions.t.sol:NoAssertionsRevertTest
+[FAIL: assertion failed: 1 != 2] testMultipleAssertFailures() ([GAS])
+Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]
+
+Ran 2 test suites [ELAPSED]: 1 tests passed, 2 failed, 0 skipped (3 total tests)
+
+Failing tests:
+Encountered 1 failing test in test/LegacyAssertions.t.sol:LegacyAssertionsTest
+[FAIL] testFlagSetFailure() ([GAS])
+
+Encountered 1 failing test in test/LegacyAssertions.t.sol:NoAssertionsRevertTest
+[FAIL: assertion failed: 1 != 2] testMultipleAssertFailures() ([GAS])
+
+Encountered a total of 2 failing tests, 1 tests succeeded
+
+Tip: Run `forge test --rerun` to retry only the 2 failed tests
+
+"#]]);
+});
+
+forgetest_init!(payment_failure, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.add_test(
+        "PaymentFailure.t.sol",
+        r#"
+import "forge-std/Test.sol";
+
+contract Payable {
+    function pay() public payable {}
+}
+
+contract PaymentFailureTest is Test {
+    function testCantPay() public {
+        Payable target = new Payable();
+        vm.prank(address(1));
+        target.pay{value: 1}();
+    }
+}
+"#,
+    );
+
+    cmd.arg("test").assert_failure().stdout_eq(str![[r#"
+[COMPILING_FILES] with [SOLC_VERSION]
+[SOLC_VERSION] [ELAPSED]
+Compiler run successful!
+
+Ran 1 test for test/PaymentFailure.t.sol:PaymentFailureTest
+[FAIL: EvmError: Revert] testCantPay() ([GAS])
+Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED]
+
+Ran 1 test suite [ELAPSED]: 0 tests passed, 1 failed, 0 skipped (1 total tests)
+
+Failing tests:
+Encountered 1 failing test in test/PaymentFailure.t.sol:PaymentFailureTest
+[FAIL: EvmError: Revert] testCantPay() ([GAS])
+
+Encountered a total of 1 failing tests, 0 tests succeeded
+
+Tip: Run `forge test --rerun` to retry only the 1 failed test
+
+"#]]);
+});