From e50b90cf0a8ca4f431e537847bb26df85a88bb6c Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 15:44:51 +0200 Subject: [PATCH 01/19] Gambit action draft --- .github/workflows/gambit.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/gambit.yml diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml new file mode 100644 index 000000000..ae2a9f09e --- /dev/null +++ b/.github/workflows/gambit.yml @@ -0,0 +1,17 @@ +name: Gambit mutation testing + +on: + workflow_dispatch: + +jobs: + run-mutation-tests: + runs-on: ubuntu-latest + steps: + - name: Install Gambit + run: | + wget -O gambit https://github.com/Certora/gambit/releases/download/v1.0.5/gambit-linux-v1.0.5 + chmod +x gambit + sudo mv gambit /usr/local/bin/ + + - name: Verify Gambit Installation + run: gambit --help From 61d9758850efe7af7161eeb8fa9a272710e125ba Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 15:49:49 +0200 Subject: [PATCH 02/19] Add on pull_request --- .github/workflows/gambit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index ae2a9f09e..fac59a8a9 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -2,6 +2,7 @@ name: Gambit mutation testing on: workflow_dispatch: + pull_request: jobs: run-mutation-tests: From 3e455acd3b82f62390ad6c537c3f445f5f8200b7 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 16:02:19 +0200 Subject: [PATCH 03/19] Run basic --- .github/workflows/gambit.yml | 11 ++++++ test-mutation/config.json | 75 +----------------------------------- 2 files changed, 13 insertions(+), 73 deletions(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index fac59a8a9..9b86a9efa 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -16,3 +16,14 @@ jobs: - name: Verify Gambit Installation run: gambit --help + + - name: Setup node/yarn + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: '**/yarn.lock' + + - name: Run mutation tests + run: | + yarn test:mutation diff --git a/test-mutation/config.json b/test-mutation/config.json index 85bc60d01..a907d28c5 100644 --- a/test-mutation/config.json +++ b/test-mutation/config.json @@ -5,78 +5,7 @@ "solc_remappings": [ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2CustomGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ERC20Gateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2GatewayRouter.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ReverseCustomGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2WethGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/L2ArbitrumMessenger.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/L2AtomicTokenBridgeFactory.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/ReverseArbToken.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/StandardArbERC20.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] + ], + "num_mutants": 3 } ] From a6ac4880b97b4d4682aa1af25c49ceaaeb364922 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 16:11:18 +0200 Subject: [PATCH 04/19] Update action --- .github/workflows/gambit.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 9b86a9efa..eb4c9ff8f 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -8,6 +8,17 @@ jobs: run-mutation-tests: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup node/yarn + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: '**/yarn.lock' + - name: Install Gambit run: | wget -O gambit https://github.com/Certora/gambit/releases/download/v1.0.5/gambit-linux-v1.0.5 @@ -17,13 +28,6 @@ jobs: - name: Verify Gambit Installation run: gambit --help - - name: Setup node/yarn - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: 'yarn' - cache-dependency-path: '**/yarn.lock' - - name: Run mutation tests run: | yarn test:mutation From 6685a48c463c361cec03cc7a02addeced557dceb Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 16:14:24 +0200 Subject: [PATCH 05/19] Install packages --- .github/workflows/gambit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index eb4c9ff8f..e66aa1e49 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -28,6 +28,9 @@ jobs: - name: Verify Gambit Installation run: gambit --help + - name: Install packages + run: yarn + - name: Run mutation tests run: | yarn test:mutation From fad89359416e1101d2dc4648b0833cff4ed04c0a Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 16:19:31 +0200 Subject: [PATCH 06/19] Install FOundry --- .github/workflows/gambit.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index e66aa1e49..5634213e8 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -19,6 +19,11 @@ jobs: cache: 'yarn' cache-dependency-path: '**/yarn.lock' + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly + - name: Install Gambit run: | wget -O gambit https://github.com/Certora/gambit/releases/download/v1.0.5/gambit-linux-v1.0.5 From 9a3db2107ca2be6b2d2e6a3d69467edc05f002a1 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Mon, 12 Aug 2024 16:23:40 +0200 Subject: [PATCH 07/19] Debug solc --- .github/workflows/gambit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 5634213e8..8dc0075ab 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -36,6 +36,9 @@ jobs: - name: Install packages run: yarn + - name: Verify solc Installation + run: solc --version + - name: Run mutation tests run: | yarn test:mutation From 97dd42ebc7149573f6c32cfe1eb0b67426f35ffc Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Tue, 13 Aug 2024 11:50:31 +0200 Subject: [PATCH 08/19] Install solc --- .github/workflows/gambit.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 8dc0075ab..2b74e89fd 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -36,6 +36,12 @@ jobs: - name: Install packages run: yarn + - name: Install solc-select + run: | + pip install solc-select + solc-select install 0.8.16 + solc-select use 0.8.16 + - name: Verify solc Installation run: solc --version From 17397bf47598c3a45ebe824b1c4976c0f216aa8d Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Tue, 13 Aug 2024 12:05:09 +0200 Subject: [PATCH 09/19] Exit non-zero if any mutants survived --- test-mutation/gambitTester.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 495f07482..165cfa708 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -61,6 +61,11 @@ async function runMutationTesting() { console.log( `\n====== Done in ${((endTime - startTime) / (60 * 1000)).toFixed(2)} min` ) + + // Exit with non-zero if any mutants survived + if (results.some(result => result.status === MutantStatus.SURVIVED)) { + process.exit(1) + } } async function _generateMutants(): Promise { From f420936c6f86870dc373cc2adb966d4f408500fa Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Tue, 13 Aug 2024 12:11:38 +0200 Subject: [PATCH 10/19] Add more files to config and use rand seed --- test-mutation/config.json | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test-mutation/config.json b/test-mutation/config.json index a907d28c5..aef8d2450 100644 --- a/test-mutation/config.json +++ b/test-mutation/config.json @@ -6,6 +6,27 @@ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" ], - "num_mutants": 3 + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/arbitrum/gateway/L2CustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ERC20Gateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true } ] From 6a875997710fa92d54c13158a2244d0ecf60e7ce Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Tue, 13 Aug 2024 12:29:46 +0200 Subject: [PATCH 11/19] Prepare CI specific config file --- .github/workflows/gambit.yml | 4 +- test-mutation/config-templates/config-ci.json | 132 ++++++++++++++++++ .../config.single.json | 0 .../config.tokenbridge-arbitrum.json | 0 .../config.tokenbridge-ethereum.json | 0 test-mutation/config.json | 24 +--- 6 files changed, 135 insertions(+), 25 deletions(-) create mode 100644 test-mutation/config-templates/config-ci.json rename test-mutation/{all-configs => config-templates}/config.single.json (100%) rename test-mutation/{all-configs => config-templates}/config.tokenbridge-arbitrum.json (100%) rename test-mutation/{all-configs => config-templates}/config.tokenbridge-ethereum.json (100%) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 2b74e89fd..a07ff9e45 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -42,8 +42,8 @@ jobs: solc-select install 0.8.16 solc-select use 0.8.16 - - name: Verify solc Installation - run: solc --version + - name: Set config file + run: cp ./test-mutation/config-templates/config-ci.json ./test-mutation/config.json - name: Run mutation tests run: | diff --git a/test-mutation/config-templates/config-ci.json b/test-mutation/config-templates/config-ci.json new file mode 100644 index 000000000..716e935c6 --- /dev/null +++ b/test-mutation/config-templates/config-ci.json @@ -0,0 +1,132 @@ +[ + { + "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1CustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ERC20Gateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ForceOnlyReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1GatewayRouter.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitERC20Gateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitGatewayRouter.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1WethGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + } +] diff --git a/test-mutation/all-configs/config.single.json b/test-mutation/config-templates/config.single.json similarity index 100% rename from test-mutation/all-configs/config.single.json rename to test-mutation/config-templates/config.single.json diff --git a/test-mutation/all-configs/config.tokenbridge-arbitrum.json b/test-mutation/config-templates/config.tokenbridge-arbitrum.json similarity index 100% rename from test-mutation/all-configs/config.tokenbridge-arbitrum.json rename to test-mutation/config-templates/config.tokenbridge-arbitrum.json diff --git a/test-mutation/all-configs/config.tokenbridge-ethereum.json b/test-mutation/config-templates/config.tokenbridge-ethereum.json similarity index 100% rename from test-mutation/all-configs/config.tokenbridge-ethereum.json rename to test-mutation/config-templates/config.tokenbridge-ethereum.json diff --git a/test-mutation/config.json b/test-mutation/config.json index aef8d2450..efdf720af 100644 --- a/test-mutation/config.json +++ b/test-mutation/config.json @@ -5,28 +5,6 @@ "solc_remappings": [ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" - ], - "num_mutants": 4, - "random_seed": true - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2CustomGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ], - "num_mutants": 4, - "random_seed": true - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ERC20Gateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ], - "num_mutants": 4, - "random_seed": true + ] } ] From b9b143e17d104c091ba9af7cf36dbd99c36ba3f8 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Tue, 13 Aug 2024 13:02:41 +0200 Subject: [PATCH 12/19] Print mutant overview --- test-mutation/config.json | 16 ++++++++++++++-- test-mutation/gambitTester.ts | 5 ++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/test-mutation/config.json b/test-mutation/config.json index efdf720af..e5a290ade 100644 --- a/test-mutation/config.json +++ b/test-mutation/config.json @@ -1,10 +1,22 @@ [ { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ArbitrumGateway.sol", + "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol", "sourceroot": "..", "solc_remappings": [ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" - ] + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true } ] diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 165cfa708..4cc12c188 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -46,7 +46,7 @@ async function runMutationTesting() { console.log('====== Generating mutants') const mutants: Mutant[] = await _generateMutants() - console.log('\n====== Test mutants') + console.log('\n====== Testing mutants') const results = await _testAllMutants(mutants) // Print summary @@ -74,6 +74,9 @@ async function _generateMutants(): Promise { fs.readFileSync(`${GAMBIT_OUT}/gambit_results.json`, 'utf8') ) console.log(`Generated ${mutants.length} mutants in ${GAMBIT_OUT}`) + console.log('----------------------------------------------') + console.log('Mutants overview:') + console.log(fs.readFileSync(`${GAMBIT_OUT}/mutants.log`, 'utf8')) return mutants } From 328602aaac3f6a9566c2a814dc8be55664d43e45 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Fri, 16 Aug 2024 16:51:33 +0200 Subject: [PATCH 13/19] Add helper func to collect sol files for testing --- test-mutation/gambitTester.ts | 42 +++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 4cc12c188..3b2a88a1d 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -5,9 +5,11 @@ import os from 'os' import * as path from 'path' import * as fs from 'fs' import * as fsExtra from 'fs-extra' +import { readdir } from 'node:fs/promises' const GAMBIT_OUT = 'gambit_out/' -const TEST_TIMES = [ +const CONTRACTS_DIR = 'contracts' +const TEST_ITEMS = [ 'contracts', 'foundry.toml', 'remappings.txt', @@ -104,7 +106,7 @@ async function _testMutant(mutant: Mutant): Promise { await fsExtra.ensureDir(testDirectory) // copy necessary files - for (const item of TEST_TIMES) { + for (const item of TEST_ITEMS) { const sourcePath = path.join(__dirname, '..', item) const destPath = path.join(testDirectory, item) await fsExtra.copy(sourcePath, destPath) @@ -186,3 +188,39 @@ function _printResults(results: TestResult[]) { console.log(`Killed: ${killedCount} (${killedPercentage}%)`) console.log(`Survived: ${survivedCount} (${survivedPercentage}%)`) } + +async function _generateConfigForGithubCI() { + const solidityDirs = [ + path.join(__dirname, '../contracts/tokenbridge/ethereum'), + path.join(__dirname, '../contracts/tokenbridge/arbitrum'), + path.join(__dirname, '../contracts/tokenbridge/libraries'), + ] + const solidityFiles = await _findSolidityFiles(solidityDirs) + + console.log(solidityFiles) +} + +async function _findSolidityFiles(directories: string[]): Promise { + const solidityFiles: string[] = [] + + async function exploreDirectory(dir: string): Promise { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name) + if (entry.isDirectory()) { + // Recursively explore subdirectory + await exploreDirectory(fullPath) + } else if (entry.isFile() && path.extname(fullPath) === '.sol') { + // If it's a .sol file, add to the list + solidityFiles.push(fullPath) + } + } + } + + for (const directory of directories) { + await exploreDirectory(directory) + } + + return solidityFiles +} From 847c436b9d2eb6fb3c72b55bfa0e8f5e2135f336 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Fri, 16 Aug 2024 17:09:15 +0200 Subject: [PATCH 14/19] Generate the config from collected files --- test-mutation/gambitTester.ts | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 3b2a88a1d..68d794394 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -5,10 +5,8 @@ import os from 'os' import * as path from 'path' import * as fs from 'fs' import * as fsExtra from 'fs-extra' -import { readdir } from 'node:fs/promises' const GAMBIT_OUT = 'gambit_out/' -const CONTRACTS_DIR = 'contracts' const TEST_ITEMS = [ 'contracts', 'foundry.toml', @@ -45,6 +43,8 @@ runMutationTesting().catch(error => { async function runMutationTesting() { const startTime = Date.now() + await _generateConfigForGithubCI() + console.log('====== Generating mutants') const mutants: Mutant[] = await _generateMutants() @@ -189,6 +189,12 @@ function _printResults(results: TestResult[]) { console.log(`Survived: ${survivedCount} (${survivedPercentage}%)`) } +//////////////////////////////////////////////////////////////////// + +// GAMBIT CONFIG GENERATION + +//////////////////////////////////////////////////////////////////// + async function _generateConfigForGithubCI() { const solidityDirs = [ path.join(__dirname, '../contracts/tokenbridge/ethereum'), @@ -197,7 +203,26 @@ async function _generateConfigForGithubCI() { ] const solidityFiles = await _findSolidityFiles(solidityDirs) - console.log(solidityFiles) + // Construct the JSON array + const jsonConfig = solidityFiles.map(file => ({ + filename: file, + sourceroot: '..', + solc_remappings: [ + '@openzeppelin=../node_modules/@openzeppelin', + '@arbitrum=../node_modules/@arbitrum', + ], + num_mutants: 4, + random_seed: true, + })) + + // Write the result to a JSON file + const outputFilePath = path.join(__dirname, 'config_for_github_ci.json') + await fs.promises.writeFile( + outputFilePath, + JSON.stringify(jsonConfig, null, 2), + 'utf-8' + ) + console.log(`Generated JSON file: ${outputFilePath}`) } async function _findSolidityFiles(directories: string[]): Promise { @@ -212,8 +237,8 @@ async function _findSolidityFiles(directories: string[]): Promise { // Recursively explore subdirectory await exploreDirectory(fullPath) } else if (entry.isFile() && path.extname(fullPath) === '.sol') { - // If it's a .sol file, add to the list - solidityFiles.push(fullPath) + const relativePath = path.relative(__dirname, fullPath) + solidityFiles.push(relativePath) } } } From fe530c202fa11cc650b155a2fd17240a9b718035 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Wed, 21 Aug 2024 11:13:31 +0200 Subject: [PATCH 15/19] Use constants --- test-mutation/gambitTester.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 68d794394..d9b3c503d 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -7,6 +7,9 @@ import * as fs from 'fs' import * as fsExtra from 'fs-extra' const GAMBIT_OUT = 'gambit_out/' +const NUM_OF_MUTANTS_PER_FILE = 3 +const CONFIG_FOR_CI_FILE = 'config_for_github_ci.json' + const TEST_ITEMS = [ 'contracts', 'foundry.toml', @@ -211,12 +214,12 @@ async function _generateConfigForGithubCI() { '@openzeppelin=../node_modules/@openzeppelin', '@arbitrum=../node_modules/@arbitrum', ], - num_mutants: 4, + num_mutants: NUM_OF_MUTANTS_PER_FILE, random_seed: true, })) // Write the result to a JSON file - const outputFilePath = path.join(__dirname, 'config_for_github_ci.json') + const outputFilePath = path.join(__dirname, CONFIG_FOR_CI_FILE) await fs.promises.writeFile( outputFilePath, JSON.stringify(jsonConfig, null, 2), From cf405c8cfe104620dcdc09bd8de6b0ed85179ec9 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Wed, 21 Aug 2024 11:27:01 +0200 Subject: [PATCH 16/19] Generate specific gambit config for CI --- .github/workflows/gambit.yml | 7 +-- package.json | 1 + test-mutation/gambitTester.ts | 65 ----------------------- test-mutation/generateGambitConfig.ts | 75 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 70 deletions(-) create mode 100644 test-mutation/generateGambitConfig.ts diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index a07ff9e45..336fa11b2 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -30,9 +30,6 @@ jobs: chmod +x gambit sudo mv gambit /usr/local/bin/ - - name: Verify Gambit Installation - run: gambit --help - - name: Install packages run: yarn @@ -42,8 +39,8 @@ jobs: solc-select install 0.8.16 solc-select use 0.8.16 - - name: Set config file - run: cp ./test-mutation/config-templates/config-ci.json ./test-mutation/config.json + - name: Prepare config file + run: yarn prepare:mutation:ci-config - name: Run mutation tests run: | diff --git a/package.json b/package.json index 2714ab098..b3c62eef7 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "test:e2e:local-env": "yarn hardhat test test-e2e/*", "test:storage": "./scripts/storage_layout_test.bash", "test:signatures": "./scripts/signatures_test.bash", + "prepare:mutation:ci-config": "ts-node test-mutation/generateGambitConfig.ts", "test:mutation": "ts-node test-mutation/gambitTester.ts", "test:unused:errors": "./test/unused-errors/find_unused_errors.sh", "deploy:local:token-bridge": "ts-node ./scripts/local-deployment/deployCreatorAndCreateTokenBridge.ts", diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index d9b3c503d..4c0df19b6 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -7,8 +7,6 @@ import * as fs from 'fs' import * as fsExtra from 'fs-extra' const GAMBIT_OUT = 'gambit_out/' -const NUM_OF_MUTANTS_PER_FILE = 3 -const CONFIG_FOR_CI_FILE = 'config_for_github_ci.json' const TEST_ITEMS = [ 'contracts', @@ -46,8 +44,6 @@ runMutationTesting().catch(error => { async function runMutationTesting() { const startTime = Date.now() - await _generateConfigForGithubCI() - console.log('====== Generating mutants') const mutants: Mutant[] = await _generateMutants() @@ -191,64 +187,3 @@ function _printResults(results: TestResult[]) { console.log(`Killed: ${killedCount} (${killedPercentage}%)`) console.log(`Survived: ${survivedCount} (${survivedPercentage}%)`) } - -//////////////////////////////////////////////////////////////////// - -// GAMBIT CONFIG GENERATION - -//////////////////////////////////////////////////////////////////// - -async function _generateConfigForGithubCI() { - const solidityDirs = [ - path.join(__dirname, '../contracts/tokenbridge/ethereum'), - path.join(__dirname, '../contracts/tokenbridge/arbitrum'), - path.join(__dirname, '../contracts/tokenbridge/libraries'), - ] - const solidityFiles = await _findSolidityFiles(solidityDirs) - - // Construct the JSON array - const jsonConfig = solidityFiles.map(file => ({ - filename: file, - sourceroot: '..', - solc_remappings: [ - '@openzeppelin=../node_modules/@openzeppelin', - '@arbitrum=../node_modules/@arbitrum', - ], - num_mutants: NUM_OF_MUTANTS_PER_FILE, - random_seed: true, - })) - - // Write the result to a JSON file - const outputFilePath = path.join(__dirname, CONFIG_FOR_CI_FILE) - await fs.promises.writeFile( - outputFilePath, - JSON.stringify(jsonConfig, null, 2), - 'utf-8' - ) - console.log(`Generated JSON file: ${outputFilePath}`) -} - -async function _findSolidityFiles(directories: string[]): Promise { - const solidityFiles: string[] = [] - - async function exploreDirectory(dir: string): Promise { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }) - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name) - if (entry.isDirectory()) { - // Recursively explore subdirectory - await exploreDirectory(fullPath) - } else if (entry.isFile() && path.extname(fullPath) === '.sol') { - const relativePath = path.relative(__dirname, fullPath) - solidityFiles.push(relativePath) - } - } - } - - for (const directory of directories) { - await exploreDirectory(directory) - } - - return solidityFiles -} diff --git a/test-mutation/generateGambitConfig.ts b/test-mutation/generateGambitConfig.ts new file mode 100644 index 000000000..d244ce17f --- /dev/null +++ b/test-mutation/generateGambitConfig.ts @@ -0,0 +1,75 @@ +import * as path from 'path' +import * as fs from 'fs' + +const NUM_OF_MUTANTS_PER_FILE = 3 +const CONFIG_FOR_CI_FILE = 'config_for_github_ci.json' + +genGambitConfigForCI().catch(error => { + console.error('Error while generating Galbit config for CI:', error) +}) + +async function genGambitConfigForCI() { + // Generate the JSON config file to config_for_github_ci.json + await _generateConfigForGithubCI() + + // Move config to config.json + const src = path.join(__dirname, CONFIG_FOR_CI_FILE) + const dest = path.join(__dirname, 'config.json') + await fs.promises.rename(src, dest) + console.log(`Moved ${src} to ${dest}`) +} + +async function _generateConfigForGithubCI() { + const solidityDirs = [ + path.join(__dirname, '../contracts/tokenbridge/ethereum'), + path.join(__dirname, '../contracts/tokenbridge/arbitrum'), + path.join(__dirname, '../contracts/tokenbridge/libraries'), + ] + const solidityFiles = await _findSolidityFiles(solidityDirs) + + // Construct the JSON array + const jsonConfig = solidityFiles.map(file => ({ + filename: file, + sourceroot: '..', + solc_remappings: [ + '@openzeppelin=../node_modules/@openzeppelin', + '@arbitrum=../node_modules/@arbitrum', + ], + num_mutants: NUM_OF_MUTANTS_PER_FILE, + random_seed: true, + })) + + // Write the result to a JSON file + const outputFilePath = path.join(__dirname, CONFIG_FOR_CI_FILE) + await fs.promises.writeFile( + outputFilePath, + JSON.stringify(jsonConfig, null, 2), + 'utf-8' + ) + console.log(`Generated JSON file: ${outputFilePath}`) +} + +async function _findSolidityFiles(directories: string[]): Promise { + const solidityFiles: string[] = [] + + async function exploreDirectory(dir: string): Promise { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name) + if (entry.isDirectory()) { + // Recursively explore subdirectory + await exploreDirectory(fullPath) + } else if (entry.isFile() && path.extname(fullPath) === '.sol') { + const relativePath = path.relative(__dirname, fullPath) + solidityFiles.push(relativePath) + } + } + } + + for (const directory of directories) { + await exploreDirectory(directory) + } + + return solidityFiles +} From 05e8898d7c14d759e8da7d70e4674eac5f69c9db Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Wed, 21 Aug 2024 14:15:08 +0200 Subject: [PATCH 17/19] Only manual trigger --- .github/workflows/gambit.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index 336fa11b2..a9c216091 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -2,7 +2,6 @@ name: Gambit mutation testing on: workflow_dispatch: - pull_request: jobs: run-mutation-tests: From 9896960c9664e7f6180be6ad01897033336b1fbc Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Wed, 21 Aug 2024 14:20:55 +0200 Subject: [PATCH 18/19] 4 mutants per file --- test-mutation/generateGambitConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-mutation/generateGambitConfig.ts b/test-mutation/generateGambitConfig.ts index d244ce17f..0ac0c2ecd 100644 --- a/test-mutation/generateGambitConfig.ts +++ b/test-mutation/generateGambitConfig.ts @@ -1,7 +1,7 @@ import * as path from 'path' import * as fs from 'fs' -const NUM_OF_MUTANTS_PER_FILE = 3 +const NUM_OF_MUTANTS_PER_FILE = 4 const CONFIG_FOR_CI_FILE = 'config_for_github_ci.json' genGambitConfigForCI().catch(error => { From 8d034d7ec6262877d9f5c095e43829646b8f7419 Mon Sep 17 00:00:00 2001 From: Goran Vladika Date: Wed, 21 Aug 2024 14:25:47 +0200 Subject: [PATCH 19/19] Format --- .github/workflows/gambit.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml index a9c216091..46c9c3964 100644 --- a/.github/workflows/gambit.yml +++ b/.github/workflows/gambit.yml @@ -42,5 +42,4 @@ jobs: run: yarn prepare:mutation:ci-config - name: Run mutation tests - run: | - yarn test:mutation + run: yarn test:mutation