diff --git a/.github/workflows/gambit.yml b/.github/workflows/gambit.yml new file mode 100644 index 000000000..46c9c3964 --- /dev/null +++ b/.github/workflows/gambit.yml @@ -0,0 +1,45 @@ +name: Gambit mutation testing + +on: + workflow_dispatch: + +jobs: + run-mutation-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup node/yarn + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: '**/yarn.lock' + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly + + - name: Install Gambit + run: | + wget -O gambit https://github.com/Certora/gambit/releases/download/v1.0.5/gambit-linux-v1.0.5 + chmod +x gambit + sudo mv gambit /usr/local/bin/ + + - name: Install packages + run: yarn + + - name: Install solc-select + run: | + pip install solc-select + solc-select install 0.8.16 + solc-select use 0.8.16 + + - name: Prepare config file + run: yarn prepare:mutation:ci-config + + - name: Run mutation tests + run: yarn test:mutation diff --git a/package.json b/package.json index 2714ab098..b3c62eef7 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "test:e2e:local-env": "yarn hardhat test test-e2e/*", "test:storage": "./scripts/storage_layout_test.bash", "test:signatures": "./scripts/signatures_test.bash", + "prepare:mutation:ci-config": "ts-node test-mutation/generateGambitConfig.ts", "test:mutation": "ts-node test-mutation/gambitTester.ts", "test:unused:errors": "./test/unused-errors/find_unused_errors.sh", "deploy:local:token-bridge": "ts-node ./scripts/local-deployment/deployCreatorAndCreateTokenBridge.ts", diff --git a/test-mutation/config-templates/config-ci.json b/test-mutation/config-templates/config-ci.json new file mode 100644 index 000000000..716e935c6 --- /dev/null +++ b/test-mutation/config-templates/config-ci.json @@ -0,0 +1,132 @@ +[ + { + "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1CustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ERC20Gateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ForceOnlyReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1GatewayRouter.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitERC20Gateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitGatewayRouter.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ReverseCustomGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + }, + { + "filename": "../contracts/tokenbridge/ethereum/gateway/L1WethGateway.sol", + "sourceroot": "..", + "solc_remappings": [ + "@openzeppelin=../node_modules/@openzeppelin", + "@arbitrum=../node_modules/@arbitrum" + ], + "num_mutants": 4, + "random_seed": true + } +] diff --git a/test-mutation/all-configs/config.single.json b/test-mutation/config-templates/config.single.json similarity index 100% rename from test-mutation/all-configs/config.single.json rename to test-mutation/config-templates/config.single.json diff --git a/test-mutation/all-configs/config.tokenbridge-arbitrum.json b/test-mutation/config-templates/config.tokenbridge-arbitrum.json similarity index 100% rename from test-mutation/all-configs/config.tokenbridge-arbitrum.json rename to test-mutation/config-templates/config.tokenbridge-arbitrum.json diff --git a/test-mutation/all-configs/config.tokenbridge-ethereum.json b/test-mutation/config-templates/config.tokenbridge-ethereum.json similarity index 100% rename from test-mutation/all-configs/config.tokenbridge-ethereum.json rename to test-mutation/config-templates/config.tokenbridge-ethereum.json diff --git a/test-mutation/config.json b/test-mutation/config.json index 85bc60d01..e5a290ade 100644 --- a/test-mutation/config.json +++ b/test-mutation/config.json @@ -1,82 +1,22 @@ [ { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ArbitrumGateway.sol", + "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol", "sourceroot": "..", "solc_remappings": [ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" - ] + ], + "num_mutants": 4, + "random_seed": true }, { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2CustomGateway.sol", + "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol", "sourceroot": "..", "solc_remappings": [ "@openzeppelin=../node_modules/@openzeppelin", "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ERC20Gateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2GatewayRouter.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ReverseCustomGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/gateway/L2WethGateway.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/L2ArbitrumMessenger.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/L2AtomicTokenBridgeFactory.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/ReverseArbToken.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] - }, - { - "filename": "../contracts/tokenbridge/arbitrum/StandardArbERC20.sol", - "sourceroot": "..", - "solc_remappings": [ - "@openzeppelin=../node_modules/@openzeppelin", - "@arbitrum=../node_modules/@arbitrum" - ] + ], + "num_mutants": 4, + "random_seed": true } ] diff --git a/test-mutation/gambitTester.ts b/test-mutation/gambitTester.ts index 495f07482..4c0df19b6 100644 --- a/test-mutation/gambitTester.ts +++ b/test-mutation/gambitTester.ts @@ -7,7 +7,8 @@ import * as fs from 'fs' import * as fsExtra from 'fs-extra' const GAMBIT_OUT = 'gambit_out/' -const TEST_TIMES = [ + +const TEST_ITEMS = [ 'contracts', 'foundry.toml', 'remappings.txt', @@ -46,7 +47,7 @@ async function runMutationTesting() { console.log('====== Generating mutants') const mutants: Mutant[] = await _generateMutants() - console.log('\n====== Test mutants') + console.log('\n====== Testing mutants') const results = await _testAllMutants(mutants) // Print summary @@ -61,6 +62,11 @@ async function runMutationTesting() { console.log( `\n====== Done in ${((endTime - startTime) / (60 * 1000)).toFixed(2)} min` ) + + // Exit with non-zero if any mutants survived + if (results.some(result => result.status === MutantStatus.SURVIVED)) { + process.exit(1) + } } async function _generateMutants(): Promise { @@ -69,6 +75,9 @@ async function _generateMutants(): Promise { fs.readFileSync(`${GAMBIT_OUT}/gambit_results.json`, 'utf8') ) console.log(`Generated ${mutants.length} mutants in ${GAMBIT_OUT}`) + console.log('----------------------------------------------') + console.log('Mutants overview:') + console.log(fs.readFileSync(`${GAMBIT_OUT}/mutants.log`, 'utf8')) return mutants } @@ -96,7 +105,7 @@ async function _testMutant(mutant: Mutant): Promise { await fsExtra.ensureDir(testDirectory) // copy necessary files - for (const item of TEST_TIMES) { + for (const item of TEST_ITEMS) { const sourcePath = path.join(__dirname, '..', item) const destPath = path.join(testDirectory, item) await fsExtra.copy(sourcePath, destPath) diff --git a/test-mutation/generateGambitConfig.ts b/test-mutation/generateGambitConfig.ts new file mode 100644 index 000000000..0ac0c2ecd --- /dev/null +++ b/test-mutation/generateGambitConfig.ts @@ -0,0 +1,75 @@ +import * as path from 'path' +import * as fs from 'fs' + +const NUM_OF_MUTANTS_PER_FILE = 4 +const CONFIG_FOR_CI_FILE = 'config_for_github_ci.json' + +genGambitConfigForCI().catch(error => { + console.error('Error while generating Galbit config for CI:', error) +}) + +async function genGambitConfigForCI() { + // Generate the JSON config file to config_for_github_ci.json + await _generateConfigForGithubCI() + + // Move config to config.json + const src = path.join(__dirname, CONFIG_FOR_CI_FILE) + const dest = path.join(__dirname, 'config.json') + await fs.promises.rename(src, dest) + console.log(`Moved ${src} to ${dest}`) +} + +async function _generateConfigForGithubCI() { + const solidityDirs = [ + path.join(__dirname, '../contracts/tokenbridge/ethereum'), + path.join(__dirname, '../contracts/tokenbridge/arbitrum'), + path.join(__dirname, '../contracts/tokenbridge/libraries'), + ] + const solidityFiles = await _findSolidityFiles(solidityDirs) + + // Construct the JSON array + const jsonConfig = solidityFiles.map(file => ({ + filename: file, + sourceroot: '..', + solc_remappings: [ + '@openzeppelin=../node_modules/@openzeppelin', + '@arbitrum=../node_modules/@arbitrum', + ], + num_mutants: NUM_OF_MUTANTS_PER_FILE, + random_seed: true, + })) + + // Write the result to a JSON file + const outputFilePath = path.join(__dirname, CONFIG_FOR_CI_FILE) + await fs.promises.writeFile( + outputFilePath, + JSON.stringify(jsonConfig, null, 2), + 'utf-8' + ) + console.log(`Generated JSON file: ${outputFilePath}`) +} + +async function _findSolidityFiles(directories: string[]): Promise { + const solidityFiles: string[] = [] + + async function exploreDirectory(dir: string): Promise { + const entries = await fs.promises.readdir(dir, { withFileTypes: true }) + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name) + if (entry.isDirectory()) { + // Recursively explore subdirectory + await exploreDirectory(fullPath) + } else if (entry.isFile() && path.extname(fullPath) === '.sol') { + const relativePath = path.relative(__dirname, fullPath) + solidityFiles.push(relativePath) + } + } + } + + for (const directory of directories) { + await exploreDirectory(directory) + } + + return solidityFiles +}