Add Gambit mutation test to CI

.github/workflows/gambit.yml

@@ -0,0 +1,45 @@
+name: Gambit mutation testing
+
+on:
+  workflow_dispatch:
+
+jobs:
+  run-mutation-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup node/yarn
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+        with:
+          version: nightly
+
+      - name: Install Gambit
+        run: |
+          wget -O gambit https://github.com/Certora/gambit/releases/download/v1.0.5/gambit-linux-v1.0.5
+          chmod +x gambit
+          sudo mv gambit /usr/local/bin/
+
+      - name: Install packages
+        run: yarn
+
+      - name: Install solc-select
+        run: |
+          pip install solc-select
+          solc-select install 0.8.16
+          solc-select use 0.8.16
+
+      - name: Prepare config file
+        run: yarn prepare:mutation:ci-config
+
+      - name: Run mutation tests
+        run: yarn test:mutation

package.json

@@ -21,6 +21,7 @@
     "test:e2e:local-env": "yarn hardhat test test-e2e/*",
     "test:storage": "./scripts/storage_layout_test.bash",
     "test:signatures": "./scripts/signatures_test.bash",
+    "prepare:mutation:ci-config": "ts-node test-mutation/generateGambitConfig.ts",
     "test:mutation": "ts-node test-mutation/gambitTester.ts",
     "test:unused:errors": "./test/unused-errors/find_unused_errors.sh",
     "deploy:local:token-bridge": "ts-node ./scripts/local-deployment/deployCreatorAndCreateTokenBridge.ts",

test-mutation/config-templates/config-ci.json

@@ -0,0 +1,132 @@
+[
+  {
+    "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1CustomGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ERC20Gateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ForceOnlyReverseCustomGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1GatewayRouter.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitCustomGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitERC20Gateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitGatewayRouter.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1OrbitReverseCustomGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ReverseCustomGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  },
+  {
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1WethGateway.sol",
+    "sourceroot": "..",
+    "solc_remappings": [
+      "@openzeppelin=../node_modules/@openzeppelin",
+      "@arbitrum=../node_modules/@arbitrum"
+    ],
+    "num_mutants": 4,
+    "random_seed": true
+  }
+]

test-mutation/config.json

@@ -1,82 +1,22 @@
 [
   {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ArbitrumGateway.sol",
+    "filename": "../contracts/tokenbridge/ethereum/L1ArbitrumMessenger.sol",
     "sourceroot": "..",
     "solc_remappings": [
       "@openzeppelin=../node_modules/@openzeppelin",
       "@arbitrum=../node_modules/@arbitrum"
-    ]
+    ],
+    "num_mutants": 4,
+    "random_seed": true
   },
   {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2CustomGateway.sol",
+    "filename": "../contracts/tokenbridge/ethereum/gateway/L1ArbitrumExtendedGateway.sol",
     "sourceroot": "..",
     "solc_remappings": [
       "@openzeppelin=../node_modules/@openzeppelin",
       "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ERC20Gateway.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2GatewayRouter.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2ReverseCustomGateway.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/gateway/L2WethGateway.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/L2ArbitrumMessenger.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/L2AtomicTokenBridgeFactory.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/ReverseArbToken.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
-  },
-  {
-    "filename": "../contracts/tokenbridge/arbitrum/StandardArbERC20.sol",
-    "sourceroot": "..",
-    "solc_remappings": [
-      "@openzeppelin=../node_modules/@openzeppelin",
-      "@arbitrum=../node_modules/@arbitrum"
-    ]
+    ],
+    "num_mutants": 4,
+    "random_seed": true
   }
 ]

test-mutation/gambitTester.ts

@@ -7,7 +7,8 @@ import * as fs from 'fs'
 import * as fsExtra from 'fs-extra'
 
 const GAMBIT_OUT = 'gambit_out/'
-const TEST_TIMES = [
+
+const TEST_ITEMS = [
   'contracts',
   'foundry.toml',
   'remappings.txt',
@@ -46,7 +47,7 @@ async function runMutationTesting() {
   console.log('====== Generating mutants')
   const mutants: Mutant[] = await _generateMutants()
 
-  console.log('\n====== Test mutants')
+  console.log('\n====== Testing mutants')
   const results = await _testAllMutants(mutants)
 
   // Print summary
@@ -61,6 +62,11 @@ async function runMutationTesting() {
   console.log(
     `\n====== Done in ${((endTime - startTime) / (60 * 1000)).toFixed(2)} min`
   )
+
+  // Exit with non-zero if any mutants survived
+  if (results.some(result => result.status === MutantStatus.SURVIVED)) {
+    process.exit(1)
+  }
 }
 
 async function _generateMutants(): Promise<Mutant[]> {
@@ -69,6 +75,9 @@ async function _generateMutants(): Promise<Mutant[]> {
     fs.readFileSync(`${GAMBIT_OUT}/gambit_results.json`, 'utf8')
   )
   console.log(`Generated ${mutants.length} mutants in ${GAMBIT_OUT}`)
+  console.log('----------------------------------------------')
+  console.log('Mutants overview:')
+  console.log(fs.readFileSync(`${GAMBIT_OUT}/mutants.log`, 'utf8'))
 
   return mutants
 }
@@ -96,7 +105,7 @@ async function _testMutant(mutant: Mutant): Promise<TestResult> {
   await fsExtra.ensureDir(testDirectory)
 
   // copy necessary files
-  for (const item of TEST_TIMES) {
+  for (const item of TEST_ITEMS) {
     const sourcePath = path.join(__dirname, '..', item)
     const destPath = path.join(testDirectory, item)
     await fsExtra.copy(sourcePath, destPath)