cleanup

cardano-testnet/src/Testnet/Components/Configuration.hs

@@ -168,7 +168,7 @@ createSPOGenesisAndFiles numTestnetNodes numDelReps maxSupply sbe shelleyGenesis
     , "--spec-conway",  inputGenesisConwayFp
     , "--testnet-magic", show testnetMagic
     , "--pools", show numTestnetNodes
-    , "--total-supply",     show maxSupply
+    , "--total-supply",     show maxSupply -- Half of this will be delegated, see https://github.com/IntersectMBO/cardano-cli/pull/874
     , "--stake-delegators", show numStakeDelegators
     , "--utxo-keys", show numSeededUTxOKeys
     , "--drep-keys", show numDelReps

cardano-testnet/src/Testnet/Defaults.hs

@@ -385,7 +385,7 @@ defaultShelleyGenesis asbe startTime maxSupply options = do
       activeSlotsCoeff = round (genesisActiveSlotsCoeff * 100) % 100
       -- make security param k satisfy: epochLength = 10 * k / f
       -- TODO: find out why this actually degrates network stability - turned off for now
-      -- securityParam = ceiling $ fromIntegral epochLength * shelleyActiveSlotsCoeff / 10
+      -- securityParam = ceiling $ fromIntegral epochLength * cardanoActiveSlotsCoeff / 10
       pVer = eraToProtocolVersion asbe
       protocolParams = Api.sgProtocolParams Api.shelleyGenesisDefaults
       protocolParamsWithPVer = protocolParams & ppProtocolVersionL' .~ pVer
@@ -480,15 +480,19 @@ defaultSpoColdVKeyFp n = defaultSpoKeysDir n </> "cold.vkey"
 defaultSpoColdSKeyFp :: Int -> FilePath
 defaultSpoColdSKeyFp n = defaultSpoKeysDir n </> "cold.skey"
 
+-- | The name of a SPO, used in file system operations
 defaultSpoName :: Int -> String
 defaultSpoName n = "pool" <> show n
 
+-- | The name of a node (which doesn't have to be a SPO)
 defaultNodeName :: Int -> String
 defaultNodeName n = "node" <> show n
 
+-- | The relative path of the node data dir, where the database is stored
 defaultNodeDataDir :: Int -> String
 defaultNodeDataDir n = "node-data" </> defaultNodeName n
 
+-- | The relative path where the SPO keys for the node are stored
 defaultSpoKeysDir :: Int -> String
 defaultSpoKeysDir n = "pools-keys" </> defaultSpoName n
 

cardano-testnet/src/Testnet/Start/Cardano.hs

@@ -134,54 +134,73 @@ getDefaultShelleyGenesis asbe maxSupply opts = do
   startTime <- H.noteShow $ DTC.addUTCTime startTimeOffsetSeconds currentTime
   return $ Defaults.defaultShelleyGenesis asbe startTime maxSupply opts
 
--- | Setup a number of credentials and pools, like this:
+-- | Setup a number of credentials and nodes, like this:
 --
 -- > ├── byron
 -- > │   └── genesis.json
 -- > ├── byron-gen-command
--- > │   └── genesis-keys.00{0,1,2}.key
--- > ├── byron.genesis.spec.json
--- > ├── configuration.yaml
--- > ├── current-stake-pools.json
+-- > │   ├── delegate-keys.00{1,2}.key
+-- > │   ├── delegation-cert.00{1,2}.json
+-- > │   ├── genesis-keys.00{0,1,2}.key
 -- > ├── delegate-keys
 -- > │   ├── delegate{1,2,3}
 -- > │   │   ├── kes.{skey,vkey}
 -- > │   │   ├── key.{skey,vkey}
 -- > │   │   ├── opcert.{cert,counter}
--- > │   │   └── vrf.{skey,vkey}
+-- > │   │   ├── vrf.{skey,vkey}
 -- > │   └── README.md
 -- > ├── drep-keys
--- > │   └── drep{1,2,3}
--- > │       └── drep.{skey,vkey}
--- > ├── genesis.{alonzo,conway}.spec.json
+-- > │   ├── drep{1,2,3}
+-- > │   │   ├── drep.{skey,drep.vkey}
+-- > │   └── README.md
 -- > ├── genesis-keys
 -- > │   ├── genesis{1,2,3}
--- > │   │   └── key.{skey,vkey}
+-- > │   │   ├── key.{skey,vkey}
+-- > │   │   └── key.vkey
 -- > │   └── README.md
+-- > ├── logs
+-- > │   ├── node{1,2,3}
+-- > │   │   ├── {stderr,stdout}.log
+-- > │   ├── ledger-epoch-state-diffs.log
+-- > │   ├── ledger-epoch-state.log
+-- > │   ├── node-20241010121635.log
+-- > │   └── node.log -> node-20241010121635.log
+-- > ├── node-data
+-- > │   ├── node{1,2,3}
+-- > │   │   ├── db
+-- > │   │   │   ├── <node database files>
+-- > │   │   ├── port
+-- > │   │   └── topology.json
 -- > ├── pools-keys
--- > │   ├── pool{1,2,3}
+-- > │   ├── pool1
 -- > │   │   ├── byron-delegate.key
 -- > │   │   ├── byron-delegation.cert
 -- > │   │   ├── cold.{skey,vkey}
 -- > │   │   ├── kes.{skey,vkey}
 -- > │   │   ├── opcert.{cert,counter}
 -- > │   │   ├── staking-reward.{skey,vkey}
--- > │   │   ├── topology.json
--- > │   │   └── vrf.{skey,vkey}
+-- > │   │   ├── vrf.{skey,vkey}
 -- > │   └── README.md
--- > ├── shelley
--- > │   └── genesis.{alonzo,conway,shelley}.json
 -- > ├── socket
--- > │   └── pool{1,2,3}
--- > │       └── sock
+-- > │   ├── node{1,2,3}
+-- > │   │   └── sock
 -- > ├── stake-delegators
--- > │   └── delegator{1,2,3}
--- > │       ├── payment.{skey,vkey}
--- > │       └── staking.{skey,vkey}
--- > └─── utxo-keys
--- >     ├── README.md
--- >     └── utxo{1,2,3}
--- >         └── utxo.{addr,skey,vkey}
+-- > │   ├── delegator{1,2,3}
+-- > │   │   ├── payment.{skey,vkey}
+-- > │   │   ├── staking.{skey,vkey}
+-- > ├── utxo-keys
+-- > │   ├── utxo{1,2,3}
+-- > │   │   ├── utxo.{addr,skey,vkey}
+-- > │   └── README.md
+-- > ├── alonzo-genesis.json
+-- > ├── byron.genesis.spec.json
+-- > ├── configuration.yaml
+-- > ├── conway-genesis.json
+-- > ├── current-stake-pools.json
+-- > ├── genesis.{alonzo,conway}.spec.json
+-- > ├── module
+-- > └── shelley-genesis.json
+--
 cardanoTestnet :: ()
   => HasCallStack
   => CardanoTestnetOptions -- ^ The options to use
@@ -205,14 +224,13 @@ cardanoTestnet
       startTime = sgSystemStart shelleyGenesis
       testnetMagic = fromIntegral $ sgNetworkMagic shelleyGenesis
       nPools = cardanoNumPools testnetOptions
-      nRelays = cardanoNumRelays testnetOptions
   AnyShelleyBasedEra sbe <- pure asbe
 
   testMinimumConfigurationRequirements testnetOptions
 
   H.note_ OS.os
 
-  when (all isJust [mconfig | TestnetNodeOptions _ mconfig _ <- cardanoNodes]) $
+  when (all isJust [mConfig | TestnetNodeOptions _ mConfig _ <- cardanoNodes]) $
     -- TODO: We need a very simple non-obscure way of generating the files necessary
     -- to run a testnet. "create-staked" is not a good way to do this especially because it
     -- makes assumptions about where things should go and where genesis template files should be.
@@ -283,14 +301,11 @@ cardanoTestnet
   config <- createConfigJson (TmpAbsolutePath tmpAbsPath) sbe
   H.evalIO $ LBS.writeFile (unFile configurationFile) config
 
-  let nodesRoles = replicate (fromIntegral nPools) TestnetNodeRoleSpo
-                   <> replicate (fromIntegral nRelays) TestnetNodeRoleRelay
-
-  portNumbersWithNodeRoles <- forM nodesRoles $ \nodeRole -> (nodeRole,) <$> H.randomPort testnetDefaultIpv4Address
-  let portNumbers = snd <$> portNumbersWithNodeRoles
+  portNumbersWithNodeOptions <- forM cardanoNodes $ \nodeOption -> (nodeOption,) <$> H.randomPort testnetDefaultIpv4Address
+  let portNumbers = snd <$> portNumbersWithNodeOptions
 
   -- Byron related
-  forM_ (zip [1..] portNumbersWithNodeRoles) $ \(i, (nodeRole, portNumber)) -> do
+  forM_ (zip [1..] portNumbersWithNodeOptions) $ \(i, (TestnetNodeOptions nodeRole _ _, portNumber)) -> do
     let iStr = printf "%03d" (i - 1)
         nodeDataDir = tmpAbsPath </> Defaults.defaultNodeDataDir i
         nodePoolKeysDir = tmpAbsPath </> Defaults.defaultSpoKeysDir i
@@ -313,8 +328,7 @@ cardanoTestnet
     H.lbsWriteFile (tmpAbsPath </> Defaults.defaultNodeDataDir i </> "topology.json") . encode $
       RealNodeTopology producers
 
-  let keysWithPorts = zip [1..] portNumbersWithNodeRoles
-  eTestnetNodes <- H.forConcurrently keysWithPorts $ \(i, (nodeRole, port)) -> do
+  eTestnetNodes <- H.forConcurrently (zip [1..] portNumbersWithNodeOptions) $ \(i, (TestnetNodeOptions nodeRole _ extraNodeCliArgs', port)) -> do
     let nodeName = Defaults.defaultNodeName i
         nodeDataDir = tmpAbsPath </> Defaults.defaultNodeDataDir i
         nodePoolKeysDir = tmpAbsPath </> Defaults.defaultSpoKeysDir i
@@ -324,6 +338,7 @@ cardanoTestnet
             TestnetNodeRoleRelay -> (Nothing, [])
             TestnetNodeRoleSpo -> do
               let keys@TestnetNodeKeys{poolNodeKeysVrf} = mkTestnetNodeKeyPaths i
+                  -- provide keys' locations for SPO nodes
                   cliArgs =
                     [ "--shelley-kes-key", nodePoolKeysDir </> "kes.skey"
                     , "--shelley-vrf-key", unFile $ signingKey poolNodeKeysVrf
@@ -339,7 +354,9 @@ cardanoTestnet
         , "--config", unFile configurationFile
         , "--topology", nodeDataDir </> "topology.json"
         , "--database-path", nodeDataDir </> "db"
-        ] <> spoNodeCliArgs
+        ]
+        <> spoNodeCliArgs
+        <> extraNodeCliArgs'
     pure $ flip TestnetNode mKeys <$> eRuntime
 
   let (failedNodes, testnetNodes') = partitionEithers eTestnetNodes

cardano-testnet/src/Testnet/Start/Types.hs

@@ -120,22 +120,22 @@ instance Default GenesisOptions where
     , genesisActiveSlotsCoeff = 0.05
     }
 
--- | Specify a SPO (Shelley era onwards only) or Relay node
+-- | Specify a SPO (Shelley era onwards only) or a Relay node
 data TestnetNodeOptions
   = TestnetNodeOptions TestnetNodeRole (Maybe NodeConfigurationYaml) [String]
     -- ^ These arguments will be appended to the default set of CLI options when
     -- starting the node.
   deriving (Eq, Show)
 
+extraNodeCliArgs :: TestnetNodeOptions -> [String]
+extraNodeCliArgs (TestnetNodeOptions _ _ args) = args
+
 -- | Determines the role of the node
 data TestnetNodeRole
   = TestnetNodeRoleSpo -- ^ Stake pool node, producing blocks
   | TestnetNodeRoleRelay -- ^ Relay node
   deriving (Eq, Show)
 
-extraNodeCliArgs :: TestnetNodeOptions -> [String]
-extraNodeCliArgs (TestnetNodeOptions _ _ args) = args
-
 cardanoDefaultTestnetNodeOptions :: [TestnetNodeOptions]
 cardanoDefaultTestnetNodeOptions =
   [ TestnetNodeOptions TestnetNodeRoleSpo Nothing []

cardano-testnet/src/Testnet/Types.hs

@@ -133,7 +133,7 @@ relayNodes = fmap testnetNodeRuntime . filter (not . isTestnetNodeRoleSpo) . tes
 
 data TestnetNode = TestnetNode
   { testnetNodeRuntime :: !NodeRuntime
-  , poolKeys :: Maybe TestnetNodeKeys -- ^ Keys are provided for SPO nodes, not relay nodes
+  , poolKeys :: Maybe TestnetNodeKeys -- ^ Keys are only present for SPO nodes
   }
 
 poolNodeStdout :: TestnetNode -> FilePath
@@ -142,6 +142,7 @@ poolNodeStdout = nodeStdout . testnetNodeRuntime
 isTestnetNodeRoleSpo :: TestnetNode -> Bool
 isTestnetNodeRoleSpo = isJust . poolKeys
 
+-- | Node process runtime parameters
 data NodeRuntime = NodeRuntime
   { nodeName :: !String
   , nodeIpv4 :: !HostAddress

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Conway/Plutus.hs

@@ -139,7 +139,6 @@ hprop_plutus_v3 = integrationWorkspace "all-plutus-script-purposes" $ \tempAbsBa
     ]
 
   -- 2. Successfully spend conway spending script
-  _ <- waitForBlocks epochStateView 2
   txinCollateral <- findLargestUtxoForPaymentKey epochStateView sbe wallet1
   plutusScriptTxIn <- fmap fst . retryUntilJustM epochStateView (WaitForBlocks 3) $
     findLargestUtxoWithAddress epochStateView sbe $ Text.pack plutusSpendingScriptAddr

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/LeadershipSchedule.hs

@@ -71,15 +71,14 @@ hprop_leadershipSchedule = integrationRetryWorkspace 0 "leadership-schedule" $ \
           , TestnetNodeOptions TestnetNodeRoleSpo Nothing []
           ]
         }
-      shelleyOptions = def { genesisEpochLength = 200 }
       eraString = eraToString sbe
 
   tr@TestnetRuntime
     { testnetMagic
     , wallets=wallet0:_
     , configurationFile
     , testnetNodes
-    } <- cardanoTestnetDefault cTestnetOptions shelleyOptions conf
+    } <- cardanoTestnetDefault cTestnetOptions def conf
 
   node1sprocket <- H.headM $ testnetSprockets tr
   execConfig <- mkExecConfig tempBaseAbsPath node1sprocket testnetMagic

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Query.hs

@@ -69,7 +69,7 @@ import           Testnet.Process.Cli.Transaction (TxOutAddress (..), mkSimpleSpe
 import           Testnet.Process.Run (execCli', execCliStdoutToJson, mkExecConfig)
 import           Testnet.Property.Assert (assertErasEqual)
 import           Testnet.Property.Util (integrationWorkspace)
-import           Testnet.Start.Types (GenesisOptions (..))
+import           Testnet.Start.Types (GenesisOptions (..), NumPools (..), cardanoNumPools)
 import           Testnet.TestQueryCmds (TestQueryCmds (..), forallQueryCommands)
 import           Testnet.Types
 
@@ -102,6 +102,7 @@ hprop_cli_queries = integrationWorkspace "cli-queries" $ \tempAbsBasePath' -> H.
         -- securityParam * 10 / slotCoeff
         , genesisActiveSlotsCoeff = 0.5
         }
+      nPools = cardanoNumPools fastTestnetOptions
 
   TestnetRuntime
     { testnetMagic
@@ -188,8 +189,7 @@ hprop_cli_queries = integrationWorkspace "cli-queries" $ \tempAbsBasePath' -> H.
         -- to stdout
         stakePoolsOut <- execCli' execConfig [ eraName, "query", "stake-pools" ]
         H.assertWith stakePoolsOut $ \pools ->
-          -- FIXME: make configurable, and read config here
-          length (lines pools) == 1
+          NumPools (length $ lines pools) == nPools
         -- Light test of the query's answer, the ids should exist:
         forM_ (lines stakePoolsOut) $ \stakePoolId -> do
           execCli' execConfig [ eraName, "query", "pool-state"
@@ -219,8 +219,7 @@ hprop_cli_queries = integrationWorkspace "cli-queries" $ \tempAbsBasePath' -> H.
               . lines
               $ stakeDistrOut
         H.assertWith stakeAddresses $ \sa ->
-          -- FIXME: make configurable and read config here
-          length sa == 1
+          NumPools (length sa) == nPools
         -- Light test of the query's answer, the ids should exist:
         forM_ stakeAddresses $ \(stakePoolId, _) -> do
           execCli' execConfig [ eraName, "query", "pool-state"
@@ -266,7 +265,7 @@ hprop_cli_queries = integrationWorkspace "cli-queries" $ \tempAbsBasePath' -> H.
     TestQueryStakeAddressInfoCmd -> pure ()
       -- stake-address-info
       {-
-         FIXME: this test is flaky - needs investigation : the reward account balance is changing e.g.
+         FIXME: this test is flaky - needs investigation : the reward account balance is changing between multiple executions e.g.
                  │ Reading file: /home/runner/work/_temp/cli-queries-test-bbd8d6517639a66e/stake-address-info-out-redacted.json
                  │ Reading file: test/cardano-testnet-test/files/golden/queries/stakeAddressInfoOut.json
                  │ Golden test failed against the golden file.

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/CommitteeAddNew.hs

@@ -44,10 +44,11 @@ import qualified Testnet.Process.Cli.SPO as SPO
 import           Testnet.Process.Cli.Transaction
 import           Testnet.Process.Run (execCli', mkExecConfig)
 import           Testnet.Property.Util (integrationWorkspace)
-import           Testnet.Start.Types (GenesisOptions (..))
+import           Testnet.Start.Types (GenesisOptions (..), cardanoNumPools)
 import           Testnet.Types
 
 import           Hedgehog
+import qualified Hedgehog as H
 import qualified Hedgehog.Extras as H
 
 -- | Execute me with:
@@ -62,15 +63,15 @@ hprop_constitutional_committee_add_new = integrationWorkspace "constitutional-co
 
   -- how many votes to cast
   let drepVotes, spoVotes :: [(String, Int)]
-      drepVotes = zip (concatMap (uncurry replicate) [(5, "yes"), (3, "no"), (2, "abstain")]) [1..]
-      spoVotes = zip (replicate 1 "yes") [1..]
-  H.noteShow_ drepVotes
-
-  let nDrepVotes :: Int
+      drepVotes = mkVotes [(5, "yes"), (3, "no"), (2, "abstain")]
+      spoVotes = mkVotes [(nSpos, "yes")]
+      -- replicate votes requested number of times
+      mkVotes :: [(Int, String)] -- ^ [(count, vote)]
+              -> [(String, Int)] -- ^ [(vote, ordering number)]
+      mkVotes votes = zip (concatMap (uncurry replicate) votes) [1..]
       nDrepVotes = length drepVotes
-  H.noteShow_ nDrepVotes
-
-  let ceo = ConwayEraOnwardsConway
+      nSpos = fromIntegral $ cardanoNumPools fastTestnetOptions
+      ceo = ConwayEraOnwardsConway
       sbe = conwayEraOnwardsToShelleyBasedEra ceo
       era = toCardanoEra sbe
       cEra = AnyCardanoEra era
@@ -80,6 +81,8 @@ hprop_constitutional_committee_add_new = integrationWorkspace "constitutional-co
         , cardanoNumDReps = fromIntegral nDrepVotes
         }
       shelleyOptions = def { genesisEpochLength = 200 }
+  H.annotateShow drepVotes
+  H.noteShow_ nDrepVotes
 
   runtime@TestnetRuntime
     { testnetMagic

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs

@@ -32,13 +32,10 @@ import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Stock.IO.Network.Sprocket as H
 import qualified Hedgehog.Extras.Test as H
 
-wCfg :: H.WatchdogConfig
-wCfg = H.WatchdogConfig { H.watchdogTimeout = 120 }
-
 -- | Execute me with:
 -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Treasury Growth/"'@
 prop_check_if_treasury_is_growing :: H.Property
-prop_check_if_treasury_is_growing = integrationRetryWorkspace 0 "growing-treasury" $ \tempAbsBasePath' -> H.runWithWatchdog_ wCfg $ do
+prop_check_if_treasury_is_growing = integrationRetryWorkspace 0 "growing-treasury" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
   -- Start testnet
   conf@Conf{tempAbsPath=TmpAbsolutePath tempAbsPath'} <- TN.mkConf tempAbsBasePath'
   let tempBaseAbsPath = makeTmpBaseAbsPath $ tempAbsPath conf

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Node/Shutdown.hs

@@ -197,8 +197,6 @@ hprop_shutdownOnSlotSynced = integrationRetryWorkspace 0 "shutdown-on-slot-synce
   let fastTestnetOptions = def
         { cardanoNodes =
           [ TestnetNodeOptions TestnetNodeRoleSpo Nothing ["--shutdown-on-slot-synced", show maxSlot]
-          , TestnetNodeOptions TestnetNodeRoleRelay Nothing []
-          , TestnetNodeOptions TestnetNodeRoleRelay Nothing []
           ]
         }
       shelleyOptions = def