Fix flaky tests by using retryUntilJustM which should provide more reliable wait

Author: carbolymer

cardano-testnet/changelog.d/20260414_154349_mgalazyn_fix_flaky_tests.md

@@ -0,0 +1,10 @@
+
+### Tests
+
+- Replaced `waitForBlocks 1` followed by immediate assertions with `retryUntilJustM` polling in seven test sites.
+  The old pattern was flaky on aarch64-linux because `waitForBlocks` only guarantees the block number advanced, not that a submitted transaction was included in that block.
+  Affected tests: Simple Script Mint, Simple Script CostCalculation, TxReferenceInputDatum (two sites), RegisterDeregisterStakeAddress (two sites), and MultiAssetReturnCollateral.
+
+- Switched all remaining `integrationWorkspace` tests to `integrationRetryWorkspace 2` so that transient failures are retried automatically.
+  Affected tests: FoldEpochState, Simple Script Mint, DRep Deposits, DRep Activity, Predefined Abstain DRep, PlutusV3 purposes, PlutusV2 two script certs, Collateral With Multiassets, Committee Add New, No Confidence, and Transaction Build Estimate.
+

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs

@@ -164,20 +164,18 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs
     expectTxSubmissionSuccess =<< submitTx sbe connectionInfo tx
 
     -- wait till transaction gets included in the block
-    _ <- waitForBlocks epochStateView 1
-
-    -- test if it's in UTxO set
-    utxo1 <- findAllUtxos epochStateView sbe
-    let txUtxo = Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') utxo1
-    (length txOuts + 1) === length (toList txUtxo)
+    txUtxo <- retryUntilJustM epochStateView (WaitForBlocks 5) $ do
+      utxo1 <- findAllUtxos epochStateView sbe
+      let txUtxo = Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') utxo1
+      pure $ txUtxo <$ guard (length (toList txUtxo) == length txOuts + 1)
 
     let chainTxOuts =
           reverse
             . drop 1
             . reverse
             . map snd
             . toList
-            $ Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') utxo1
+            $ Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') txUtxo
 
     -- check that the transaction's outputs are the same as we've submitted them
     -- i.e. check the datums
@@ -227,11 +225,10 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs
     expectTxSubmissionSuccess =<< submitTx sbe connectionInfo tx
 
     -- wait till transaction gets included in the block
-    _ <- waitForBlocks epochStateView 1
-
-    -- test if the transaction is visible in UTxO set
-    utxo1 <- findAllUtxos epochStateView sbe
-    let txUtxo = Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') utxo1
+    txUtxo <- retryUntilJustM epochStateView (WaitForBlocks 5) $ do
+      utxo1 <- findAllUtxos epochStateView sbe
+      let txUtxo = Utxo.filterWithKey (\(TxIn txId' _) _ -> txId == txId') utxo1
+      pure $ txUtxo <$ guard (not . null $ toList txUtxo)
     [toCtxUTxOTxOut txOut] === (snd <$> toList txUtxo)
     pure txUtxo
 

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Plutus/MultiAssetReturnCollateral.hs

@@ -4,16 +4,18 @@
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 
-module Cardano.Testnet.Test.Cli.Plutus.MultiAssetReturnCollateral 
+module Cardano.Testnet.Test.Cli.Plutus.MultiAssetReturnCollateral
   ( hprop_collateral_with_tokens
-  ) where 
+  ) where
 
 import           Cardano.Api
+
 import           Cardano.Testnet
 
 import           Prelude
-import qualified Data.Aeson as Aeson
+
 import           Control.Monad (void)
+import qualified Data.Aeson as Aeson
 import           Data.Default.Class
 import qualified Data.Text as T
 import           System.FilePath ((</>))
@@ -22,7 +24,7 @@ import           Testnet.Components.Configuration
 import           Testnet.Components.Query
 import           Testnet.Defaults
 import           Testnet.Process.Run (execCli', mkExecConfig)
-import           Testnet.Property.Util (integrationWorkspace)
+import           Testnet.Property.Util (integrationRetryWorkspace)
 import           Testnet.Types
 
 import           Hedgehog (Property)
@@ -31,7 +33,7 @@ import qualified Hedgehog.Extras as H
 
 -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Collateral With Multiassets/"'@
 hprop_collateral_with_tokens :: Property
-hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
+hprop_collateral_with_tokens = integrationRetryWorkspace 2 "collateral-with-tokens" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
   conf@Conf { tempAbsPath } <- mkConf tempAbsBasePath'
   let tempAbsPath' = unTmpAbsPath tempAbsPath
   work <- H.createDirectoryIfMissing $ tempAbsPath' </> "work"
@@ -71,22 +73,22 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
     ]
   utxo1 <- findUtxosWithAddress epochStateView sbe $ paymentKeyInfoAddr wallet0
 
-  H.noteShow_ utxo1 
+  H.noteShow_ utxo1
     -- Create a simple always-succeeds Plutus V3 script
   plutusScript <- H.note $ work </> "always-succeeds-script.plutusV3"
   H.writeFile plutusScript $ T.unpack plutusV3Script
 
-  
+
   -- Get the policy ID
   mintingPolicyId <- filter (/= '\n') <$>
     execCli' execConfig
       [ anyEraToString anyEra, "transaction"
       , "policyid"
       , "--script-file", plutusScript
       ]
-      
+
   let assetName = "7161636f696e" -- "qacoin" in hex
-  
+
   -- Create a Plutus script address
   plutusSpendingScriptAddr <-
     execCli' execConfig
@@ -113,7 +115,7 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
     , "--tx-in-collateral", T.unpack $ renderTxIn txinCollateral
     , "--tx-out-return-collateral", adaOnlyCollateralValue
     , "--witness-override", show @Int 2
-    , "--tx-out", collateralToBeValue 
+    , "--tx-out", collateralToBeValue
     , "--tx-out", fundPlutusScriptAddrVal
     , "--tx-out-datum-hash-value", "0"
     , "--mint", mintValue
@@ -144,14 +146,11 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
 
   -- STEP 2: Attempt to spend from script with collateral containing tokens
   -- This will fail because collateral cannot contain non-ADA tokens
-  
+
   -- Wait for transactions to be processed and find UTxOs
-  _ <- waitForBlocks epochStateView 1
-  
-  -- Find the UTxO with tokens at wallet1 (for collateral)
-  txinCollateralWithTokensM <- 
-    findLargestMultiAssetUtxoWithAddress epochStateView sbe $ T.pack maCollateralAddress
-  (txinCollateralWithTokens, collateralTxOut) <- H.evalMaybe txinCollateralWithTokensM
+  (txinCollateralWithTokens, collateralTxOut) <-
+    retryUntilJustM epochStateView (WaitForBlocks 5) $
+      findLargestMultiAssetUtxoWithAddress epochStateView sbe $ T.pack maCollateralAddress
   H.note_ "Collateral TxOut"
   H.noteShow_  collateralTxOut
   -- Find the UTxO at the script address
@@ -173,8 +172,8 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
     , "--out-file", spendScriptUTxOTxBody
     ]
   let prettyTxBodyFile = work </> "spend-script-utxo-tx-body-view.json"
-  void $ execCli' execConfig ["debug", "transaction", "view", "--tx-body-file", spendScriptUTxOTxBody, "--out-file", prettyTxBodyFile] 
-  
+  void $ execCli' execConfig ["debug", "transaction", "view", "--tx-body-file", spendScriptUTxOTxBody, "--out-file", prettyTxBodyFile]
+
   txBodyPrettyJson :: Aeson.Value <- H.leftFailM . H.readJsonFile $ prettyTxBodyFile
   H.note_ "Tx body"
   H.noteShowPretty_ txBodyPrettyJson
@@ -186,10 +185,10 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
     , "--signing-key-file", signingKeyFp $ paymentKeyInfoPair wallet2
     , "--out-file", spendScriptUTxOTx
     ]
-  
+
   let prettyTxFile = work </> "spend-script-utxo-tx-view.json"
-  void $ execCli' execConfig ["debug", "transaction", "view", "--tx-file", spendScriptUTxOTx, "--out-file", prettyTxFile] 
-  
+  void $ execCli' execConfig ["debug", "transaction", "view", "--tx-file", spendScriptUTxOTx, "--out-file", prettyTxFile]
+
   txPrettyJson :: Aeson.Value <- H.leftFailM . H.readJsonFile $ prettyTxFile
   H.noteShowPretty_ txPrettyJson
 
@@ -198,4 +197,4 @@ hprop_collateral_with_tokens = integrationWorkspace "collateral-with-tokens" $ \
     , "--tx-file", spendScriptUTxOTx
     ]
 
-  H.success
+  H.success

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Plutus/Scripts.hs

@@ -28,7 +28,7 @@ import           Testnet.Components.Query
 import           Testnet.Defaults
 import           Testnet.Process.Cli.SPO
 import           Testnet.Process.Run (execCli', execCliAny, mkExecConfig)
-import           Testnet.Property.Util (integrationWorkspace)
+import           Testnet.Property.Util (integrationRetryWorkspace)
 import           Testnet.Types
 
 import           Hedgehog (Property)
@@ -46,7 +46,7 @@ import qualified Hedgehog.Extras as H
 -- Execute me with:
 -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/PlutusV3 purposes/"'@
 hprop_plutus_purposes_v3 :: Property
-hprop_plutus_purposes_v3 = integrationWorkspace "all-plutus-script-purposes" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
+hprop_plutus_purposes_v3 = integrationRetryWorkspace 2 "all-plutus-script-purposes" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
   conf@Conf { tempAbsPath } <- mkConf tempAbsBasePath'
   let tempAbsPath' = unTmpAbsPath tempAbsPath
   work <- H.createDirectoryIfMissing $ tempAbsPath' </> "work"
@@ -196,7 +196,7 @@ hprop_plutus_purposes_v3 = integrationWorkspace "all-plutus-script-purposes" $ \
 -- Execute me with:
 -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/PlutusV2 transaction with two script certs/"'@
 hprop_tx_two_script_certs_v2 :: Property
-hprop_tx_two_script_certs_v2 = integrationWorkspace "tx-2-script-certs" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
+hprop_tx_two_script_certs_v2 = integrationRetryWorkspace 2 "tx-2-script-certs" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
   conf@Conf { tempAbsPath } <- mkConf tempAbsBasePath'
   let tempAbsPath' = unTmpAbsPath tempAbsPath
   work <- H.createDirectoryIfMissing $ tempAbsPath' </> "work"
@@ -257,8 +257,8 @@ hprop_tx_two_script_certs_v2 = integrationWorkspace "tx-2-script-certs" $ \tempA
   let txbody = work </> "two-certs-tx-body"
       tx = work </> "two-certs-tx"
       txout = mconcat [ utxoAddr, "+", show @Int 2_000_000 ]
-  
-  s <- execCli' execConfig [anyEraToString anyEra, "transaction", "policyid", "--script-file", plutusScript] 
+
+  s <- execCli' execConfig [anyEraToString anyEra, "transaction", "policyid", "--script-file", plutusScript]
   H.note_ $ "Script hash: " <> s
   let txBuildArgs =
         [ anyEraToString anyEra, "transaction", "build"

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Scripts/Simple/CostCalculation.hs

@@ -11,26 +11,27 @@ import           Cardano.Api hiding (Value)
 import           Cardano.Api.Experimental (Some (Some))
 import           Cardano.Api.Ledger (EpochInterval (..))
 import           Cardano.Api.UTxO (difference, size)
+
 import           Cardano.Testnet
 
 import           Prelude
-import           Testnet.Types
-
-import           Control.Monad (void)
 
+import           Control.Monad (guard, void)
 import           Data.Default.Class (Default (def))
 import qualified Data.Text as Text
 import           System.FilePath ((</>))
 import qualified System.Info as SYS
 
-import           Testnet.Components.Query (getEpochStateView, getTxIx, waitForBlocks,
-                   watchEpochStateUpdate, findAllUtxos)
-import qualified Testnet.Defaults  as Defaults
+import           Testnet.Components.Query (TestnetWaitPeriod (..), findAllUtxos, getEpochStateView,
+                   getTxIx, retryUntilJustM, watchEpochStateUpdate)
+import qualified Testnet.Defaults as Defaults
 import           Testnet.Process.Cli.Transaction (TxOutAddress (..), mkSpendOutputsOnlyTx,
                    retrieveTransactionId, signTx, submitTx)
 import           Testnet.Process.Run (execCli', mkExecConfig)
 import           Testnet.Property.Util (integrationRetryWorkspace)
 import           Testnet.Start.Types (eraToString)
+import           Testnet.Types
+
 import           Hedgehog (Property)
 import qualified Hedgehog as H
 import qualified Hedgehog.Extras.Test.Base as H
@@ -60,7 +61,7 @@ hprop_ref_simple_script_mint = integrationRetryWorkspace 2 "ref-simple-script" $
     , wallets = wallet0 : wallet1 : _
     } <-
     createAndRunTestnet options def conf
-  
+
   poolNode1 <- H.headM testnetNodes
   poolSprocket1 <- H.noteShow $ nodeSprocket poolNode1
   execConfig <- mkExecConfig tempBaseAbsPath poolSprocket1 testnetMagic
@@ -69,7 +70,7 @@ hprop_ref_simple_script_mint = integrationRetryWorkspace 2 "ref-simple-script" $
   refScriptSizeWork <- H.createDirectoryIfMissing $ work </> "ref-script-publish"
 
   let utxoVKeyFile2 = verificationKeyFp $ paymentKeyInfoPair wallet1
-      
+
   reqSignerHash <- filter (/= '\n') <$>
     execCli' execConfig
       [ eraName, "address", "key-hash"
@@ -148,7 +149,7 @@ hprop_ref_simple_script_mint = integrationRetryWorkspace 2 "ref-simple-script" $
   -- Attempt to mint from a refernce script
   let assetName = "4D696C6C6172436F696E"
 
- 
+
   simpleMintingPolicyId <- filter (/= '\n') <$>
      execCli' execConfig
        [ eraName, "transaction"
@@ -186,14 +187,12 @@ hprop_ref_simple_script_mint = integrationRetryWorkspace 2 "ref-simple-script" $
       "signed-tx"
       unsignedUnlockTx
       [Some $ paymentKeyInfoPair wallet1]
-  
+
   utxoPre <- findAllUtxos epochStateView sbe
 
 
   submitTx execConfig cEra signedUnlockTx
-  void $ waitForBlocks epochStateView 1 
-  utxoPost <- findAllUtxos epochStateView sbe
-
-  let diff = difference utxoPost utxoPre
-      
-  size diff H.=== 2
+  void $ retryUntilJustM epochStateView (WaitForBlocks 5) $ do
+    utxoPost <- findAllUtxos epochStateView sbe
+    let diff = difference utxoPost utxoPre
+    pure . guard $ size diff == 2

cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Scripts/Simple/Mint.hs

@@ -19,7 +19,7 @@ import           Cardano.Testnet
 
 import           Prelude
 
-import           Control.Monad (void)
+import           Control.Monad (guard, void)
 import           Data.Default.Class
 import qualified Data.Text as T
 import           System.FilePath ((</>))
@@ -29,16 +29,15 @@ import           Testnet.Components.Query
 import           Testnet.Defaults
 import           Testnet.Process.Cli.SPO
 import           Testnet.Process.Run (execCli', mkExecConfig)
-import           Testnet.Property.Util (integrationWorkspace)
+import           Testnet.Property.Util (integrationRetryWorkspace)
 import           Testnet.Types
 
 import           Hedgehog (Property)
-import qualified Hedgehog as H
 import qualified Hedgehog.Extras as H
 
 -- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Simple Script.Simple Script Mint/"'@
 hprop_simple_script_mint :: Property
-hprop_simple_script_mint = integrationWorkspace "simple-script-mint" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
+hprop_simple_script_mint = integrationRetryWorkspace 2 "simple-script-mint" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do
   conf@Conf { tempAbsPath } <- mkConf tempAbsBasePath'
   let tempAbsPath' = unTmpAbsPath tempAbsPath
   work <- H.createDirectoryIfMissing $ tempAbsPath' </> "work"
@@ -138,10 +137,10 @@ hprop_simple_script_mint = integrationWorkspace "simple-script-mint" $ \tempAbsB
 
   let spendScriptUTxOTxBody = work </> "spend-script-utxo-tx-body"
       spendScriptUTxOTx = work </> "spend-script-utxo-tx"
- 
+
       txoutWithSupplementalDatum = mconcat [utxoAddr, "+", show @Int 1_000_000]
 
-  -- Mint with a simple script 
+  -- Mint with a simple script
   reqSignerHash <- filter (/= '\n') <$>
     execCli' execConfig
       [ anyEraToString anyEra, "address", "key-hash"
@@ -168,7 +167,7 @@ hprop_simple_script_mint = integrationWorkspace "simple-script-mint" $ \tempAbsB
        ]
 
   H.note_ plutusScriptTestPolId
-  
+
   let mintValue = mconcat ["5 ", simpleMintingPolicyId, ".", assetName]
       txout = mconcat [ utxoAddr, "+", show @Int 2_000_000
                        , "+", mintValue
@@ -203,9 +202,7 @@ hprop_simple_script_mint = integrationWorkspace "simple-script-mint" $ \tempAbsB
     [ "latest", "transaction", "submit"
     , "--tx-file", spendScriptUTxOTx
     ]
-  void $ waitForBlocks epochStateView 1 
-  utxoPost <- findAllUtxos epochStateView sbe
-
-  let diff = difference  utxoPost utxoPre
-
-  size diff H.=== 3
+  void $ retryUntilJustM epochStateView (WaitForBlocks 5) $ do
+    utxoPost <- findAllUtxos epochStateView sbe
+    let diff = difference utxoPost utxoPre
+    pure . guard $ size diff == 3