Annotate counterexample output of potentially flaky tests (#1511)

# Description

Annotate known-flaky tests with counterexamples describing their
flakiness (and their respective tickets about the flakiness)

fixes #1443

Author: fraser-iohk

ouroboros-consensus-cardano/test/byron-test/Test/ThreadNet/Byron.hs

@@ -862,6 +862,7 @@ prop_simple_real_pbft_convergence TestSetup
     tabulate "Ref.PBFT result" [Ref.resultConstrName refResult] $
     tabulate "proposed protocol version was adopted" [show aPvuRequired] $
     tabulate "proposed software version was adopted" [show aSvuRequired] $
+    counterexample flakyTestCopy $
     counterexample ("params: " <> show params) $
     counterexample ("Ref.PBFT result: " <> show refResult) $
     counterexample
@@ -1055,6 +1056,9 @@ prop_simple_real_pbft_convergence TestSetup
     genesisSecrets :: Genesis.GeneratedSecrets
     (genesisConfig, genesisSecrets) = generateGenesisConfig slotLength params
 
+    flakyTestCopy :: String
+    flakyTestCopy = "This test may be flaky, and its failure may not be indicative of an actual problem: see https://github.com/IntersectMBO/ouroboros-consensus/issues/1294 and / or https://github.com/IntersectMBO/ouroboros-consensus/issues/582"
+
 byronForgeEbbEnv :: ForgeEbbEnv ByronBlock
 byronForgeEbbEnv = ForgeEbbEnv
     { forgeEBB = Byron.forgeEBB . configBlock

ouroboros-consensus-cardano/test/cardano-test/Test/ThreadNet/Cardano.hs

@@ -176,6 +176,7 @@ prop_simple_cardano_convergence TestSetup
   , setupTestConfig
   , setupVersion
   } =
+    counterexample flakyTestCopy $
     prop_general_semisync pga testOutput .&&.
     prop_inSync testOutput .&&.
     prop_ReachesEra2 reachesEra2 .&&.
@@ -436,6 +437,9 @@ prop_simple_cardano_convergence TestSetup
         counterexample "CP violation in final chains!" $
         property $ unNonZero (maxRollbacks setupK) >= finalIntersectionDepth
 
+    flakyTestCopy :: String
+    flakyTestCopy = "This test may be flaky, and its failure may not be indicative of an actual problem: see https://github.com/IntersectMBO/ouroboros-consensus/issues/545"
+
 mkProtocolCardanoAndHardForkTxs ::
      forall c m. (IOLike m, c ~ StandardCrypto)
      -- Byron

ouroboros-consensus-diffusion/test/mock-test/Test/ThreadNet/Praos.hs

@@ -108,6 +108,7 @@ prop_simple_praos_convergence TestSetup
   , setupTestConfig    = testConfig
   , setupEvolvingStake = evolvingStake
   } =
+    counterexample flakyTestCopy $
     counterexample (tracesToDot testOutputNodes) $
     prop_general PropGeneralArgs
       { pgaBlockProperty       = prop_validSimpleBlock
@@ -155,3 +156,5 @@ prop_simple_praos_convergence TestSetup
                                   (blockForgingPraos numCoreNodes nid)
             , mkRekeyM = Nothing
             }
+
+    flakyTestCopy = "This test may be flaky, and its failure may not be indicative of an actual problem: see https://github.com/IntersectMBO/ouroboros-consensus/issues/1105"

ouroboros-consensus/bench/mempool-bench/Main.hs

@@ -23,6 +23,7 @@ import           Main.Utf8 (withStdTerminalHandles)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (ByteSize32)
 import qualified Ouroboros.Consensus.Mempool.Capacity as Mempool
 import           System.Exit (die, exitFailure)
+import           System.IO (hPutStrLn, stderr)
 import qualified Test.Consensus.Mempool.Mocked as Mocked
 import           Test.Consensus.Mempool.Mocked (MockedMempool)
 import           Test.Tasty (withResource)
@@ -46,7 +47,9 @@ main = withStdTerminalHandles $ do
           Nothing               -> exitFailure
           Just    runIngredient -> do
             success <- runIngredient
-            unless success exitFailure
+            unless success $ do
+              hPutStrLn stderr "This benchmark is flaky in GitHub Actions due to CI runner load, which can it to run significantly slower than expected. It may be useful to try to re-run the job if it fails. See https://github.com/IntersectMBO/ouroboros-consensus/issues/313"
+              exitFailure
       where
         benchmarkJustAddingTransactions =
             bgroup "Just adding" $

ouroboros-consensus/test/consensus-test/Test/Consensus/BlockchainTime/Simple.hs

@@ -112,7 +112,7 @@ instance Arbitrary TestDelayIO where
 -- test gives us a more useful property. Also see issue #3894.
 prop_delayNextSlot :: TestDelayIO -> Property
 prop_delayNextSlot TestDelayIO{..} =
-    ioProperty test
+    counterexample flakyTestCopy $ ioProperty test
   where
     test :: IO Property
     test = do
@@ -261,6 +261,7 @@ instance Arbitrary Schedule where
 
 prop_delayClockShift :: Schedule -> Property
 prop_delayClockShift schedule =
+    counterexample flakyTestCopy $
     tabulate "schedule length"    [show $ range (length (getSchedule schedule))] $
     tabulate "schedule goes back" [show $ scheduleGoesBack schedule]             $
     tabulate "schedule skips"     [show $ range (scheduleCountSkips schedule)]   $
@@ -312,15 +313,16 @@ prop_delayClockShift schedule =
 -- | Just as a sanity check, verify that this works in IO
 prop_delayNoClockShift :: Property
 prop_delayNoClockShift =
-    ioProperty $ do
-      now   <- getCurrentTime
-      slots <- originalDelay $
-                 testOverrideDelay
-                   (SystemStart now)
-                   (slotLengthFromMillisec 100)
-                   (secondsToNominalDiffTime 20)
-                   5
-      pure $ slots === [SlotNo n | n <- [0..4]]
+    counterexample flakyTestCopy $
+      ioProperty $ do
+        now   <- getCurrentTime
+        slots <- originalDelay $
+                   testOverrideDelay
+                     (SystemStart now)
+                     (slotLengthFromMillisec 100)
+                     (secondsToNominalDiffTime 20)
+                     5
+        pure $ slots === [SlotNo n | n <- [0..4]]
 
 -- | Note that that under load, the returned list could be missing certain slots
 -- or contain more slots than requested. This means that tests using this
@@ -624,3 +626,6 @@ overrideDelay start schedule ma = runSim $ do
 
 originalDelay :: OverrideDelay IO a -> IO a
 originalDelay ma = runReaderT (unOverrideDelay ma) (error "schedule unused")
+
+flakyTestCopy :: String
+flakyTestCopy = "This test may be flaky, and its failure may not be indicative of an actual problem: see https://github.com/IntersectMBO/ouroboros-consensus/issues/567"

ouroboros-consensus/test/infra-test/Test/Ouroboros/Consensus/ChainGenerator/Tests/Adversarial.hs

@@ -459,6 +459,7 @@ instance QC.Arbitrary SomeTestAdversarialMutation where
 -- TODO this did fail after >500,000 tests. Is that amount of flakiness acceptable?
 prop_adversarialChainMutation :: SomeTestAdversarialMutation -> QCGen -> QC.Property
 prop_adversarialChainMutation (SomeTestAdversarialMutation Proxy Proxy testAdversarialMut) testSeedAsSeed0 =
+  TT.counterexample flakyTestCopy $
   QC.ioProperty $ do
     A.SomeCheckedAdversarialRecipe Proxy recipeA' <- pure someRecipeA'
 
@@ -507,6 +508,8 @@ prop_adversarialChainMutation (SomeTestAdversarialMutation Proxy Proxy testAdver
                 A.BadDensity{} -> pure $ QC.property ()
                 A.BadRace{}    -> pure $ QC.property ()
 
+    flakyTestCopy = "This test may be flaky, and its failure may not be indicative of an actual problem: see https://github.com/IntersectMBO/ouroboros-consensus/issues/1442"
+
 -----
 
 -- | A nice rendering for failures of 'prop_adversarialChainMutation'