Add support for measures to LocalTxMonitor

cabal.project

@@ -44,3 +44,28 @@ package ouroboros-network
 if(os(windows))
   constraints:
     bitvec -simd
+
+
+allow-newer:
+  network-mux:network,
+  ouroboros-network-framework:network,
+  ouroboros-network:network,
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network.git
+  tag: 4075f20101c002fdc6322c7b515a3ba36a03444f
+  --sha256: 094d7vbk1q24ajc5nf8a4i54n3y9zpv5q301npdizy897j9jklhb
+  subdir:
+    -- cardano-client
+    -- cardano-ping
+    -- monoidal-synchronisation
+    -- network-mux
+    -- ntp-client
+    -- ouroboros-network
+    ouroboros-network-api
+    -- ouroboros-network-framework
+    -- ouroboros-network-mock
+    ouroboros-network-protocols
+    -- ouroboros-network-testing
+    -- quickcheck-monoids

ouroboros-consensus-cardano/changelog.d/20250213_115925_fraser.murray_localtxmonitor_measures.md

@@ -0,0 +1,4 @@
+### Non-Breaking
+
+- Add instances for `TxMeasureMetrics` to Cardano block types
+

ouroboros-consensus-cardano/src/ouroboros-consensus-cardano/Ouroboros/Consensus/Cardano/Node.hs

@@ -341,6 +341,7 @@ instance CardanoHardForkConstraints c
       , (NodeToClientV_17, CardanoNodeToClientVersion13)
       , (NodeToClientV_18, CardanoNodeToClientVersion14)
       , (NodeToClientV_19, CardanoNodeToClientVersion15)
+      , (NodeToClientV_20, CardanoNodeToClientVersion15)
       ]
 
   latestReleasedNodeVersion _prx = (Just NodeToNodeV_14, Just NodeToClientV_19)

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/Mempool.hs

@@ -41,7 +41,7 @@ import qualified Cardano.Ledger.Allegra.Rules as AllegraEra
 import           Cardano.Ledger.Alonzo.Core (Tx, TxSeq, bodyTxL, eraDecoder,
                      fromTxSeq, ppMaxBBSizeL, ppMaxBlockExUnitsL, sizeTxF)
 import qualified Cardano.Ledger.Alonzo.Rules as AlonzoEra
-import           Cardano.Ledger.Alonzo.Scripts (ExUnits, ExUnits',
+import           Cardano.Ledger.Alonzo.Scripts (ExUnits, ExUnits' (..),
                      pointWiseExUnits, unWrapExUnits)
 import           Cardano.Ledger.Alonzo.Tx (totExUnits)
 import qualified Cardano.Ledger.Api as L
@@ -429,6 +429,20 @@ data AlonzoMeasure = AlonzoMeasure {
 instance HasByteSize AlonzoMeasure where
   txMeasureByteSize = unIgnoringOverflow . byteSize
 
+instance Semigroup AlonzoMeasure where
+  AlonzoMeasure b1 e1 <> AlonzoMeasure b2 e2 =
+    AlonzoMeasure (b1 <> b2) (e1 <> e2)
+
+instance Monoid AlonzoMeasure where
+  mappend = (<>)
+  mempty = AlonzoMeasure mempty mempty
+
+instance TxMeasureMetrics AlonzoMeasure where
+  txMeasureMetricTxSizeBytes = txMeasureMetricTxSizeBytes . byteSize
+  txMeasureMetricExUnitsMemory = exUnitsMem' . exUnits
+  txMeasureMetricExUnitsSteps = exUnitsSteps' . exUnits
+  txMeasureMetricRefScriptsSizeBytes _ = mempty
+
 fromExUnits :: ExUnits -> ExUnits' Natural
 fromExUnits = unWrapExUnits
 
@@ -522,9 +536,24 @@ data ConwayMeasure = ConwayMeasure {
     deriving (Measure)
          via (InstantiatedAt Generic ConwayMeasure)
 
+instance Semigroup ConwayMeasure where
+  ConwayMeasure a1 r1 <> ConwayMeasure a2 r2 =
+    ConwayMeasure (a1 <> a2) (r1 <> r2)
+
+instance Monoid ConwayMeasure where
+  mappend = (<>)
+  mempty = ConwayMeasure mempty mempty
+
 instance HasByteSize ConwayMeasure where
   txMeasureByteSize = txMeasureByteSize . alonzoMeasure
 
+instance TxMeasureMetrics ConwayMeasure where
+  txMeasureMetricTxSizeBytes = txMeasureMetricTxSizeBytes . alonzoMeasure
+  txMeasureMetricExUnitsMemory = txMeasureMetricExUnitsMemory . alonzoMeasure
+  txMeasureMetricExUnitsSteps = txMeasureMetricExUnitsSteps . alonzoMeasure
+  txMeasureMetricRefScriptsSizeBytes =
+    unIgnoringOverflow . refScriptsSize
+
 blockCapacityConwayMeasure ::
      forall proto era.
      ( ShelleyCompatible proto era

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/NetworkProtocolVersion.hs

@@ -43,6 +43,7 @@ instance SupportedNetworkProtocolVersion (ShelleyBlock proto era) where
       , (NodeToClientV_17, ShelleyNodeToClientVersion9)
       , (NodeToClientV_18, ShelleyNodeToClientVersion10)
       , (NodeToClientV_19, ShelleyNodeToClientVersion11)
+      , (NodeToClientV_20, ShelleyNodeToClientVersion11)
       ]
 
   latestReleasedNodeVersion = latestReleasedNodeVersionDefault

ouroboros-consensus-diffusion/src/ouroboros-consensus-diffusion/Ouroboros/Consensus/NodeKernel.hs

@@ -765,9 +765,9 @@ getMempoolReader mempool = MempoolReader.TxSubmissionMempoolReader
         { mempoolTxIdsAfter = \idx ->
             [ ( txId (txForgetValidated tx)
               , idx'
-              , SizeInBytes $ unByteSize32 byteSize
+              , SizeInBytes $ unByteSize32 $ txMeasureByteSize msr
               )
-            | (tx, idx', byteSize) <- snapshotTxsAfter idx
+            | (tx, idx', msr) <- snapshotTxsAfter idx
             ]
         , mempoolLookupTx   = snapshotLookupTx
         , mempoolHasTx      = snapshotHasTx

ouroboros-consensus/changelog.d/20250213_115925_fraser.murray_localtxmonitor_measures.md

@@ -0,0 +1,4 @@
+### Breaking
+
+- Add `TxMeasureMetrics (TxMeasure blk)` constraint to `CanHardFork`
+

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/HardFork/Combinator/Abstract/CanHardFork.hs

@@ -35,6 +35,7 @@ class ( All SingleEraBlock xs
       , HasByteSize (HardForkTxMeasure xs)
       , NoThunks    (HardForkTxMeasure xs)
       , Show        (HardForkTxMeasure xs)
+      , TxMeasureMetrics (HardForkTxMeasure xs)
       ) => CanHardFork xs where
   -- | A measure that can accurately represent the 'TxMeasure' of any era.
   --

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/Query/Version.hs

@@ -24,3 +24,4 @@ nodeToClientVersionToQueryVersion x = case x of
   NodeToClientV_17 -> QueryVersion2
   NodeToClientV_18 -> QueryVersion2
   NodeToClientV_19 -> QueryVersion2
+  NodeToClientV_20 -> QueryVersion2

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Ledger/SupportsMempool.hs

@@ -19,6 +19,7 @@ module Ouroboros.Consensus.Ledger.SupportsMempool (
   , LedgerSupportsMempool (..)
   , TxId
   , TxLimits (..)
+  , TxMeasureMetrics (..)
   , Validated
   , WhetherToIntervene (..)
   ) where
@@ -35,6 +36,7 @@ import qualified Data.Measure
 import           Data.Word (Word32)
 import           GHC.Stack (HasCallStack)
 import           NoThunks.Class
+import           Numeric.Natural
 import           Ouroboros.Consensus.Block.Abstract
 import           Ouroboros.Consensus.Ledger.Abstract
 import           Ouroboros.Consensus.Ticked
@@ -172,10 +174,11 @@ class HasTxs blk where
 -- bit more complex as it had to take other factors into account (like
 -- execution units). For details please see the individual instances for the
 -- TxLimits.
-class ( Measure     (TxMeasure blk)
-      , HasByteSize (TxMeasure blk)
-      , NoThunks    (TxMeasure blk)
-      , Show        (TxMeasure blk)
+class ( Measure          (TxMeasure blk)
+      , HasByteSize      (TxMeasure blk)
+      , NoThunks         (TxMeasure blk)
+      , TxMeasureMetrics (TxMeasure blk)
+      , Show             (TxMeasure blk)
       ) => TxLimits blk where
   -- | The (possibly multi-dimensional) size of a transaction in a block.
   type TxMeasure blk
@@ -271,6 +274,7 @@ newtype IgnoringOverflow a = IgnoringOverflow { unIgnoringOverflow :: a }
   deriving newtype  (Monoid, Semigroup)
   deriving newtype  (NoThunks)
   deriving newtype  (HasByteSize)
+  deriving newtype  (TxMeasureMetrics)
 
 instance Measure (IgnoringOverflow ByteSize32) where
   zero = coerce (0 :: Word32)
@@ -284,3 +288,15 @@ class HasByteSize a where
 
 instance HasByteSize ByteSize32 where
   txMeasureByteSize = id
+
+class TxMeasureMetrics msr where
+  txMeasureMetricTxSizeBytes :: msr -> ByteSize32
+  txMeasureMetricExUnitsMemory :: msr -> Natural
+  txMeasureMetricExUnitsSteps :: msr -> Natural
+  txMeasureMetricRefScriptsSizeBytes :: msr -> ByteSize32
+
+instance TxMeasureMetrics ByteSize32 where
+  txMeasureMetricTxSizeBytes = id
+  txMeasureMetricExUnitsMemory _ = 0
+  txMeasureMetricExUnitsSteps _ = 0
+  txMeasureMetricRefScriptsSizeBytes _ = mempty

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Mempool/API.hs

@@ -320,13 +320,12 @@ data ForgeLedgerState blk =
 data MempoolSnapshot blk = MempoolSnapshot {
     -- | Get all transactions (oldest to newest) in the mempool snapshot along
     -- with their ticket number.
-    snapshotTxs         :: [(Validated (GenTx blk), TicketNo, ByteSize32)]
+    snapshotTxs         :: [(Validated (GenTx blk), TicketNo, TxMeasure blk)]
 
     -- | Get all transactions (oldest to newest) in the mempool snapshot,
     -- along with their ticket number, which are associated with a ticket
     -- number greater than the one provided.
-  , snapshotTxsAfter    ::
-      TicketNo -> [(Validated (GenTx blk), TicketNo, ByteSize32)]
+  , snapshotTxsAfter    :: TicketNo -> [(Validated (GenTx blk), TicketNo, TxMeasure blk)]
 
     -- | Get the greatest prefix (oldest to newest) that respects the given
     -- block capacity.

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Mempool/Impl/Common.hs

@@ -431,12 +431,12 @@ snapshotFromIS is = MempoolSnapshot {
     }
  where
   implSnapshotGetTxs :: InternalState blk
-                     -> [(Validated (GenTx blk), TicketNo, ByteSize32)]
+                     -> [(Validated (GenTx blk), TicketNo, TxMeasure blk)]
   implSnapshotGetTxs = flip implSnapshotGetTxsAfter TxSeq.zeroTicketNo
 
   implSnapshotGetTxsAfter :: InternalState blk
                           -> TicketNo
-                          -> [(Validated (GenTx blk), TicketNo, ByteSize32)]
+                          -> [(Validated (GenTx blk), TicketNo, TxMeasure blk)]
   implSnapshotGetTxsAfter IS{isTxs} =
     TxSeq.toTuples . snd . TxSeq.splitAfterTicketNo isTxs
 

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Mempool/TxSeq.hs

@@ -38,8 +38,6 @@ import qualified Data.Measure as Measure
 import           Data.Word (Word64)
 import           GHC.Generics (Generic)
 import           NoThunks.Class (NoThunks)
-import           Ouroboros.Consensus.Ledger.SupportsMempool (ByteSize32,
-                     HasByteSize, txMeasureByteSize)
 
 {-------------------------------------------------------------------------------
   Mempool transaction sequence as a finger tree
@@ -256,13 +254,13 @@ toList :: TxSeq sz tx -> [TxTicket sz tx]
 toList (TxSeq ftree) = Foldable.toList ftree
 
 -- | Convert a 'TxSeq' to a list of pairs of transactions and their
--- associated 'TicketNo's and 'ByteSize32's.
-toTuples :: HasByteSize sz => TxSeq sz tx -> [(tx, TicketNo, ByteSize32)]
+-- associated 'TicketNo's and sizes.
+toTuples :: TxSeq sz tx -> [(tx, TicketNo, sz)]
 toTuples (TxSeq ftree) = fmap
     (\ticket ->
        ( txTicketTx ticket
        , txTicketNo ticket
-       , txMeasureByteSize (txTicketSize ticket)
+       , txTicketSize ticket
        )
     )
     (Foldable.toList ftree)

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/MiniProtocol/LocalTxMonitor/Server.hs

@@ -1,10 +1,16 @@
-{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE FlexibleContexts    #-}
+{-# LANGUAGE NamedFieldPuns      #-}
+{-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE TypeFamilies #-}
-{-# LANGUAGE ViewPatterns #-}
+{-# LANGUAGE TypeFamilies        #-}
+{-# LANGUAGE ViewPatterns        #-}
+{-# LANGUAGE PatternSynonyms     #-}
 
 module Ouroboros.Consensus.MiniProtocol.LocalTxMonitor.Server (localTxMonitorServer) where
 
+import           Data.Map.Strict (Map)
+import qualified Data.Map.Strict as Map
+import qualified Data.Measure as Measure
 import           Ouroboros.Consensus.Block
 import           Ouroboros.Consensus.Ledger.SupportsMempool
 import           Ouroboros.Consensus.Mempool
@@ -33,20 +39,20 @@ localTxMonitorServer mempool =
       , recvMsgAcquire = do
           s <- atomically $
                 (,)
-            <$> (txMeasureByteSize <$> getCapacity mempool)
+            <$> getCapacity mempool
             <*> getSnapshot mempool
           pure $ serverStAcquiring s
       }
 
     serverStAcquiring
-      :: (ByteSize32, MempoolSnapshot blk)
+      :: (TxMeasure blk, MempoolSnapshot blk)
       -> ServerStAcquiring (GenTxId blk) (GenTx blk) SlotNo m ()
     serverStAcquiring s@(_, snapshot) =
       SendMsgAcquired (snapshotSlotNo snapshot) (serverStAcquired s (snapshotTxs snapshot))
 
     serverStAcquired
-      :: (ByteSize32, MempoolSnapshot blk)
-      -> [(Validated (GenTx blk), idx, ByteSize32)]
+      :: (TxMeasure blk, MempoolSnapshot blk)
+      -> [(Validated (GenTx blk), idx, TxMeasure blk)]
       -> ServerStAcquired (GenTxId blk) (GenTx blk) SlotNo m ()
     serverStAcquired s@(capacity, snapshot) txs =
       ServerStAcquired
@@ -61,16 +67,25 @@ localTxMonitorServer mempool =
       , recvMsgGetSizes = do
           let MempoolSize{msNumTxs,msNumBytes} = snapshotMempoolSize snapshot
           let sizes = MempoolSizeAndCapacity
-                { capacityInBytes = unByteSize32 capacity
-                , sizeInBytes     = unByteSize32 msNumBytes
+                { capacityInBytes = unByteSize32 $ txMeasureByteSize capacity
+                , sizeInBytes     = unByteSize32 $ txMeasureByteSize msNumBytes
                 , numberOfTxs     = msNumTxs
                 }
           pure $ SendMsgReplyGetSizes sizes (serverStAcquired s txs)
+      , recvMsgGetMeasures = do
+          let txsMeasures =
+                foldl (\acc (_, _, m) -> Measure.plus acc m) Measure.zero txs
+              measures = MempoolMeasures
+                { txCount = fromIntegral $ length txs
+                , measuresMap =
+                    mkMeasuresMap (Proxy :: Proxy blk) txsMeasures capacity
+                } -- TODO what to do about overflow?
+          pure $ SendMsgReplyGetMeasures measures (serverStAcquired s txs)
       , recvMsgAwaitAcquire = do
           s' <- atomically $ do
             s'@(_, snapshot') <-
                   (,)
-              <$> (txMeasureByteSize <$> getCapacity mempool)
+              <$> getCapacity mempool
               <*> getSnapshot mempool
             s' <$ check (not (snapshot `isSameSnapshot` snapshot'))
           pure $ serverStAcquiring s'
@@ -89,3 +104,31 @@ localTxMonitorServer mempool =
       snapshotSlotNo a == snapshotSlotNo b
 
     tno (_a, b, _c) = b :: TicketNo
+
+mkMeasuresMap :: TxMeasureMetrics (TxMeasure blk)
+              => Proxy blk
+              -> TxMeasure blk
+              -> TxMeasure blk
+              -> Map MeasureName (SizeAndCapacity Integer)
+mkMeasuresMap Proxy size capacity =
+  Map.fromList
+    [ (TransactionBytes, SizeAndCapacity (byteSizeInteger $ txMeasureMetricTxSizeBytes size) (byteSizeInteger $ txMeasureMetricTxSizeBytes capacity))
+    , (ExUnitsMemory, SizeAndCapacity (fromIntegral $ txMeasureMetricExUnitsMemory size) (fromIntegral $ txMeasureMetricExUnitsMemory capacity))
+    , (ExUnitsSteps, SizeAndCapacity (fromIntegral $ txMeasureMetricExUnitsSteps size) (fromIntegral $ txMeasureMetricExUnitsSteps capacity))
+    , (ReferenceScriptsBytes, SizeAndCapacity (byteSizeInteger $ txMeasureMetricRefScriptsSizeBytes size) (byteSizeInteger $ txMeasureMetricRefScriptsSizeBytes capacity))
+    ]
+  where
+    byteSizeInteger :: ByteSize32 -> Integer
+    byteSizeInteger = fromIntegral . unByteSize32
+
+pattern TransactionBytes :: MeasureName
+pattern TransactionBytes = MeasureName "transaction_bytes"
+
+pattern ExUnitsSteps :: MeasureName
+pattern ExUnitsSteps = MeasureName "ex_units_steps"
+
+pattern ExUnitsMemory :: MeasureName
+pattern ExUnitsMemory = MeasureName "ex_units_memory"
+
+pattern ReferenceScriptsBytes :: MeasureName
+pattern ReferenceScriptsBytes = MeasureName "reference_scripts_bytes"

ouroboros-consensus/test/consensus-test/Test/Consensus/Mempool.hs

@@ -305,7 +305,7 @@ prop_Mempool_TraceRemovedTxs setup =
       ]
 
 prjTx ::
-     (Validated (GenTx TestBlock), TicketNo, ByteSize32)
+     (Validated (GenTx TestBlock), TicketNo, TxMeasure TestBlock)
   -> Validated (GenTx TestBlock)
 prjTx (a, _b, _c) = a