cli: read/write byron tx files in the right binary format

This is sadly a little confusing. In Byron there are 4 kinds of on-chain
objects, and only one of those is normal txs. So we have a type called
GenTx ByronBlock that is the "generalised tx" that covers all four
cases. This is the type that is sent between mempools.

But the actual on-chain Byron tx type is a TxAux. The cli code for
generating txs and writing them out to file was using the GenTx format.
And them same for reading them back in again. While this is self
consistent this is also very unhelpful, since we should use the real
format as it exists on the chain, not the intermediate wrapper format.

Doing it this way means we can submit them directly to the tx-submission
service, We can decode and print them, check their hashes etc.

The code isn't perfect. We can do better by handling this GenTx vs Tx
distinction in a more general way so that we can write the CLI code more
generically between Byron, Shelley and mock protocols & ledgers.

cardano-node/src/Cardano/CLI/Ops.hs

@@ -32,6 +32,7 @@ import qualified Data.Text as T
 import qualified Text.JSON.Canonical as CanonicalJSON
 
 import           Cardano.Crypto (RequiresNetworkMagic, SigningKey (..))
+import           Cardano.Binary (DecoderError)
 import           Codec.CBOR.Read (DeserialiseFailure, deserialiseFromBytes)
 import           Codec.CBOR.Write (toLazyByteString)
 import qualified Cardano.Crypto.Signing as Crypto
@@ -101,7 +102,7 @@ data CliError
   | SigningKeyDeserialisationFailed !FilePath !DeserialiseFailure
   | VerificationKeyDeserialisationFailed !FilePath !Text
   | DlgCertificateDeserialisationFailed !FilePath !Text
-  | TxDeserialisationFailed !FilePath !DeserialiseFailure
+  | TxDeserialisationFailed !FilePath !DecoderError
   -- TODO:  sadly, VerificationKeyParseError isn't exported from Cardano.Crypto.Signing/*
   -- Inconsistencies
   | DelegationError !Genesis.GenesisDelegationError

cardano-node/src/Cardano/CLI/Run.hs

@@ -29,7 +29,6 @@ module Cardano.CLI.Run (
 
 import           Cardano.Prelude hiding (option, trace)
 
-import           Codec.Serialise (serialise)
 import           Control.Monad.Trans.Except (ExceptT)
 import           Control.Monad.Trans.Except.Extra (hoistEither, firstExceptT)
 import qualified Data.ByteString.Lazy as LB
@@ -272,7 +271,7 @@ runCommand (SpendGenesisUTxO ptcl genFile genHash (NewTxFile ctTx) ctKey genRich
                 update
                 ptcl
                 sk
-    liftIO . ensureNewFileLBS ctTx $ serialise tx
+    liftIO . ensureNewFileLBS ctTx $ toCborTxAux tx
 
 runCommand (SpendUTxO ptcl genFile genHash (NewTxFile ctTx) ctKey ins outs) = do
     sk <- readSigningKey ptcl ctKey
@@ -292,7 +291,7 @@ runCommand (SpendUTxO ptcl genFile genHash (NewTxFile ctTx) ctKey ins outs) = do
                  update
                  ptcl
                  sk
-    liftIO . ensureNewFileLBS ctTx $ serialise gTx
+    liftIO . ensureNewFileLBS ctTx $ toCborTxAux gTx
 
 runCommand (GenerateTxs
                logConfigFp

cardano-node/src/Cardano/CLI/Tx.hs

@@ -9,20 +9,25 @@ module Cardano.CLI.Tx
   , NewTxFile(..)
   , prettyAddress
   , readByronTx
+  , normalByronTxToGenTx
   , txSpendGenesisUTxOByronPBFT
   , issueGenesisUTxOExpenditure
   , txSpendUTxOByronPBFT
   , issueUTxOExpenditure
   , nodeSubmitTx
+
+    --TODO: remove when they are exported from the ledger
+  , fromCborTxAux
+  , toCborTxAux
   )
 where
 
-import           Prelude (error, show)
-import           Cardano.Prelude hiding (option, show, trace, (%))
+import           Prelude (error)
+import           Cardano.Prelude hiding (option, trace, (%))
 
-import           Codec.Serialise (deserialiseOrFail)
-import           Control.Monad.Trans.Except.Extra (left, right)
+import           Control.Monad.Trans.Except.Extra (right)
 import qualified Data.ByteString.Lazy as LB
+import qualified Data.ByteString as B
 import qualified Data.Map.Strict as Map
 import           Data.String (IsString)
 import           Data.Text (Text)
@@ -31,10 +36,11 @@ import           Formatting ((%), sformat)
 
 import           Control.Tracer (traceWith, stdoutTracer)
 
+import qualified Cardano.Binary as Binary
+
 import           Cardano.Chain.Common (Address)
 import qualified Cardano.Chain.Common as Common
 import           Cardano.Chain.Genesis as Genesis
-import qualified Cardano.Chain.MempoolPayload as CC.Mempool
 import           Cardano.Chain.UTxO ( mkTxAux, annotateTxAux
                                     , Tx(..), TxId, TxIn, TxOut)
 import qualified Cardano.Chain.UTxO as UTxO
@@ -77,9 +83,14 @@ prettyAddress addr = sformat
 readByronTx :: TxFile -> IO (GenTx ByronBlock)
 readByronTx (TxFile fp) = do
   txBS <- LB.readFile fp
-  case deserialiseOrFail txBS of
+  case fromCborTxAux txBS of
     Left e -> throwIO $ TxDeserialisationFailed fp e
-    Right tx -> pure tx
+    Right tx -> pure (normalByronTxToGenTx tx)
+
+-- | The 'GenTx' is all the kinds of transactions that can be submitted
+-- and \"normal\" Byron transactions are just one of the kinds.
+normalByronTxToGenTx :: UTxO.ATxAux ByteString -> GenTx ByronBlock
+normalByronTxToGenTx tx' = Byron.ByronTx (Byron.byronIdTx tx') tx'
 
 -- | Given a Tx id, produce a UTxO Tx input witness, by signing it
 --   with respect to a given protocol magic.
@@ -139,10 +150,9 @@ txSpendGenesisUTxOByronPBFT
   -> SigningKey
   -> Address
   -> NonEmpty TxOut
-  -> GenTx ByronBlock
+  -> UTxO.ATxAux ByteString
 txSpendGenesisUTxOByronPBFT gc sk genAddr outs =
-    Byron.fromMempoolPayload
-      $ CC.Mempool.MempoolTx $ annotateTxAux $ mkTxAux tx (pure wit)
+    annotateTxAux $ mkTxAux tx (pure wit)
   where
     tx = UnsafeTx (pure txIn) outs txattrs
 
@@ -167,7 +177,7 @@ issueGenesisUTxOExpenditure
   -> Update
   -> Protocol
   -> Crypto.SigningKey
-  -> ExceptT RealPBFTError IO (GenTx ByronBlock)
+  -> ExceptT RealPBFTError IO (UTxO.ATxAux ByteString)
 issueGenesisUTxOExpenditure
   genRichAddr
   outs
@@ -182,14 +192,9 @@ issueGenesisUTxOExpenditure
   sk =
     withRealPBFT gHash genFile nMagic sigThresh delCertFp sKeyFp update ptcl
       $ \(Consensus.ProtocolRealPBFT gc _ _ _ _)-> do
-          case txSpendGenesisUTxOByronPBFT gc sk genRichAddr outs of
-            tx@(ByronTx txid _) -> do
-              putStrLn $ sformat ("TxId: "%Crypto.hashHexF) txid
-              right tx
-            x -> left . InvariantViolation
-                      . T.pack
-                      $ "A non-ByronTx GenTx out of 'txSpendUTxOByronPBFT': "
-                      <> show x
+          let tx = txSpendGenesisUTxOByronPBFT gc sk genRichAddr outs
+          traceWith stdoutTracer ("TxId: " ++ condense (Byron.byronIdTx tx))
+          right tx
 
 -- | Generate a transaction from given Tx inputs to outputs,
 --   signed by the given key.
@@ -198,10 +203,9 @@ txSpendUTxOByronPBFT
   -> SigningKey
   -> NonEmpty TxIn
   -> NonEmpty TxOut
-  -> GenTx ByronBlock
+  -> UTxO.ATxAux ByteString
 txSpendUTxOByronPBFT gc sk ins outs =
-    Byron.fromMempoolPayload
-      $ CC.Mempool.MempoolTx $ annotateTxAux $ mkTxAux tx (pure wit)
+    annotateTxAux $ mkTxAux tx (pure wit)
   where
     tx = UnsafeTx ins outs txattrs
 
@@ -223,7 +227,7 @@ issueUTxOExpenditure
   -> Update
   -> Protocol
   -> Crypto.SigningKey
-  -> ExceptT RealPBFTError IO (GenTx ByronBlock)
+  -> ExceptT RealPBFTError IO (UTxO.ATxAux ByteString)
 issueUTxOExpenditure
   ins
   outs
@@ -238,15 +242,9 @@ issueUTxOExpenditure
   key = do
     withRealPBFT gHash genFile nMagic sigThresh delCertFp sKeyFp update ptcl $
       \(Consensus.ProtocolRealPBFT gc _ _ _ _)-> do
-        case txSpendUTxOByronPBFT gc key ins outs of
-          tx@(ByronTx txid _) -> do
-            putStrLn $ sformat ("TxId: "%Crypto.hashHexF) txid
-            pure tx
-          x ->
-            left . InvariantViolation
-                 . T.pack
-                 $ "A non-ByronTx GenTx out of 'txSpendUTxOByronPBFT': "
-                 <> show x
+        let tx = txSpendUTxOByronPBFT gc key ins outs
+        traceWith stdoutTracer ("TxId: " ++ condense (Byron.byronIdTx tx))
+        pure tx
 
 -- | Submit a transaction to a node specified by topology info.
 nodeSubmitTx
@@ -283,3 +281,17 @@ nodeSubmitTx
                  (node topology)
                  gentx
                  stdoutTracer
+
+--TODO: remove these local definitions when the updated ledger lib is available
+fromCborTxAux :: LB.ByteString ->  Either Binary.DecoderError (UTxO.ATxAux B.ByteString)
+fromCborTxAux lbs =
+    fmap (annotationBytes lbs)
+      $ Binary.decodeFullDecoder "Cardano.Chain.UTxO.TxAux.fromCborTxAux"
+                                 Binary.fromCBOR lbs
+  where
+    annotationBytes :: Functor f => LB.ByteString -> f Binary.ByteSpan -> f B.ByteString
+    annotationBytes bytes = fmap (LB.toStrict . Binary.slice bytes)
+
+toCborTxAux :: UTxO.ATxAux ByteString -> LB.ByteString
+toCborTxAux = LB.fromStrict . UTxO.aTaAnnotation -- The ByteString anotation is the CBOR encoded version.
+

cardano-node/src/Cardano/CLI/Tx/Generation.hs

@@ -66,7 +66,7 @@ import qualified Cardano.Crypto as Crypto
 import           Cardano.Config.Topology (NodeAddress (..),
                                           NodeHostAddress(..))
 import           Cardano.CLI.Ops
-import           Cardano.CLI.Tx (txSpendGenesisUTxOByronPBFT)
+import           Cardano.CLI.Tx (txSpendGenesisUTxOByronPBFT, normalByronTxToGenTx)
 import           Cardano.CLI.Tx.BenchmarkingTxSubmission (ROEnv (..),
                                                           TraceBenchTxSubmit (..),
                                                           bulkSubmission)
@@ -407,7 +407,8 @@ prepareInitialFunds llTracer
         }
 
   let genesisTx :: GenTx ByronBlock
-      genesisTx = txSpendGenesisUTxOByronPBFT genesisConfig
+      genesisTx = normalByronTxToGenTx $
+                  txSpendGenesisUTxOByronPBFT genesisConfig
                                               signingKey
                                               genesisAddress
                                               (NE.fromList [outForBig])