fix tests (#1996)

Change-Id: I4ce486756b840818bf55fe7c20b4495ed7c02d3d

chainweb.cabal

@@ -133,6 +133,7 @@ library
     cc-options: -DSQLITE_CORE
     exposed-modules:
           Chainweb.Backup
+        , Chainweb.Block
         , Chainweb.BlockCreationTime
         , Chainweb.BlockHash
         , Chainweb.BlockHeader

src/Chainweb/Block.hs

@@ -0,0 +1,12 @@
+module Chainweb.Block
+    (Block(..))
+    where
+
+import Chainweb.BlockHeader
+import Chainweb.Payload
+
+data Block = Block
+    { _blockHeader :: !BlockHeader
+    , _blockPayloadWithOutputs :: !PayloadWithOutputs
+    }
+    deriving (Eq, Show)

src/Chainweb/BlockHeader.hs

@@ -685,7 +685,7 @@ makeGenesisBlockHeaders v = HM.fromList [ (cid, makeGenesisBlockHeader v cid) |
 
 makeGenesisBlockHeader :: ChainwebVersion -> ChainId -> BlockHeader
 makeGenesisBlockHeader v cid =
-    makeGenesisBlockHeader' v cid (_genesisTime (_versionGenesis v) ^?! onChain cid) (Nonce 0)
+    makeGenesisBlockHeader' v cid (_genesisTime (_versionGenesis v) ^?! atChain cid) (Nonce 0)
 
 -- this version does not rely on the genesis block headers, but just the version graphs
 genesisHeightSlow :: HasCallStack => ChainwebVersion -> ChainId -> BlockHeight
@@ -717,7 +717,7 @@ makeGenesisBlockHeader' v p ct@(BlockCreationTime t) n =
         $ mkFeatureFlags
         :+: ct
         :+: genesisParentBlockHash v cid
-        :+: (v ^?! versionGenesis . genesisBlockTarget . onChain cid)
+        :+: (v ^?! versionGenesis . genesisBlockTarget . atChain cid)
         :+: genesisBlockPayloadHash v cid
         :+: cid
         :+: BlockWeight 0

src/Chainweb/BlockHeader/Validation.hs

@@ -693,7 +693,7 @@ prop_block_genesis_parent b
 
 prop_block_genesis_target :: BlockHeader -> Bool
 prop_block_genesis_target b = isGenesisBlockHeader b
-    ==> _blockTarget b == _chainwebVersion b ^?! versionGenesis . genesisBlockTarget . onChain (_chainId b)
+    ==> _blockTarget b == _chainwebVersion b ^?! versionGenesis . genesisBlockTarget . atChain (_chainId b)
 
 prop_block_current :: Time Micros -> BlockHeader -> Bool
 prop_block_current t b = BlockCreationTime t >= _blockCreationTime b

src/Chainweb/BlockHeaderDB/RestAPI.hs

@@ -49,7 +49,6 @@ module Chainweb.BlockHeaderDB.RestAPI
 -- * API types
   BlockHashPage
 , BlockHeaderPage
-, Block(..)
 , BlockPage
 
 -- * Encodings
@@ -101,6 +100,7 @@ import Network.HTTP.Media ((//), (/:))
 import Servant.API
 
 -- internal modules
+import Chainweb.Block
 import Chainweb.BlockHash
 import Chainweb.BlockHeader
 import Chainweb.BlockHeaderDB
@@ -120,12 +120,6 @@ type BlockHashPage = Page (NextItem BlockHash) BlockHash
 
 type BlockHeaderPage = Page (NextItem BlockHash) BlockHeader
 
-data Block = Block
-    { _blockHeader :: !BlockHeader
-    , _blockPayloadWithOutputs :: !PayloadWithOutputs
-    }
-    deriving (Eq, Show)
-
 -- because this endpoint is only used on the service API, we assume clients
 -- want object-encoded block headers.
 blockProperties :: KeyValue e kv => Block -> [kv]

src/Chainweb/BlockHeaderDB/RestAPI/Server.hs

@@ -76,6 +76,7 @@ import Chainweb.RestAPI.Utils
 import Chainweb.TreeDB
 import Chainweb.Utils.Paging
 import Chainweb.Version
+import Chainweb.Block
 
 -- -------------------------------------------------------------------------- --
 -- Handler Tools

src/Chainweb/ChainId.hs

@@ -59,8 +59,9 @@ module Chainweb.ChainId
 
 -- * Mapping from chain IDs to values
 , ChainMap(..)
-, onChain
+, atChain
 , onChains
+, onChain
 , chainZip
 ) where
 
@@ -276,6 +277,10 @@ data ChainMap a = AllChains a | OnChains (HashMap ChainId a)
 onChains :: [(ChainId, a)] -> ChainMap a
 onChains = OnChains . HM.fromList
 
+-- | A smart constructor, @onChain c a = OnChains (HM.singleton c a)@.
+onChain :: ChainId -> a -> ChainMap a
+onChain c a = OnChains (HM.singleton c a)
+
 -- | Zips two `ChainMap`s on their chain IDs.
 chainZip :: (a -> a -> a) -> ChainMap a -> ChainMap a -> ChainMap a
 chainZip f (OnChains l) (OnChains r) = OnChains $ HM.unionWith f l r
@@ -296,7 +301,7 @@ instance FromJSON a => FromJSON (ChainMap a) where
 makePrisms ''ChainMap
 
 -- | Provides access to the value at a `ChainId`, if it exists.
-onChain :: ChainId -> Fold (ChainMap a) a
-onChain cid = folding $ \case
+atChain :: ChainId -> Fold (ChainMap a) a
+atChain cid = folding $ \case
     OnChains m -> m ^. at cid
     AllChains a -> Just a

src/Chainweb/Chainweb/Configuration.hs

@@ -617,10 +617,10 @@ parseVersion = constructVersion
             maybe (_versionUpgrades winningVersion) (\fub' ->
                 OnChains $ HM.mapWithKey
                     (\cid _ ->
-                        case winningVersion ^?! versionForks . at fub' . _Just . onChain cid of
+                        case winningVersion ^?! versionForks . at fub' . _Just . atChain cid of
                             ForkNever -> error "Chainweb.Chainweb.Configuration.parseVersion: the fork upper bound never occurs in this version."
-                            ForkAtBlockHeight fubHeight -> HM.filterWithKey (\bh _ -> bh <= fubHeight) (winningVersion ^?! versionUpgrades . onChain cid)
-                            ForkAtGenesis -> winningVersion ^?! versionUpgrades . onChain cid
+                            ForkAtBlockHeight fubHeight -> HM.filterWithKey (\bh _ -> bh <= fubHeight) (winningVersion ^?! versionUpgrades . atChain cid)
+                            ForkAtGenesis -> winningVersion ^?! versionUpgrades . atChain cid
                     )
                     (HS.toMap (chainIds winningVersion))
             ) fub

src/Chainweb/Pact/Backend/ChainwebPactCoreDb.hs

@@ -613,7 +613,7 @@ doCommit = view blockHandlerMode >>= \case
     m -> do
         txrs <- if m == Transactional
         then do
-            modify' $ over latestTxId (\(TxId tid) -> TxId (tid + 1))
+            modify' $ over latestTxId (\(TxId tid) -> TxId (succ tid))
             pending <- use (bsPendingTxOrError "commit")
             persistIntraBlockWrites <- view blockHandlerPersistIntraBlockWrites
             -- merge pending tx into pending block data

src/Chainweb/Pact/PactService.hs

@@ -254,7 +254,7 @@ initialPayloadState
 initialPayloadState v cid
     | v ^. versionCheats . disablePact = pure ()
     | otherwise = initializeCoinContract v cid $
-        v ^?! versionGenesis . genesisBlockPayload . onChain cid
+        v ^?! versionGenesis . genesisBlockPayload . atChain cid
 
 initializeCoinContract
     :: forall tbl logger. (CanReadablePayloadCas tbl, Logger logger)

src/Chainweb/Pact/PactService/Pact5/ExecBlock.hs

@@ -141,10 +141,9 @@ pact5TransactionsFromPayload
     :: PayloadData
     -> IO (Vector Pact5.Transaction)
 pact5TransactionsFromPayload plData = do
-    vtrans <- fmap V.fromList $
-              mapM toCWTransaction $
+    vtrans <- mapM toCWTransaction $
               toList (_payloadDataTransactions plData)
-    let (theLefts, theRights) = partitionEithers $ V.toList vtrans
+    let (theLefts, theRights) = partitionEithers vtrans
     unless (null theLefts) $ do
         let ls = map T.pack theLefts
         throwM $ TransactionDecodeFailure $ "Failed to decode pact transactions: "
@@ -167,6 +166,7 @@ continueBlock
     -> BlockInProgress Pact5
     -> PactBlockM logger tbl (BlockInProgress Pact5)
 continueBlock mpAccess blockInProgress = do
+  pbBlockHandle .= _blockInProgressHandle blockInProgress
   liftPactServiceM $ logDebug "starting continueBlock"
   -- update the mempool, ensuring that we reintroduce any transactions that
   -- were removed due to being completed in a block on a different fork.
@@ -207,15 +207,18 @@ continueBlock mpAccess blockInProgress = do
     (V.fromList $ fmap pact5RequestKeyToTransactionHash $ concat invalids)
 
   let !blockInProgress' = blockInProgress
-        & blockInProgressHandle .~ finalBlockHandle
-        & blockInProgressTransactions . transactionPairs %~ (\txs -> txs <> V.fromList (concat $ reverse valids))
-        & blockInProgressRemainingGasLimit .~ finalGasLimit
+        & blockInProgressHandle .~
+          finalBlockHandle
+        & blockInProgressTransactions . transactionPairs .~
+          startTxs <> V.fromList (concat valids)
+        & blockInProgressRemainingGasLimit .~
+          finalGasLimit
 
   return blockInProgress'
 
   where
   blockParentHeader = _parentHeader $ _blockInProgressParentHeader blockInProgress
-  refill fetchLimit txTimeLimit = go [] []
+  refill fetchLimit txTimeLimit blockFillState = over _2 reverse <$> go [] [] blockFillState
     where
     go
       :: [CompletedTransactions] -> [InvalidTransactions]
@@ -239,15 +242,15 @@ continueBlock mpAccess blockInProgress = do
         else do
           -- all request keys from mempool
           -- badlist vs included
-          (newCompletedTransactions, newInvalidTransactions, newGasLimit, timedOut) <-
+          (newCompletedTransactions, newInvalidTransactions, newBlockGasLimit, timedOut) <-
             execNewTransactions (_blockInProgressMiner blockInProgress) prevRemainingGas txTimeLimit newTxs
           liftPactServiceM $ do
             logDebug $ "Refill: included request keys: " <> sshow @[Hash] (fmap (unRequestKey . _crReqKey . snd) newCompletedTransactions)
             logDebug $ "Refill: badlisted request keys: " <> sshow @[Hash] (fmap unRequestKey newInvalidTransactions)
 
           let newBlockFillState = BlockFill
                 { _bfCount = succ prevFillCount
-                , _bfGasLimit = newGasLimit
+                , _bfGasLimit = newBlockGasLimit
                 , _bfTxHashes =
                   flip
                     (foldr (S.insert . pact5RequestKeyToTransactionHash . view (_2 . Pact5.crReqKey)))

src/Chainweb/Pact/Types.hs

@@ -1335,7 +1335,7 @@ pact5CommandToBytes tx = Transaction
 
 pact5CommandResultToBytes :: Pact5.CommandResult Pact5.Hash TxFailedError -> TransactionOutput
 pact5CommandResultToBytes cr = TransactionOutput
-  { _transactionOutputBytes = traceShowId $
+  { _transactionOutputBytes =
     -- TODO: pact5, error codes
     J.encodeStrict (fmap (sshow @_ @Text) cr)
   }

src/Chainweb/Pact4/ModuleCache.hs

@@ -120,7 +120,7 @@ moduleCacheKeys (ModuleCache a) = fst <$> LHM.toList a
 -- it uses genesisHeight which is from BlockHeader which imports Guards
 cleanModuleCache :: ChainwebVersion -> ChainId -> BlockHeight -> Bool
 cleanModuleCache v cid bh =
-    case v ^?! versionForks . at Chainweb217Pact . _Just . onChain cid of
+    case v ^?! versionForks . at Chainweb217Pact . _Just . atChain cid of
         ForkAtBlockHeight bh' -> bh == bh'
         ForkAtGenesis -> bh == genesisHeight v cid
         ForkNever -> False

src/Chainweb/Pact4/TransactionExec.hs

@@ -805,7 +805,7 @@ applyUpgrades
   -> TransactionM logger p (Maybe ModuleCache)
 applyUpgrades v cid height
     | Just (ForSomePactVersion Pact4T upg) <-
-        v ^? versionUpgrades . onChain cid . ix height = applyUpgrade upg
+        v ^? versionUpgrades . atChain cid . ix height = applyUpgrade upg
     | cleanModuleCache v cid height = filterModuleCache
     | otherwise = return Nothing
   where

src/Chainweb/Pact5/TransactionExec.hs

@@ -467,7 +467,7 @@ applyCmd logger maybeGasLogger pactDb txCtx spv initialGas cmd = do
                 , _crTxId = _erTxId payloadResult
                 , _crResult =
                     -- TODO: don't use `last` here for GHC 9.10 compat
-                    PactResultOk $ compileValueToPactValue $ last $ traceShowId (_erOutput payloadResult)
+                    PactResultOk $ compileValueToPactValue $ last $ _erOutput payloadResult
                 , _crGas = gasUsed
                 , _crLogs = Just $ _erLogs buyGasResult <> _erLogs payloadResult <> _erLogs redeemGasResult
                 , _crContinuation = _erExec payloadResult
@@ -570,7 +570,7 @@ applyUpgrades
 applyUpgrades logger db txCtx
      | Just (ForPact5 upg) <- _chainwebVersion txCtx
           ^? versionUpgrades
-          . onChain (_chainId txCtx)
+          . atChain (_chainId txCtx)
           . ix (ctxCurrentBlockHeight txCtx)
          = applyUpgrade upg
      | otherwise = return ()
@@ -705,6 +705,8 @@ buyGas
   -> Command (Payload PublicMeta ParsedCode)
   -> IO (Either Pact5BuyGasError EvalResult)
 buyGas logger db txCtx cmd = do
+  logFunctionText logger L.Debug $
+    "buying gas for " <> sshow (_cmdHash cmd)
   -- TODO: use quirked gas?
   let gasPayerCaps =
         [ cap
@@ -795,6 +797,8 @@ redeemGas :: (Logger logger)
   -> IO (Either Pact5RedeemGasError EvalResult)
 redeemGas logger pactDb txCtx gasUsed maybeFundTxPactId cmd
     | isChainweb224Pact, Nothing <- maybeFundTxPactId = do
+      logFunctionText logger L.Debug $
+        "redeeming gas (post-2.24) for " <> sshow (_cmdHash cmd)
       -- if we're past chainweb 2.24, we don't use defpacts for gas; see 'pact/coin-contract/coin.pact#redeem-gas'
       let (redeemGasTerm, redeemGasData) = mkRedeemGasTerm mid mks sender gasTotal gasFee
 
@@ -817,6 +821,8 @@ redeemGas logger pactDb txCtx gasUsed maybeFundTxPactId cmd
             pure $ Right evalResult
 
     | not isChainweb224Pact, Just fundTxPactId <- maybeFundTxPactId = do
+      logFunctionText logger L.Debug $
+        "redeeming gas (pre-2.24) for " <> sshow (_cmdHash cmd)
       -- before chainweb 2.24, we use defpacts for gas; see: 'pact/coin-contract/coin.pact#fund-tx'
       let redeemGasData = PObject $ Map.singleton "fee" (PDecimal $ _pact5GasSupply gasFee)
       evalContinuation Transactional

src/Chainweb/Payload.hs

@@ -737,7 +737,7 @@ decodePayloadDataList = runGetS $ do
 encodePayloadWithOutputsList :: PayloadWithOutputsList -> B.ByteString
 encodePayloadWithOutputsList (PayloadWithOutputsList xs) = runPutS $ do
     putWord64be (fromIntegral $ length xs)
-    forM_ xs putPayloadWithOutputs 
+    forM_ xs putPayloadWithOutputs
 
 decodePayloadWithOutputsList :: (MonadThrow m) => B.ByteString -> m PayloadWithOutputsList
 decodePayloadWithOutputsList = runGetS $ do

src/Chainweb/Payload/PayloadStore.hs

@@ -334,7 +334,7 @@ initializePayloadDb
 initializePayloadDb v db = traverse_ initForChain $ chainIds v
   where
     initForChain cid =
-        addNewPayload db (genesisHeight v cid) $ v ^?! versionGenesis . genesisBlockPayload . onChain cid
+        addNewPayload db (genesisHeight v cid) $ v ^?! versionGenesis . genesisBlockPayload . atChain cid
 
 -- -------------------------------------------------------------------------- --
 -- Insert new Payload

src/Chainweb/Rosetta/Internal.hs

@@ -124,7 +124,7 @@ matchLogs
     -> ExceptT RosettaFailure Handler tx
 matchLogs typ bh logs coinbase txs
   | bheight == genesisHeight v cid = matchGenesis
-  | Just (ForPact4 upg) <- v ^? versionUpgrades . onChain cid . at bheight . _Just = matchRemediation upg
+  | Just (ForPact4 upg) <- v ^? versionUpgrades . atChain cid . at bheight . _Just = matchRemediation upg
   -- TODO: integrate pact 5?
   | otherwise = matchRest
   where

src/Chainweb/Version.hs

@@ -539,7 +539,7 @@ makeLensesWith (lensRules & generateLazyPatterns .~ True) 'VersionDefaults
 makeLensesWith (lensRules & generateLazyPatterns .~ True) 'VersionQuirks
 
 genesisBlockPayloadHash :: ChainwebVersion -> ChainId -> BlockPayloadHash
-genesisBlockPayloadHash v cid = v ^?! versionGenesis . genesisBlockPayload . onChain cid . to _payloadWithOutputsPayloadHash
+genesisBlockPayloadHash v cid = v ^?! versionGenesis . genesisBlockPayload . atChain cid . to _payloadWithOutputsPayloadHash
 
 instance HasTextRepresentation ChainwebVersionName where
     toText = getChainwebVersionName
@@ -669,8 +669,8 @@ indexByForkHeights v = OnChains . foldl' go (HM.empty <$ HS.toMap (chainIds v))
         newTxs = HM.fromList $
             [ (cid, HM.singleton forkHeight upg)
             | cid <- HM.keys acc
-            , Just upg <- [txsPerChain ^? onChain cid]
-            , ForkAtBlockHeight forkHeight <- [v ^?! versionForks . at fork . _Just . onChain cid]
+            , Just upg <- [txsPerChain ^? atChain cid]
+            , ForkAtBlockHeight forkHeight <- [v ^?! versionForks . at fork . _Just . atChain cid]
             , forkHeight /= maxBound
             ]
 

src/Chainweb/Version/Guards.hs

@@ -74,7 +74,7 @@ import Chainweb.Version
 import Chainweb.Utils.Rule
 
 getForkHeight :: Fork -> ChainwebVersion -> ChainId -> ForkHeight
-getForkHeight fork v cid = v ^?! versionForks . at fork . _Just . onChain cid
+getForkHeight fork v cid = v ^?! versionForks . at fork . _Just . atChain cid
 
 checkFork
     :: (BlockHeight -> ForkHeight -> Bool)

src/Chainweb/Version/Mainnet.hs

@@ -159,7 +159,7 @@ mainnet = ChainwebVersion
     , _versionWindow = WindowWidth 120
     , _versionHeaderBaseSizeBytes = 318 - 110
     , _versionMaxBlockGasLimit =
-        (succ $ mainnet ^?! versionForks . at Chainweb216Pact . _Just . onChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above`
+        (succ $ mainnet ^?! versionForks . at Chainweb216Pact . _Just . atChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above`
         End Nothing
     , _versionBootstraps = domainAddr2PeerInfo mainnetBootstrapHosts
     , _versionGenesis = VersionGenesis

src/Chainweb/Version/Testnet.hs

@@ -139,7 +139,7 @@ testnet = ChainwebVersion
     , _versionWindow = WindowWidth 120
     , _versionHeaderBaseSizeBytes = 318 - 110
     , _versionMaxBlockGasLimit =
-        (succ $ testnet ^?! versionForks . at Chainweb216Pact . _Just . onChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above`
+        (succ $ testnet ^?! versionForks . at Chainweb216Pact . _Just . atChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above`
         End Nothing
     , _versionBootstraps = domainAddr2PeerInfo testnetBootstrapHosts
     , _versionGenesis = VersionGenesis

src/Chainweb/Version/Utils.hs

@@ -463,7 +463,7 @@ verifiersAt v cid bh =
     M.restrictKeys allVerifierPlugins activeVerifierNames
     where
     activeVerifierNames =
-        case measureRule bh $ _versionVerifierPluginNames v ^?! onChain cid of
+        case measureRule bh $ _versionVerifierPluginNames v ^?! atChain cid of
             Bottom vs -> vs
             Top (_, vs) -> vs
             Between (_, vs) _ -> vs