diff --git a/cabal.project b/cabal.project index d7c9798d49..6ff9b0da80 100644 --- a/cabal.project +++ b/cabal.project @@ -26,7 +26,7 @@ source-repository-package source-repository-package type: git location: https://github.com/kadena-io/rosetta.git - tag: d20b0012c6f5e1d28843b89a4edd5ea652ade0c3 + tag: 1ccb68d7aec0414f494fb06f591214e7cf845627 package vault documentation: false diff --git a/chainweb.cabal b/chainweb.cabal index 755bf2f12b..79e31a59fa 100644 --- a/chainweb.cabal +++ b/chainweb.cabal @@ -211,8 +211,9 @@ library , Chainweb.RestAPI.Utils , Chainweb.Rosetta.Internal , Chainweb.Rosetta.RestAPI + , Chainweb.Rosetta.RestAPI.Client , Chainweb.Rosetta.RestAPI.Server - , Chainweb.Rosetta.Util + , Chainweb.Rosetta.Utils , Chainweb.SPV , Chainweb.SPV.CreateProof , Chainweb.SPV.VerifyProof @@ -466,7 +467,9 @@ test-suite chainweb-tests Chainweb.Test.Pact.Utils Chainweb.Test.RestAPI Chainweb.Test.RestAPI.Client_ + Chainweb.Test.RestAPI.Utils Chainweb.Test.Rosetta + Chainweb.Test.Rosetta.RestAPI Chainweb.Test.Roundtrips Chainweb.Test.SPV Chainweb.Test.Store.CAS @@ -666,6 +669,7 @@ executable cwtool , chainweb-storage >= 0.1 , clock >= 0.7 , configuration-tools >= 0.5 + , connection >=0.2 , containers >= 0.5 , cryptonite >= 0.25 , deepseq >= 1.4 @@ -675,6 +679,7 @@ executable cwtool , file-embed , errors >= 2.3 , exceptions >= 0.8 + , extra >= 1.6 , file-embed >= 0.0 , formatting >= 6.3 , http-client >= 0.5 diff --git a/src/Chainweb/Pact/TransactionExec.hs b/src/Chainweb/Pact/TransactionExec.hs index 190418fb12..3e1098c274 100644 --- a/src/Chainweb/Pact/TransactionExec.hs +++ b/src/Chainweb/Pact/TransactionExec.hs @@ -434,6 +434,18 @@ applyUpgrades v cid height applyTxs txsIO = do infoLog "Applying upgrade!" txs <- map (fmap payloadObj) <$> liftIO txsIO + + -- + -- Note (emily): the historical use of 'mapM_' here means that we are not + -- threading the updated module cache from tx to tx in our upgrades. + -- This means that the outputed module cache is the set of modules + -- loaded in the /last/ tx, and that result will go into the hash. + -- + -- Whether this can be fixed in the future should be explored, but we've + -- already built fixes to address this problem that also affect the + -- hashes of later blocks. + -- + local (set txExecutionConfig def) $ mapM_ applyTx txs mc <- use txCache @@ -464,6 +476,13 @@ applyTwentyChainUpgrade v cid bh infoLog $ "Applying 20-chain upgrades on chain " <> sshow cid let txs = fmap payloadObj <$> txlist + + -- + -- Note (emily): This function does not need to care about + -- module caching, because it is already seeded with the correct cache + -- state, and is not updating the module cache, unlike 'applyUpgrades'. + -- + traverse_ applyTx txs | otherwise = return () where diff --git a/src/Chainweb/Rosetta/Internal.hs b/src/Chainweb/Rosetta/Internal.hs index 8855bf2b0d..78cd34c0b0 100644 --- a/src/Chainweb/Rosetta/Internal.hs +++ b/src/Chainweb/Rosetta/Internal.hs @@ -52,13 +52,12 @@ import Chainweb.Pact.Service.Types (Domain'(..), BlockTxHistory(..)) import Chainweb.Payload hiding (Transaction(..)) import Chainweb.Payload.PayloadStore import Chainweb.Rosetta.RestAPI +import Chainweb.Rosetta.Utils import Chainweb.TreeDB (seekAncestor) import Chainweb.Utils import Chainweb.Version import Chainweb.WebPactExecutionService (PactExecutionService(..)) -import Chainweb.Rosetta.Util - --- -------------------------------------------------------------------------------- diff --git a/src/Chainweb/Rosetta/RestAPI.hs b/src/Chainweb/Rosetta/RestAPI.hs index 97d47f3c3c..24e7a7c8fe 100644 --- a/src/Chainweb/Rosetta/RestAPI.hs +++ b/src/Chainweb/Rosetta/RestAPI.hs @@ -1,6 +1,7 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE KindSignatures #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeOperators #-} @@ -14,8 +15,29 @@ -- module Chainweb.Rosetta.RestAPI ( -- * Endpoints - RosettaApi_ - , RosettaApi + RosettaApi + , rosettaApi + -- * Standalone APIs for client derivation + , RosettaAccountBalanceApi + , rosettaAccountBalanceApi + , RosettaBlockTransactionApi + , rosettaBlockTransactionApi + , RosettaBlockApi + , rosettaBlockApi + , RosettaConstructionMetadataApi + , rosettaConstructionMetadataApi + , RosettaConstructionSubmitApi + , rosettaConstructionSubmitApi + , RosettaMempoolTransactionApi + , rosettaMempoolTransactionApi + , RosettaMempoolApi + , rosettaMempoolApi + , RosettaNetworkListApi + , rosettaNetworkListApi + , RosettaNetworkOptionsApi + , rosettaNetworkOptionsApi + , RosettaNetworkStatusApi + , rosettaNetworkStatusApi -- * Errors , RosettaFailure(..) , rosettaError @@ -25,7 +47,6 @@ module Chainweb.Rosetta.RestAPI import Control.Error.Util import Control.Monad (when) -import Control.Monad.Except (throwError) import Control.Monad.Trans.Except (ExceptT) import Data.Aeson (encode) @@ -33,8 +54,7 @@ import qualified Data.Text as T import Rosetta -import Servant.API -import Servant.Server +import Servant import Text.Read (readMaybe) @@ -46,46 +66,165 @@ import Chainweb.Version --- -type RosettaApi_ = - -- Accounts -- - "rosetta" :> "account" :> "balance" - :> ReqBody '[JSON] AccountBalanceReq - :> Post '[JSON] AccountBalanceResp - -- Blocks -- - :<|> "rosetta" :> "block" :> "transaction" - :> ReqBody '[JSON] BlockTransactionReq - :> Post '[JSON] BlockTransactionResp - :<|> "rosetta" :> "block" - :> ReqBody '[JSON] BlockReq - :> Post '[JSON] BlockResp - -- Construction -- - :<|> "rosetta" :> "construction" :> "metadata" - :> ReqBody '[JSON] ConstructionMetadataReq - :> Post '[JSON] ConstructionMetadataResp - :<|> "rosetta" :> "construction" :> "submit" - :> ReqBody '[JSON] ConstructionSubmitReq - :> Post '[JSON] ConstructionSubmitResp - -- Mempool -- - :<|> "rosetta" :> "mempool" :> "transaction" - :> ReqBody '[JSON] MempoolTransactionReq - :> Post '[JSON] MempoolTransactionResp - :<|> "rosetta" :> "mempool" - :> ReqBody '[JSON] MempoolReq - :> Post '[JSON] MempoolResp - -- Network -- - :<|> "rosetta" :> "network" :> "list" - :> ReqBody '[JSON] MetadataReq - :> Post '[JSON] NetworkListResp - :<|> "rosetta" :> "network" :> "options" - :> ReqBody '[JSON] NetworkReq - :> Post '[JSON] NetworkOptionsResp - :<|> "rosetta" :> "network" :> "status" - :> ReqBody '[JSON] NetworkReq - :> Post '[JSON] NetworkStatusResp +-- ------------------------------------------------------------------ -- +-- Rosetta Api type RosettaApi (v :: ChainwebVersionT) = 'ChainwebEndpoint v :> Reassoc RosettaApi_ --- TODO: Investigate if Rosetta Erros can be dynamic? +type RosettaApi_ = "rosetta" :> + ( -- Accounts -- + RosettaAccountBalanceApi_ + -- Blocks -- + :<|> RosettaBlockTransactionApi_ + :<|> RosettaBlockApi_ + -- Construction -- + :<|> RosettaConstructionMetadataApi_ + :<|> RosettaConstructionSubmitApi_ + -- Mempool -- + :<|> RosettaMempoolTransactionApi_ + :<|> RosettaMempoolApi_ + -- Network -- + :<|> RosettaNetworkListApi_ + :<|> RosettaNetworkOptionsApi_ + :<|> RosettaNetworkStatusApi_ + ) + +rosettaApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaApi v) +rosettaApi = Proxy + +-- ------------------------------------------------------------------ -- +-- Standalone Endpoints + Witnesses + +type RosettaApiEndpoint (v :: ChainwebVersionT) api + = 'ChainwebEndpoint v + :> "rosetta" + :> api + +type RosettaAccountBalanceApi v = RosettaApiEndpoint v RosettaAccountBalanceApi_ +type RosettaBlockTransactionApi v = RosettaApiEndpoint v RosettaBlockTransactionApi_ +type RosettaBlockApi v = RosettaApiEndpoint v RosettaBlockApi_ +type RosettaConstructionSubmitApi v = RosettaApiEndpoint v RosettaConstructionSubmitApi_ +type RosettaConstructionMetadataApi v = RosettaApiEndpoint v RosettaConstructionMetadataApi_ +type RosettaMempoolTransactionApi v = RosettaApiEndpoint v RosettaMempoolTransactionApi_ +type RosettaMempoolApi v = RosettaApiEndpoint v RosettaMempoolApi_ +type RosettaNetworkListApi v = RosettaApiEndpoint v RosettaNetworkListApi_ +type RosettaNetworkOptionsApi v = RosettaApiEndpoint v RosettaNetworkOptionsApi_ +type RosettaNetworkStatusApi v = RosettaApiEndpoint v RosettaNetworkStatusApi_ + +type RosettaAccountBalanceApi_ + = "account" + :> "balance" + :> ReqBody '[JSON] AccountBalanceReq + :> Post '[JSON] AccountBalanceResp + +type RosettaBlockTransactionApi_ + = "block" + :> "transaction" + :> ReqBody '[JSON] BlockTransactionReq + :> Post '[JSON] BlockTransactionResp + +type RosettaBlockApi_ + = "block" + :> ReqBody '[JSON] BlockReq + :> Post '[JSON] BlockResp + +type RosettaConstructionMetadataApi_ + = "construction" + :> "metadata" + :> ReqBody '[JSON] ConstructionMetadataReq + :> Post '[JSON] ConstructionMetadataResp + +type RosettaConstructionSubmitApi_ + = "construction" + :> "submit" + :> ReqBody '[JSON] ConstructionSubmitReq + :> Post '[JSON] ConstructionSubmitResp + +type RosettaMempoolTransactionApi_ + = "mempool" + :> "transaction" + :> ReqBody '[JSON] MempoolTransactionReq + :> Post '[JSON] MempoolTransactionResp + +type RosettaMempoolApi_ + = "mempool" + :> ReqBody '[JSON] MempoolReq + :> Post '[JSON] MempoolResp + +type RosettaNetworkListApi_ + = "network" + :> "list" + :> ReqBody '[JSON] MetadataReq + :> Post '[JSON] NetworkListResp + +type RosettaNetworkOptionsApi_ + = "network" + :> "options" + :> ReqBody '[JSON] NetworkReq + :> Post '[JSON] NetworkOptionsResp + +type RosettaNetworkStatusApi_ + = "network" + :> "status" + :> ReqBody '[JSON] NetworkReq + :> Post '[JSON] NetworkStatusResp + +rosettaAccountBalanceApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaAccountBalanceApi v) +rosettaAccountBalanceApi = Proxy + +rosettaBlockTransactionApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaBlockTransactionApi v) +rosettaBlockTransactionApi = Proxy + +rosettaBlockApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaBlockApi v) +rosettaBlockApi = Proxy + +rosettaConstructionMetadataApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaConstructionMetadataApi v) +rosettaConstructionMetadataApi = Proxy + +rosettaConstructionSubmitApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaConstructionSubmitApi v) +rosettaConstructionSubmitApi = Proxy + +rosettaMempoolTransactionApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaMempoolTransactionApi v) +rosettaMempoolTransactionApi = Proxy + +rosettaMempoolApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaMempoolApi v) +rosettaMempoolApi = Proxy + +rosettaNetworkListApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaNetworkListApi v) +rosettaNetworkListApi = Proxy + +rosettaNetworkOptionsApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaNetworkOptionsApi v) +rosettaNetworkOptionsApi = Proxy + +rosettaNetworkStatusApi + :: forall (v :: ChainwebVersionT) + . Proxy (RosettaNetworkStatusApi v) +rosettaNetworkStatusApi = Proxy + +-- ------------------------------------------------------------------ -- +-- Rosetta Exceptions + +-- TODO: Investigate if Rosetta Erros can be dynamic data RosettaFailure = RosettaChainUnspecified | RosettaInvalidChain @@ -167,4 +306,3 @@ readChainIdText :: ChainwebVersion -> T.Text -> Maybe ChainId readChainIdText v c = do cid <- readMaybe @Word (T.unpack c) mkChainId v maxBound cid - diff --git a/src/Chainweb/Rosetta/RestAPI/Client.hs b/src/Chainweb/Rosetta/RestAPI/Client.hs new file mode 100644 index 0000000000..b744c8681b --- /dev/null +++ b/src/Chainweb/Rosetta/RestAPI/Client.hs @@ -0,0 +1,235 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE KindSignatures #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE RankNTypes #-} +-- | +-- Module: Chainweb.Rosetta.RestAPI.Client +-- Copyright: Copyright © 2018 - 2020 Kadena LLC. +-- License: MIT +-- Maintainer: Emily Pillmore +-- Stability: experimental +-- +-- This module defines the client API for the Chainweb Rosetta +-- integration. +-- +module Chainweb.Rosetta.RestAPI.Client +( -- * AccounT Endpoints + rosettaAccountBalanceApiClient + -- * Block Endpoints +, rosettaBlockTransactionApiClient +, rosettaBlockApiClient + -- * Construction Endpoints +, rosettaConstructionMetadataApiClient +, rosettaConstructionSubmitApiClient + -- * Mempool Endpoints +, rosettaMempoolApiClient +, rosettaMempoolTransactionApiClient + -- * Network Endpoints +, rosettaNetworkListApiClient +, rosettaNetworkOptionsApiClient +, rosettaNetworkStatusApiClient +) +where + + +import Rosetta + +import Servant.Client + +-- internal chainweb modules + +import Chainweb.ChainId +import Chainweb.Rosetta.RestAPI +import Chainweb.Version + + +-- -------------------------------------------------------------------------- -- +-- Accounts Endpoints + +rosettaAccountBalanceApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => AccountBalanceReq + -- ^ Contains a network id, account id, and a partial block identifier + -- which is not populated. + -> ClientM AccountBalanceResp +rosettaAccountBalanceApiClient_ = client (rosettaAccountBalanceApi @v) + +rosettaAccountBalanceApiClient + :: ChainwebVersion + -> AccountBalanceReq + -- ^ Contains a network id, account id, and a partial block identifier + -- which is not populated. + -> ClientM AccountBalanceResp +rosettaAccountBalanceApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaAccountBalanceApiClient_ @v + +-- -------------------------------------------------------------------------- -- +-- Block Endpoints + +rosettaBlockTransactionApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => BlockTransactionReq + -- ^ Contains a network id, a block id, and a transaction id + -> ClientM BlockTransactionResp +rosettaBlockTransactionApiClient_ = client (rosettaBlockTransactionApi @v) + +rosettaBlockTransactionApiClient + :: ChainwebVersion + -> BlockTransactionReq + -- ^ Contains a network id, a block id, and a transaction id + -> ClientM BlockTransactionResp +rosettaBlockTransactionApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaBlockTransactionApiClient_ @v + +rosettaBlockApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => BlockReq + -- ^ Contains a network id and a partial block id + -> ClientM BlockResp +rosettaBlockApiClient_ = client (rosettaBlockApi @v) + +rosettaBlockApiClient + :: ChainwebVersion + -> BlockReq + -- ^ Contains a network id and a partial block id + -> ClientM BlockResp +rosettaBlockApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaBlockApiClient_ @v + +-- -------------------------------------------------------------------------- -- +-- Construction Endpoints + +rosettaConstructionMetadataApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => ConstructionMetadataReq + -- ^ contains a network id and a metadata object which specifies the + -- metadata to return. + -> ClientM ConstructionMetadataResp +rosettaConstructionMetadataApiClient_ = client (rosettaConstructionMetadataApi @v) + +rosettaConstructionMetadataApiClient + :: ChainwebVersion + -> ConstructionMetadataReq + -- ^ contains a network id and a metadata object which specifies the + -- metadata to return. + -> ClientM ConstructionMetadataResp +rosettaConstructionMetadataApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaConstructionMetadataApiClient_ @v + +rosettaConstructionSubmitApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => ConstructionSubmitReq + -- ^ Contains a network id and a signed transaction + -> ClientM ConstructionSubmitResp +rosettaConstructionSubmitApiClient_ = client (rosettaConstructionSubmitApi @v) + +rosettaConstructionSubmitApiClient + :: ChainwebVersion + -> ConstructionSubmitReq + -- ^ Contains a network id and a signed transaction + -> ClientM ConstructionSubmitResp +rosettaConstructionSubmitApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaConstructionSubmitApiClient_ @v + +-- -------------------------------------------------------------------------- -- +-- Mempool Endpoints + +rosettaMempoolTransactionApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => MempoolTransactionReq + -- ^ Contains a network id and a transaction id + -> ClientM MempoolTransactionResp +rosettaMempoolTransactionApiClient_ = client (rosettaMempoolTransactionApi @v) + +rosettaMempoolTransactionApiClient + :: ChainwebVersion + -> MempoolTransactionReq + -- ^ Contains a network id and a transaction id + -> ClientM MempoolTransactionResp +rosettaMempoolTransactionApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaMempoolTransactionApiClient_ @v + +rosettaMempoolApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => MempoolReq + -- ^ contains a network id + -> ClientM MempoolResp +rosettaMempoolApiClient_ = client (rosettaMempoolApi @v) + +rosettaMempoolApiClient + :: ChainwebVersion + -> MempoolReq + -- ^ contains a network id + -> ClientM MempoolResp +rosettaMempoolApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaMempoolApiClient_ @v + +-- -------------------------------------------------------------------------- -- +-- Network Endpoints + +rosettaNetworkListApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => MetadataReq + -- ^ Contains an optional object with metadata + -> ClientM NetworkListResp +rosettaNetworkListApiClient_ = client (rosettaNetworkListApi @v) + +rosettaNetworkListApiClient + :: ChainwebVersion + -> MetadataReq + -- ^ Contains an optional object with metadata + -> ClientM NetworkListResp +rosettaNetworkListApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaNetworkListApiClient_ @v + +rosettaNetworkOptionsApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => NetworkReq + -- ^ Contains a network identifier and optional object with metadata + -> ClientM NetworkOptionsResp +rosettaNetworkOptionsApiClient_ = client (rosettaNetworkOptionsApi @v) + +rosettaNetworkOptionsApiClient + :: ChainwebVersion + -> NetworkReq + -- ^ Contains a network identifier and optional object with metadata + -> ClientM NetworkOptionsResp +rosettaNetworkOptionsApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaNetworkOptionsApiClient_ @v + +rosettaNetworkStatusApiClient_ + :: forall (v :: ChainwebVersionT) + . KnownChainwebVersionSymbol v + => NetworkReq + -- ^ Contains a network identifier and optional object with metadata + -> ClientM NetworkStatusResp +rosettaNetworkStatusApiClient_ = client (rosettaNetworkStatusApi @v) + +rosettaNetworkStatusApiClient + :: ChainwebVersion + -> NetworkReq + -- ^ Contains a network identifier and optional object with metadata + -> ClientM NetworkStatusResp +rosettaNetworkStatusApiClient + (FromSingChainwebVersion (SChainwebVersion :: Sing v)) + = rosettaNetworkStatusApiClient_ @v diff --git a/src/Chainweb/Rosetta/RestAPI/Server.hs b/src/Chainweb/Rosetta/RestAPI/Server.hs index 06dd9d9cc6..de39daf025 100644 --- a/src/Chainweb/Rosetta/RestAPI/Server.hs +++ b/src/Chainweb/Rosetta/RestAPI/Server.hs @@ -56,7 +56,7 @@ import qualified Chainweb.RestAPI.NetworkID as ChainwebNetId import Chainweb.RestAPI.Utils import Chainweb.Rosetta.Internal import Chainweb.Rosetta.RestAPI -import Chainweb.Rosetta.Util +import Chainweb.Rosetta.Utils import Chainweb.Transaction (ChainwebTransaction) import Chainweb.Utils import Chainweb.Utils.Paging @@ -153,7 +153,6 @@ blockH blockH v cutDb ps crs (BlockReq net (PartialBlockId bheight bhash)) = runExceptT work >>= either throwRosetta pure where - block :: BlockHeader -> [Transaction] -> Block block bh txs = Block { _block_blockId = blockId bh diff --git a/src/Chainweb/Rosetta/Util.hs b/src/Chainweb/Rosetta/Utils.hs similarity index 99% rename from src/Chainweb/Rosetta/Util.hs rename to src/Chainweb/Rosetta/Utils.hs index 52169194fe..70a76a6f43 100644 --- a/src/Chainweb/Rosetta/Util.hs +++ b/src/Chainweb/Rosetta/Utils.hs @@ -2,14 +2,14 @@ {-# LANGUAGE ScopedTypeVariables #-} -- | --- Module: Chainweb.Rosetta.Util +-- Module: Chainweb.Rosetta.Utils -- Copyright: Copyright © 2018 - 2020 Kadena LLC. -- License: MIT -- Maintainer: Linda Ortega -- Stability: experimental -- -- -module Chainweb.Rosetta.Util where +module Chainweb.Rosetta.Utils where import Data.Aeson import Data.Decimal diff --git a/stack.yaml b/stack.yaml index 939f903247..e346ee9f5c 100644 --- a/stack.yaml +++ b/stack.yaml @@ -42,4 +42,4 @@ extra-deps: - github: kadena-io/chainweb-storage commit: 07e7eb7596c7105aee42dbdb6edd10e3f23c0d7e - github: kadena-io/rosetta - commit: d20b0012c6f5e1d28843b89a4edd5ea652ade0c3 + commit: 1ccb68d7aec0414f494fb06f591214e7cf845627 diff --git a/test/Chainweb/Test/MultiNode.hs b/test/Chainweb/Test/MultiNode.hs index 850fd8fdb8..d5a16b9669 100644 --- a/test/Chainweb/Test/MultiNode.hs +++ b/test/Chainweb/Test/MultiNode.hs @@ -42,7 +42,7 @@ import Control.Concurrent import Control.Concurrent.Async import Control.DeepSeq import Control.Exception -import Control.Lens (over, set, view) +import Control.Lens (set, view) import Control.Monad import Data.Aeson @@ -50,7 +50,6 @@ import Data.Foldable import qualified Data.HashMap.Strict as HM import qualified Data.HashSet as HS import qualified Data.List as L -import Data.Streaming.Network (HostPreference) import qualified Data.Text as T import qualified Data.Text.Encoding as T #if DEBUG_MULTINODE_TEST @@ -80,12 +79,8 @@ import Chainweb.Chainweb.PeerResources import Chainweb.Cut import Chainweb.CutDB import Chainweb.Graph -import Chainweb.HostAddress import Chainweb.Logger -import Chainweb.Miner.Config -import Chainweb.Miner.Pact import Chainweb.NodeId -import Chainweb.Test.P2P.Peer.BootstrapConfig import Chainweb.Test.Utils import Chainweb.Time (Seconds(..)) import Chainweb.Utils @@ -112,64 +107,22 @@ import P2P.Peer -- similulate a full-scale chain in a miniaturized settings. -- -host :: Hostname -host = unsafeHostnameFromText "::1" - -interface :: HostPreference -interface = "::1" - -- | Test Configuration for a scaled down Test chainweb. -- -config +multiConfig :: ChainwebVersion -> Natural -- ^ number of nodes -> NodeId -- ^ NodeId -> ChainwebConfiguration -config v n nid = defaultChainwebConfiguration v - & set configNodeId nid - -- Set the node id. - - & set (configP2p . p2pConfigPeer . peerConfigHost) host - & set (configP2p . p2pConfigPeer . peerConfigInterface) interface - -- Only listen on the loopback device. On Mac OS X this prevents the - -- firewall dialog form poping up. - - & set (configP2p . p2pConfigKnownPeers) mempty - & set (configP2p . p2pConfigIgnoreBootstrapNodes) True - -- The bootstrap peer info is set later after the bootstrap nodes - -- has started and got its port assigned. - - & set (configP2p . p2pConfigMaxPeerCount) (n * 2) - -- We make room for all test peers in peer db. - - & set (configP2p . p2pConfigMaxSessionCount) 4 - -- We set this to a low number in order to keep the network sparse (or - -- at last no being a clique) and to also limit the number of - -- port allocations - +multiConfig v n nid = config v n nid & set (configP2p . p2pConfigSessionTimeout) 20 -- Use short sessions to cover session timeouts and setup logic in the -- test. - - & set (configMining . miningInNode) miner - - & set configReintroTxs True - -- enable transaction re-introduction - - & set (configTransactionIndex . enableConfigEnabled) True - -- enable transaction index - & set configThrottling throttling -- throttling is effectively disabled to not slow down the test nodes where - miner = NodeMiningConfig - { _nodeMiningEnabled = True - , _nodeMiner = noMiner - , _nodeTestMiners = MinerCount n - } - throttling = defaultThrottlingConfig { _throttlingRate = 10_000 -- per second , _throttlingMiningRate = 10_000 -- per second @@ -177,41 +130,10 @@ config v n nid = defaultChainwebConfiguration v , _throttlingLocalRate = 10_000 -- per 10 seconds } --- | Set the bootstrap node port of a 'ChainwebConfiguration' --- -setBootstrapPeerInfo - :: PeerInfo - -- ^ Peer info of bootstrap node - -> ChainwebConfiguration - -> ChainwebConfiguration -setBootstrapPeerInfo - = over (configP2p . p2pConfigKnownPeers) . (:) - -- The the port of the bootstrap node. Normally this is hard-coded. - -- But in test-suites that may run concurrently we want to use a port - -- that is assigned by the OS. - --- | Configure a bootstrap node --- -bootstrapConfig - :: ChainwebConfiguration - -> ChainwebConfiguration -bootstrapConfig conf = conf - & set (configP2p . p2pConfigPeer) peerConfig - & set (configP2p . p2pConfigKnownPeers) [] - where - peerConfig = (head $ bootstrapPeerConfig $ _configChainwebVersion conf) - & set peerConfigPort 0 - -- Normally, the port of bootstrap nodes is hard-coded. But in - -- test-suites that may run concurrently we want to use a port that is - -- assigned by the OS. - - & set peerConfigHost host - & set peerConfigInterface interface - -- -------------------------------------------------------------------------- -- -- Minimal Node Setup that logs conensus state to the given mvar -node +multiNode :: LogLevel -> (T.Text -> IO ()) -> MVar ConsensusState @@ -219,7 +141,7 @@ node -> ChainwebConfiguration -> RocksDb -> IO () -node loglevel write stateVar bootstrapPeerInfoVar conf rdb = do +multiNode loglevel write stateVar bootstrapPeerInfoVar conf rdb = do withChainweb conf logger nodeRocksDb Nothing False $ \cw -> do -- If this is the bootstrap node we extract the port number and @@ -275,14 +197,14 @@ runNodes loglevel write stateVar v n = forConcurrently_ [0 .. int n - 1] $ \i -> do threadDelay (500_000 * int i) - let baseConf = config v n (NodeId i) + let baseConf = multiConfig v n (NodeId i) conf <- if | i == 0 -> return $ bootstrapConfig baseConf | otherwise -> setBootstrapPeerInfo <$> readMVar bootstrapPortVar <*> pure baseConf - node loglevel write stateVar bootstrapPortVar conf rdb + multiNode loglevel write stateVar bootstrapPortVar conf rdb runNodesForSeconds :: LogLevel diff --git a/test/Chainweb/Test/Pact/RemotePactTest.hs b/test/Chainweb/Test/Pact/RemotePactTest.hs index 43017e09cd..70617c3c43 100644 --- a/test/Chainweb/Test/Pact/RemotePactTest.hs +++ b/test/Chainweb/Test/Pact/RemotePactTest.hs @@ -28,11 +28,9 @@ module Chainweb.Test.Pact.RemotePactTest , polling , sending , PollingExpectation(..) -, ChainwebNetwork(..) ) where import Control.Concurrent hiding (modifyMVar, newMVar, putMVar, readMVar) -import Control.Concurrent.Async import Control.Concurrent.MVar.Strict import Control.DeepSeq import Control.Lens @@ -46,31 +44,22 @@ import Data.Aeson.Lens hiding (values) import qualified Data.ByteString.Short as SB import Data.Decimal import Data.Default (def) -import Data.Either import Data.Foldable (toList) import qualified Data.HashMap.Strict as HashMap import qualified Data.List as L import qualified Data.List.NonEmpty as NEL import qualified Data.Map.Strict as M import Data.Maybe -import Data.Streaming.Network (HostPreference) import Data.String.Conv (toS) import Data.Text (Text) import qualified Data.Text as T -import Data.Text.Encoding (encodeUtf8) import NeatInterpolation -import Network.Connection as HTTP -import Network.HTTP.Client.TLS as HTTP - import Numeric.Natural import Servant.Client -import System.IO.Extra -import System.LogLevel - import Test.Tasty import Test.Tasty.HUnit @@ -93,22 +82,14 @@ import Pact.Types.Term import Chainweb.BlockHeight import Chainweb.ChainId -import Chainweb.Chainweb -import Chainweb.Chainweb.ChainResources -import Chainweb.Chainweb.PeerResources import Chainweb.Cut.CutHashes import Chainweb.CutDB.RestAPI.Client import Chainweb.Graph -import Chainweb.HostAddress -import Chainweb.Logger import Chainweb.Mempool.Mempool -import Chainweb.Miner.Config -import Chainweb.Miner.Pact (noMiner) -import Chainweb.NodeId import Chainweb.Pact.RestAPI.Client import Chainweb.Pact.Service.Types -import Chainweb.Test.P2P.Peer.BootstrapConfig import Chainweb.Test.Pact.Utils +import Chainweb.Test.RestAPI.Utils import Chainweb.Test.Utils import Chainweb.Time import Chainweb.Utils hiding (check) @@ -116,19 +97,10 @@ import Chainweb.Version import Data.CAS.RocksDB -import P2P.Node.Configuration -import P2P.Peer -- -------------------------------------------------------------------------- -- -- Global Settings -debug :: String -> IO () -#if DEBUG_TEST -debug = putStrLn -#else -debug = const $ return () -#endif - nNodes :: Natural nNodes = 1 @@ -155,7 +127,7 @@ gp = 0.1 -- tests :: RocksDb -> ScheduledTest tests rdb = testGroupSch "Chainweb.Test.Pact.RemotePactTest" - [ withNodes rdb nNodes $ \net -> + [ withNodes v "remotePactTest-" rdb nNodes $ \net -> withMVarResource 0 $ \iomvar -> withTime $ \iot -> testGroup "remote pact tests" @@ -193,24 +165,24 @@ tests rdb = testGroupSch "Chainweb.Test.Pact.RemotePactTest" -- about 10 seconds. Once initialization is complete even large numbers of empty -- blocks were mined almost instantaneously. -- -awaitNetworkHeight :: IO ChainwebNetwork -> CutHeight -> IO () +awaitNetworkHeight :: IO ClientEnv -> CutHeight -> IO () awaitNetworkHeight nio h = do - cenv <- _getClientEnv <$> nio + cenv <- nio ch <- awaitCutHeight cenv h debug $ "cut height: " <> sshow (_cutHashesHeight ch) -responseGolden :: IO ChainwebNetwork -> IO RequestKeys -> TestTree +responseGolden :: IO ClientEnv -> IO RequestKeys -> TestTree responseGolden networkIO rksIO = golden "remote-golden" $ do rks <- rksIO - cenv <- _getClientEnv <$> networkIO + cenv <- networkIO PollResponses theMap <- polling cid cenv rks ExpectPactResult let values = mapMaybe (\rk -> _crResult <$> HashMap.lookup rk theMap) (NEL.toList $ _rkRequestKeys rks) return $! toS $! foldMap A.encode values -localTest :: IO (Time Micros) -> IO ChainwebNetwork -> IO () +localTest :: IO (Time Micros) -> IO ClientEnv -> IO () localTest iot nio = do - cenv <- fmap _getClientEnv nio + cenv <- nio mv <- newMVar 0 SubmitBatch batch <- testBatch iot mv gp let cmd = head $ toList batch @@ -219,9 +191,9 @@ localTest iot nio = do assertEqual "expect /local to return gas for tx" (_crGas res) 5 assertEqual "expect /local to succeed and return 3" e (Right (PLiteral $ LDecimal 3)) -localContTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +localContTest :: IO (Time Micros) -> IO ClientEnv -> TestTree localContTest iot nio = testCaseSteps "local continuation test" $ \step -> do - cenv <- _getClientEnv <$> nio + cenv <- nio let sid = unsafeChainId 0 step "execute /send with initial pact continuation tx" @@ -267,9 +239,9 @@ localContTest iot nio = testCaseSteps "local continuation test" $ \step -> do $ mkCont $ mkContMsg pid 1 -localChainDataTest :: IO (Time Micros) -> IO ChainwebNetwork -> IO () +localChainDataTest :: IO (Time Micros) -> IO ClientEnv -> IO () localChainDataTest iot nio = do - cenv <- fmap _getClientEnv nio + cenv <- nio mv <- newMVar (0 :: Int) SubmitBatch batch <- localTestBatch iot mv let cmd = head $ toList batch @@ -302,19 +274,19 @@ localChainDataTest iot nio = do assert' name value = assertEqual name (M.lookup (FieldKey (toS name)) m) (Just value) expectedResult _ = assertFailure "Didn't get back an object map!" -pollingBadlistTest :: IO ChainwebNetwork -> TestTree +pollingBadlistTest :: IO ClientEnv -> TestTree pollingBadlistTest nio = testCase "/poll reports badlisted txs" $ do - cenv <- fmap _getClientEnv nio + cenv <- nio let rks = RequestKeys $ NEL.fromList [pactDeadBeef] sid <- liftIO $ mkChainId v maxBound (0 :: Int) void $ polling sid cenv rks ExpectPactError -sendValidationTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +sendValidationTest :: IO (Time Micros) -> IO ClientEnv -> TestTree sendValidationTest iot nio = testCaseSteps "/send reports validation failure" $ \step -> do step "check sending poisoned TTL batch" - cenv <- fmap _getClientEnv nio + cenv <- nio mv <- newMVar 0 SubmitBatch batch1 <- testBatch' iot 10_000 mv gp SubmitBatch batch2 <- testBatch' (return $ Time $ TimeSpan 0) 2 mv gp @@ -370,9 +342,9 @@ expectSendFailure expectErr act = tryAllSynchronous act >>= \case test er = assertSatisfies ("Expected message containing '" ++ expectErr ++ "'") er (L.isInfixOf expectErr) -spvTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +spvTest :: IO (Time Micros) -> IO ClientEnv -> TestTree spvTest iot nio = testCaseSteps "spv client tests" $ \step -> do - cenv <- fmap _getClientEnv nio + cenv <- nio batch <- mkTxBatch sid <- mkChainId v maxBound (1 :: Int) r <- flip runClientM cenv $ do @@ -416,9 +388,9 @@ spvTest iot nio = testCaseSteps "spv client tests" $ \step -> do , "target-chain-id" A..= tid ] -txTooBigGasTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +txTooBigGasTest :: IO (Time Micros) -> IO ClientEnv -> TestTree txTooBigGasTest iot nio = testCaseSteps "transaction size gas tests" $ \step -> do - cenv <- fmap _getClientEnv nio + cenv <- nio sid <- mkChainId v maxBound (0 :: Int) let runSend batch expectation = flip runClientM cenv $ do @@ -481,12 +453,12 @@ txTooBigGasTest iot nio = testCaseSteps "transaction size gas tests" $ \step -> txcode1 = txcode0 <> "(identity 1)" -caplistTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +caplistTest :: IO (Time Micros) -> IO ClientEnv -> TestTree caplistTest iot nio = testCaseSteps "caplist TRANSFER + FUND_TX test" $ \step -> do let testCaseStep = void . liftIO . step - cenv <- fmap _getClientEnv nio + cenv <- nio sid <- liftIO $ mkChainId v maxBound (0 :: Int) r <- flip runClientM cenv $ do @@ -552,12 +524,12 @@ allocation02KeyPair' = , "2f75b5d875dd7bf07cc1a6973232a9e53dc1d4ffde2bab0bbace65cd87e87f53" ) -allocationTest :: IO (Time Micros) -> IO ChainwebNetwork -> TestTree +allocationTest :: IO (Time Micros) -> IO ClientEnv -> TestTree allocationTest iot nio = testCaseSteps "genesis allocation tests" $ \step -> do let testCaseStep = void . liftIO . step - cenv <- fmap _getClientEnv nio + cenv <- nio sid <- liftIO $ mkChainId v maxBound (0 :: Int) step "positive allocation test: allocation00 release" @@ -692,18 +664,6 @@ data PactTransaction = PactTransaction , _pactData :: Maybe A.Value } deriving (Eq, Show) - -data PactTestFailure - = PollingFailure String - | SendFailure String - | LocalFailure String - | SpvFailure String - | SlowChain String - deriving Show - -instance Exception PactTestFailure - - mkSingletonBatch :: IO (Time Micros) -> SimpleKeyPair @@ -722,26 +682,20 @@ mkSingletonBatch iot kps (PactTransaction c d) nonce pmk clist = do withRequestKeys :: IO (Time Micros) -> IO (MVar Int) - -> IO ChainwebNetwork + -> IO ClientEnv -> (IO RequestKeys -> TestTree) -> TestTree withRequestKeys iot ioNonce networkIO f = withResource mkKeys (\_ -> return ()) f where mkKeys :: IO RequestKeys mkKeys = do - cenv <- _getClientEnv <$> networkIO + cenv <- networkIO mNonce <- ioNonce testSend iot mNonce cenv testSend :: IO (Time Micros) -> MVar Int -> ClientEnv -> IO RequestKeys testSend iot mNonce env = testBatch iot mNonce gp >>= sending cid env -getClientEnv :: BaseUrl -> IO ClientEnv -getClientEnv url = do - let mgrSettings = HTTP.mkManagerSettings (HTTP.TLSSettingsSimple True False False) Nothing - mgr <- HTTP.newTlsManagerWith mgrSettings - return $ mkClientEnv mgr url - awaitCutHeight :: ClientEnv -> CutHeight @@ -767,139 +721,6 @@ awaitCutHeight cenv i = do <> " [" <> show (view rsIterNumberL s) <> "]" return True --- | Calls to /local via the pact local api client with retry --- -local - :: ChainId - -> ClientEnv - -> Command Text - -> IO (CommandResult Hash) -local sid cenv cmd = - recovering testRetryPolicy [h] $ \s -> do - debug - $ "requesting local cmd for " <> (take 19 $ show cmd) - <> " [" <> show (view rsIterNumberL s) <> "]" - - -- send a single spv request and return the result - -- - runClientM (pactLocalApiClient v sid cmd) cenv >>= \case - Left e -> throwM $ LocalFailure (show e) - Right t -> return t - where - h _ = Handler $ \case - LocalFailure _ -> return True - _ -> return False - -localTestToRetry - :: ChainId - -> ClientEnv - -> Command Text - -> (CommandResult Hash -> Bool) - -> IO (CommandResult Hash) -localTestToRetry sid cenv cmd test = retrying testRetryPolicy check (\_ -> go) - where - go = local sid cenv cmd - check _ cr = return $ not $ test cr - --- | Request an SPV proof using exponential retry logic --- -spv - :: ChainId - -> ClientEnv - -> SpvRequest - -> IO TransactionOutputProofB64 -spv sid cenv r = - recovering testRetryPolicy [h] $ \s -> do - debug - $ "requesting spv proof for " <> show r - <> " [" <> show (view rsIterNumberL s) <> "]" - - -- send a single spv request and return the result - -- - runClientM (pactSpvApiClient v sid r) cenv >>= \case - Left e -> throwM $ SpvFailure (show e) - Right t -> return t - where - h _ = Handler $ \case - SpvFailure _ -> return True - _ -> return False - --- | Backoff up to a constant 250ms, limiting to ~40s --- (actually saw a test have to wait > 22s) -testRetryPolicy :: RetryPolicy -testRetryPolicy = stepped <> limitRetries 150 - where - stepped = retryPolicy $ \rs -> case rsIterNumber rs of - 0 -> Just 20_000 - 1 -> Just 50_000 - 2 -> Just 100_000 - _ -> Just 250_000 - --- | Send a batch with retry logic waiting for success. -sending - :: ChainId - -> ClientEnv - -> SubmitBatch - -> IO RequestKeys -sending sid cenv batch = - recovering testRetryPolicy [h] $ \s -> do - debug - $ "sending requestkeys " <> show (_cmdHash <$> toList ss) - <> " [" <> show (view rsIterNumberL s) <> "]" - - -- Send and return naively - -- - runClientM (pactSendApiClient v sid batch) cenv >>= \case - Left e -> throwM $ SendFailure (show e) - Right rs -> return rs - - where - ss = _sbCmds batch - - h _ = Handler $ \case - SendFailure _ -> return True - _ -> return False - --- | Poll with retry using an exponential backoff --- -data PollingExpectation = ExpectPactError | ExpectPactResult - -polling - :: ChainId - -> ClientEnv - -> RequestKeys - -> PollingExpectation - -> IO PollResponses -polling sid cenv rks pollingExpectation = - recovering testRetryPolicy [h] $ \s -> do - debug - $ "polling for requestkeys " <> show (toList rs) - <> " [" <> show (view rsIterNumberL s) <> "]" - - -- Run the poll cmd loop and check responses - -- by making sure results are successful and request keys - -- are sane - - runClientM (pactPollApiClient v sid $ Poll rs) cenv >>= \case - Left e -> throwM $ PollingFailure (show e) - Right r@(PollResponses mp) -> - if all (go mp) (toList rs) - then return r - else throwM $ PollingFailure $ T.unpack $ "polling check failed: " <> encodeToText r - where - h _ = Handler $ \case - PollingFailure _ -> return True - _ -> return False - - rs = _rkRequestKeys rks - - validate (PactResult a) = case pollingExpectation of - ExpectPactResult -> isRight a - ExpectPactError -> isLeft a - - go m rk = case m ^. at rk of - Just cr -> _crReqKey cr == rk && validate (_crResult cr) - Nothing -> False testBatch'' :: Pact.ChainId -> IO (Time Micros) -> Integer -> MVar Int -> GasPrice -> IO SubmitBatch testBatch'' chain iot ttl mnonce gp' = modifyMVar mnonce $ \(!nn) -> do @@ -924,122 +745,6 @@ testBatch iot mnonce = testBatch' iot ttl mnonce -- test node(s), config, etc. for this test -------------------------------------------------------------------------------- -newtype ChainwebNetwork = ChainwebNetwork { _getClientEnv :: ClientEnv } - -withNodes - :: RocksDb - -> Natural - -> (IO ChainwebNetwork -> TestTree) - -> TestTree -withNodes rdb n f = withResource start - (cancel . fst) - (f . fmap (ChainwebNetwork . snd)) - where - start :: IO (Async (), ClientEnv) - start = do - peerInfoVar <- newEmptyMVar - a <- async $ runTestNodes rdb Quiet v n peerInfoVar - i <- readMVar peerInfoVar - cwEnv <- getClientEnv $ getCwBaseUrl $ _hostAddressPort $ _peerAddr i - return (a, cwEnv) - - getCwBaseUrl :: Port -> BaseUrl - getCwBaseUrl p = BaseUrl - { baseUrlScheme = Https - , baseUrlHost = "127.0.0.1" - , baseUrlPort = fromIntegral p - , baseUrlPath = "" - } - -runTestNodes - :: RocksDb - -> LogLevel - -> ChainwebVersion - -> Natural - -> MVar PeerInfo - -> IO () -runTestNodes rdb loglevel ver n portMVar = - forConcurrently_ [0 .. int n - 1] $ \i -> do - threadDelay (1000 * int i) - let baseConf = config ver n (NodeId i) - conf <- if - | i == 0 -> - return $ bootstrapConfig baseConf - | otherwise -> - setBootstrapPeerInfo <$> readMVar portMVar <*> pure baseConf - node rdb loglevel portMVar conf - -node :: RocksDb -> LogLevel -> MVar PeerInfo -> ChainwebConfiguration -> IO () -node rdb loglevel peerInfoVar conf = do - rocksDb <- testRocksDb ("remotePactTest-" <> encodeUtf8 (toText nid)) rdb - System.IO.Extra.withTempDir $ \dir -> withChainweb conf logger rocksDb (Just dir) False $ \cw -> do - - -- If this is the bootstrap node we extract the port number and publish via an MVar. - when (nid == NodeId 0) $ do - let bootStrapInfo = view (chainwebPeer . peerResPeer . peerInfo) cw - putMVar peerInfoVar bootStrapInfo - - poisonDeadBeef cw - runChainweb cw `finally` do - logFunctionText logger Info "write sample data" - logFunctionText logger Info "shutdown node" - return () - where - nid = _configNodeId conf - logger :: GenericLogger - logger = addLabel ("node", toText nid) $ genericLogger loglevel print - - poisonDeadBeef cw = mapM_ poison crs - where - crs = map snd $ HashMap.toList $ view chainwebChains cw - poison cr = mempoolAddToBadList (view chainResMempool cr) deadbeef - -deadbeef :: TransactionHash -deadbeef = TransactionHash "deadbeefdeadbeefdeadbeefdeadbeef" - pactDeadBeef :: RequestKey pactDeadBeef = let (TransactionHash b) = deadbeef in RequestKey $ Hash $ SB.fromShort b - -host :: Hostname -host = unsafeHostnameFromText "::1" - -interface :: HostPreference -interface = "::1" - -config - :: ChainwebVersion - -> Natural - -> NodeId - -> ChainwebConfiguration -config ver n nid = defaultChainwebConfiguration ver - & set configNodeId nid - & set (configP2p . p2pConfigPeer . peerConfigHost) host - & set (configP2p . p2pConfigPeer . peerConfigInterface) interface - & set (configP2p . p2pConfigKnownPeers) mempty - & set (configP2p . p2pConfigIgnoreBootstrapNodes) True - & set (configP2p . p2pConfigMaxPeerCount) (n * 2) - & set (configP2p . p2pConfigMaxSessionCount) 4 - & set (configP2p . p2pConfigSessionTimeout) 60 - & set (configMining . miningInNode) miner - & set configReintroTxs True - & set (configTransactionIndex . enableConfigEnabled) True - & set (configBlockGasLimit) 1_000_000 - where - miner = NodeMiningConfig - { _nodeMiningEnabled = True - , _nodeMiner = noMiner - , _nodeTestMiners = MinerCount n } - -bootstrapConfig :: ChainwebConfiguration -> ChainwebConfiguration -bootstrapConfig conf = conf - & set (configP2p . p2pConfigPeer) peerConfig - & set (configP2p . p2pConfigKnownPeers) [] - where - peerConfig = head (bootstrapPeerConfig $ _configChainwebVersion conf) - & set peerConfigPort 0 - & set peerConfigHost host - -setBootstrapPeerInfo :: PeerInfo -> ChainwebConfiguration -> ChainwebConfiguration -setBootstrapPeerInfo = - over (configP2p . p2pConfigKnownPeers) . (:) diff --git a/test/Chainweb/Test/Pact/Utils.hs b/test/Chainweb/Test/Pact/Utils.hs index 731487477c..df56fd4ff8 100644 --- a/test/Chainweb/Test/Pact/Utils.hs +++ b/test/Chainweb/Test/Pact/Utils.hs @@ -38,7 +38,7 @@ module Chainweb.Test.Pact.Utils -- * Command builder , defaultCmd , mkCmd -, buildCmd +, buildRawCmd , buildCwCmd , buildTextCmd , mkExec' @@ -105,6 +105,7 @@ import Control.Monad import Control.Monad.Catch import Data.Aeson (Value(..), object, (.=)) +import Data.ByteString (ByteString) import Data.CAS.HashMap hiding (toList) import Data.CAS.RocksDB import Data.Decimal @@ -331,33 +332,30 @@ mkCmd nonce rpc = defaultCmd , _cbNonce = nonce } --- | Main builder command. -buildCmd :: CmdBuilder -> IO (Command (Payload PublicMeta ParsedCode)) -buildCmd CmdBuilder{..} = do - akps <- mapM toApiKp _cbSigners - kps <- mkKeyPairs akps - cmd <- mkCommand kps pm _cbNonce nid _cbRPC - case verifyCommand cmd of - ProcSucc r -> return r +-- | Build parsed + verified Pact command +-- +buildCwCmd :: CmdBuilder -> IO ChainwebTransaction +buildCwCmd cmd = buildRawCmd cmd >>= \c -> case verifyCommand c of + ProcSucc r -> return $ fmap mkPayloadWithText r ProcFail e -> throwM $ userError $ "buildCmd failed: " ++ e - where - nid = fmap (P.NetworkId . sshow) _cbNetworkId - cid = fromString $ show (chainIdInt _cbChainId :: Int) - pm = PublicMeta cid _cbSender _cbGasLimit _cbGasPrice _cbTTL _cbCreationTime --- | Main builder command. +-- | Build unparsed, unverified command +-- buildTextCmd :: CmdBuilder -> IO (Command Text) -buildTextCmd CmdBuilder{..} = do - akps <- mapM toApiKp _cbSigners - kps <- mkKeyPairs akps - cmd <- mkCommand kps pm _cbNonce nid _cbRPC - return $ T.decodeUtf8 <$> cmd +buildTextCmd = fmap (fmap T.decodeUtf8) . buildRawCmd + +-- | Build a raw bytestring command +-- +buildRawCmd :: CmdBuilder -> IO (Command ByteString) +buildRawCmd CmdBuilder{..} = do + akps <- mapM toApiKp _cbSigners + kps <- mkKeyPairs akps + mkCommand kps pm _cbNonce nid _cbRPC where nid = fmap (P.NetworkId . sshow) _cbNetworkId cid = fromString $ show (chainIdInt _cbChainId :: Int) pm = PublicMeta cid _cbSender _cbGasLimit _cbGasPrice _cbTTL _cbCreationTime - dieL :: MonadThrow m => [Char] -> Either [Char] a -> m a dieL msg = either (\s -> throwM $ userError $ msg ++ ": " ++ s) return @@ -368,11 +366,6 @@ toApiKp (CmdSigner Signer{..} privKey) = do return $! ApiKeyPair (PrivBS sk) (Just (PubBS pk)) _siAddress _siScheme (Just _siCapList) --- | 'buildCmd' variant for 'ChainwebTransaction' -buildCwCmd :: CmdBuilder -> IO ChainwebTransaction -buildCwCmd = fmap (fmap mkPayloadWithText) . buildCmd - - -- ----------------------------------------------------------------------- -- -- Service creation utilities @@ -413,7 +406,7 @@ testPactCtxSQLite -> SQLiteEnv -> PactServiceConfig -> IO (TestPactCtx cas,PactDbEnv') -testPactCtxSQLite v cid bhdb pdb sqlenv config = do +testPactCtxSQLite v cid bhdb pdb sqlenv conf = do (dbSt,cpe) <- initRelationalCheckpointer' initialBlockState sqlenv logger v cid let rs = readRewards ph = ParentHeader $ genesisBlockHeader v cid @@ -433,11 +426,11 @@ testPactCtxSQLite v cid bhdb pdb sqlenv config = do , _psBlockHeaderDb = bhdb , _psGasModel = constGasModel 0 , _psMinerRewards = rs - , _psReorgLimit = fromIntegral $ _pactReorgLimit config + , _psReorgLimit = fromIntegral $ _pactReorgLimit conf , _psOnFatalError = defaultOnFatalError mempty , _psVersion = v - , _psValidateHashesOnReplay = _pactRevalidate config - , _psAllowReadsInLocal = _pactAllowReadsInLocal config + , _psValidateHashesOnReplay = _pactRevalidate conf + , _psAllowReadsInLocal = _pactAllowReadsInLocal conf , _psIsBatch = False , _psCheckpointerDepth = 0 } @@ -526,7 +519,7 @@ withPactCtxSQLite -> PactServiceConfig -> (WithPactCtxSQLite cas -> TestTree) -> TestTree -withPactCtxSQLite v bhdbIO pdbIO config f = +withPactCtxSQLite v bhdbIO pdbIO conf f = withResource initializeSQLite freeSQLiteResource $ \io -> @@ -540,13 +533,7 @@ withPactCtxSQLite v bhdbIO pdbIO config f = bhdb <- bhdbIO pdb <- pdbIO (_,s) <- ios - testPactCtxSQLite v cid bhdb pdb s config - -withMVarResource :: a -> (IO (MVar a) -> TestTree) -> TestTree -withMVarResource value = withResource (newMVar value) mempty - -withTime :: (IO (Time Micros) -> TestTree) -> TestTree -withTime = withResource getCurrentTimeIntegral mempty + testPactCtxSQLite v cid bhdb pdb s conf toTxCreationTime :: Integral a => Time a -> TxCreationTime toTxCreationTime (Time timespan) = TxCreationTime $ fromIntegral $ timeSpanToSeconds timespan diff --git a/test/Chainweb/Test/RestAPI/Utils.hs b/test/Chainweb/Test/RestAPI/Utils.hs new file mode 100644 index 0000000000..d373783d18 --- /dev/null +++ b/test/Chainweb/Test/RestAPI/Utils.hs @@ -0,0 +1,432 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +module Chainweb.Test.RestAPI.Utils +( -- * Retry Policies + testRetryPolicy + + -- * Debugging +, debug + + -- * Utils +, repeatUntil + + -- * Pact client DSL +, PactTestFailure(..) +, PollingExpectation(..) +, local +, localTestToRetry +, spv +, sending +, polling + + -- * Rosetta client DSL +, RosettaTestException(..) +, accountBalance +, blockTransaction +, block +, constructionMetadata +, constructionSubmit +, mempoolTransaction +, mempool +, networkOptions +, networkList +, networkStatus +) where + + +import Control.Lens +import Control.Monad.Catch +import Control.Retry + +import Data.Either +import Data.Foldable (toList) +import Data.Text (Text) +import qualified Data.Text as T + +import Rosetta + +import Servant.Client + +-- internal chainweb modules + +import Chainweb.ChainId +import Chainweb.Graph +import Chainweb.Pact.RestAPI.Client +import Chainweb.Pact.Service.Types +import Chainweb.Rosetta.RestAPI.Client +import Chainweb.Utils +import Chainweb.Version + +-- internal pact modules + +import Pact.Types.API +import Pact.Types.Command +import Pact.Types.Hash + +-- ------------------------------------------------------------------ -- +-- Defaults + +debug :: String -> IO () +#if DEBUG_TEST +debug = putStrLn +#else +debug = const $ return () +#endif + +v :: ChainwebVersion +v = FastTimedCPM petersonChainGraph + +-- | Backoff up to a constant 250ms, limiting to ~40s +-- (actually saw a test have to wait > 22s) +testRetryPolicy :: RetryPolicy +testRetryPolicy = stepped <> limitRetries 150 + where + stepped = retryPolicy $ \rs -> case rsIterNumber rs of + 0 -> Just 20_000 + 1 -> Just 50_000 + 2 -> Just 100_000 + _ -> Just 250_000 + +-- ------------------------------------------------------------------ -- +-- Pact api client utils w/ retry + + +data PactTestFailure + = PollingFailure String + | SendFailure String + | LocalFailure String + | SpvFailure String + | SlowChain String + deriving Show + +instance Exception PactTestFailure + +-- | Retry an IO action until it satisfies a predicate +-- +repeatUntil :: (a -> IO Bool) -> IO a -> IO a +repeatUntil test action = retrying testRetryPolicy + (\_ b -> not <$> test b) + (const action) + +-- | Calls to /local via the pact local api client with retry +-- +local + :: ChainId + -> ClientEnv + -> Command Text + -> IO (CommandResult Hash) +local sid cenv cmd = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting local cmd for " <> (take 19 $ show cmd) + <> " [" <> show (view rsIterNumberL s) <> "]" + + -- send a single spv request and return the result + -- + runClientM (pactLocalApiClient v sid cmd) cenv >>= \case + Left e -> throwM $ LocalFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + LocalFailure _ -> return True + _ -> return False + +localTestToRetry + :: ChainId + -> ClientEnv + -> Command Text + -> (CommandResult Hash -> Bool) + -> IO (CommandResult Hash) +localTestToRetry sid cenv cmd test = + repeatUntil (return . test) (local sid cenv cmd) + +-- | Request an SPV proof using exponential retry logic +-- +spv + :: ChainId + -> ClientEnv + -> SpvRequest + -> IO TransactionOutputProofB64 +spv sid cenv r = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting spv proof for " <> show r + <> " [" <> show (view rsIterNumberL s) <> "]" + + -- send a single spv request and return the result + -- + runClientM (pactSpvApiClient v sid r) cenv >>= \case + Left e -> throwM $ SpvFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + SpvFailure _ -> return True + _ -> return False + +-- | Send a batch with retry logic waiting for success. +sending + :: ChainId + -> ClientEnv + -> SubmitBatch + -> IO RequestKeys +sending sid cenv batch = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "sending requestkeys " <> show (_cmdHash <$> toList ss) + <> " [" <> show (view rsIterNumberL s) <> "]" + + -- Send and return naively + -- + runClientM (pactSendApiClient v sid batch) cenv >>= \case + Left e -> throwM $ SendFailure (show e) + Right rs -> return rs + + where + ss = _sbCmds batch + + h _ = Handler $ \case + SendFailure _ -> return True + _ -> return False + +-- | Poll with retry using an exponential backoff +-- +data PollingExpectation = ExpectPactError | ExpectPactResult + +polling + :: ChainId + -> ClientEnv + -> RequestKeys + -> PollingExpectation + -> IO PollResponses +polling sid cenv rks pollingExpectation = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "polling for requestkeys " <> show (toList rs) + <> " [" <> show (view rsIterNumberL s) <> "]" + + -- Run the poll cmd loop and check responses + -- by making sure results are successful and request keys + -- are sane + + runClientM (pactPollApiClient v sid $ Poll rs) cenv >>= \case + Left e -> throwM $ PollingFailure (show e) + Right r@(PollResponses mp) -> + if all (go mp) (toList rs) + then return r + else throwM $ PollingFailure $ T.unpack $ "polling check failed: " <> encodeToText r + where + h _ = Handler $ \case + PollingFailure _ -> return True + _ -> return False + + rs = _rkRequestKeys rks + + validate (PactResult a) = case pollingExpectation of + ExpectPactResult -> isRight a + ExpectPactError -> isLeft a + + go m rk = case m ^. at rk of + Just cr -> _crReqKey cr == rk && validate (_crResult cr) + Nothing -> False + + +-- ------------------------------------------------------------------ -- +-- Rosetta api client utils w/ retry + +data RosettaTestException + = AccountBalanceFailure String + | BlockTransactionFailure String + | BlockFailure String + | ConstructionMetadataFailure String + | ConstructionSubmitFailure String + | MempoolTransactionFailure String + | MempoolFailure String + | NetworkListFailure String + | NetworkOptionsFailure String + | NetworkStatusFailure String + deriving Show + +instance Exception RosettaTestException + +accountBalance + :: ClientEnv + -> AccountBalanceReq + -> IO AccountBalanceResp +accountBalance cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting account balance for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaAccountBalanceApiClient v req) cenv >>= \case + Left e -> throwM $ AccountBalanceFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + AccountBalanceFailure _ -> return True + _ -> return False + +blockTransaction + :: ClientEnv + -> BlockTransactionReq + -> IO BlockTransactionResp +blockTransaction cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting block transaction for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaBlockTransactionApiClient v req) cenv >>= \case + Left e -> throwM $ BlockTransactionFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + BlockTransactionFailure _ -> return True + _ -> return False + +block + :: ClientEnv + -> BlockReq + -> IO BlockResp +block cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting block for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaBlockApiClient v req) cenv >>= \case + Left e -> throwM $ BlockFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + BlockFailure _ -> return True + _ -> return False + +constructionMetadata + :: ClientEnv + -> ConstructionMetadataReq + -> IO ConstructionMetadataResp +constructionMetadata cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting construction metadata for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaConstructionMetadataApiClient v req) cenv >>= \case + Left e -> throwM $ ConstructionMetadataFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + ConstructionMetadataFailure _ -> return True + _ -> return False + +constructionSubmit + :: ClientEnv + -> ConstructionSubmitReq + -> IO ConstructionSubmitResp +constructionSubmit cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting construction submit for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaConstructionSubmitApiClient v req) cenv >>= \case + Left e -> throwM $ ConstructionSubmitFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + ConstructionSubmitFailure _ -> return True + _ -> return False + +mempoolTransaction + :: ClientEnv + -> MempoolTransactionReq + -> IO MempoolTransactionResp +mempoolTransaction cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting mempool transaction for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaMempoolTransactionApiClient v req) cenv >>= \case + Left e -> throwM $ MempoolTransactionFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + MempoolTransactionFailure _ -> return True + _ -> return False + +mempool + :: ClientEnv + -> MempoolReq + -> IO MempoolResp +mempool cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting mempool for " <> (show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaMempoolApiClient v req) cenv >>= \case + Left e -> throwM $ MempoolFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + MempoolFailure _ -> return True + _ -> return False + +networkList + :: ClientEnv + -> MetadataReq + -> IO NetworkListResp +networkList cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting network list for " <> (take 10 $ show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaNetworkListApiClient v req) cenv >>= \case + Left e -> throwM $ NetworkListFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + NetworkListFailure _ -> return True + _ -> return False + +networkOptions + :: ClientEnv + -> NetworkReq + -> IO NetworkOptionsResp +networkOptions cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting network options for " <> (take 10 $ show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaNetworkOptionsApiClient v req) cenv >>= \case + Left e -> throwM $ NetworkOptionsFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + NetworkOptionsFailure _ -> return True + _ -> return False + +networkStatus + :: ClientEnv + -> NetworkReq + -> IO NetworkStatusResp +networkStatus cenv req = + recovering testRetryPolicy [h] $ \s -> do + debug + $ "requesting network status for " <> (take 10 $ show req) + <> " [" <> show (view rsIterNumberL s) <> "]" + + runClientM (rosettaNetworkStatusApiClient v req) cenv >>= \case + Left e -> throwM $ NetworkStatusFailure (show e) + Right t -> return t + where + h _ = Handler $ \case + NetworkStatusFailure _ -> return True + _ -> return False diff --git a/test/Chainweb/Test/Rosetta.hs b/test/Chainweb/Test/Rosetta.hs index bb494b23b5..a1884e1ffa 100644 --- a/test/Chainweb/Test/Rosetta.hs +++ b/test/Chainweb/Test/Rosetta.hs @@ -38,7 +38,7 @@ import Test.Tasty.HUnit import Chainweb.Rosetta.Internal import Chainweb.Rosetta.RestAPI -import Chainweb.Rosetta.Util +import Chainweb.Rosetta.Utils import Chainweb.Version --- @@ -265,7 +265,7 @@ matchNonGenesisSingleTransactionsToLogs = do expectMatch rk3 rk3Exp expectMatch rk4 rk4Exp expectMissing "request key should not be present" missingRk - + where run :: T.Text -> Either String (Maybe Transaction) run trk = getActual cases f @@ -273,7 +273,7 @@ matchNonGenesisSingleTransactionsToLogs = do targets = [ "ReqKey1", "ReqKey0", "ReqKey3", "ReqKey2", "ReqKey4", "RandomReqKey"] - + expectMatch actual (msg, expect) = case actual of Left err -> assertFailure $ adjust msg err @@ -381,7 +381,7 @@ checkValidateNetwork = do where run :: (ChainwebVersion, NetworkId) -> Either RosettaFailure T.Text run (v,net) = runExceptT (validateNetwork v net) >>= either Left (pure . chainIdToText) - + validNetId = (Development, NetworkId { _networkId_blockchain = "kadena" , _networkId_network = "development" diff --git a/test/Chainweb/Test/Rosetta/RestAPI.hs b/test/Chainweb/Test/Rosetta/RestAPI.hs new file mode 100644 index 0000000000..626da1ea6a --- /dev/null +++ b/test/Chainweb/Test/Rosetta/RestAPI.hs @@ -0,0 +1,513 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE TypeFamilies #-} +module Chainweb.Test.Rosetta.RestAPI +( tests +) where + + +import Control.Concurrent.Async +import Control.Concurrent.MVar +import Control.Lens + +import qualified Data.Aeson as A +import Data.Functor (void) +import qualified Data.HashMap.Strict as HM +import Data.IORef +import qualified Data.List.NonEmpty as NEL +import Data.Text (Text) +import Data.Foldable + +import GHC.Natural +import GHC.Word + +import Servant.Client + +import Test.Tasty +import Test.Tasty.HUnit + +-- internal pact modules + +import Pact.Types.API +import Pact.Types.Command + +-- internal chainweb modules + +import Chainweb.Graph +import Chainweb.Pact.Utils (aeson) +import Chainweb.Rosetta.RestAPI +import Chainweb.Rosetta.Utils +import Chainweb.Test.Pact.Utils +import Chainweb.Test.RestAPI.Utils +import Chainweb.Test.Utils +import Chainweb.Time (Time(..), Micros(..)) +import Chainweb.Utils +import Chainweb.Version + +import Data.CAS.RocksDB + +import Rosetta + +import System.IO.Unsafe (unsafePerformIO) + + +-- -------------------------------------------------------------------------- -- +-- Global Settings + +v :: ChainwebVersion +v = FastTimedCPM petersonChainGraph + +nodes:: Natural +nodes = 1 + +cid :: ChainId +cid = unsafeChainId 0 + +cids :: [Text] +cids = chainIds v ^.. folded . to chainIdInt . to (sshow @Int) + +nonceRef :: IORef Natural +nonceRef = unsafePerformIO $ newIORef 0 + +type RosettaTest = IO (Time Micros) -> IO ClientEnv -> ScheduledTest + +-- -------------------------------------------------------------------------- -- +-- Test Tree + +tests :: RocksDb -> ScheduledTest +tests rdb = testGroupSch "Chainweb.Test.Rosetta.RestAPI" go + where + go = return $ + withNodes v "rosettaRemoteTests-" rdb nodes $ \envIo -> + withTime $ \tio -> testGroup "Rosetta Api tests" $ + schedule Sequential (tgroup tio envIo) + + -- Not supported: + -- + -- * Mempool Transaction: cant test reasonably without DOS'ing the mempool + -- * Construction Metadata: N/A + -- + -- Note: + -- + -- * Tests run in sequence, but still interact with each other because + -- confirmation depths are not validated for each tx. Checking account + -- balances between two different tests is futile. + -- + + tgroup tio envIo = fmap (\test -> test tio envIo) + [ accountBalanceTests + , blockTransactionTests + , blockTests + , constructionSubmitTests + , mempoolTests + , networkListTests + , networkOptionsTests + , networkStatusTests + ] + +-- | Rosetta account balance endpoint tests +-- +accountBalanceTests :: RosettaTest +accountBalanceTests tio envIo = + testCaseSchSteps "Account Balance Tests" $ \step -> do + step "check initial balance" + cenv <- envIo + resp0 <- accountBalance cenv req + checkBalance resp0 100000000.000 + + step "send 1.0 tokens to sender00 from sender01" + void $! transferOneAsync_ tio cenv (void . return) + + step "check post-transfer balance" + resp1 <- accountBalance cenv req + checkBalance resp1 99999998.9453 + where + req = AccountBalanceReq nid (AccountId "sender00" Nothing Nothing) Nothing + + checkBalance resp bal1 = do + let b0 = head $ _accountBalanceResp_balances resp + b1 = kdaToRosettaAmount bal1 + curr = _amount_currency b0 + + b1 @=? b0 + curr @=? kda + +-- | Rosetta block transaction endpoint tests +-- +blockTransactionTests :: RosettaTest +blockTransactionTests tio envIo = + testCaseSchSteps "Block Transaction Tests" $ \step -> do + cenv <- envIo + rkmv <- newEmptyMVar @RequestKeys + + step "send 1.0 from sender00 to sender01 and extract block tx request" + prs <- transferOneAsync tio cenv (putMVar rkmv) + req <- mkTxReq rkmv prs + + step "send in block tx request" + resp <- blockTransaction cenv req + + (fundtx,cred,deb,redeem,reward) <- + case _transaction_operations $ _blockTransactionResp_transaction resp of + [a,b,c,d,e] -> return (a,b,c,d,e) + _ -> assertFailure "every transfer should result in 5 transactions" + + + step "validate initial gas buy at index 0" + validateOp 0 "FundTx" "sender00" fundtx + + step "validate sender01 credit at index 1" + validateOp 1 "TransferOrCreateAcct" "sender01" cred + + step "validate sender00 debit at index 2" + validateOp 2 "TransferOrCreateAcct" "sender00" deb + + step "validate sender00 gas redemption at index 3" + validateOp 3 "GasPayment" "sender00" redeem + + step "validate miner gas reward at index 4" + validateOp 4 "GasPayment" "NoMiner" reward + + where + mkTxReq rkmv prs = do + rk <- NEL.head . _rkRequestKeys <$> takeMVar rkmv + meta <- extractMetadata rk prs + bh <- meta ^?! mix "blockHeight" + bhash <- meta ^?! mix "blockHash" + + let bid = BlockId bh bhash + tid = rkToTransactionId rk + + return $ BlockTransactionReq nid bid tid + + +-- | Rosetta block endpoint tests +-- +blockTests :: RosettaTest +blockTests tio envIo = testCaseSchSteps "Block Tests" $ \step -> do + cenv <- envIo + rkmv <- newEmptyMVar @RequestKeys + + step "fetch genesis block" + resp0 <- block cenv (req 0) + (_block_blockId $ _blockResp_block resp0) @?= genesisId + + step "send transaction" + prs <- transferOneAsync tio cenv (putMVar rkmv) + rk <- NEL.head . _rkRequestKeys <$> takeMVar rkmv + cmdMeta <- extractMetadata rk prs + bh <- cmdMeta ^?! mix "blockHeight" + + step "check tx at block height matches sent tx + remediations" + resp1 <- block cenv (req bh) + validateTransferResp bh resp1 + where + req h = BlockReq nid $ PartialBlockId (Just h) Nothing + + validateTransferResp bh resp = do + _blockResp_otherTransactions resp @?= Nothing + + let validateBlock b = do + _block_metadata b @?= Nothing + _blockId_index (_block_blockId b) @?= bh + _blockId_index (_block_parentBlockId b) @?= (bh - 1) + + case _block_transactions b of + [x,y] -> case _transaction_operations x <> _transaction_operations y of + [a,r1, r2, b',c,d,e,f] -> validateTxs (Just (r1,r2)) a b' c d e f + [a,b',c,d,e,f] -> validateTxs Nothing a b' c d e f + _ -> assertFailure "total tx # should be >= 6: coinbase + possible remeds + 5 for every tx" + _ -> assertFailure "every block should result in at least 2 transactions: coinbase + txs" + + validateBlock $ _blockResp_block resp + + validateTxs remeds cbase fundtx cred deb redeem reward = do + + -- coinbase is considered a separate tx list + validateOp 0 "CoinbaseReward" "NoMiner" cbase + + case remeds of + Just (rem1, rem2) -> do + + -- TODO: this case preserves Linda's txlog bug when + -- txs and remeds are present + + validateOp 0 "TransferOrCreateAcct" "sender09" rem1 + validateOp 1 "TransferOrCreateAcct" "sender07" rem2 + validateOp 2 "TransferOrCreateAcct" "sender00" fundtx + validateOp 3 "TransferOrCreateAcct" "sender01" cred + validateOp 4 "TransferOrCreateAcct" "sender00" deb + validateOp 5 "TransferOrCreateAcct" "sender00" redeem + validateOp 6 "TransferOrCreateAcct" "NoMiner" reward + Nothing -> do + validateOp 0 "FundTx" "sender00" fundtx + validateOp 1 "TransferOrCreateAcct" "sender01" cred + validateOp 2 "TransferOrCreateAcct" "sender00" deb + validateOp 3 "GasPayment" "sender00" redeem + validateOp 4 "GasPayment" "NoMiner" reward + + +-- | Rosetta construction submit endpoint tests (i.e. tx submission directly to mempool) +-- +constructionSubmitTests :: RosettaTest +constructionSubmitTests tio envIo = + testCaseSchSteps "Construction Submit Tests" $ \step -> do + cenv <- envIo + + step "build one-off construction submit request" + SubmitBatch (c NEL.:| []) <- mkTransfer tio + + let rk = cmdToRequestKey c + req = ConstructionSubmitReq nid (encodeToText c) + + step "send construction submit request and poll on request key" + resp0 <- constructionSubmit cenv req + + _constructionSubmitResp_transactionId resp0 @?= rkToTransactionId rk + _constructionSubmitResp_metadata resp0 @?= Nothing + + step "confirm transaction details via poll" + PollResponses prs <- polling cid cenv (RequestKeys $ pure rk) ExpectPactResult + + case HM.lookup rk prs of + Nothing -> assertFailure $ "unable to find poll response for: " <> show rk + Just cr -> _crReqKey cr @?= rk + +-- | Rosetta mempool endpoint tests +-- +mempoolTests :: RosettaTest +mempoolTests tio envIo = testCaseSchSteps "Mempool Tests" $ \step -> do + cenv <- envIo + rkmv <- newEmptyMVar @RequestKeys + + step "execute transfer and wait on mempool data" + void $! async $ transferOneAsync_ tio cenv (putMVar rkmv) + rk NEL.:| [] <- _rkRequestKeys <$> takeMVar rkmv + + let tid = rkToTransactionId rk + let test (MempoolResp ts) = return $ elem tid ts + + step "compare requestkey against mempool responses" + void $! repeatUntil test $ mempool cenv req + where + req = MempoolReq nid + +-- | Rosetta network list endpoint tests +-- +networkListTests :: RosettaTest +networkListTests _ envIo = + testCaseSchSteps "Network List Tests" $ \step -> do + cenv <- envIo + + step "send network list request" + resp <- networkList cenv req + + for_ (_networkListResp_networkIds resp) $ \n -> do + _networkId_blockchain n @=? "kadena" + _networkId_network n @=? "fastTimedCPM-peterson" + assertBool "chain id of subnetwork is valid" + $ maybe False (\a -> elem (_subNetworkId_network a) cids) + $ _networkId_subNetworkId n + where + req = MetadataReq Nothing + +-- | Rosetta network options tests +-- +networkOptionsTests :: RosettaTest +networkOptionsTests _ envIo = + testCaseSchSteps "Network Options Tests" $ \step -> do + cenv <- envIo + + step "send network options request" + resp <- networkOptions cenv req0 + + let allow = _networkOptionsResp_allow resp + version = _networkOptionsResp_version resp + + step "check options responses against allowable data and versions" + version @=? rosettaVersion + + step "Check that response errors are a subset of valid errors" + respErrors resp @?= rosettaFailures + + step "Check that response statuses are a subset of valid statuses" + _allow_operationStatuses allow @?= operationStatuses + + step "Check that response op types are a subset of op types" + _allow_operationTypes allow @?= operationTypes + where + req0 = NetworkReq nid Nothing + respErrors = _allow_errors . _networkOptionsResp_allow + +-- | Rosetta network status tests +-- +networkStatusTests :: RosettaTest +networkStatusTests tio envIo = + testCaseSchSteps "Network Status Tests" $ \step -> do + cenv <- envIo + + step "send network status request" + resp0 <- networkStatus cenv req + + step "check status response against genesis" + genesisId @=? _networkStatusResp_genesisBlockId resp0 + + step "send in a transaction and update current block" + transferOneAsync_ tio cenv (void . return) + resp1 <- networkStatus cenv req + + step "check status response genesis and block height" + genesisId @=? _networkStatusResp_genesisBlockId resp1 + (blockIdOf resp1 > blockIdOf resp0) @? "current block id heights must increment" + where + req = NetworkReq nid Nothing + + blockIdOf = _blockId_index . _networkStatusResp_currentBlockId + +-- ------------------------------------------------------------------ -- +-- Test Data + +kda :: Currency +kda = Currency "KDA" 12 Nothing + +nid :: NetworkId +nid = NetworkId + { _networkId_blockchain = "kadena" + , _networkId_network = "fastTimedCPM-peterson" + , _networkId_subNetworkId = Just (SubNetworkId "0" Nothing) + } + +genesisId :: BlockId +genesisId = BlockId 0 "rdfJIktp_WL0oMr8Wr6lH49YkERAJ9MlFp0RPLMXPDE" + +rosettaVersion :: RosettaNodeVersion +rosettaVersion = RosettaNodeVersion + { _version_rosettaVersion = "1.3.1" + , _version_nodeVersion = "2.0" + , _version_middlewareVersion = Nothing + , _version_metadata = Just $ HM.fromList + [ "node-api-version" A..= ("0.0" :: Text) + , "chainweb-version" A..= ("fastTimedCPM-peterson" :: Text) + ] + } + +rosettaFailures :: [RosettaError] +rosettaFailures = rosettaError <$> enumFrom RosettaChainUnspecified + +operationStatuses :: [OperationStatus] +operationStatuses = + [ OperationStatus "Successful" True + , OperationStatus "Remediation" True + ] + +operationTypes :: [Text] +operationTypes = + [ "CoinbaseReward" + , "FundTx" + , "GasPayment" + , "TransferOrCreateAcct" + ] + +-- | Validate all useful data for a tx operation +-- +validateOp + :: Word64 + -- ^ op idx + -> Text + -- ^ operation type + -> Text + -- ^ operation account name + -> Operation + -- ^ the op + -> Assertion +validateOp idx opType acct o = do + _operation_operationId o @?= OperationId idx Nothing + _operation_type o @?= opType + _operation_status o @?= "Successful" + _operation_account o @?= Just (AccountId acct Nothing Nothing) + +-- ------------------------------------------------------------------ -- +-- Test Pact Cmds + +-- | Build a simple transfer from sender00 to sender01 +-- +mkTransfer :: IO (Time Micros) -> IO SubmitBatch +mkTransfer tio = do + t <- toTxCreationTime <$> tio + n <- readIORef nonceRef + c <- buildTextCmd + $ set cbSigners + [ mkSigner' sender00 + [ mkTransferCap "sender00" "sender01" 1.0 + , mkGasCap + ] + ] + $ set cbCreationTime t + $ set cbNetworkId (Just v) + $ mkCmd ("nonce-transfer-" <> sshow t <> "-" <> sshow n) + $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" + + modifyIORef' nonceRef (+1) + return $ SubmitBatch (pure c) + +-- | Transfer one token from sender00 to sender01, applying some callback to +-- the command batch before sending. +-- +transferOneAsync + :: IO (Time Micros) + -> ClientEnv + -> (RequestKeys -> IO ()) + -> IO PollResponses +transferOneAsync tio cenv callback = do + batch0 <- mkTransfer tio + void $! callback (f batch0) + rks <- sending cid cenv batch0 + prs <- polling cid cenv rks ExpectPactResult + return prs + where + f (SubmitBatch cs) = RequestKeys (cmdToRequestKey <$> cs) + +-- | Transfer one token from sender00 to sender01 asynchronously applying some +-- callback (usually putting the requestkeys into some 'MVar'), and forgetting +-- the poll response results. We use this when we want to just execute and poll +-- and do not need the responses. +-- +transferOneAsync_ + :: IO (Time Micros) + -> ClientEnv + -> (RequestKeys -> IO ()) + -> IO () +transferOneAsync_ tio cenv callback + = void $! transferOneAsync tio cenv callback + +-- ------------------------------------------------------------------ -- +-- Utils + +-- | Extract poll response metadata at some request key +-- +extractMetadata :: RequestKey -> PollResponses -> IO (HM.HashMap Text A.Value) +extractMetadata rk (PollResponses pr) = case HM.lookup rk pr of + Just cr -> case _crMetaData cr of + Just (A.Object o) -> return o + _ -> assertFailure "impossible: empty metadata" + _ -> assertFailure "test transfer did not succeed" + +-- | A composition of an index into a k-v structure with aeson values +-- and conversion to non-JSONified structured, asserting test failure if +-- it fails to decode as the give type @a@. +-- +mix + :: forall a m + . ( A.FromJSON a + , Ixed m + , IxValue m ~ A.Value + ) + => Index m + -> Fold m (IO a) +mix i = ix i . to A.fromJSON . to (aeson assertFailure return) diff --git a/test/Chainweb/Test/Utils.hs b/test/Chainweb/Test/Utils.hs index 209fbfea0f..b5f4781f6a 100644 --- a/test/Chainweb/Test/Utils.hs +++ b/test/Chainweb/Test/Utils.hs @@ -1,6 +1,7 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE RankNTypes #-} @@ -91,6 +92,7 @@ module Chainweb.Test.Utils , ScheduledTest(..) , schedule , testCaseSch +, testCaseSchSteps , testGroupSch , testPropertySch @@ -103,13 +105,27 @@ module Chainweb.Test.Utils -- * Misc , genEnum + +-- * Multi-node testing utils +, withNodes +, runTestNodes +, node +, deadbeef +, config +, bootstrapConfig +, setBootstrapPeerInfo +, host +, interface +, withTime +, withMVarResource ) where import Control.Concurrent import Control.Concurrent.Async import Control.Exception (bracket) -import Control.Lens (deep, filtered, toListOf) -import Control.Monad.Catch (MonadThrow) +import Control.Lens +import Control.Monad +import Control.Monad.Catch (MonadThrow, finally) import Control.Monad.IO.Class import qualified Data.ByteString.Lazy as BL @@ -120,13 +136,17 @@ import Data.Bytes.Put import qualified Data.ByteString as B import Data.Coerce (coerce) import Data.Foldable +import qualified Data.HashMap.Strict as HashMap import Data.List (sortOn,isInfixOf) import qualified Data.Text as T +import qualified Data.Text.Encoding as T import Data.Tree import qualified Data.Tree.Lens as LT import Data.Word (Word64) +import qualified Network.Connection as HTTP import qualified Network.HTTP.Client as HTTP +import qualified Network.HTTP.Client.TLS as HTTP import Network.Socket (close) import qualified Network.Wai as W import qualified Network.Wai.Handler.Warp as W @@ -138,7 +158,9 @@ import Servant.Client (BaseUrl(..), ClientEnv, Scheme(..), mkClientEnv) import System.Directory import System.Environment (withArgs) +import qualified System.IO.Extra as Extra import System.IO.Temp +import System.LogLevel import System.Random (randomIO) import Test.QuickCheck.Property (Property, Testable, (===)) @@ -162,18 +184,25 @@ import Chainweb.BlockHeaderDB.Internal import Chainweb.BlockHeight import Chainweb.BlockWeight import Chainweb.ChainId +import Chainweb.Chainweb +import Chainweb.Chainweb.ChainResources import Chainweb.Chainweb.MinerResources (MiningCoordination) +import Chainweb.Chainweb.PeerResources import Chainweb.Crypto.MerkleLog hiding (header) import Chainweb.CutDB import Chainweb.Difficulty (targetToDifficulty) import Chainweb.Graph -import Chainweb.Logger (Logger, GenericLogger) -import Chainweb.Mempool.Mempool (MempoolBackend(..)) +import Chainweb.HostAddress +import Chainweb.Logger +import Chainweb.Mempool.Mempool (MempoolBackend(..), TransactionHash(..)) +import Chainweb.Miner.Config +import Chainweb.Miner.Pact +import Chainweb.NodeId import Chainweb.Payload.PayloadStore import Chainweb.RestAPI import Chainweb.RestAPI.NetworkID import Chainweb.Test.P2P.Peer.BootstrapConfig - (bootstrapCertificate, bootstrapKey) + (bootstrapCertificate, bootstrapKey, bootstrapPeerConfig) import Chainweb.Test.Utils.BlockHeader import Chainweb.Time import Chainweb.TreeDB @@ -186,6 +215,8 @@ import Data.CAS.RocksDB import Network.X509.SelfSigned import qualified P2P.Node.PeerDB as P2P +import P2P.Node.Configuration +import P2P.Peer -- -------------------------------------------------------------------------- -- -- Misc @@ -818,6 +849,9 @@ data ScheduledTest = ScheduledTest { _schLabel :: String , _schTest :: TestTree testCaseSch :: String -> Assertion -> ScheduledTest testCaseSch l a = ScheduledTest l $ testCase l a +testCaseSchSteps :: String -> ((String -> IO ()) -> Assertion) -> ScheduledTest +testCaseSchSteps l a = ScheduledTest l $ testCaseSteps l a + testGroupSch :: String -> [TestTree] -> ScheduledTest testGroupSch l ts = ScheduledTest l $ testGroup l ts @@ -849,3 +883,144 @@ runSchedRocks test = withTempRocksDb "chainweb-tests" $ \rdb -> runSched (test r -- > matchTest "myTest" $ runSched tests matchTest :: String -> IO a -> IO a matchTest pat = withArgs ["-p",pat] + +-- ------------------------------------------------------------------------ -- +-- Multi-node network utils + +withNodes + :: ChainwebVersion + -> B.ByteString + -> RocksDb + -> Natural + -> (IO ClientEnv -> TestTree) + -> TestTree +withNodes v label rdb n f = withResource start + (cancel . fst) + (f . fmap snd) + where + start :: IO (Async (), ClientEnv) + start = do + peerInfoVar <- newEmptyMVar + a <- async $ runTestNodes label rdb Quiet v n peerInfoVar + i <- readMVar peerInfoVar + cwEnv <- getClientEnv $ getCwBaseUrl $ _hostAddressPort $ _peerAddr i + return (a, cwEnv) + + getCwBaseUrl :: Port -> BaseUrl + getCwBaseUrl p = BaseUrl + { baseUrlScheme = Https + , baseUrlHost = "127.0.0.1" + , baseUrlPort = fromIntegral p + , baseUrlPath = "" + } + +runTestNodes + :: B.ByteString + -> RocksDb + -> LogLevel + -> ChainwebVersion + -> Natural + -> MVar PeerInfo + -> IO () +runTestNodes label rdb loglevel ver n portMVar = + forConcurrently_ [0 .. int n - 1] $ \i -> do + threadDelay (1000 * int i) + let baseConf = config ver n (NodeId i) + conf <- if + | i == 0 -> + return $ bootstrapConfig baseConf + | otherwise -> + setBootstrapPeerInfo <$> readMVar portMVar <*> pure baseConf + node label rdb loglevel portMVar conf + +node + :: B.ByteString + -> RocksDb + -> LogLevel + -> MVar PeerInfo + -> ChainwebConfiguration + -> IO () +node label rdb loglevel peerInfoVar conf = do + rocksDb <- testRocksDb (label <> T.encodeUtf8 (toText nid)) rdb + Extra.withTempDir $ \dir -> withChainweb conf logger rocksDb (Just dir) False $ \cw -> do + + -- If this is the bootstrap node we extract the port number and publish via an MVar. + when (nid == NodeId 0) $ do + let bootStrapInfo = view (chainwebPeer . peerResPeer . peerInfo) cw + putMVar peerInfoVar bootStrapInfo + + poisonDeadBeef cw + runChainweb cw `finally` do + logFunctionText logger Info "write sample data" + logFunctionText logger Info "shutdown node" + return () + where + nid = _configNodeId conf + logger :: GenericLogger + logger = addLabel ("node", toText nid) $ genericLogger loglevel print + + poisonDeadBeef cw = mapM_ poison crs + where + crs = map snd $ HashMap.toList $ view chainwebChains cw + poison cr = mempoolAddToBadList (view chainResMempool cr) deadbeef + +deadbeef :: TransactionHash +deadbeef = TransactionHash "deadbeefdeadbeefdeadbeefdeadbeef" + +config + :: ChainwebVersion + -> Natural + -> NodeId + -> ChainwebConfiguration +config ver n nid = defaultChainwebConfiguration ver + & set configNodeId nid + & set (configP2p . p2pConfigPeer . peerConfigHost) host + & set (configP2p . p2pConfigPeer . peerConfigInterface) interface + & set (configP2p . p2pConfigKnownPeers) mempty + & set (configP2p . p2pConfigIgnoreBootstrapNodes) True + & set (configP2p . p2pConfigMaxPeerCount) (n * 2) + & set (configP2p . p2pConfigMaxSessionCount) 4 + & set (configP2p . p2pConfigSessionTimeout) 60 + & set (configMining . miningInNode) miner + & set configReintroTxs True + & set (configTransactionIndex . enableConfigEnabled) True + & set configBlockGasLimit 1_000_000 + & set configRosetta True + where + miner = NodeMiningConfig + { _nodeMiningEnabled = True + , _nodeMiner = noMiner + , _nodeTestMiners = MinerCount n } + +bootstrapConfig :: ChainwebConfiguration -> ChainwebConfiguration +bootstrapConfig conf = conf + & set (configP2p . p2pConfigPeer) peerConfig + & set (configP2p . p2pConfigKnownPeers) [] + where + peerConfig = head (bootstrapPeerConfig $ _configChainwebVersion conf) + & set peerConfigPort 0 + & set peerConfigHost host + +setBootstrapPeerInfo :: PeerInfo -> ChainwebConfiguration -> ChainwebConfiguration +setBootstrapPeerInfo = + over (configP2p . p2pConfigKnownPeers) . (:) + + +host :: Hostname +host = unsafeHostnameFromText "::1" + +interface :: W.HostPreference +interface = "::1" + +getClientEnv :: BaseUrl -> IO ClientEnv +getClientEnv url = flip mkClientEnv url <$> HTTP.newTlsManagerWith mgrSettings + where + mgrSettings = HTTP.mkManagerSettings + (HTTP.TLSSettingsSimple True False False) + Nothing + +withMVarResource :: a -> (IO (MVar a) -> TestTree) -> TestTree +withMVarResource value = withResource (newMVar value) mempty + +withTime :: (IO (Time Micros) -> TestTree) -> TestTree +withTime = withResource getCurrentTimeIntegral mempty diff --git a/test/ChainwebTests.hs b/test/ChainwebTests.hs index 150e9b12be..1acae46d20 100644 --- a/test/ChainwebTests.hs +++ b/test/ChainwebTests.hs @@ -45,6 +45,7 @@ import qualified Chainweb.Test.Pact.TransactionTests import qualified Chainweb.Test.Pact.TTL import qualified Chainweb.Test.RestAPI import qualified Chainweb.Test.Rosetta +import qualified Chainweb.Test.Rosetta.RestAPI import qualified Chainweb.Test.Roundtrips import qualified Chainweb.Test.SPV import qualified Chainweb.Test.Store.CAS.FS @@ -72,6 +73,7 @@ main = $ testGroup "Chainweb Tests" . schedule Sequential $ pactTestSuite rdb : mempoolTestSuite db h0 + : rosettaTestSuite rdb : suite rdb where adj NoTimeout = Timeout (1_000_000 * 60 * 10) "10m" @@ -95,6 +97,11 @@ pactTestSuite rdb = testGroupSch "Chainweb-Pact Tests" , Chainweb.Test.Pact.NoCoinbase.tests ] +rosettaTestSuite :: RocksDb -> ScheduledTest +rosettaTestSuite rdb = testGroupSch "Chainweb-Rosetta API Tests" $ schedule Sequential + [ Chainweb.Test.Rosetta.RestAPI.tests rdb + ] + suite :: RocksDb -> [ScheduledTest] suite rdb = [ testGroupSch "Chainweb Unit Tests"