From 5dbc7da5bbeb5d1f274fd0b70bf0948dd645331f Mon Sep 17 00:00:00 2001 From: chessai Date: Thu, 16 Nov 2023 14:23:45 -0600 Subject: [PATCH 1/8] PactDB Compaction (#1680) * PactDB: compute sha3 checksums * fix build * compact system tables * refactor * docs * test verifies hashes * return global hash * dedicated test * Code-complete with cw-tool * test comments * cleanup sql * cleanup * cleanup * fix build * Hashing no longer in checkpointer, internal to compaction * cleanup * fix drop table, add vacuum * test TODOs * optparse cleanups * remove flag addition in chainweb.cabal * clean up imports * make relevant fields strict * minor src cleanups * add some docs to compaction functions * first test stuff * more compaction haddocks/add blockheight to invalid block height exception * remove redundant import * add flag to not drop new tables * compact concurrently, add log directory for per-chain logs * debugging doRestore * compaction: just don't drop new tables at all, ever * change compact log file name prefix from compact-log- to compact-chain- * Revert "compaction: just don't drop new tables at all, ever" This reverts commit 06210f1a62277563d91717666db74909475587d7. * undo residual changes from rebase * undo redundant logging * use new chainweb version code * print compaction hash in base64 * fix compactSystemTables * Add --threads option and use pooledMapConcurrentlyN_ in compaction * compaction logging: create directory and files before attempting to log. not sure why this stopped working? * add new test idea from John * add NoGrandHash flag * fix type error * fix compact flag for no grand hash * don't verifyTable when NoGrandHash is set * fix compile error in tests from prior change * make PactService exit if rosetta is enabled and we don't have the full history (#1736) * make PactService exit if rosetta is enabled and we don't have the full history * Compaction.hs: delete extra newline * PactService: temporarily putStrLn some crude output for testing purposes * refactor withPactTestBlockDb to give access to the SQLiteEnv * doGetEarliest: fix callDb debug string * fix tests * _pactRosettaEnabled -> _pactFullHistoryRequired * informative error message if you try to rewind before min(blockheight) (#1726) * informative error message if you try to rewind before min(blockheight) * earliestBlock returning nothing should use genesisHeight * attempt FastTimedCPM10to19; failure * Revert "attempt FastTimedCPM10to19; failure" This reverts commit 66c807b2d2df8ba9b8bfddc911fee39355835c1c. * amend test * add test * remove new exception type and revert changes in failOnTooLowRequestedHeight * remove redundant test in PactMultiChainTest * cleanups after test debugging --------- Co-authored-by: Edmund Noble * MultiNode compaction test (#1742) * rewrite StartedChainweb into GADT syntax * add multi node compaction test * Compaction: log every query * refactor compaction code * STM-based chain selection algo for mining * compact-resume test: move to singletonChainGraph * compaction idempotency test (#1759) * compaction idempotency test * compaction pre-and-post active row test (#1756) * add pre-and-post compaction test * compactTable: log the table name * udpate to latest patience library * benchmark newBlock after compaction (#1774) * [test] compaction tables created after target blockheight are dropped (#1768) * add drop tables test * add latest option for chain compaction * Revert "STM-based chain selection algo for mining" This reverts commit d6cd8e8a5a3a9dc4696311f5a59faf195dc0a101. * compaction compare pact state tool (#1764) * add pact-diff tool * flush handle in withPerChainFileLogger * remove redundant/bad old Compaction.hs test * respond to some review * finish responding to Austin's review * respond to first round of Edmund's reviews * respond to Edmund's second wave of review * Revert "compact-resume test: move to singletonChainGraph" This reverts commit f4fc0e9716f74ebeaa3223b9ffd615937ba0b1d5. * make compact function take a TargetBlockHeight instead of a BlockHeight * get rid of redundant ITxId type * fix index of getLatestBlockHeight query log * respond to more review * compaction test txlogs (#1780) * txlog compaction test --------- Co-authored-by: Lars Kuhtz Co-authored-by: Stuart Popejoy Co-authored-by: Evgenii Akentev Co-authored-by: Edmund Noble Co-authored-by: Edmund Noble --- bench/Chainweb/Pact/Backend/ForkingBench.hs | 86 ++- cabal.project | 6 + chainweb.cabal | 6 +- src/Chainweb/Chainweb.hs | 7 +- src/Chainweb/Chainweb/Configuration.hs | 2 +- src/Chainweb/Pact/Backend/Compaction.hs | 713 ++++++++++++++++++ src/Chainweb/Pact/Backend/PactState.hs | 428 +++++++++++ .../Pact/Backend/RelationalCheckpointer.hs | 8 +- src/Chainweb/Pact/Backend/Types.hs | 4 +- src/Chainweb/Pact/PactService.hs | 37 +- src/Chainweb/Pact/PactService/Checkpointer.hs | 10 +- src/Chainweb/Pact/PactService/ExecBlock.hs | 4 +- src/Chainweb/Pact/Service/Types.hs | 11 + src/Chainweb/Pact/Types.hs | 2 +- src/Chainweb/Rosetta/Internal.hs | 4 - src/Chainweb/Utils.hs | 12 + src/Chainweb/WebPactExecutionService.hs | 1 - test/Chainweb/Test/MultiNode.hs | 65 +- test/Chainweb/Test/Pact/PactMultiChainTest.hs | 18 +- test/Chainweb/Test/Pact/PactReplay.hs | 24 +- .../Chainweb/Test/Pact/PactSingleChainTest.hs | 540 +++++++++++-- test/Chainweb/Test/Pact/RemotePactTest.hs | 235 +++++- test/Chainweb/Test/Pact/TTL.hs | 2 +- test/Chainweb/Test/Pact/Utils.hs | 131 +++- test/Chainweb/Test/Rosetta/RestAPI.hs | 2 +- test/Chainweb/Test/Utils.hs | 112 ++- test/SlowTests.hs | 1 + tools/cwtool/CwTool.hs | 11 + 28 files changed, 2263 insertions(+), 219 deletions(-) create mode 100644 src/Chainweb/Pact/Backend/Compaction.hs create mode 100644 src/Chainweb/Pact/Backend/PactState.hs diff --git a/bench/Chainweb/Pact/Backend/ForkingBench.hs b/bench/Chainweb/Pact/Backend/ForkingBench.hs index fc5d8b819b..df780f3292 100644 --- a/bench/Chainweb/Pact/Backend/ForkingBench.hs +++ b/bench/Chainweb/Pact/Backend/ForkingBench.hs @@ -1,10 +1,12 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedRecordDot #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} @@ -33,6 +35,7 @@ import Data.FileEmbed import Data.Foldable (toList) import Data.IORef import Data.List (uncons) +import Data.List qualified as List import Data.List.NonEmpty (NonEmpty) import qualified Data.List.NonEmpty as NEL import Data.Map.Strict (Map) @@ -49,6 +52,7 @@ import qualified Data.Yaml as Y import GHC.Generics hiding (from, to) import System.Environment +import System.Logger.Types qualified import System.LogLevel import System.Random @@ -74,11 +78,13 @@ import Chainweb.BlockCreationTime import Chainweb.BlockHeader import Chainweb.BlockHeaderDB import Chainweb.BlockHeaderDB.Internal +import Chainweb.BlockHeight (BlockHeight(..)) import Chainweb.ChainId import Chainweb.Graph import Chainweb.Logger import Chainweb.Mempool.Mempool (BlockFill(..)) import Chainweb.Miner.Pact +import Chainweb.Pact.Backend.Compaction qualified as C import Chainweb.Pact.Backend.Types import Chainweb.Pact.Backend.Utils import Chainweb.Pact.PactService @@ -90,7 +96,7 @@ import Chainweb.Pact.Utils (toTxCreationTime) import Chainweb.Payload import Chainweb.Payload.PayloadStore import Chainweb.Payload.PayloadStore.InMemory -import Chainweb.Test.TestVersions +import Chainweb.Test.TestVersions (slowForkingCpmTestVersion) import Chainweb.Time import Chainweb.Transaction import Chainweb.Utils @@ -111,28 +117,48 @@ _run args = withTempRocksDb "forkingbench" $ \rdb -> -- -------------------------------------------------------------------------- -- -- Benchmarks +data BenchConfig = BenchConfig + { numPriorBlocks :: Word64 + -- ^ number of blocks to create prior to benchmarking + , validate :: Validate + -- ^ whether or not to validate the blocks as part of the benchmark + , compact :: Compact + -- ^ whether or not to compact the pact database prior to benchmarking + } + +defBenchConfig :: BenchConfig +defBenchConfig = BenchConfig + { numPriorBlocks = 100 + , validate = DontValidate + , compact = DontCompact + } + +data Compact = DoCompact | DontCompact + deriving stock (Eq) + +data Validate = DoValidate | DontValidate + deriving stock (Eq) + bench :: RocksDb -> C.Benchmark -bench rdb = C.bgroup "PactService" +bench rdb = C.bgroup "PactService" $ [ forkingBench , doubleForkingBench - , oneBlock True 1 - , oneBlock True 10 - , oneBlock True 50 - , oneBlock True 100 - , oneBlock False 0 - , oneBlock False 1 - , oneBlock False 10 - , oneBlock False 50 - , oneBlock False 100 - ] + ] ++ map (oneBlock defBenchConfig) [1, 10, 50, 100] + ++ map (oneBlock validateCfg) [0, 1, 10, 50, 100] + ++ map (oneBlock compactCfg) [0, 1, 10, 50, 100] + ++ map (oneBlock compactValidateCfg) [1, 10, 50, 100] where - forkingBench = withResources rdb 10 Quiet + validateCfg = defBenchConfig { validate = DoValidate } + compactCfg = defBenchConfig { compact = DoCompact } + compactValidateCfg = compactCfg { validate = DoValidate } + + forkingBench = withResources rdb 10 Quiet DontCompact $ \mainLineBlocks pdb bhdb nonceCounter pactQueue _ -> C.bench "forkingBench" $ C.whnfIO $ do let (T3 _ join1 _) = mainLineBlocks !! 5 void $ playLine pdb bhdb 5 join1 pactQueue nonceCounter - doubleForkingBench = withResources rdb 10 Quiet + doubleForkingBench = withResources rdb 10 Quiet DontCompact $ \mainLineBlocks pdb bhdb nonceCounter pactQueue _ -> C.bench "doubleForkingBench" $ C.whnfIO $ do let (T3 _ join1 _) = mainLineBlocks !! 5 @@ -141,15 +167,21 @@ bench rdb = C.bgroup "PactService" void $ playLine pdb bhdb forkLength1 join1 pactQueue nonceCounter void $ playLine pdb bhdb forkLength2 join1 pactQueue nonceCounter - oneBlock validate txCount = withResources rdb 3 Error go + oneBlock :: BenchConfig -> Int -> C.Benchmark + oneBlock cfg txCount = withResources rdb cfg.numPriorBlocks Error cfg.compact go where - go mainLineBlocks _pdb _bhdb _nonceCounter pactQueue txsPerBlock = + go mainLineBlocks _pdb _bhdb _nonceCounter pactQueue txsPerBlock = do C.bench name $ C.whnfIO $ do writeIORef txsPerBlock txCount let (T3 _ join1 _) = last mainLineBlocks - createBlock validate (ParentHeader join1) (Nonce 1234) pactQueue - name = "block-new" ++ (if validate then "-validated" else "") ++ - "[" ++ show txCount ++ "]" + createBlock cfg.validate (ParentHeader join1) (Nonce 1234) pactQueue + name = "block-new [" + ++ List.intercalate "," + [ "txCount=" ++ show txCount + , "validate=" ++ show (cfg.validate == DoValidate) + , "compact=" ++ show (cfg.compact == DoCompact) + ] + ++ "]" -- -------------------------------------------------------------------------- -- -- Benchmark Function @@ -188,14 +220,14 @@ mineBlock -> PactQueue -> IO (T3 ParentHeader BlockHeader PayloadWithOutputs) mineBlock parent nonce pdb bhdb pact = do - !r@(T3 _ newHeader payload) <- createBlock True parent nonce pact + r@(T3 _ newHeader payload) <- createBlock DoValidate parent nonce pact addNewPayload pdb payload -- NOTE: this doesn't validate the block header, which is fine in this test case unsafeInsertBlockHeaderDb bhdb newHeader return r createBlock - :: Bool + :: Validate -> ParentHeader -> Nonce -> PactQueue @@ -216,7 +248,7 @@ createBlock validate parent nonce pact = do creationTime parent - when validate $ do + when (validate == DoValidate) $ do mv' <- validateBlock bh (payloadWithOutputsToPayloadData payload) pact void $ assertNotLeft =<< takeMVar mv' @@ -250,9 +282,10 @@ withResources :: () => RocksDb -> Word64 -> LogLevel + -> Compact -> RunPactService -> C.Benchmark -withResources rdb trunkLength logLevel f = C.envWithCleanup create destroy unwrap +withResources rdb trunkLength logLevel compact f = C.envWithCleanup create destroy unwrap where unwrap ~(NoopNFData (Resources {..})) = @@ -270,6 +303,13 @@ withResources rdb trunkLength logLevel f = C.envWithCleanup create destroy unwra startPact testVer logger blockHeaderDb payloadDb mp sqlEnv mainTrunkBlocks <- playLine payloadDb blockHeaderDb trunkLength genesisBlock (snd pactService) nonceCounter + when (compact == DoCompact) $ do + C.withDefaultLogger System.Logger.Types.Error $ \lgr -> do + let flags = [C.NoGrandHash] + let db = _sConn sqlEnv + let bh = BlockHeight trunkLength + void $ C.compact (C.Target bh) lgr db flags + return $ NoopNFData $ Resources {..} destroy (NoopNFData (Resources {..})) = do diff --git a/cabal.project b/cabal.project index 39554ed52e..b358111d00 100644 --- a/cabal.project +++ b/cabal.project @@ -125,6 +125,12 @@ source-repository-package tag: 174af3523616c8fe01449da5ccbb9f16df097ac3 --sha256: sha256-kVFIy+Aj3TNJpsM1Cs/5uGmzeWwHKYWjjCQ+L1/XOj8= +source-repository-package + type: git + location: https://github.com/chessai/patience + tag: 2f67d546ea6608fc6ebe5f2f6976503cbf340442 + --sha256: 0x137akvbh4kr3qagksw74xdj2xz5vjnx1fbr41bb54a0lkcb8mm + -- -------------------------------------------------------------------------- -- -- Relaxed Bounds diff --git a/chainweb.cabal b/chainweb.cabal index 72cd8399ec..fbbbc916ac 100644 --- a/chainweb.cabal +++ b/chainweb.cabal @@ -274,6 +274,8 @@ library -- pact , Chainweb.Pact.Backend.ChainwebPactDb , Chainweb.Pact.Backend.DbCache + , Chainweb.Pact.Backend.Compaction + , Chainweb.Pact.Backend.PactState , Chainweb.Pact.Backend.RelationalCheckpointer , Chainweb.Pact.Backend.SQLite.DirectV2 , Chainweb.Pact.Backend.SQLite.V2 @@ -403,6 +405,7 @@ library , tls-session-manager >= 0.0 , token-bucket >= 0.1 , transformers >= 0.5 + , unliftio ^>= 0.2 , unordered-containers >= 0.2.16 , uuid >= 1.3.15 , wai >= 3.2.2.1 @@ -506,7 +509,7 @@ test-suite chainweb-tests build-depends: -- internal - chainweb + , chainweb -- external , Decimal >= 0.4.2 @@ -540,6 +543,7 @@ test-suite chainweb-tests , merkle-log >=0.2 , mtl >= 2.3 , network >= 3.1.2 + , patience >= 0.3 , http-client-tls >=0.3 , pact , pact-json >= 0.1 diff --git a/src/Chainweb/Chainweb.hs b/src/Chainweb/Chainweb.hs index 6033adddef..c8e018507e 100644 --- a/src/Chainweb/Chainweb.hs +++ b/src/Chainweb/Chainweb.hs @@ -328,9 +328,9 @@ validatingMempoolConfig cid v gl gp mv = Mempool.InMemConfig f (This _) = Left (T2 (Mempool.TransactionHash "") (Mempool.InsertErrorOther "preInsertBatch: align mismatch 1")) -data StartedChainweb logger - = forall cas. (CanReadablePayloadCas cas, Logger logger) => StartedChainweb !(Chainweb logger cas) - | Replayed !Cut !Cut +data StartedChainweb logger where + StartedChainweb :: (CanReadablePayloadCas cas, Logger logger) => !(Chainweb logger cas) -> StartedChainweb logger + Replayed :: !Cut -> !Cut -> StartedChainweb logger data ChainwebStatus = ProcessStarted @@ -427,6 +427,7 @@ withChainwebInternal conf logger peer serviceSock rocksDb pactDbDir backupDir re , _pactBlockGasLimit = maybe id min maxGasLimit (_configBlockGasLimit conf) , _pactLogGas = _configLogGas conf , _pactModuleCacheLimit = _configModuleCacheLimit conf + , _pactFullHistoryRequired = _configRosetta conf -- this could be OR'd with other things that require full history } pruningLogger :: T.Text -> logger diff --git a/src/Chainweb/Chainweb/Configuration.hs b/src/Chainweb/Chainweb/Configuration.hs index a0d1fbc0e0..ecfaaceb58 100644 --- a/src/Chainweb/Chainweb/Configuration.hs +++ b/src/Chainweb/Chainweb/Configuration.hs @@ -448,7 +448,7 @@ defaultChainwebConfiguration v = ChainwebConfiguration , _configP2p = defaultP2pConfiguration , _configThrottling = defaultThrottlingConfig , _configMempoolP2p = defaultEnableConfig defaultMempoolP2pConfig - , _configBlockGasLimit = 150000 + , _configBlockGasLimit = 150_000 , _configLogGas = False , _configMinGasPrice = 1e-8 , _configPactQueueSize = 2000 diff --git a/src/Chainweb/Pact/Backend/Compaction.hs b/src/Chainweb/Pact/Backend/Compaction.hs new file mode 100644 index 0000000000..9df0dbf12a --- /dev/null +++ b/src/Chainweb/Pact/Backend/Compaction.hs @@ -0,0 +1,713 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE InstanceSigs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ViewPatterns #-} + +-- | +-- Module: Chainweb.Pact.Backend.Compaction +-- Copyright: Copyright © 2023 Kadena LLC. +-- License: see LICENSE.md +-- +-- Compact Checkpointer PactDbs by culling old journal rows. +-- + +module Chainweb.Pact.Backend.Compaction + ( CompactFlag(..) + , TargetBlockHeight(..) + , CompactM + , compact + , compactAll + , main + , withDefaultLogger + , withPerChainFileLogger + ) where + +import UnliftIO.Async (pooledMapConcurrentlyN_) +import Control.Concurrent (forkIO, threadDelay) +import Control.Concurrent.MVar (swapMVar, readMVar, newMVar) +import Control.Exception (Exception, SomeException(..)) +import Control.Lens (makeLenses, set, over, view, (^.)) +import Control.Monad (forM_, unless, void, when) +import Control.Monad.Catch (MonadCatch(catch), MonadThrow(throwM)) +import Control.Monad.IO.Class (MonadIO(liftIO)) +import Control.Monad.Reader (MonadReader, ReaderT, runReaderT, local) +import Control.Monad.Trans.Control (MonadBaseControl, liftBaseOp) +import Data.ByteString (ByteString) +import Data.Foldable qualified as F +import Data.Function (fix) +import Data.List qualified as List +import Data.Map.Strict qualified as M +import Data.Set (Set) +import Data.Set qualified as Set +import Data.String (IsString) +import Data.Text (Text) +import Data.Text qualified as Text +import Data.Text.Encoding qualified as Text +import Data.Vector qualified as V +import Data.Vector (Vector) +import Database.SQLite3.Direct (Utf8(..), Database) +import GHC.Stack (HasCallStack) +import Options.Applicative +import System.Directory (createDirectoryIfMissing) +import System.FilePath (()) +import System.IO qualified as IO +import System.IO (Handle) + +import Chainweb.BlockHeight (BlockHeight) +import Chainweb.Logger (setComponent) +import Chainweb.Utils (sshow, HasTextRepresentation, fromText, toText, int) +import Chainweb.Version (ChainId, ChainwebVersion(..), ChainwebVersionName, unsafeChainId, chainIdToText) +import Chainweb.Version.Mainnet (mainnet) +import Chainweb.Version.Registry (lookupVersionByName) +import Chainweb.Version.Utils (chainIdsAt) +import Chainweb.Pact.Backend.Types (SQLiteEnv(..)) +import Chainweb.Pact.Backend.Utils (fromUtf8, withSqliteDb) +import Chainweb.Utils (encodeB64Text) + +import System.Logger +import System.Logger.Backend.ColorOption (useColor) +import Data.LogMessage + +import Pact.Types.Persistence (TxId(..)) +import Pact.Types.SQLite (SType(..), RType(..)) +import Pact.Types.SQLite qualified as Pact + +newtype TableName = TableName { getTableName :: Utf8 } + deriving stock (Show) + deriving newtype (Eq, IsString) + +data CompactException + = CompactExceptionInternal !Text + | CompactExceptionDb !SomeException + | CompactExceptionInvalidBlockHeight !BlockHeight + | CompactExceptionTableVerificationFailure !TableName + | CompactExceptionNoLatestBlockHeight + deriving stock (Show) + deriving anyclass (Exception) + +data CompactFlag + = KeepCompactTables + -- ^ Keep compaction tables post-compaction for inspection. + | NoVacuum + -- ^ Don't VACUUM database + | NoDropNewTables + -- ^ Don't drop new tables created after the compaction height. + | NoGrandHash + -- ^ Don't compute the grand hash. + deriving stock (Eq,Show,Read,Enum,Bounded) + +internalError :: MonadThrow m => Text -> m a +internalError = throwM . CompactExceptionInternal + +data CompactEnv = CompactEnv + { _ceLogger :: !(Logger SomeLogMessage) + , _ceDb :: !Database + , _ceFlags :: ![CompactFlag] + } +makeLenses ''CompactEnv + +withDefaultLogger :: LogLevel -> (Logger SomeLogMessage -> IO a) -> IO a +withDefaultLogger ll f = withHandleBackend_ logText defaultHandleBackendConfig $ \b -> + withLogger defaultLoggerConfig b $ \l -> f (set setLoggerLevel ll l) + +withPerChainFileLogger :: FilePath -> ChainId -> LogLevel -> (Logger SomeLogMessage -> IO a) -> IO a +withPerChainFileLogger logDir chainId ll f = do + createDirectoryIfMissing False {- don't create parents -} logDir + let logFile = logDir ("chain-" <> cid <> ".log") + !_ <- writeFile logFile "" + let handleConfig = defaultHandleBackendConfig + { _handleBackendConfigHandle = FileHandle logFile + } + withHandleBackend_' logText handleConfig $ \h b -> do + + done <- newMVar False + void $ forkIO $ fix $ \go -> do + doneYet <- readMVar done + when (not doneYet) $ do + IO.hFlush h + threadDelay 5_000_000 + go + IO.hFlush h + + withLogger defaultLoggerConfig b $ \l -> do + let logger = setComponent "compaction" + $ over setLoggerScope (("chain", chainIdToText chainId) :) + $ set setLoggerLevel ll l + a <- f logger + void $ swapMVar done True + pure a + where + cid = Text.unpack (chainIdToText chainId) + +withHandleBackend_' :: (MonadIO m, MonadBaseControl IO m) + => (msg -> Text) + -> HandleBackendConfig + -> (Handle -> LoggerBackend msg -> m a) + -> m a +withHandleBackend_' format conf inner = + case conf ^. handleBackendConfigHandle of + StdErr -> run IO.stderr + StdOut -> run IO.stdout + FileHandle file -> liftBaseOp (IO.withFile file IO.AppendMode) run + where + run h = do + colored <- liftIO $ useColor (conf ^. handleBackendConfigColor) h + inner h (handleBackend_ format h colored) + +newtype CompactM a = CompactM { unCompactM :: ReaderT CompactEnv IO a } + deriving newtype (Functor,Applicative,Monad,MonadReader CompactEnv,MonadIO,MonadThrow,MonadCatch) + +instance MonadLog Text CompactM where + localScope :: (LogScope -> LogScope) -> CompactM x -> CompactM x + localScope f = local (over (ceLogger . setLoggerScope) f) + + logg :: LogLevel -> Text -> CompactM () + logg ll m = do + l <- view ceLogger + liftIO $ loggerFunIO l ll $ toLogMessage $ TextLog m + + withLevel :: LogLevel -> CompactM x -> CompactM x + withLevel l = local (set (ceLogger.setLoggerLevel) l) + + withPolicy :: LogPolicy -> CompactM x -> CompactM x + withPolicy p = local (set (ceLogger.setLoggerPolicy) p) + +-- | Run compaction monad +runCompactM :: CompactEnv -> CompactM a -> IO a +runCompactM e a = runReaderT (unCompactM a) e + +-- | Prepare/Execute a "$VTABLE$"-templated query. +execM_ :: () + => Text -- ^ query name (for logging purposes) + -> TableName -- ^ table name + -> Text -- ^ "$VTABLE$"-templated query + -> CompactM () +execM_ msg tbl q = do + db <- view ceDb + logQueryDebug msg + q' <- templateStmt tbl q + liftIO $ Pact.exec_ db q' + +execNoTemplateM_ :: () + => Text -- ^ query name (for logging purposes) + -> Utf8 -- ^ query + -> CompactM () +execNoTemplateM_ msg qry = do + db <- view ceDb + logQueryDebug msg + liftIO $ Pact.exec_ db qry + +-- | Prepare/Execute a "$VTABLE$"-templated, parameterised query. +-- The parameters are the results of the 'CompactM' 'SType' computations. +execM' :: () + => Text -- ^ query name (for logging purposes) + -> TableName -- ^ table name + -> Text -- ^ "$VTABLE$"-templated query + -> [SType] -- ^ parameters + -> CompactM () +execM' msg tbl stmt ps = do + db <- view ceDb + logQueryDebug msg + stmt' <- templateStmt tbl stmt + liftIO $ Pact.exec' db stmt' ps + +exec_ :: () + => Text + -> Utf8 + -> CompactM () +exec_ msg qry = do + db <- view ceDb + logQueryDebug msg + liftIO $ Pact.exec_ db qry + +qry_ :: () + => Text + -> Utf8 + -> [RType] + -> CompactM [[SType]] +qry_ msg qry rs = do + db <- view ceDb + logQueryDebug msg + liftIO $ Pact.qry_ db qry rs + +-- | Prepare/Execute a "$VTABLE$"-templated, parameterised query. +-- 'RType's are the expected results. +qryM :: () + => Text -- ^ query name (for logging purposes) + -> TableName -- ^ table name + -> Text -- ^ "$VTABLE$"-templated query + -> [SType] -- ^ parameters + -> [RType] -- ^ result types + -> CompactM [[SType]] +qryM msg tbl q ins outs = do + db <- view ceDb + logQueryDebug msg + q' <- templateStmt tbl q + liftIO $ Pact.qry db q' ins outs + +qryNoTemplateM :: () + => Text -- ^ query name (for logging purposes) + -> Utf8 -- ^ query + -> [SType] -- ^ parameters + -> [RType] -- ^ results + -> CompactM [[SType]] +qryNoTemplateM msg q ins outs = do + db <- view ceDb + logQueryDebug msg + liftIO $ Pact.qry db q ins outs + +logQueryDebug :: Text -> CompactM () +logQueryDebug msg = do + logg Info ("Query: " <> msg) + +-- | Statements are templated with "$VTABLE$" substituted +-- with the currently-focused versioned table. +templateStmt :: TableName -> Text -> CompactM Utf8 +templateStmt (TableName (Utf8 tblName)) s = + pure $ Utf8 $ Text.encodeUtf8 $ + Text.replace "$VTABLE$" ("[" <> Text.decodeUtf8 tblName <> "]") s + +-- | Execute a SQLite transaction, rolling back on failure. +-- Throws a 'CompactExceptionDb' on failure. +withTx :: HasCallStack => CompactM a -> CompactM a +withTx a = do + exec_ "withTx.0" "SAVEPOINT compact_tx" + catch (a >>= \r -> exec_ "withTx.1" "RELEASE SAVEPOINT compact_tx" >> pure r) $ + \e@SomeException {} -> do + exec_ "withTx.2" "ROLLBACK TRANSACTION TO SAVEPOINT compact_tx" + throwM $ CompactExceptionDb e + +unlessFlagSet :: CompactFlag -> CompactM () -> CompactM () +unlessFlagSet f x = do + yeahItIs <- isFlagSet f + unless yeahItIs x + +isFlagSet :: CompactFlag -> CompactM Bool +isFlagSet f = view ceFlags >>= \fs -> pure (f `elem` fs) + +isFlagNotSet :: CompactFlag -> CompactM Bool +isFlagNotSet f = not <$> isFlagSet f + +withTables :: Vector TableName -> (TableName -> CompactM a) -> CompactM () +withTables ts a = do + V.iforM_ ts $ \i u@(TableName (Utf8 t')) -> do + let lbl = Text.decodeUtf8 t' <> " (" <> sshow (i + 1) <> " of " <> sshow (V.length ts) <> ")" + localScope (("table",lbl):) $ a u + +-- | Takes a bunch of singleton tablename rows, sorts them, returns them as +-- @TableName@ +sortedTableNames :: [[SType]] -> [TableName] +sortedTableNames rows = M.elems $ M.fromListWith const $ flip List.map rows $ \case + [SText n@(Utf8 s)] -> (Text.toLower (Text.decodeUtf8 s), TableName n) + _ -> error "sortedTableNames: expected text" + +-- | CompactGrandHash associates table name with grand hash of its versioned rows, +-- and NULL with grand hash of all table hashes. +createCompactGrandHash :: CompactM () +createCompactGrandHash = do + logg Info "createTables" + execNoTemplateM_ "createTable: CompactGrandHash" + " CREATE TABLE IF NOT EXISTS CompactGrandHash \ + \ ( tablename TEXT \ + \ , hash BLOB \ + \ , UNIQUE (tablename) ); " + + execNoTemplateM_ "deleteFrom: CompactGrandHash" + "DELETE FROM CompactGrandHash" + +-- | CompactActiveRow collects all active rows from all tables. +createCompactActiveRow :: CompactM () +createCompactActiveRow = do + execNoTemplateM_ "createTable: CompactActiveRow" + " CREATE TABLE IF NOT EXISTS CompactActiveRow \ + \ ( tablename TEXT NOT NULL \ + \ , rowkey TEXT NOT NULL \ + \ , vrowid INTEGER NOT NULL \ + \ , hash BLOB \ + \ , UNIQUE (tablename,rowkey) ); " + + execNoTemplateM_ "deleteFrom: CompactActiveRow" + "DELETE FROM CompactActiveRow" + +locateTarget :: TargetBlockHeight -> CompactM BlockHeight +locateTarget = \case + Target bh -> do + ensureBlockHeightExists bh + pure bh + Latest -> do + getLatestBlockHeight + +ensureBlockHeightExists :: BlockHeight -> CompactM () +ensureBlockHeightExists bh = do + r <- qryNoTemplateM + "ensureBlockHeightExists.0" + "SELECT blockheight FROM BlockHistory WHERE blockheight = ?1" + [bhToSType bh] + [RInt] + case r of + [[SInt rBH]] -> do + when (fromIntegral bh /= rBH) $ do + throwM $ CompactExceptionInvalidBlockHeight bh + _ -> do + error "ensureBlockHeightExists.0: impossible" + +getLatestBlockHeight :: CompactM BlockHeight +getLatestBlockHeight = do + r <- qryNoTemplateM + "getLatestBlockHeight.0" + "SELECT blockheight FROM BlockHistory ORDER BY blockheight DESC LIMIT 1" + [] + [RInt] + case r of + [[SInt bh]] -> do + pure (fromIntegral bh) + _ -> do + throwM CompactExceptionNoLatestBlockHeight + +getEndingTxId :: BlockHeight -> CompactM TxId +getEndingTxId bh = do + r <- qryNoTemplateM + "getTxId.0" + "SELECT endingtxid FROM BlockHistory WHERE blockheight=?" + [bhToSType bh] + [RInt] + case r of + [] -> do + throwM (CompactExceptionInvalidBlockHeight bh) + [[SInt t]] -> do + pure (TxId (fromIntegral t)) + _ -> do + internalError "initialize: expected single-row int" + +getVersionedTables :: BlockHeight -> CompactM (Vector TableName) +getVersionedTables bh = do + logg Info "getVersionedTables" + rs <- qryNoTemplateM + "getVersionedTables.0" + " SELECT DISTINCT tablename FROM VersionedTableMutation \ + \ WHERE blockheight <= ? ORDER BY blockheight; " + [bhToSType bh] + [RText] + pure (V.fromList (sortedTableNames rs)) + +tableRowCount :: TableName -> Text -> CompactM () +tableRowCount tbl label = + qryM "tableRowCount.0" tbl "SELECT COUNT(*) FROM $VTABLE$" [] [RInt] >>= \case + [[SInt r]] -> logg Info $ label <> ":rowcount=" <> sshow r + _ -> internalError "count(*) failure" + +-- | For a given table, collect all active rows into CompactActiveRow, +-- and compute+store table grand hash in CompactGrandHash. +collectTableRows :: TxId -> TableName -> CompactM () +collectTableRows txId tbl = do + tableRowCount tbl "collectTableRows" + let vt = tableNameToSType tbl + let txid = txIdToSType txId + + doGrandHash <- isFlagNotSet NoGrandHash + + let collectInsert = Text.concat + [ "INSERT INTO CompactActiveRow " + , "SELECT ?1,rowkey,rowid," <> if doGrandHash + then "sha3_256('T',?1,'K',rowkey,'I',txid,'D',rowdata) " + else "NULL " + , "FROM $VTABLE$ t1 " + , "WHERE txid=(SELECT MAX(txid) FROM $VTABLE$ t2 " + , "WHERE t2.rowkey=t1.rowkey AND t2.txid>= \case + [[SBlob h]] -> pure h + _ -> throwM $ CompactExceptionInternal "computeGlobalHash: bad result" + +-- | Delete non-active rows from given table. +compactTable :: TableName -> CompactM () +compactTable tbl = do + logg Info $ "compactTable: " <> fromUtf8 (getTableName tbl) + + execM' + "compactTable.0" + tbl + " DELETE FROM $VTABLE$ WHERE rowid NOT IN \ + \ (SELECT t.rowid FROM $VTABLE$ t \ + \ LEFT JOIN CompactActiveRow v \ + \ WHERE t.rowid = v.vrowid AND v.tablename=?1); " + [tableNameToSType tbl] + +-- | For given table, re-compute table grand hash and compare +-- with stored grand hash in CompactGrandHash. +verifyTable :: TableName -> CompactM ByteString +verifyTable tbl = do + logg Info "verifyTable" + curr <- computeTableHash tbl + rs <- qryNoTemplateM "verifyTable.0" + "SELECT hash FROM CompactGrandHash WHERE tablename=?1" + [tableNameToSType tbl] + [RBlob] + case rs of + [[SBlob prev]] + | prev == curr -> do + tableRowCount tbl "verifyTable" + pure curr + | otherwise -> + throwM (CompactExceptionTableVerificationFailure tbl) + _ -> throwM $ CompactExceptionInternal "verifyTable: bad result" + +-- | For given table, compute table grand hash for max txid. +computeTableHash :: TableName -> CompactM ByteString +computeTableHash tbl = do + rs <- qryM "computeTableHash.0" tbl + " SELECT sha3a_256(hash) FROM \ + \ (SELECT sha3_256('T',?1,'K',rowkey,'I',txid,'D',rowdata) as hash \ + \ FROM $VTABLE$ t1 \ + \ WHERE txid=(select max(txid) FROM $VTABLE$ t2 \ + \ WHERE t2.rowkey=t1.rowkey) GROUP BY rowkey); " + [tableNameToSType tbl] + [RBlob] + case rs of + [[SBlob curr]] -> pure curr + _ -> throwM $ CompactExceptionInternal "checksumTable: bad result" + +-- | Drop any versioned tables created after target blockheight. +dropNewTables :: BlockHeight -> CompactM () +dropNewTables bh = do + logg Info "dropNewTables" + nts <- fmap (V.fromList . sortedTableNames) $ qryNoTemplateM "dropNewTables.0" + " SELECT tablename FROM VersionedTableCreation \ + \ WHERE createBlockheight > ?1 ORDER BY createBlockheight; " + [bhToSType bh] + [RText] + + withTables nts $ \tbl -> do + execM_ "dropNewTables.1" tbl "DROP TABLE IF EXISTS $VTABLE$" + +-- | Delete all rows from Checkpointer system tables that are not for the target blockheight. +compactSystemTables :: BlockHeight -> CompactM () +compactSystemTables bh = do + let systemTables = ["BlockHistory", "VersionedTableMutation", "TransactionIndex", "VersionedTableCreation"] + forM_ systemTables $ \tbl -> do + let tblText = fromUtf8 (getTableName tbl) + logg Info $ "Compacting system table " <> tblText + let column = + if tbl == "VersionedTableCreation" + then "createBlockheight" + else "blockheight" + execM' + ("compactSystemTables: " <> tblText) + tbl + ("DELETE FROM $VTABLE$ WHERE " <> column <> " != ?1;") + [bhToSType bh] + +dropCompactTables :: CompactM () +dropCompactTables = do + execNoTemplateM_ "dropCompactTables.0" + " DROP TABLE CompactGrandHash; \ + \ DROP TABLE CompactActiveRow; " + +compact :: () + => TargetBlockHeight + -> Logger SomeLogMessage + -> Database + -> [CompactFlag] + -> IO (Maybe ByteString) +compact tbh logger db flags = runCompactM (CompactEnv logger db flags) $ do + logg Info "Beginning compaction" + + doGrandHash <- isFlagNotSet NoGrandHash + + withTx $ do + createCompactGrandHash + createCompactActiveRow + + blockHeight <- locateTarget tbh + txId <- getEndingTxId blockHeight + + logg Info $ "Target blockheight: " <> sshow blockHeight + logg Info $ "Ending TxId: " <> sshow txId + + versionedTables <- getVersionedTables blockHeight + + gh <- withTx $ do + withTables versionedTables $ \tbl -> collectTableRows txId tbl + if doGrandHash + then Just <$> computeGlobalHash + else pure Nothing + + withTx $ do + withTables versionedTables $ \tbl -> do + compactTable tbl + unlessFlagSet NoGrandHash $ void $ verifyTable tbl + unlessFlagSet NoDropNewTables $ do + logg Info "Dropping new tables" + dropNewTables blockHeight + compactSystemTables blockHeight + + unlessFlagSet KeepCompactTables $ do + logg Info "Dropping compact-specific tables" + withTx dropCompactTables + + unlessFlagSet NoVacuum $ do + logg Info "Vacuum" + execNoTemplateM_ "VACUUM" "VACUUM;" + + case gh of + Just h -> do + logg Info $ "Compaction complete, hash=" <> encodeB64Text h + Nothing -> do + logg Info "Compaction complete" + + pure gh + +data TargetBlockHeight + = Target !BlockHeight + -- ^ compact to this blockheight across all chains + | Latest + -- ^ for each chain, compact to its latest blockheight + deriving stock (Eq, Show) + +data CompactConfig = CompactConfig + { ccBlockHeight :: TargetBlockHeight + , ccDbDir :: FilePath + , ccVersion :: ChainwebVersion + , ccFlags :: [CompactFlag] + , ccChains :: Maybe (Set ChainId) + , logDir :: FilePath + , ccThreads :: Int + } + deriving stock (Eq, Show) + +compactAll :: CompactConfig -> IO () +compactAll CompactConfig{..} = do + latestBlockHeightChain0 <- do + let cid = unsafeChainId 0 + withDefaultLogger Debug $ \logger -> do + let resetDb = False + withSqliteDb cid logger ccDbDir resetDb $ \(SQLiteEnv db _) -> do + runCompactM (CompactEnv logger db []) getLatestBlockHeight + + let allCids = Set.fromList $ F.toList $ chainIdsAt ccVersion latestBlockHeightChain0 + let targetCids = Set.toList $ maybe allCids (Set.intersection allCids) ccChains + + flip (pooledMapConcurrentlyN_ ccThreads) targetCids $ \cid -> do + withPerChainFileLogger logDir cid Debug $ \logger -> do + let resetDb = False + withSqliteDb cid logger ccDbDir resetDb $ \(SQLiteEnv db _) -> do + void $ compact ccBlockHeight logger db ccFlags + +main :: IO () +main = do + config <- execParser opts + compactAll config + where + opts :: ParserInfo CompactConfig + opts = info (parser <**> helper) + (fullDesc <> progDesc "Pact DB Compaction tool") + + collapseSum :: [Parser [a]] -> Parser [a] + collapseSum = foldr (\x y -> (++) <$> x <*> y) (pure []) + + maybeList :: [a] -> Maybe [a] + maybeList = \case + [] -> Nothing + xs -> Just xs + + parser :: Parser CompactConfig + parser = CompactConfig + <$> (fmap Target (fromIntegral @Int <$> option auto + (short 'b' + <> long "target-blockheight" + <> metavar "BLOCKHEIGHT" + <> help "Target blockheight")) <|> pure Latest) + <*> strOption + (short 'd' + <> long "pact-database-dir" + <> metavar "DBDIR" + <> help "Pact database directory") + <*> ((lookupVersionByName . fromTextSilly @ChainwebVersionName) <$> strOption + (short 'v' + <> long "graph-version" + <> metavar "VERSION" + <> help "Chainweb version for graph. Only needed for non-standard graphs." + <> value (toText (_versionName mainnet)) + <> showDefault)) + <*> collapseSum + [ flag [] [KeepCompactTables] + (long "keep-compact-tables" + <> help "Keep compaction tables post-compaction, for inspection.") + , flag [] [NoVacuum] + (long "no-vacuum" + <> help "Don't VACUUM database.") + , flag [] [NoDropNewTables] + (long "no-drop-new-tables" + <> help "Don't drop new tables.") + , flag [] [NoGrandHash] + (long "no-grand-hash" + <> help "Don't compute the compact grand hash.") + ] + <*> fmap (fmap Set.fromList . maybeList) (many (unsafeChainId <$> option auto + (short 'c' + <> long "chain" + <> metavar "CHAINID" + <> help "Add this chain to the target set of ones to compact."))) + <*> strOption + (long "log-dir" + <> metavar "DIRECTORY" + <> help "Directory where logs will be placed" + <> value ".") + <*> option auto + (short 't' + <> long "threads" + <> metavar "THREADS" + <> value 4 + <> help "Number of threads for compaction processing") + +fromTextSilly :: HasTextRepresentation a => Text -> a +fromTextSilly t = case fromText t of + Just a -> a + Nothing -> error "fromText failed" + +bhToSType :: BlockHeight -> SType +bhToSType bh = SInt (int bh) + +txIdToSType :: TxId -> SType +txIdToSType (TxId txid) = SInt (fromIntegral txid) + +tableNameToSType :: TableName -> SType +tableNameToSType (TableName tbl) = SText tbl diff --git a/src/Chainweb/Pact/Backend/PactState.hs b/src/Chainweb/Pact/Backend/PactState.hs new file mode 100644 index 0000000000..2f2ad4ba81 --- /dev/null +++ b/src/Chainweb/Pact/Backend/PactState.hs @@ -0,0 +1,428 @@ +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE OverloadedRecordDot #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE StrictData #-} +{-# LANGUAGE TypeApplications #-} + +-- | +-- Module: Chainweb.Pact.Backend.PactState +-- Copyright: Copyright © 2023 Kadena LLC. +-- License: see LICENSE.md +-- +-- Diff Pact state between two databases. +-- +-- There are other utilities provided by this module whose purpose is either +-- to get the pact state. +-- +-- The code in this module operates primarily on 'Stream's, because the amount +-- of user data can grow quite large. by comparing one table at a time, we can +-- keep maximum memory utilisation in check. +-- + +module Chainweb.Pact.Backend.PactState + ( getPactTableNames + , getPactTables + , getLatestPactState + , getLatestBlockHeight + + , PactRow(..) + , Table(..) + , TableDiffable(..) + + , pactDiffMain + ) + where + +import Data.IORef (newIORef, readIORef, atomicModifyIORef') +import Control.Exception (bracket) +import Control.Monad (forM, forM_, when, void) +import Control.Monad.IO.Class (MonadIO(liftIO)) +import Control.Monad.Trans.Class (lift) +import Control.Monad.Except (ExceptT(..), runExceptT, throwError) +import Data.Aeson (ToJSON(..), (.=)) +import Data.Aeson qualified as Aeson +import Data.Vector (Vector) +import Data.Vector qualified as Vector +import Data.ByteString (ByteString) +import Data.Foldable qualified as F +import Data.Int (Int64) +import Data.List qualified as List +import Data.Map (Map) +import Data.Map.Merge.Strict qualified as Merge +import Data.Map.Strict qualified as M +import Data.Text (Text) +import Data.Text qualified as Text +import Data.Text.IO qualified as Text +import Data.Text.Encoding qualified as Text +import Database.SQLite3.Direct (Utf8(..), Database) +import Database.SQLite3.Direct qualified as SQL +import Options.Applicative + +import Chainweb.BlockHeight (BlockHeight(..)) +import Chainweb.Logger (logFunctionText, logFunctionJson) +import Chainweb.Utils (HasTextRepresentation, fromText, toText, int) +import Chainweb.Version (ChainwebVersion(..), ChainwebVersionName, ChainId, chainIdToText) +import Chainweb.Version.Mainnet (mainnet) +import Chainweb.Version.Registry (lookupVersionByName) +import Chainweb.Version.Utils (chainIdsAt) +import Chainweb.Pact.Backend.Types (SQLiteEnv(..)) +import Chainweb.Pact.Backend.Utils (fromUtf8, withSqliteDb) +import Chainweb.Pact.Backend.Compaction qualified as C + +import System.Directory (doesFileExist) +import System.FilePath (()) +import System.Exit (exitFailure) +import System.Logger (LogLevel(..)) +import System.LogLevel qualified as LL + +import Pact.Types.SQLite (SType(..), RType(..)) +import Pact.Types.SQLite qualified as Pact +import Streaming.Prelude (Stream, Of) +import Streaming.Prelude qualified as S + +excludedTables :: [Utf8] +excludedTables = checkpointerTables ++ compactionTables + where + checkpointerTables = ["BlockHistory", "VersionedTableCreation", "VersionedTableMutation", "TransactionIndex"] + compactionTables = ["CompactGrandHash", "CompactActiveRow"] + +getLatestBlockHeight :: Database -> IO BlockHeight +getLatestBlockHeight db = do + let qryText = "SELECT MAX(blockheight) FROM BlockHistory" + Pact.qry db qryText [] [RInt] >>= \case + [[SInt bh]] -> pure (BlockHeight (int bh)) + _ -> error "getLatestBlockHeight: expected int" + +getPactTableNames :: Database -> IO (Vector Utf8) +getPactTableNames db = do + let sortedTableNames :: [[SType]] -> [Utf8] + sortedTableNames rows = M.elems $ M.fromListWith const $ flip List.map rows $ \case + [SText u] -> (Text.toLower (fromUtf8 u), u) + _ -> error "getPactTableNames.sortedTableNames: expected text" + + tables <- fmap sortedTableNames $ do + let qryText = + "SELECT name FROM sqlite_schema \ + \WHERE \ + \ type = 'table' \ + \AND \ + \ name NOT LIKE 'sqlite_%'" + Pact.qry db qryText [] [RText] + + pure (Vector.fromList tables) + +-- | Get all of the rows for each table. The tables will be sorted +-- lexicographically by name. +getPactTables :: Database -> Stream (Of Table) IO () +getPactTables db = do + let fmtTable x = "\"" <> x <> "\"" + + tables <- liftIO $ getPactTableNames db + + forM_ tables $ \tbl -> do + if tbl `notElem` excludedTables + then do + let qryText = "SELECT rowkey, rowdata, txid FROM " + <> fmtTable tbl + userRows <- liftIO $ Pact.qry db qryText [] [RText, RBlob, RInt] + shapedRows <- forM userRows $ \case + [SText (Utf8 rowKey), SBlob rowData, SInt txId] -> do + pure $ PactRow {..} + _ -> error "getPactTableNames: unexpected shape of user table row" + S.yield $ Table (fromUtf8 tbl) shapedRows + else do + pure () + +stepStatement :: SQL.Statement -> [RType] -> Stream (Of [SType]) IO (Either SQL.Error ()) +stepStatement stmt rts = runExceptT $ do + -- todo: rename from acc + let acc :: SQL.StepResult -> ExceptT SQL.Error (Stream (Of [SType]) IO) () + acc = \case + SQL.Done -> do + pure () + SQL.Row -> do + as <- forM (List.zip [0..] rts) $ \(colIx, expectedColType) -> do + liftIO $ case expectedColType of + RInt -> SInt <$> SQL.columnInt64 stmt colIx + RDouble -> SDouble <$> SQL.columnDouble stmt colIx + RText -> SText <$> SQL.columnText stmt colIx + RBlob -> SBlob <$> SQL.columnBlob stmt colIx + lift $ S.yield as + liftIO (SQL.step stmt) >>= \case + Left err -> do + throwError err + Right sr -> do + acc sr + + -- maybe use stepNoCB + ExceptT (liftIO (SQL.step stmt)) >>= acc + +-- | Prepare/execute query with params +qry :: () + => Database + -> Utf8 + -> [SType] + -> [RType] + -> (Stream (Of [SType]) IO (Either SQL.Error ()) -> IO x) + -> IO x +qry db qryText args returnTypes k = do + bracket (Pact.prepStmt db qryText) SQL.finalize $ \stmt -> do + Pact.bindParams stmt args + k (stepStatement stmt returnTypes) + +getLatestPactState :: Database -> Stream (Of TableDiffable) IO () +getLatestPactState db = do + let fmtTable x = "\"" <> x <> "\"" + + tables <- liftIO $ getPactTableNames db + + forM_ tables $ \tbl -> do + when (tbl `notElem` excludedTables) $ do + let qryText = "SELECT rowkey, rowdata, txid FROM " + <> fmtTable tbl + latestState <- fmap (M.map (\prc -> prc.rowData)) $ liftIO $ qry db qryText [] [RText, RBlob, RInt] $ \rows -> do + let go :: Map ByteString PactRowContents -> [SType] -> Map ByteString PactRowContents + go m = \case + [SText (Utf8 rowKey), SBlob rowData, SInt txId] -> + M.insertWith (\prc1 prc2 -> if prc1.txId > prc2.txId then prc1 else prc2) rowKey (PactRowContents rowData txId) m + _ -> error "getLatestPactState: unexpected shape of user table row" + S.fold_ go M.empty id rows + S.yield (TableDiffable (fromUtf8 tbl) latestState) + +-- This assumes the same tables (essentially zipWith). +-- Note that this assumes we got the state from `getLatestPactState`, +-- because `getPactTableNames` sorts the table names, and `getLatestPactState` +-- sorts the [PactRow] by rowKey. +-- +-- If we ever step across two tables that do not have the same name, we throw an error. +-- +-- This diminishes the utility of comparing two pact states that are known to be +-- at different heights, but that hurts our ability to perform the diff in +-- constant memory. +-- +-- TODO: maybe inner stream should be a ByteStream +diffLatestPactState :: () + => Stream (Of TableDiffable) IO () + -> Stream (Of TableDiffable) IO () + -> Stream (Of (Text, Stream (Of RowKeyDiffExists) IO ())) IO () +diffLatestPactState = go + where + go :: Stream (Of TableDiffable) IO () -> Stream (Of TableDiffable) IO () -> Stream (Of (Text, Stream (Of RowKeyDiffExists) IO ())) IO () + go s1 s2 = do + e1 <- liftIO $ S.next s1 + e2 <- liftIO $ S.next s2 + + case (e1, e2) of + (Left (), Left ()) -> do + pure () + (Right _, Left ()) -> do + error "left stream longer than right" + (Left (), Right _) -> do + error "right stream longer than left" + (Right (t1, next1), Right (t2, next2)) -> do + when (t1.name /= t2.name) $ do + error "diffLatestPactState: mismatched table names" + S.yield (t1.name, diffTables t1 t2) + go next1 next2 + +-- | We don't include the entire rowdata in the diff, only the rowkey. +-- This is just a space-saving measure. +data RowKeyDiffExists + = Old ByteString + -- ^ The rowkey exists in the same table of the first db, but not the second. + | New ByteString + -- ^ The rowkey exists in the same table of the second db, but not the first. + | Delta ByteString + -- ^ The rowkey exists in the same table of both dbs, but the rowdata + -- differs. + +diffTables :: TableDiffable -> TableDiffable -> Stream (Of RowKeyDiffExists) IO () +diffTables t1 t2 = do + void $ Merge.mergeA + (Merge.traverseMaybeMissing $ \rk _rd -> do + S.yield (Old rk) + pure Nothing + ) + (Merge.traverseMaybeMissing $ \rk _rd -> do + S.yield (New rk) + pure Nothing + ) + (Merge.zipWithMaybeAMatched $ \rk rd1 rd2 -> do + when (rd1 /= rd2) $ do + S.yield (Delta rk) + pure Nothing + ) + t1.rows + t2.rows + +rowKeyDiffExistsToObject :: RowKeyDiffExists -> Aeson.Value +rowKeyDiffExistsToObject = \case + Old rk -> Aeson.object + [ "old" .= Text.decodeUtf8 rk + ] + New rk -> Aeson.object + [ "new" .= Text.decodeUtf8 rk + ] + Delta rk -> Aeson.object + [ "delta" .= Text.decodeUtf8 rk + ] + +-- | A pact table - just its name and its rows. +data Table = Table + { name :: Text + , rows :: [PactRow] + } + deriving stock (Eq, Show) + +-- | A diffable pact table - its name and the _active_ pact state +-- as a Map from RowKey to RowData. +data TableDiffable = TableDiffable + { name :: Text + , rows :: Map ByteString ByteString -- Map RowKey RowData + } + deriving stock (Eq, Ord, Show) + +data PactRow = PactRow + { rowKey :: ByteString + , rowData :: ByteString + , txId :: Int64 + } + deriving stock (Eq, Show) + +instance Ord PactRow where + compare pr1 pr2 = + compare pr1.txId pr2.txId + <> compare pr1.rowKey pr2.rowKey + <> compare pr1.rowData pr2.rowData + +instance ToJSON PactRow where + toJSON pr = Aeson.object + [ "row_key" .= Text.decodeUtf8 pr.rowKey + , "row_data" .= Text.decodeUtf8 pr.rowData + , "tx_id" .= pr.txId + ] + +data PactRowContents = PactRowContents + { rowData :: ByteString + , txId :: Int64 + } + deriving stock (Eq, Show) + +data PactDiffConfig = PactDiffConfig + { firstDbDir :: FilePath + , secondDbDir :: FilePath + , chainwebVersion :: ChainwebVersion + , logDir :: FilePath + } + +data Diffy = Difference | NoDifference + deriving stock (Eq) + +instance Semigroup Diffy where + Difference <> _ = Difference + _ <> Difference = Difference + _ <> _ = NoDifference + +instance Monoid Diffy where + mempty = NoDifference + +pactDiffMain :: IO () +pactDiffMain = do + cfg <- execParser opts + + when (cfg.firstDbDir == cfg.secondDbDir) $ do + Text.putStrLn "Source and target Pact database directories cannot be the same." + exitFailure + + let cids = List.sort $ F.toList $ chainIdsAt cfg.chainwebVersion (BlockHeight maxBound) + + diffyRef <- newIORef @(Map ChainId Diffy) M.empty + + forM_ cids $ \cid -> do + C.withPerChainFileLogger cfg.logDir cid Info $ \logger -> do + let logText = logFunctionText logger + + sqliteFileExists1 <- doesPactDbExist cid cfg.firstDbDir + sqliteFileExists2 <- doesPactDbExist cid cfg.secondDbDir + + if | not sqliteFileExists1 -> do + logText LL.Warn $ "[SQLite for chain in " <> Text.pack cfg.firstDbDir <> " doesn't exist. Skipping]" + | not sqliteFileExists2 -> do + logText LL.Warn $ "[SQLite for chain in " <> Text.pack cfg.secondDbDir <> " doesn't exist. Skipping]" + | otherwise -> do + let resetDb = False + withSqliteDb cid logger cfg.firstDbDir resetDb $ \(SQLiteEnv db1 _) -> do + withSqliteDb cid logger cfg.secondDbDir resetDb $ \(SQLiteEnv db2 _) -> do + logText LL.Info "[Starting diff]" + let diff = diffLatestPactState (getLatestPactState db1) (getLatestPactState db2) + diffy <- S.foldMap_ id $ flip S.mapM diff $ \(tblName, tblDiff) -> do + logText LL.Info $ "[Starting table " <> tblName <> "]" + d <- S.foldMap_ id $ flip S.mapM tblDiff $ \d -> do + logFunctionJson logger LL.Warn $ rowKeyDiffExistsToObject d + pure Difference + logText LL.Info $ "[Finished table " <> tblName <> "]" + pure d + + logText LL.Info $ case diffy of + Difference -> "[Non-empty diff]" + NoDifference -> "[Empty diff]" + logText LL.Info $ "[Finished chain " <> chainIdToText cid <> "]" + + atomicModifyIORef' diffyRef $ \m -> (M.insert cid diffy m, ()) + + diffy <- readIORef diffyRef + case M.foldMapWithKey (\_ d -> d) diffy of + Difference -> do + Text.putStrLn "Diff complete. Differences found." + exitFailure + NoDifference -> do + Text.putStrLn "Diff complete. No differences found." + where + opts :: ParserInfo PactDiffConfig + opts = info (parser <**> helper) + (fullDesc <> progDesc "Compare two Pact databases") + + parser :: Parser PactDiffConfig + parser = PactDiffConfig + <$> strOption + (long "first-database-dir" + <> metavar "PACT_DB_DIRECTORY" + <> help "First Pact database directory") + <*> strOption + (long "second-database-dir" + <> metavar "PACT_DB_DIRECTORY" + <> help "Second Pact database directory") + <*> (fmap (lookupVersionByName . fromTextSilly @ChainwebVersionName) $ strOption + (long "graph-version" + <> metavar "CHAINWEB_VERSION" + <> help "Chainweb version for graph. Only needed for non-standard graphs." + <> value (toText (_versionName mainnet)) + <> showDefault)) + <*> strOption + (long "log-dir" + <> metavar "LOG_DIRECTORY" + <> help "Directory where logs will be placed" + <> value ".") + +fromTextSilly :: HasTextRepresentation a => Text -> a +fromTextSilly t = case fromText t of + Just a -> a + Nothing -> error "fromText failed" + +doesPactDbExist :: ChainId -> FilePath -> IO Bool +doesPactDbExist cid dbDir = do + let chainDbFileName = mconcat + [ "pact-v1-chain-" + , Text.unpack (chainIdToText cid) + , ".sqlite" + ] + let file = dbDir chainDbFileName + doesFileExist file diff --git a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs index 8022013eda..ec8d7a17b4 100644 --- a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs +++ b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs @@ -233,13 +233,13 @@ doDiscard dbenv = runBlockEnv dbenv $ do -- commitSavepoint Block -doGetEarliest :: HasCallStack => Db logger -> IO (BlockHeight, BlockHash) +doGetEarliest :: HasCallStack => Db logger -> IO (Maybe (BlockHeight, BlockHash)) doGetEarliest dbenv = - runBlockEnv dbenv $ callDb "getLatestBlock" $ \db -> do + runBlockEnv dbenv $ callDb "getEarliestBlock" $ \db -> do r <- qry_ db qtext [RInt, RBlob] >>= mapM go case r of - [] -> fail "Chainweb.Pact.Backend.RelationalCheckpointer.doGetEarliest: no earliest block. This is a bug in chainweb-node." - (!o:_) -> return o + [] -> return Nothing + (!o:_) -> return (Just o) where qtext = "SELECT blockheight, hash FROM BlockHistory \ \ ORDER BY blockheight ASC LIMIT 1" diff --git a/src/Chainweb/Pact/Backend/Types.hs b/src/Chainweb/Pact/Backend/Types.hs index 5ecdc59ce9..b69b277222 100644 --- a/src/Chainweb/Pact/Backend/Types.hs +++ b/src/Chainweb/Pact/Backend/Types.hs @@ -297,9 +297,9 @@ data Checkpointer logger = Checkpointer -- ^ commits pending modifications to block, with the given blockhash , _cpDiscard :: !(IO ()) -- ^ discard pending block changes - , _cpGetEarliestBlock :: !(IO (BlockHeight, BlockHash)) + , _cpGetEarliestBlock :: !(IO (Maybe (BlockHeight, BlockHash))) -- ^ get the checkpointer's idea of the earliest block. The block height - -- is the height of the block of the block hash. + -- is the height of the block of the block hash. , _cpGetLatestBlock :: !(IO (Maybe (BlockHeight, BlockHash))) -- ^ get the checkpointer's idea of the latest block. The block height is -- is the height of the block of the block hash. diff --git a/src/Chainweb/Pact/PactService.hs b/src/Chainweb/Pact/PactService.hs index ee058adc08..15d9c45578 100644 --- a/src/Chainweb/Pact/PactService.hs +++ b/src/Chainweb/Pact/PactService.hs @@ -72,6 +72,7 @@ import Prelude hiding (lookup) import qualified Pact.Gas as P import Pact.Gas.Table +import qualified Pact.JSON.Encode as J import qualified Pact.Interpreter as P import qualified Pact.Types.ChainMeta as P import qualified Pact.Types.Command as P @@ -127,8 +128,7 @@ runPactService ver cid chainwebLogger reqQ mempoolAccess bhDb pdb sqlenv config serviceRequests mempoolAccess reqQ withPactService - :: Logger logger - => CanReadablePayloadCas tbl + :: (Logger logger, CanReadablePayloadCas tbl) => ChainwebVersion -> ChainId -> logger @@ -141,8 +141,8 @@ withPactService withPactService ver cid chainwebLogger bhDb pdb sqlenv config act = withProdRelationalCheckpointer checkpointerLogger initialBlockState sqlenv ver cid $ \checkpointer -> do let !rs = readRewards - !initialParentHeader = ParentHeader $ genesisBlockHeader ver cid - !pse = PactServiceEnv + let !initialParentHeader = ParentHeader $ genesisBlockHeader ver cid + let !pse = PactServiceEnv { _psMempoolAccess = Nothing , _psCheckpointer = checkpointer , _psPdb = pdb @@ -162,9 +162,31 @@ withPactService ver cid chainwebLogger bhDb pdb sqlenv config act = , _psBlockGasLimit = _pactBlockGasLimit config , _psChainId = cid } - !pst = PactServiceState Nothing mempty initialParentHeader P.noSPVSupport - runPactServiceM pst pse $ do + let !pst = PactServiceState Nothing mempty initialParentHeader P.noSPVSupport + + when (_pactFullHistoryRequired config) $ do + mEarliestBlock <- _cpGetEarliestBlock checkpointer + case mEarliestBlock of + Nothing -> do + pure () + Just (earliestBlockHeight, _) -> do + let gHeight = genesisHeight ver cid + when (gHeight /= earliestBlockHeight) $ do + let e = FullHistoryRequired + { _earliestBlockHeight = earliestBlockHeight + , _genesisHeight = gHeight + } + let msg = J.object + [ "details" J..= e + , "message" J..= J.text "Your node has been configured\ + \ to require the full Pact history; however, the full\ + \ history is not available. Perhaps you have compacted\ + \ your Pact state?" + ] + logError_ chainwebLogger (J.encodeText msg) + throwM e + runPactServiceM pst pse $ do -- If the latest header that is stored in the checkpointer was on an -- orphaned fork, there is no way to recover it in the call of -- 'initalPayloadState.readContracts'. We therefore rewind to the latest @@ -906,7 +928,6 @@ chainweb213GasModel = modifiedGasModel _ -> P.milliGasToGas $ fullRunFunction name ga modifiedGasModel = defGasModel { P.runGasModel = \t g -> P.gasToMilliGas (modifiedRunFunction t g) } - getGasModel :: TxContext -> P.GasModel getGasModel ctx | chainweb213Pact (ctxVersion ctx) (ctxChainId ctx) (ctxCurrentBlockHeight ctx) = chainweb213GasModel @@ -914,3 +935,5 @@ getGasModel ctx pactLabel :: (Logger logger) => Text -> PactServiceM logger tbl x -> PactServiceM logger tbl x pactLabel lbl x = localLabel ("pact-request", lbl) x + + diff --git a/src/Chainweb/Pact/PactService/Checkpointer.hs b/src/Chainweb/Pact/PactService/Checkpointer.hs index 5ce2773802..02a3d9e8d0 100644 --- a/src/Chainweb/Pact/PactService/Checkpointer.hs +++ b/src/Chainweb/Pact/PactService/Checkpointer.hs @@ -41,10 +41,10 @@ module Chainweb.Pact.PactService.Checkpointer -- There are two function for restoring the checkpointer for evaluation of back -- code: -- - -- * 'withCheckPointerRewind' and + -- * 'withCheckpointerRewind' and -- * 'withCurrentCheckpointer'. -- - -- 'withCheckPointerRewind' rewinds the checkpointer to the provided parent + -- 'withCheckpointerRewind' rewinds the checkpointer to the provided parent -- header. 'withCurrentCheckpointer' evaluates the pact transaction within the -- context of the current checkpointer state. Both functions update the value of -- '_psParentHeader' at the beginning and the end of each call. @@ -171,7 +171,7 @@ data WithCheckpointerResult a -- -- This function assumes that '_psParentHeader' has been updated to match the -- latest block in the checkpointers. This is guaranteed to be the case after --- calling any of 'rewindTo', 'syncParentHeader', 'withCheckPointerRewind', +-- calling any of 'rewindTo', 'syncParentHeader', 'withCheckpointerRewind', -- 'withCheckPointerWithoutRewind', or 'withCurrentCheckpointer'. -- -- /NOTE:/ @@ -273,7 +273,7 @@ withCheckpointerRewind rewindLimit p caller act = do -- | Run a batch of checkpointer operations, possibly involving the evaluation -- transactions accross several blocks using more than a single call of --- 'withCheckPointerRewind' or 'withCurrentCheckpointer', and persist the final +-- 'withCheckpointerRewind' or 'withCurrentCheckpointer', and persist the final -- state. In case of an failure, the checkpointer is reverted to the initial -- state. -- @@ -308,7 +308,7 @@ withBatchIO runPact act = mask $ \umask -> do -- | Run a batch of checkpointer operations, possibly involving the evaluation -- transactions accross several blocks using more than a single call of --- 'withCheckPointerRewind' or 'withCurrentCheckpointer', and discard the final +-- 'withCheckpointerRewind' or 'withCurrentCheckpointer', and discard the final -- state at the end. -- withDiscardedBatch :: PactServiceM logger tbl a -> PactServiceM logger tbl a diff --git a/src/Chainweb/Pact/PactService/ExecBlock.hs b/src/Chainweb/Pact/PactService/ExecBlock.hs index cd9a6b6d13..08fafb1091 100644 --- a/src/Chainweb/Pact/PactService/ExecBlock.hs +++ b/src/Chainweb/Pact/PactService/ExecBlock.hs @@ -102,8 +102,8 @@ setParentHeader msg ph@(ParentHeader bh) = do -- /NOTE:/ -- -- Any call of this function must occur within a dedicated call to --- 'withChwithCheckpointerRewind', 'withCurrentCheckpointer' or --- 'withCheckPointerWithoutRewind'. +-- 'withCheckpointerRewind', 'withCurrentCheckpointer' or +-- 'withCheckpointerWithoutRewind'. -- execBlock :: (CanReadablePayloadCas tbl, Logger logger) diff --git a/src/Chainweb/Pact/Service/Types.hs b/src/Chainweb/Pact/Service/Types.hs index 9763143fa2..55df2d595d 100644 --- a/src/Chainweb/Pact/Service/Types.hs +++ b/src/Chainweb/Pact/Service/Types.hs @@ -103,6 +103,9 @@ data PactServiceConfig = PactServiceConfig -- ^ whether to write transaction gas logs at INFO , _pactModuleCacheLimit :: !DbCacheLimitBytes -- ^ limit of the database module cache in bytes of corresponding row data + , _pactFullHistoryRequired :: !Bool + -- ^ Whether or not the node requires that the full Pact history be + -- available. Compaction can remove history. } deriving (Eq,Show) data GasPurchaseFailure = GasPurchaseFailure TransactionHash PactError @@ -210,6 +213,10 @@ data PactException { _localRewindExceededLimit :: !RewindLimit , _localRewindRequestedDepth :: !RewindDepth } | LocalRewindGenesisExceeded + | FullHistoryRequired + { _earliestBlockHeight :: !BlockHeight + , _genesisHeight :: !BlockHeight + } deriving (Eq,Generic) instance Show PactException where @@ -239,6 +246,10 @@ instance J.Encode PactException where , "_localRewindRequestedDepth" J..= J.Aeson @Int (fromIntegral $ _rewindDepth $ _localRewindRequestedDepth o) ] build LocalRewindGenesisExceeded = tagged "LocalRewindGenesisExceeded" J.null + build o@(FullHistoryRequired{}) = tagged "FullHistoryRequired" $ J.object + [ "_fullHistoryRequiredEarliestBlockHeight" J..= J.Aeson @Int (fromIntegral $ _earliestBlockHeight o) + , "_fullHistoryRequiredGenesisHeight" J..= J.Aeson @Int (fromIntegral $ _genesisHeight o) + ] tagged :: J.Encode v => Text -> v -> J.Builder tagged t v = J.object diff --git a/src/Chainweb/Pact/Types.hs b/src/Chainweb/Pact/Types.hs index 072ea5ba28..44d59b1f0e 100644 --- a/src/Chainweb/Pact/Types.hs +++ b/src/Chainweb/Pact/Types.hs @@ -479,6 +479,7 @@ testPactServiceConfig = PactServiceConfig , _pactBlockGasLimit = testBlockGasLimit , _pactLogGas = False , _pactModuleCacheLimit = defaultModuleCacheLimit + , _pactFullHistoryRequired = False } -- | This default value is only relevant for testing. In a chainweb-node the @GasLimit@ @@ -668,7 +669,6 @@ execPactServiceM execPactServiceM st env act = execStateT (runReaderT (_unPactServiceM act) env) st - getCheckpointer :: PactServiceM logger tbl (Checkpointer logger) getCheckpointer = view psCheckpointer diff --git a/src/Chainweb/Rosetta/Internal.hs b/src/Chainweb/Rosetta/Internal.hs index 54507281c5..e84e12ff3a 100644 --- a/src/Chainweb/Rosetta/Internal.hs +++ b/src/Chainweb/Rosetta/Internal.hs @@ -186,7 +186,6 @@ genesisTransactions genesisTransactions logs cid txs = pure $ V.toList $ V.map (getGenesisLog logs cid) txs - -- | Matches a single genesis transaction to its coin contract logs. genesisTransaction :: Map TxId [AccountLog] @@ -200,7 +199,6 @@ genesisTransaction logs cid rest target = do V.find (\c -> _crReqKey c == target) rest pure $ getGenesisLog logs cid cr - ------------------------ -- Coinbase Helpers -- ------------------------ @@ -510,7 +508,6 @@ getLatestBlockHeader cutDb cid = do c <- liftIO $ _cut cutDb HM.lookup cid (_cutMap c) ?? RosettaInvalidChain - findBlockHeaderInCurrFork :: CutDb tbl -> ChainId @@ -545,7 +542,6 @@ findBlockHeaderInCurrFork cutDb cid someHeight someHash = do somebh <- liftIO $ seekAncestor db latest (int hi) somebh ?? RosettaInvalidBlockHeight - getBlockOutputs :: forall tbl . CanReadablePayloadCas tbl diff --git a/src/Chainweb/Utils.hs b/src/Chainweb/Utils.hs index b7d7834e3e..a20988ed2e 100644 --- a/src/Chainweb/Utils.hs +++ b/src/Chainweb/Utils.hs @@ -522,6 +522,18 @@ instance HasTextRepresentation Integer where fromText = treadM {-# INLINE fromText #-} +instance HasTextRepresentation Word where + toText = sshow + {-# INLINE toText #-} + fromText = treadM + {-# INLINE fromText #-} + +instance HasTextRepresentation Word64 where + toText = sshow + {-# INLINE toText #-} + fromText = treadM + {-# INLINE fromText #-} + instance HasTextRepresentation UTCTime where toText = T.pack . formatTime defaultTimeLocale iso8601DateTimeFormat {-# INLINE toText #-} diff --git a/src/Chainweb/WebPactExecutionService.hs b/src/Chainweb/WebPactExecutionService.hs index 348db4b5b8..5d0604de2a 100644 --- a/src/Chainweb/WebPactExecutionService.hs +++ b/src/Chainweb/WebPactExecutionService.hs @@ -145,7 +145,6 @@ mkWebPactExecutionService hm = WebPactExecutionService $ PactExecutionService $ "PactExecutionService: Invalid chain ID: " ++ show cid - mkPactExecutionService :: PactQueue -> PactExecutionService diff --git a/test/Chainweb/Test/MultiNode.hs b/test/Chainweb/Test/MultiNode.hs index ae526ebafd..ebb1da26cf 100644 --- a/test/Chainweb/Test/MultiNode.hs +++ b/test/Chainweb/Test/MultiNode.hs @@ -4,6 +4,7 @@ {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE MultiWayIf #-} @@ -32,13 +33,13 @@ -- The configuration defines a scaled down, accelerated chain that tries to -- similulate a full-scale chain in a miniaturized settings. -- -module Chainweb.Test.MultiNode ( test, replayTest ) where +module Chainweb.Test.MultiNode ( test, replayTest, compactAndResumeTest ) where import Control.Concurrent import Control.Concurrent.Async import Control.DeepSeq import Control.Exception -import Control.Lens (set, view) +import Control.Lens (set, view, over) import Control.Monad import Data.Aeson @@ -58,6 +59,7 @@ import qualified Streaming.Prelude as S import System.FilePath import System.IO.Temp +import System.Logger.Types qualified as YAL import System.LogLevel import System.Timeout @@ -73,6 +75,9 @@ import Chainweb.Chainweb import Chainweb.Chainweb.Configuration import Chainweb.Chainweb.CutResources import Chainweb.Chainweb.PeerResources +import Chainweb.Pact.Backend.Compaction qualified as C +import Chainweb.Pact.Backend.Types (_sConn) +import Chainweb.Pact.Backend.Utils (withSqliteDb) import Chainweb.Cut import Chainweb.CutDB import Chainweb.Graph @@ -283,6 +288,62 @@ runNodesForSeconds loglevel write baseConf n (Seconds seconds) rdb pactDbDir inn void $ timeout (int seconds * 1_000_000) $ runNodes loglevel write baseConf n rdb pactDbDir inner +-- | Run nodes +-- Each node creates blocks +-- We wait until they've made a sufficient amount of blocks +-- We stop the nodes +-- We open sqlite connections to some of the database dirs and compact them +-- We restart all nodes with the same database dirs +-- We observe that they can make progress +compactAndResumeTest :: () + => LogLevel + -> ChainwebVersion + -> Natural + -> TestTree +compactAndResumeTest logLevel v n = + let + name = "compact-resume" + in + after AllFinish "ConsensusNetwork" $ testCaseSteps name $ \step -> + withTempRocksDb "compact-resume-test-rocks" $ \rdb -> + withSystemTempDirectory "compact-resume-test-pact" $ \pactDbDir -> do + let logFun = step . T.unpack + let logger = genericLogger logLevel logFun + + logFun "phase 1... creating blocks" + -- N.B: This consensus state stuff counts the number of blocks + -- in RocksDB, rather than the number of blocks in all chains + -- on the current cut. This is fine because we ultimately just want + -- to make sure that we are making progress (i.e, new blocks). + stateVar <- newMVar (emptyConsensusState v) + let ct :: Int -> StartedChainweb logger -> IO () + ct = harvestConsensusState logger stateVar + runNodesForSeconds logLevel logFun (multiConfig v n) n 60 rdb pactDbDir ct + Just stats1 <- consensusStateSummary <$> swapMVar stateVar (emptyConsensusState v) + assertGe "average block count before compaction" (Actual $ _statBlockCount stats1) (Expected 50) + logFun $ sshow stats1 + + logFun "phase 2... compacting" + let cid = unsafeChainId 0 + -- compact only half of them + let nids = filter even [0 .. int @_ @Int n - 1] + forM_ nids $ \nid -> do + let dir = pactDbDir show nid + withSqliteDb cid logger dir False $ \sqlEnv -> do + C.withDefaultLogger YAL.Warn $ \cLogger -> do + let cLogger' = over YAL.setLoggerScope (\scope -> ("nodeId",sshow nid) : ("chainId",sshow cid) : scope) cLogger + let flags = [C.NoVacuum, C.NoGrandHash] + let db = _sConn sqlEnv + let bh = BlockHeight 5 + void $ C.compact (C.Target bh) cLogger' db flags + + logFun "phase 3... restarting nodes and ensuring progress" + runNodesForSeconds logLevel logFun (multiConfig v n) n 60 rdb pactDbDir ct + Just stats2 <- consensusStateSummary <$> swapMVar stateVar (emptyConsensusState v) + -- We ensure that we've gotten to at least 1.5x the previous block count + assertGe "average block count post-compaction" (Actual $ _statBlockCount stats2) (Expected (3 * _statBlockCount stats1 `div` 2)) + logFun $ sshow stats2 + replayTest :: LogLevel -> ChainwebVersion diff --git a/test/Chainweb/Test/Pact/PactMultiChainTest.hs b/test/Chainweb/Test/Pact/PactMultiChainTest.hs index 649df5ad04..df18fff525 100644 --- a/test/Chainweb/Test/Pact/PactMultiChainTest.hs +++ b/test/Chainweb/Test/Pact/PactMultiChainTest.hs @@ -360,9 +360,14 @@ runLocal cid' cmd = runLocalWithDepth Nothing cid' cmd runLocalWithDepth :: Maybe RewindDepth -> ChainId -> CmdBuilder -> PactTestM (Either PactException LocalResult) runLocalWithDepth depth cid' cmd = do + pact <- getPactService cid' + cwCmd <- buildCwCmd cmd + liftIO $ _pactLocal pact Nothing Nothing depth cwCmd + +getPactService :: ChainId -> PactTestM PactExecutionService +getPactService cid' = do HM.lookup cid' <$> view menvPacts >>= \case - Just pact -> buildCwCmd cmd >>= - liftIO . _pactLocal pact Nothing Nothing depth + Just pact -> return pact Nothing -> liftIO $ assertFailure $ "No pact service found at chain id " ++ show cid' assertLocalFailure @@ -484,8 +489,6 @@ pact43UpgradeTest = do ]) $ mkKeySetData "k" [sender00] - - chainweb215Test :: PactTestM () chainweb215Test = do @@ -578,8 +581,6 @@ chainweb215Test = do , mkXResumeEvent "sender00" "sender00" 0.0123 sender00Ks "pact" h' (toText cid) "0" ] - - pact431UpgradeTest :: PactTestM () pact431UpgradeTest = do @@ -1360,7 +1361,6 @@ buildXSend caps = MempoolCmdBuilder $ \(MempoolInput _ bh) -> "(coin.transfer-crosschain 'sender00 'sender00 (read-keyset 'k) \"0\" 0.0123)" $ mkKeySetData "k" [sender00] - chain0 :: ChainId chain0 = unsafeChainId 0 @@ -1398,7 +1398,6 @@ setFromHeader bh = set cbChainId (_blockChainId bh) . set cbCreationTime (toTxCreationTime $ _bct $ _blockCreationTime bh) - buildBasic :: PactRPC T.Text -> MempoolCmdBuilder @@ -1417,9 +1416,6 @@ buildBasic' f r = MempoolCmdBuilder $ \(MempoolInput _ bh) -> $ setFromHeader bh $ mkCmd (sshow bh) r - - - -- | Get output on latest cut for chain getPWO :: ChainId -> PactTestM (PayloadWithOutputs,BlockHeader) getPWO chid = do diff --git a/test/Chainweb/Test/Pact/PactReplay.hs b/test/Chainweb/Test/Pact/PactReplay.hs index a393c72523..c62c97d606 100644 --- a/test/Chainweb/Test/Pact/PactReplay.hs +++ b/test/Chainweb/Test/Pact/PactReplay.hs @@ -86,12 +86,12 @@ tests rdb = onRestart :: IO (IORef MemPoolAccess) - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> (String -> IO ()) -> Assertion onRestart mpio iop step = do setOneShotMempool mpio testMemPoolAccess - bdb <- snd <$> iop + (_, _, bdb) <- iop bhdb' <- getBlockHeaderDb cid bdb block <- maxEntry bhdb' step $ "max block has height " <> sshow (_blockHeight block) @@ -160,7 +160,7 @@ dupegenMemPoolAccess = do serviceInitializationAfterFork :: IO (IORef MemPoolAccess) -> BlockHeader - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> Assertion serviceInitializationAfterFork mpio genesisBlock iop = do setOneShotMempool mpio testMemPoolAccess @@ -188,11 +188,11 @@ serviceInitializationAfterFork mpio genesisBlock iop = do restartPact :: IO () restartPact = do - q <- fst <$> iop + (_, q, _) <- iop addRequest q CloseMsg pruneDbs = forM_ cids $ \c -> do - dbs <- snd <$> iop + (_, _, dbs) <- iop db <- getBlockHeaderDb c dbs h <- maxEntry db tableDelete (_chainDbCas db) (casKey $ RankedBlockHeader h) @@ -202,7 +202,7 @@ serviceInitializationAfterFork mpio genesisBlock iop = do firstPlayThrough :: IO (IORef MemPoolAccess) -> BlockHeader - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> Assertion firstPlayThrough mpio genesisBlock iop = do setOneShotMempool mpio testMemPoolAccess @@ -228,7 +228,7 @@ firstPlayThrough mpio genesisBlock iop = do testDupes :: IO (IORef MemPoolAccess) -> BlockHeader - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> Assertion testDupes mpio genesisBlock iop = do setMempool mpio =<< dupegenMemPoolAccess @@ -259,12 +259,12 @@ testDupes mpio genesisBlock iop = do testDeepForkLimit :: IO (IORef MemPoolAccess) -> RewindLimit - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue,TestBlockDb) -> (String -> IO ()) -> Assertion testDeepForkLimit mpio (RewindLimit deepForkLimit) iop step = do setOneShotMempool mpio testMemPoolAccess - bdb <- snd <$> iop + (_, _, bdb) <- iop bhdb <- getBlockHeaderDb cid bdb step "query max db entry" maxblock <- maxEntry bhdb @@ -302,7 +302,7 @@ testDeepForkLimit mpio (RewindLimit deepForkLimit) iop step = do mineBlock :: ParentHeader -> Nonce - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> IO (T3 ParentHeader BlockHeader PayloadWithOutputs) mineBlock ph nonce iop = timeout 5000000 go >>= \case Nothing -> error "PactReplay.mineBlock: Test timeout. Most likely a test case caused a pact service failure that wasn't caught, and the test was blocked while waiting for the result" @@ -311,7 +311,7 @@ mineBlock ph nonce iop = timeout 5000000 go >>= \case go = do -- assemble block without nonce and timestamp - let r = fst <$> iop + let r = (\(_, q, _) -> q) <$> iop mv <- r >>= newBlock noMiner ph payload <- assertNotLeft =<< takeMVar mv @@ -325,7 +325,7 @@ mineBlock ph nonce iop = timeout 5000000 go >>= \case mv' <- r >>= validateBlock bh (payloadWithOutputsToPayloadData payload) void $ assertNotLeft =<< takeMVar mv' - bdb <- snd <$> iop + (_, _, bdb) <- iop let pdb = _bdbPayloadDb bdb addNewPayload pdb payload diff --git a/test/Chainweb/Test/Pact/PactSingleChainTest.hs b/test/Chainweb/Test/Pact/PactSingleChainTest.hs index d0494bedfc..f8942d0edb 100644 --- a/test/Chainweb/Test/Pact/PactSingleChainTest.hs +++ b/test/Chainweb/Test/Pact/PactSingleChainTest.hs @@ -4,6 +4,7 @@ {-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedRecordDot #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE RecordWildCards #-} @@ -17,18 +18,24 @@ module Chainweb.Test.Pact.PactSingleChainTest ) where import Control.Arrow ((&&&)) +import Control.Concurrent (forkIO) import Control.Concurrent.MVar import Control.DeepSeq -import Control.Lens hiding ((.=)) +import Control.Lens hiding ((.=), matching) import Control.Monad import Control.Monad.Catch +import Patience qualified as PatienceL +import Patience.Map qualified as PatienceM +import Patience.Map (Delta(..)) import Data.Aeson (object, (.=), Value(..), decodeStrict, eitherDecode) import qualified Data.ByteString.Lazy as BL import Data.Either (isRight, fromRight) import Data.IORef import qualified Data.Map.Strict as M +import Data.Maybe (isJust, isNothing) import qualified Data.Text as T +import Data.Text (Text) import qualified Data.Text.IO as T import qualified Data.Text.Encoding as T import qualified Data.Vector as V @@ -54,29 +61,39 @@ import Pact.JSON.Yaml import Chainweb.BlockCreationTime import Chainweb.BlockHeader +import Chainweb.BlockHeight (BlockHeight(..)) import Chainweb.Graph +import Chainweb.Logger (genericLogger) import Chainweb.Mempool.Mempool import Chainweb.MerkleLogHash (unsafeMerkleLogHash) import Chainweb.Miner.Pact +import Chainweb.Pact.Backend.Compaction qualified as C import Chainweb.Pact.Backend.Types import Chainweb.Pact.Service.BlockValidation hiding (local) -import Chainweb.Pact.Service.PactQueue (PactQueue) +import Chainweb.Pact.Service.PactQueue (PactQueue, newPactQueue) import Chainweb.Pact.Service.Types +import Chainweb.Pact.PactService (runPactService) import Chainweb.Pact.PactService.ExecBlock import Chainweb.Pact.Types import Chainweb.Pact.Utils (emptyPayload) import Chainweb.Payload import Chainweb.Test.Cut.TestBlockDb -import Chainweb.Test.Pact.Utils +import Chainweb.Test.Pact.Utils hiding (compact) +import Chainweb.Test.Pact.Utils qualified as Utils import Chainweb.Test.Utils import Chainweb.Test.TestVersions import Chainweb.Time +import Chainweb.Transaction (ChainwebTransaction) import Chainweb.Utils import Chainweb.Version import Chainweb.Version.Utils +import Chainweb.WebBlockHeaderDB (getWebBlockHeaderDb) import Chainweb.Storage.Table.RocksDB +import System.Logger.Types qualified as LL +import System.LogLevel (LogLevel(..)) + testVersion :: ChainwebVersion testVersion = slowForkingCpmTestVersion petersonChainGraph @@ -103,16 +120,16 @@ tests rdb = testGroup testName , test mempoolRefillTest , test blockGasLimitTest , testTimeout preInsertCheckTimeoutTest + , rosettaFailsWithoutFullHistory rdb + , rewindPastMinBlockHeightFails rdb + , pactStateSamePreAndPostCompaction rdb + , compactionIsIdempotent rdb + , compactionUserTablesDropped rdb ] where testName = "Chainweb.Test.Pact.PactSingleChainTest" - testWithConf f conf = - withDelegateMempool $ \dm -> - withPactTestBlockDb testVersion cid rdb (snd <$> dm) conf $ - f (fst <$> dm) - - test f = testWithConf f testPactServiceConfig - testTimeout f = testWithConf f (testPactServiceConfig { _pactPreInsertCheckTimeout = 1 }) + test = test' rdb + testTimeout = testTimeout' rdb testHistLookup1 = getHistoricalLookupNoTxs "sender00" (assertSender00Bal 100_000_000 "check latest entry for sender00 after a no txs block") @@ -121,8 +138,29 @@ tests rdb = testGroup testName testHistLookup3 = getHistoricalLookupWithTxs "sender00" (assertSender00Bal 9.999998051e7 "check latest entry for sender00 after block with txs") - -forSuccess :: NFData a => String -> IO (MVar (Either PactException a)) -> IO a +testWithConf' :: () + => RocksDb + -> (IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree) + -> PactServiceConfig + -> TestTree +testWithConf' rdb f conf = + withDelegateMempool $ \dm -> + withPactTestBlockDb testVersion cid rdb (snd <$> dm) conf $ + f (fst <$> dm) + +test' :: () + => RocksDb + -> (IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree) + -> TestTree +test' rdb f = testWithConf' rdb f testPactServiceConfig + +testTimeout' :: () + => RocksDb + -> (IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree) + -> TestTree +testTimeout' rdb f = testWithConf' rdb f (testPactServiceConfig { _pactPreInsertCheckTimeout = 5 }) + +forSuccess :: (NFData a, HasCallStack) => String -> IO (MVar (Either PactException a)) -> IO a forSuccess msg mvio = (`catchAllSynchronous` handler) $ do mv <- mvio takeMVar mv >>= \case @@ -131,12 +169,11 @@ forSuccess msg mvio = (`catchAllSynchronous` handler) $ do where handler e = assertFailure $ msg ++ ": exception thrown: " ++ show e - -runBlock :: PactQueue -> TestBlockDb -> TimeSpan Micros -> String -> IO PayloadWithOutputs -runBlock q bdb timeOffset msg = do +runBlock :: (HasCallStack) => PactQueue -> TestBlockDb -> TimeSpan Micros -> IO PayloadWithOutputs +runBlock q bdb timeOffset = do ph <- getParentTestBlockDb bdb cid let blockTime = add timeOffset $ _bct $ _blockCreationTime ph - nb <- forSuccess (msg <> ": newblock") $ + nb <- forSuccess "newBlock" $ newBlock noMiner (ParentHeader ph) q forM_ (chainIds testVersion) $ \c -> do let o | c == cid = nb @@ -146,15 +183,15 @@ runBlock q bdb timeOffset msg = do forSuccess "newBlockAndValidate: validate" $ validateBlock nextH (payloadWithOutputsToPayloadData nb) q -newBlockAndValidate :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +newBlockAndValidate :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree newBlockAndValidate refIO reqIO = testCase "newBlockAndValidate" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO setOneShotMempool refIO goldenMemPool - void $ runBlock q bdb second "newBlockAndValidate" + void $ runBlock q bdb second -newBlockAndValidationFailure :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +newBlockAndValidationFailure :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree newBlockAndValidationFailure refIO reqIO = testCase "newBlockAndValidationFailure" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO setOneShotMempool refIO goldenMemPool ph <- getParentTestBlockDb bdb cid @@ -190,11 +227,327 @@ toRowData v = case eitherDecode encV of where encV = J.encode v -getHistory :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +-- Test that PactService fails if Rosetta is enabled and we don't have all of +-- the history. +-- +-- We do this in two stages: +-- +-- 1: +-- - Start PactService with Rosetta disabled +-- - Run some blocks +-- - Compact to some arbitrary greater-than-genesis height +-- 2: +-- - Start PactService with Rosetta enabled +-- - Catch the exception that should arise at the start of PactService, +-- when performing the history check +rosettaFailsWithoutFullHistory :: () + => RocksDb + -> TestTree +rosettaFailsWithoutFullHistory rdb = + withTemporaryDir $ \iodir -> + withSqliteDb cid iodir $ \sqlEnvIO -> + withDelegateMempool $ \dm -> + sequentialTestGroup "rosettaFailsWithoutFullHistory" AllSucceed + [ + -- Run some blocks and then compact + withPactTestBlockDb' testVersion cid rdb sqlEnvIO mempty testPactServiceConfig $ \reqIO -> + testCase "runBlocksAndCompact" $ do + (sqlEnv, q, bdb) <- reqIO + + mempoolRef <- fmap (pure . fst) dm + + setOneShotMempool mempoolRef goldenMemPool + replicateM_ 10 $ void $ runBlock q bdb second + + Utils.compact LL.Error [C.NoVacuum, C.NoGrandHash] sqlEnv (C.Target (BlockHeight 5)) + + -- This needs to run after the previous test + -- Annoyingly, we must inline the PactService util starts here. + -- ResourceT will help clean all this up + , testCase "PactService Should fail" $ do + pactQueue <- newPactQueue 2000 + blockDb <- mkTestBlockDb testVersion rdb + bhDb <- getWebBlockHeaderDb (_bdbWebBlockHeaderDb blockDb) cid + sqlEnv <- sqlEnvIO + mempool <- fmap snd dm + let payloadDb = _bdbPayloadDb blockDb + let cfg = testPactServiceConfig { _pactFullHistoryRequired = True } + let logger = genericLogger System.LogLevel.Error (\_ -> return ()) + e <- try $ runPactService testVersion cid logger pactQueue mempool bhDb payloadDb sqlEnv cfg + case e of + Left (FullHistoryRequired {}) -> do + pure () + Left err -> do + assertFailure $ "Expected FullHistoryRequired exception, instead got: " ++ show err + Right _ -> do + assertFailure "Expected FullHistoryRequired exception, instead there was no exception at all." + ] + +rewindPastMinBlockHeightFails :: () + => RocksDb + -> TestTree +rewindPastMinBlockHeightFails rdb = + compactionSetup "rewindPastMinBlockHeightFails" rdb testPactServiceConfig $ \cr -> do + setOneShotMempool cr.mempoolRef goldenMemPool + replicateM_ 10 $ runBlock cr.pactQueue cr.blockDb second + + Utils.compact LL.Error [C.NoVacuum, C.NoGrandHash] cr.sqlEnv (C.Target (BlockHeight 5)) + + -- Genesis block header; compacted away by now + let bh = genesisBlockHeader testVersion cid + + syncResult <- readMVar =<< pactSyncToBlock bh cr.pactQueue + case syncResult of + Left (PactInternalError {}) -> do + return () + Left err -> do + assertFailure $ "Expected a PactInternalError, but got: " ++ show err + Right _ -> do + assertFailure "Expected an exception, but didn't encounter one." + +pactStateSamePreAndPostCompaction :: () + => RocksDb + -> TestTree +pactStateSamePreAndPostCompaction rdb = + compactionSetup "pactStateSamePreAndPostCompaction" rdb testPactServiceConfig $ \cr -> do + let numBlocks :: Num a => a + numBlocks = 100 + + setOneShotMempool cr.mempoolRef goldenMemPool + + let makeTx :: Int -> BlockHeader -> IO ChainwebTransaction + makeTx nth bh = buildCwCmd + $ set cbSigners [mkSigner' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] + $ setFromHeader bh + $ mkCmd (sshow (nth, bh)) + $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" + + supply <- newIORef @Int 0 + madeTx <- newIORef @Bool False + replicateM_ numBlocks $ do + setMempool cr.mempoolRef $ mempty { + mpaGetBlock = \_ _ _ _ bh -> do + madeTxYet <- readIORef madeTx + if madeTxYet + then do + pure mempty + else do + n <- atomicModifyIORef' supply $ \a -> (a + 1, a) + tx <- makeTx n bh + writeIORef madeTx True + pure $ V.fromList [tx] + } + void $ runBlock cr.pactQueue cr.blockDb second + writeIORef madeTx False + + let db = _sConn cr.sqlEnv + + statePreCompaction <- getLatestPactState db + + Utils.compact LL.Error [C.NoVacuum, C.NoGrandHash] cr.sqlEnv (C.Target (BlockHeight numBlocks)) + + statePostCompaction <- getLatestPactState db + + let stateDiff = M.filter (not . PatienceM.isSame) (PatienceM.diff statePreCompaction statePostCompaction) + when (not (null stateDiff)) $ do + T.putStrLn "" + forM_ (M.toList stateDiff) $ \(tbl, delta) -> do + T.putStrLn "" + T.putStrLn tbl + case delta of + Same _ -> do + pure () + Old x -> do + putStrLn $ "a pre-only value appeared in the pre- and post-compaction diff: " ++ show x + New x -> do + putStrLn $ "a post-only value appeared in the pre- and post-compaction diff: " ++ show x + Delta x1 x2 -> do + let daDiff = M.filter (not . PatienceM.isSame) (PatienceM.diff x1 x2) + forM_ daDiff $ \item -> do + case item of + Old x -> do + putStrLn $ "old: " ++ show x + New x -> do + putStrLn $ "new: " ++ show x + Same _ -> do + pure () + Delta x y -> do + putStrLn $ "old: " ++ show x + putStrLn $ "new: " ++ show y + putStrLn "" + assertFailure "pact state check failed" + +compactionIsIdempotent :: () + => RocksDb + -> TestTree +compactionIsIdempotent rdb = + compactionSetup "compactionIdempotent" rdb testPactServiceConfig $ \cr -> do + let numBlocks :: Num a => a + numBlocks = 100 + + setOneShotMempool cr.mempoolRef goldenMemPool + + let makeTx :: Int -> BlockHeader -> IO ChainwebTransaction + makeTx nth bh = buildCwCmd + $ set cbSigners [mkSigner' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] + $ setFromHeader bh + $ mkCmd (sshow (nth, bh)) + $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" + + supply <- newIORef @Int 0 + madeTx <- newIORef @Bool False + replicateM_ numBlocks $ do + setMempool cr.mempoolRef $ mempty { + mpaGetBlock = \_ _ _ _ bh -> do + madeTxYet <- readIORef madeTx + if madeTxYet + then do + pure mempty + else do + n <- atomicModifyIORef' supply $ \a -> (a + 1, a) + tx <- makeTx n bh + writeIORef madeTx True + pure $ V.fromList [tx] + } + void $ runBlock cr.pactQueue cr.blockDb second + writeIORef madeTx False + + let db = _sConn cr.sqlEnv + + let compact h = + Utils.compact LL.Error [C.NoVacuum, C.NoGrandHash] cr.sqlEnv h + + let compactionHeight = C.Target (BlockHeight numBlocks) + compact compactionHeight + statePostCompaction1 <- getPactUserTables db + compact compactionHeight + statePostCompaction2 <- getPactUserTables db + + let stateDiff = M.filter (not . PatienceM.isSame) (PatienceM.diff statePostCompaction1 statePostCompaction2) + when (not (null stateDiff)) $ do + T.putStrLn "" + forM_ (M.toList stateDiff) $ \(tbl, delta) -> do + T.putStrLn "" + T.putStrLn tbl + case delta of + Same _ -> do + pure () + Old x -> do + putStrLn $ "a pre-only value appeared in the compaction idempotency diff: " ++ show x + New x -> do + putStrLn $ "a post-only value appeared in the compaction idempotency diff: " ++ show x + Delta x1 x2 -> do + let daDiff = PatienceL.pairItems (\a b -> rowKey a == rowKey b) (PatienceL.diff x1 x2) + forM_ daDiff $ \item -> do + case item of + Old x -> do + putStrLn $ "old: " ++ show x + New x -> do + putStrLn $ "new: " ++ show x + Same _ -> do + pure () + Delta x y -> do + putStrLn $ "old: " ++ show x + putStrLn $ "new: " ++ show y + putStrLn "" + assertFailure "pact state check failed" + +-- | Test that user tables created before the compaction height are kept, +-- while those created after the compaction height are dropped. +compactionUserTablesDropped :: () + => RocksDb + -> TestTree +compactionUserTablesDropped rdb = + let + -- creating a module uses about 60k gas. this is + -- that plus some change. + gasLimit :: GasLimit + gasLimit = 70_000 + + pactCfg = testPactServiceConfig { + _pactBlockGasLimit = gasLimit + } + in + compactionSetup "compactionUserTablesDropped" rdb pactCfg $ \cr -> do + let numBlocks :: Num a => a + numBlocks = 100 + let halfwayPoint :: Integral a => a + halfwayPoint = numBlocks `div` 2 + + setOneShotMempool cr.mempoolRef goldenMemPool + + let createTable :: Int -> Text -> IO ChainwebTransaction + createTable n tblName = do + let tx = T.unlines + [ "(namespace 'free)" + , "(module m" <> sshow n <> " G" + , " (defcap G () true)" + , " (defschema empty-schema)" + , " (deftable " <> tblName <> ":{empty-schema})" + , ")" + , "(create-table " <> tblName <> ")" + ] + buildCwCmd + $ signSender00 + $ set cbGasLimit gasLimit + $ mkCmd ("createTable-" <> tblName <> "-" <> sshow n) + $ mkExec tx + $ mkKeySetData "sender00" [sender00] + + let beforeTable = "test_before" + let afterTable = "test_after" + + supply <- newIORef @Int 0 + madeBeforeTable <- newIORef @Bool False + madeAfterTable <- newIORef @Bool False + replicateM_ numBlocks $ do + setMempool cr.mempoolRef $ mempty { + mpaGetBlock = \_ _ mBlockHeight _ _ -> do + let mkTable madeRef tbl = do + madeYet <- readIORef madeRef + if madeYet + then do + pure mempty + else do + n <- atomicModifyIORef' supply $ \a -> (a + 1, a) + tx <- createTable n tbl + writeIORef madeRef True + pure (V.fromList [tx]) + + if mBlockHeight <= halfwayPoint + then do + mkTable madeBeforeTable beforeTable + else do + mkTable madeAfterTable afterTable + } + void $ runBlock cr.pactQueue cr.blockDb second + + let freeBeforeTbl = "free.m0_" <> beforeTable + let freeAfterTbl = "free.m1_" <> afterTable + + let db = _sConn cr.sqlEnv + + statePre <- getPactUserTables db + let assertExists tbl = do + let msg = "Table " ++ T.unpack tbl ++ " should exist pre-compaction, but it doesn't." + assertBool msg (isJust (M.lookup tbl statePre)) + assertExists freeBeforeTbl + assertExists freeAfterTbl + + Utils.compact LL.Error [C.NoVacuum, C.NoGrandHash] cr.sqlEnv (C.Target (BlockHeight halfwayPoint)) + + statePost <- getPactUserTables db + flip assertBool (isJust (M.lookup freeBeforeTbl statePost)) $ + T.unpack beforeTable ++ " was dropped; it wasn't supposed to be." + + flip assertBool (isNothing (M.lookup freeAfterTbl statePost)) $ + T.unpack afterTable ++ " wasn't dropped; it was supposed to be." + +getHistory :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree getHistory refIO reqIO = testCase "getHistory" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO setOneShotMempool refIO goldenMemPool - void $ runBlock q bdb second "getHistory" + void $ runBlock q bdb second h <- getParentTestBlockDb bdb cid mv <- pactBlockTxHistory h (UserTables "coin_coin-table") q @@ -208,7 +561,7 @@ getHistory refIO reqIO = testCase "getHistory" $ do , "keys" .= ["368820f80c324bbc7c2b0610688a7da43e39f91d118732671cd9c7500ff43cca" :: T.Text] ] - , "balance" .= (Number 99999900.0) + , "balance" .= Number 99_999_900.0 ])]) (M.lookup 10 hist) -- and transaction txids @@ -236,30 +589,29 @@ getHistoricalLookupNoTxs :: T.Text -> (Maybe (TxLog RowData) -> IO ()) -> IO (IORef MemPoolAccess) - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree -getHistoricalLookupNoTxs key assertF refIO reqIO = testCase msg $ do - (q,bdb) <- reqIO - setOneShotMempool refIO mempty - void $ runBlock q bdb second msg - h <- getParentTestBlockDb bdb cid - histLookup q h key >>= assertF - where msg = T.unpack $ "getHistoricalLookupNoTxs: " <> key +getHistoricalLookupNoTxs key assertF refIO reqIO = + testCase (T.unpack ("getHistoricalLookupNoTxs: " <> key)) $ do + (_, q, bdb) <- reqIO + setOneShotMempool refIO mempty + void $ runBlock q bdb second + h <- getParentTestBlockDb bdb cid + histLookup q h key >>= assertF getHistoricalLookupWithTxs :: T.Text -> (Maybe (TxLog RowData) -> IO ()) -> IO (IORef MemPoolAccess) - -> IO (PactQueue,TestBlockDb) + -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree -getHistoricalLookupWithTxs key assertF refIO reqIO = testCase msg $ do - (q,bdb) <- reqIO - setOneShotMempool refIO goldenMemPool - void $ runBlock q bdb second msg - h <- getParentTestBlockDb bdb cid - histLookup q h key >>= assertF - where msg = T.unpack $ "getHistoricalLookupWithTxs: " <> key - +getHistoricalLookupWithTxs key assertF refIO reqIO = + testCase (T.unpack ("getHistoricalLookupWithTxs: " <> key)) $ do + (_, q, bdb) <- reqIO + setOneShotMempool refIO goldenMemPool + void $ runBlock q bdb second + h <- getParentTestBlockDb bdb cid + histLookup q h key >>= assertF histLookup :: PactQueue -> BlockHeader -> T.Text -> IO (Maybe (TxLog RowData)) histLookup q bh k = do @@ -280,23 +632,23 @@ assertSender00Bal bal msg hist = ]))) hist -newBlockRewindValidate :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +newBlockRewindValidate :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree newBlockRewindValidate mpRefIO reqIO = testCase "newBlockRewindValidate" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO setOneShotMempool mpRefIO chainDataMemPool cut0 <- readMVar $ _bdbCut bdb -- genesis cut -- cut 1a - void $ runBlock q bdb second "newBlockRewindValidate-1a" + void $ runBlock q bdb second cut1a <- readMVar $ _bdbCut bdb -- rewind, cut 1b void $ swapMVar (_bdbCut bdb) cut0 - void $ runBlock q bdb second "newBlockRewindValidate-1b" + void $ runBlock q bdb second -- rewind to cut 1a to trigger replay with chain data bug void $ swapMVar (_bdbCut bdb) cut1a - void $ runBlock q bdb (secondsToTimeSpan 2) "newBlockRewindValidate-2" + void $ runBlock q bdb (secondsToTimeSpan 2) where @@ -309,7 +661,6 @@ newBlockRewindValidate mpRefIO reqIO = testCase "newBlockRewindValidate" $ do $ mkExec' "(chain-data)" } - signSender00 :: CmdBuilder -> CmdBuilder signSender00 = set cbSigners [mkSigner' sender00 []] @@ -324,9 +675,9 @@ pattern BlockGasLimitError <- Left (PactInternalError (decodeStrict . T.encodeUtf8 -> Just (PactExceptionTag "BlockGasLimitExceeded"))) -- this test relies on block gas errors being thrown before other Pact errors. -blockGasLimitTest :: HasCallStack => IO (IORef MemPoolAccess) -> IO (PactQueue, TestBlockDb) -> TestTree +blockGasLimitTest :: HasCallStack => IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree blockGasLimitTest _ reqIO = testCase "blockGasLimitTest" $ do - (q,_) <- reqIO + (_, q, _) <- reqIO let useGas g = do @@ -379,26 +730,26 @@ blockGasLimitTest _ reqIO = testCase "blockGasLimitTest" $ do _ -> return () -mempoolRefillTest :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +mempoolRefillTest :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree mempoolRefillTest mpRefIO reqIO = testCase "mempoolRefillTest" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO supply <- newMVar (0 :: Int) mp supply [ ( 0, [goodTx, goodTx] ), ( 1, [badTx] ) ] - runBlock q bdb second "mempoolRefillTest-1" >>= checkCount 2 + runBlock q bdb second >>= checkCount 2 mp supply [ ( 0, [goodTx, goodTx] ), ( 1, [goodTx, badTx] ) ] - runBlock q bdb second "mempoolRefillTest-2" >>= checkCount 3 + runBlock q bdb second >>= checkCount 3 mp supply [ ( 0, [badTx, goodTx] ), ( 1, [goodTx, badTx] ) ] - runBlock q bdb second "mempoolRefillTest-3" >>= checkCount 2 + runBlock q bdb second >>= checkCount 2 mp supply [ ( 0, [badTx] ), ( 1, [goodTx, goodTx] ) ] - runBlock q bdb second "mempoolRefillTest-4" >>= checkCount 2 + runBlock q bdb second >>= checkCount 2 mp supply [ ( 0, [goodTx, goodTx] ), ( 1, [badTx, badTx] ) ] - runBlock q bdb second "mempoolRefillTest-5" >>= checkCount 2 + runBlock q bdb second >>= checkCount 2 where @@ -432,26 +783,24 @@ mempoolRefillTest mpRefIO reqIO = testCase "mempoolRefillTest" $ do setFromHeader bh . mkCmd nonce - - -moduleNameFork :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +moduleNameFork :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree moduleNameFork mpRefIO reqIO = testCase "moduleNameFork" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO -- install in free in block 1 setOneShotMempool mpRefIO (moduleNameMempool "free" "test") - void $ runBlock q bdb second "moduleNameFork-1" + void $ runBlock q bdb second -- install in user in block 2 setOneShotMempool mpRefIO (moduleNameMempool "user" "test") - void $ runBlock q bdb second "moduleNameFork-1" + void $ runBlock q bdb second -- do something else post-fork setOneShotMempool mpRefIO (moduleNameMempool "free" "test2") - void $ runBlock q bdb second "moduleNameFork-1" + void $ runBlock q bdb second setOneShotMempool mpRefIO (moduleNameMempool "user" "test2") - void $ runBlock q bdb second "moduleNameFork-1" + void $ runBlock q bdb second -- TODO this test doesn't actually validate, I turn on Debug and make sure it -- goes well. @@ -474,16 +823,16 @@ moduleNameMempool ns mn = mempty mkExec' code -mempoolCreationTimeTest :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +mempoolCreationTimeTest :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree mempoolCreationTimeTest mpRefIO reqIO = testCase "mempoolCreationTimeTest" $ do - (q,bdb) <- reqIO + (_, q, bdb) <- reqIO let start@(Time startSpan) :: Time Micros = Time (TimeSpan (Micros 100_000_000)) s30 = scaleTimeSpan (30 :: Int) second s15 = scaleTimeSpan (15 :: Int) second -- b1 block time is start - void $ runBlock q bdb startSpan "mempoolCreationTimeTest-1" + void $ runBlock q bdb startSpan -- do pre-insert check with transaction at start + 15s @@ -493,7 +842,7 @@ mempoolCreationTimeTest mpRefIO reqIO = testCase "mempoolCreationTimeTest" $ do setOneShotMempool mpRefIO $ mp tx -- b2 will be made at start + 30s - void $ runBlock q bdb s30 "mempoolCreationTimeTest-2" + void $ runBlock q bdb s30 where @@ -514,9 +863,9 @@ mempoolCreationTimeTest mpRefIO reqIO = testCase "mempoolCreationTimeTest" $ do unless (V.and oks) $ throwM $ userError "Insert failed" return txs -preInsertCheckTimeoutTest :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +preInsertCheckTimeoutTest :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree preInsertCheckTimeoutTest _ reqIO = testCase "preInsertCheckTimeoutTest" $ do - (q,_) <- reqIO + (_, q, _) <- reqIO coinV3 <- T.readFile "pact/coin-contract/v3/coin-v3.pact" coinV4 <- T.readFile "pact/coin-contract/v4/coin-v4.pact" @@ -543,9 +892,9 @@ preInsertCheckTimeoutTest _ reqIO = testCase "preInsertCheckTimeoutTest" $ do rs <- forSuccess "preInsertCheckTimeoutTest" $ pactPreInsertCheck (V.fromList [txCoinV3, txCoinV4, txCoinV5]) q assertBool ("should be InsertErrorTimedOut but got " ++ show rs) $ V.and $ V.map (== Left InsertErrorTimedOut) rs -badlistNewBlockTest :: IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +badlistNewBlockTest :: IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree badlistNewBlockTest mpRefIO reqIO = testCase "badlistNewBlockTest" $ do - (reqQ,_) <- reqIO + (_, reqQ, _) <- reqIO let hashToTxHashList = V.singleton . requestKeyToTransactionHash . RequestKey . toUntypedHash @'Blake2b_256 badHashRef <- newIORef $ hashToTxHashList initialHash badTx <- buildCwCmd @@ -567,9 +916,9 @@ badlistNewBlockTest mpRefIO reqIO = testCase "badlistNewBlockTest" $ do } -goldenNewBlock :: String -> MemPoolAccess -> IO (IORef MemPoolAccess) -> IO (PactQueue,TestBlockDb) -> TestTree +goldenNewBlock :: String -> MemPoolAccess -> IO (IORef MemPoolAccess) -> IO (SQLiteEnv, PactQueue, TestBlockDb) -> TestTree goldenNewBlock name mp mpRefIO reqIO = golden name $ do - (reqQ,_) <- reqIO + (_, reqQ, _) <- reqIO setOneShotMempool mpRefIO mp resp <- forSuccess ("goldenNewBlock:" ++ name) $ newBlock noMiner (ParentHeader genesisHeader) reqQ @@ -624,3 +973,44 @@ goldenMemPool = mempty mkCmd ("1" <> sshow n) $ mkExec code $ mkKeySetData "test-admin-keyset" [sender00] + +data CompactionResources = CompactionResources + { mempoolRef :: IO (IORef MemPoolAccess) + , mempool :: MemPoolAccess + , sqlEnv :: SQLiteEnv + , pactQueue :: PactQueue + , blockDb :: TestBlockDb + } + +compactionSetup :: () + => String + -- ^ test pattern + -> RocksDb + -> PactServiceConfig + -> (CompactionResources -> IO ()) + -> TestTree +compactionSetup pat rdb pactCfg f = + withTemporaryDir $ \iodir -> + withSqliteDb cid iodir $ \sqlEnvIO -> + withDelegateMempool $ \dm -> + testCase pat $ do + blockDb <- mkTestBlockDb testVersion rdb + bhDb <- getWebBlockHeaderDb (_bdbWebBlockHeaderDb blockDb) cid + let payloadDb = _bdbPayloadDb blockDb + sqlEnv <- sqlEnvIO + (mempoolRef, mempool) <- do + (ref, nonRef) <- dm + pure (pure ref, nonRef) + pactQueue <- newPactQueue 2000 + + let logger = genericLogger System.LogLevel.Error (\_ -> return ()) + + void $ forkIO $ runPactService testVersion cid logger pactQueue mempool bhDb payloadDb sqlEnv pactCfg + + f $ CompactionResources + { mempoolRef = mempoolRef + , mempool = mempool + , sqlEnv = sqlEnv + , pactQueue = pactQueue + , blockDb = blockDb + } diff --git a/test/Chainweb/Test/Pact/RemotePactTest.hs b/test/Chainweb/Test/Pact/RemotePactTest.hs index 192daf5539..f6fa543e8c 100644 --- a/test/Chainweb/Test/Pact/RemotePactTest.hs +++ b/test/Chainweb/Test/Pact/RemotePactTest.hs @@ -3,6 +3,7 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} @@ -37,21 +38,25 @@ import Control.Monad.IO.Class import qualified Data.Aeson as A import Data.Aeson.Lens hiding (values) +import Data.Bifunctor (first) +import Control.Monad.Trans.Except (runExceptT, except) +import Control.Monad.Except (throwError) import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as LBS import qualified Data.ByteString.Short as SB +import Data.IORef (modifyIORef', newIORef, readIORef) import Data.Word (Word64) import Data.Default (def) import Data.Foldable (toList) -import qualified Data.HashMap.Strict as HashMap +import qualified Data.HashMap.Strict as HM import qualified Data.List as L import qualified Data.List.NonEmpty as NEL import qualified Data.Map.Strict as M import Data.Maybe import Data.Text (Text) import qualified Data.Text as T - -import Numeric.Natural +import qualified Data.Text.Encoding as T +import System.Logger.Types (LogLevel(..)) import Servant.Client @@ -72,6 +77,8 @@ import Pact.Types.Hash (Hash(..)) import qualified Pact.Types.PactError as Pact import Pact.Types.PactValue import Pact.Types.Pretty +import Pact.Types.Persistence (RowKey(..), TxLog(..)) +import Pact.Types.RowData (RowData(..)) import Pact.Types.Term -- internal modules @@ -79,11 +86,15 @@ import Pact.Types.Term import Chainweb.ChainId import Chainweb.Graph import Chainweb.Mempool.Mempool +import Chainweb.Pact.Backend.Compaction qualified as C +import Chainweb.Pact.Backend.Utils qualified as Backend +import Chainweb.Pact.Backend.Types (SQLiteEnv(..)) import Chainweb.Pact.RestAPI.Client import Chainweb.Pact.RestAPI.EthSpv import Chainweb.Pact.Service.Types import Chainweb.Pact.Validations (defaultMaxTTL) import Chainweb.Test.Pact.Utils +import Chainweb.Test.Pact.Utils qualified as Utils import Chainweb.Test.RestAPI.Utils import Chainweb.Test.Utils import Chainweb.Test.TestVersions @@ -93,11 +104,10 @@ import Chainweb.Version import Chainweb.Version.Mainnet import Chainweb.Storage.Table.RocksDB - -- -------------------------------------------------------------------------- -- -- Global Settings -nNodes :: Natural +nNodes :: Word nNodes = 1 v :: ChainwebVersion @@ -135,6 +145,16 @@ tests rdb = testGroup "Chainweb.Test.Pact.RemotePactTest" withResource' getCurrentTimeIntegral $ \(iotm :: IO (Time Micros)) -> let cenv = _getServiceClientEnv <$> net iot = toTxCreationTime <$> iotm + pactDir = do + m <- _getNodeDbDirs <$> net + -- This looks up the pactDbDir for node 0. This is + -- kind of a hack, because there is only one node in + -- this test. However, it doesn't matter much, because + -- we are dealing with both submitting /local txs + -- and compaction, so picking an arbitrary node + -- to run these two operations on is fine. + pure (fst (head m)) + in testGroup "remote pact tests" [ withResourceT (liftIO $ join $ withRequestKeys <$> iot <*> cenv) $ \reqkeys -> golden "remote-golden" $ join $ responseGolden <$> cenv <*> reqkeys @@ -153,6 +173,9 @@ tests rdb = testGroup "Chainweb.Test.Pact.RemotePactTest" , after AllSucceed "remote spv" $ testCase "trivialLocalCheck" $ join $ localTest <$> iot <*> cenv + , after AllSucceed "remote spv" $ + testCase "txlogsCompactionTest" $ + join $ txlogsCompactionTest <$> iot <*> cenv <*> pactDir , after AllSucceed "remote spv" $ testCase "localChainData" $ join $ localChainDataTest <$> iot <*> cenv @@ -186,10 +209,190 @@ tests rdb = testGroup "Chainweb.Test.Pact.RemotePactTest" responseGolden :: ClientEnv -> RequestKeys -> IO LBS.ByteString responseGolden cenv rks = do PollResponses theMap <- polling cid cenv rks ExpectPactResult - let values = mapMaybe (\rk -> _crResult <$> HashMap.lookup rk theMap) + let values = mapMaybe (\rk -> _crResult <$> HM.lookup rk theMap) (NEL.toList $ _rkRequestKeys rks) return $ foldMap J.encode values +-- | Check that txlogs don't problematically access history +-- post-compaction. +-- +-- At a high level, the test does this: +-- - Submits a tx that creates a module with a table named `persons`. +-- +-- This module exposes a few functions for reading, inserting, +-- and overwriting rows to the `persons` table. +-- +-- The tx also inserts some people into `persons` for +-- some initial state. +-- +-- This module also exposes a way to access the `txlogs` +-- of the `persons` table (what this test is concerned with). +-- +-- - Submits a tx that overwrites a row in the +-- `persons` table. +-- +-- - Compacts to the latest blockheight on each chain. This should +-- get rid of any out-of-date rows. +-- +-- - Submits a /local tx that reads the `txlogs` on the `persons` +-- table. Call this `txLogs`. +-- +-- If this read fails, Pact is doing something problematic! +-- +-- If this read doesn't fail, we need to check that `txLogs` +-- matches the latest pact state post-compaction. Because +-- compaction sweeps away the out-of-date rows, they shouldn't +-- appear in the `txLogs` anymore, and the two should be equivalent. +txlogsCompactionTest :: Pact.TxCreationTime -> ClientEnv -> FilePath -> IO () +txlogsCompactionTest t cenv pactDbDir = do + let cmd :: Text -> Text -> CmdBuilder + cmd nonce tx = do + set cbSigners [mkSigner' sender00 []] + $ set cbTTL defaultMaxTTL + $ set cbCreationTime t + $ set cbChainId cid + $ set cbNetworkId (Just v) + $ mkCmd nonce + $ mkExec tx + $ mkKeySetData "sender00" [sender00] + + createTableTx <- buildTextCmd + $ set cbGasLimit 300_000 + $ cmd "create-table-persons" + $ T.unlines + [ "(namespace 'free)" + , "(module m0 G" + , " (defcap G () true)" + , " (defschema person" + , " name:string" + , " age:integer" + , " )" + , " (deftable persons:{person})" + , " (defun read-persons (k) (read persons k))" + , " (defun insert-persons (id name age) (insert persons id { 'name:name, 'age:age }))" + , " (defun write-persons (id name age) (write persons id { 'name:name, 'age:age }))" + , " (defun persons-txlogs (i) (map (txlog persons) (txids persons i)))" + , ")" + , "(create-table persons)" + , "(insert-persons \"A\" \"Lindsey Lohan\" 42)" + , "(insert-persons \"B\" \"Nico Robin\" 30)" + , "(insert-persons \"C\" \"chessai\" 420)" + ] + + nonceSupply <- newIORef @Word 1 -- starts at 1 since 0 is always the create-table tx + let nextNonce = do + cur <- readIORef nonceSupply + modifyIORef' nonceSupply (+ 1) + pure cur + + let submitAndCheckTx tx = do + submitResult <- flip runClientM cenv $ + pactSendApiClient v cid $ SubmitBatch $ NEL.fromList [tx] + case submitResult of + Left err -> do + assertFailure $ "Error when sending tx: " ++ show err + Right rks -> do + PollResponses m <- polling cid cenv rks ExpectPactResult + case HM.lookup (NEL.head (_rkRequestKeys rks)) m of + Just cr -> do + case _crResult cr of + PactResult (Left err) -> do + assertFailure $ "validation failure on tx: " ++ show err + PactResult _ -> do + pure () + Nothing -> do + assertFailure "impossible" + + submitAndCheckTx createTableTx + + let getLatestState :: IO (M.Map RowKey RowData) + getLatestState = C.withDefaultLogger Error $ \logger -> do + Backend.withSqliteDb cid logger pactDbDir False $ \(SQLiteEnv db _) -> do + st <- Utils.getLatestPactState db + case M.lookup "free.m0_persons" st of + Just ps -> fmap M.fromList $ forM (M.toList ps) $ \(rkBytes, rdBytes) -> do + let rk = RowKey (T.decodeUtf8 rkBytes) + case A.eitherDecodeStrict' @RowData rdBytes of + Left err -> do + assertFailure $ "Failed decoding rowdata: " ++ err + Right rd -> do + pure (rk, rd) + Nothing -> error "getting state of free.m0_persons failed" + + let createTxLogsTx :: Word -> IO (Command Text) + createTxLogsTx n = do + -- cost is about 360k. + -- cost = flatCost(module) + flatCost(map) + flatCost(txIds) + numTxIds * (costOf(txlog)) + C + -- = 60_000 + 4 + 100_000 + 2 * 100_000 + C + -- = 360_004 + C + -- Note there are two transactions that write to `persons`, which is + -- why `numTxIds` = 2 (and not the number of rows). + let gasLimit = 400_000 + buildTextCmd + $ set cbGasLimit gasLimit + $ cmd ("test-txlogs-" <> sshow n) + $ T.unlines + [ "(namespace 'free)" + , "(module m" <> sshow n <> " G" + , " (defcap G () true)" + , " (defun persons-txlogs (i) (m0.persons-txlogs i))" + , ")" + , "(persons-txlogs 0)" + ] + + let createWriteTx :: Word -> IO (Command Text) + createWriteTx n = do + -- module = 60k, write = 100 + let gasLimit = 70_000 + buildTextCmd + $ set cbGasLimit gasLimit + $ cmd ("test-write-" <> sshow n) + $ T.unlines + [ "(namespace 'free)" + , "(module m" <> sshow n <> " G" + , " (defcap G () true)" + , " (defun test-write (id name age) (m0.write-persons id name age))" + , ")" + , "(test-write \"C\" \"chessai\" 69)" + ] + + let -- This can't be a Map because the RowKeys aren't + -- necessarily unique, unlike in `getLatestPactState`. + crGetTxLogs :: CommandResult Hash -> IO [(RowKey, A.Value)] + crGetTxLogs cr = do + e <- runExceptT $ do + pv0 <- except (first show (_pactResult (_crResult cr))) + case pv0 of + PList arr -> do + fmap concat $ forM arr $ \pv -> do + txLogs <- except (A.eitherDecode @[TxLog A.Value] (J.encode pv)) + pure $ flip map txLogs $ \txLog -> + (RowKey (_txKey txLog), _txValue txLog) + _ -> do + throwError "expected outermost PList when decoding TxLogs" + case e of + Left err -> do + assertFailure $ "crGetTxLogs failed: " ++ err + Right txlogs -> do + pure txlogs + + submitAndCheckTx =<< createWriteTx =<< nextNonce + + C.withDefaultLogger Error $ \logger -> do + let flags = [C.NoVacuum, C.NoGrandHash] + let resetDb = False + + Backend.withSqliteDb cid logger pactDbDir resetDb $ \(SQLiteEnv db _) -> do + void $ C.compact C.Latest logger db flags + + txLogs <- crGetTxLogs =<< local cid cenv =<< createTxLogsTx =<< nextNonce + + latestState <- getLatestState + assertEqual + "txlogs match latest state" + txLogs + (map (\(rk, rd) -> (rk, J.toJsonViaEncode (_rdData rd))) (M.toList latestState)) + localTest :: Pact.TxCreationTime -> ClientEnv -> IO () localTest t cenv = do mv <- newMVar 0 @@ -211,7 +414,7 @@ localContTest t cenv step = do PollResponses m <- polling cid' cenv rks ExpectPactResult pid <- case _rkRequestKeys rks of rk NEL.:| [] -> maybe (assertFailure "impossible") (return . _pePactId) - $ HashMap.lookup rk m >>= _crContinuation + $ HM.lookup rk m >>= _crContinuation _ -> assertFailure "continuation did not succeed" step "execute /local continuation dry run" @@ -258,7 +461,7 @@ pollingConfirmDepth t cenv step = do PollResponses m <- pollingWithDepth cid' cenv rks (Just $ ConfirmationDepth 10) ExpectPactResult afterPolling <- getCurrentBlockHeight v cenv cid' - assertBool "there are two command results" $ length (HashMap.keys m) == 2 + assertBool "there are two command results" $ length (HM.keys m) == 2 -- we are checking that we have waited at least 10 blocks using /poll for the transaction assertBool "the difference between heights should be no less than the confirmation depth" $ (afterPolling - beforePolling) >= 10 @@ -416,7 +619,7 @@ localPreflightSimTest t cenv step = do Right MetadataValidationFailure{} -> assertFailure "Preflight produced an impossible result" Right (LocalResultWithWarns cr' ws) -> do - let crbh :: Integer = fromIntegral $ fromMaybe 0 $ getBlockHeight cr' + let crbh :: Integer = fromIntegral $ fromMaybe 0 $ crGetBlockHeight cr' expectedbh = 1 + fromIntegral currentBlockHeight assertBool "Preflight's metadata should have increment block height" -- we don't control the node in remote tests and the data can get oudated, @@ -437,7 +640,7 @@ localPreflightSimTest t cenv step = do Right MetadataValidationFailure{} -> assertFailure "Preflight produced an impossible result" Right (LocalResultWithWarns cr' ws) -> do - let crbh :: Integer = fromIntegral $ fromMaybe 0 $ getBlockHeight cr' + let crbh :: Integer = fromIntegral $ fromMaybe 0 $ crGetBlockHeight cr' expectedbh = toInteger $ 1 + (fromIntegral currentBlockHeight') - rewindDepth assertBool "Preflight's metadata block height should reflect the rewind depth" -- we don't control the node in remote tests and the data can get oudated, @@ -672,7 +875,7 @@ txTooBigGasTest t cenv step = do void $ step "pollApiClient: polling for request key" PollResponses resp <- polling sid cenv rks expectation - return (HashMap.lookup (NEL.head $ _rkRequestKeys rks) resp) + return (HM.lookup (NEL.head $ _rkRequestKeys rks) resp) runLocal (SubmitBatch cmds) = do void $ step "localApiClient: submit transaction" @@ -741,7 +944,7 @@ caplistTest t cenv step = do testCaseStep "poll for transfer results" PollResponses rs <- liftIO $ polling sid cenv rks ExpectPactResult - return (HashMap.lookup (NEL.head $ _rkRequestKeys rks) rs) + return (HM.lookup (NEL.head $ _rkRequestKeys rks) rs) case r of Left e -> assertFailure $ "test failure for TRANSFER + FUND_TX: " <> show e @@ -859,10 +1062,10 @@ allocationTest t cenv step = do sid = unsafeChainId 0 localAfterPollResponse (PollResponses prs) cr = - getBlockHeight cr > getBlockHeight (snd $ head $ HashMap.toList prs) + crGetBlockHeight cr > crGetBlockHeight (snd $ head $ HM.toList prs) localAfterBlockHeight bh cr = - getBlockHeight cr > Just bh + crGetBlockHeight cr > Just bh accountInfo = Right $ PObject @@ -948,5 +1151,5 @@ pactDeadBeef = let (TransactionHash b) = deadbeef in RequestKey $ Hash b -- avoiding `scientific` dep here -getBlockHeight :: CommandResult a -> Maybe Word64 -getBlockHeight = preview (crMetaData . _Just . key "blockHeight" . _Number . to ((fromIntegral :: Integer -> Word64 ) . round . toRational)) +crGetBlockHeight :: CommandResult a -> Maybe Word64 +crGetBlockHeight = preview (crMetaData . _Just . key "blockHeight" . _Number . to ((fromIntegral :: Integer -> Word64 ) . round . toRational)) diff --git a/test/Chainweb/Test/Pact/TTL.hs b/test/Chainweb/Test/Pact/TTL.hs index 18ee6305c2..7a3cd167a7 100644 --- a/test/Chainweb/Test/Pact/TTL.hs +++ b/test/Chainweb/Test/Pact/TTL.hs @@ -283,7 +283,7 @@ withTestPact rdb test = withResource' newEmptyMVar $ \mempoolVarIO -> withPactTestBlockDb testVer cid rdb (mempool mempoolVarIO) testPactServiceConfig $ \ios -> test $ do - (pq,bdb) <- ios + (_, pq, bdb) <- ios mp <- mempoolVarIO bhdb <- getBlockHeaderDb cid bdb return $ Ctx mp pq (_bdbPayloadDb bdb) bhdb diff --git a/test/Chainweb/Test/Pact/Utils.hs b/test/Chainweb/Test/Pact/Utils.hs index abe29dd80f..0681b89717 100644 --- a/test/Chainweb/Test/Pact/Utils.hs +++ b/test/Chainweb/Test/Pact/Utils.hs @@ -1,10 +1,13 @@ {-# LANGUAGE LambdaCase #-} +{-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedRecordDot #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -80,6 +83,7 @@ module Chainweb.Test.Pact.Utils , csPrivKey -- * Pact Service creation , withPactTestBlockDb +, withPactTestBlockDb' , withWebPactExecutionService , withPactCtxSQLite , WithPactCtxSQLite @@ -90,6 +94,7 @@ module Chainweb.Test.Pact.Utils , testPactServiceConfig , withBlockHeaderDb , withTemporaryDir +, withSqliteDb -- * Mempool utils , delegateMemPoolAccess , withDelegateMempool @@ -99,6 +104,11 @@ module Chainweb.Test.Pact.Utils , runCut , Noncer , zeroNoncer +-- * Pact State +, compact +, PactRow(..) +, getLatestPactState +, getPactUserTables -- * miscellaneous , toTxCreationTime , dummyLogger @@ -113,7 +123,7 @@ module Chainweb.Test.Pact.Utils import Control.Arrow ((&&&)) import Control.Concurrent.Async import Control.Concurrent.MVar -import Control.Lens (view, _3, makeLenses) +import Control.Lens (view, _2, makeLenses) import Control.Monad import Control.Monad.Catch import Control.Monad.IO.Class @@ -126,6 +136,7 @@ import Data.Default (def) import Data.Foldable import qualified Data.HashMap.Strict as HM import Data.IORef +import Data.Map (Map) import qualified Data.Map.Strict as M import Data.Maybe import Data.Text (Text) @@ -134,11 +145,15 @@ import qualified Data.Text.Encoding as T import Data.String import qualified Data.Vector as V +import Database.SQLite3.Direct (Database) + import GHC.Generics +import Streaming.Prelude qualified as S import System.Directory import System.IO.Temp (createTempDirectory) import System.LogLevel +import System.Logger.Types qualified as LL import Test.Tasty @@ -174,11 +189,14 @@ import Chainweb.ChainId import Chainweb.Graph import Chainweb.Logger import Chainweb.Miner.Pact +import Chainweb.Pact.Backend.Compaction qualified as C +import Chainweb.Pact.Backend.PactState qualified as PactState +import Chainweb.Pact.Backend.PactState (TableDiffable(..), Table(..), PactRow(..)) import Chainweb.Pact.Backend.RelationalCheckpointer (initRelationalCheckpointer') import Chainweb.Pact.Backend.SQLite.DirectV2 import Chainweb.Pact.Backend.Types -import Chainweb.Pact.Backend.Utils +import Chainweb.Pact.Backend.Utils hiding (withSqliteDb) import Chainweb.Pact.PactService import Chainweb.Pact.Service.PactQueue import Chainweb.Pact.Service.Types @@ -805,6 +823,67 @@ withTemporaryDir = withResource (getTemporaryDirectory >>= \d -> createTempDirectory d "test-pact") removeDirectoryRecursive +-- | Single-chain Pact via service queue. +-- +-- The difference between this and 'withPactTestBlockDb' is that, +-- this function takes a `SQLiteEnv` resource which it then exposes +-- to the test function. +-- +-- TODO: Consolidate these two functions. +withPactTestBlockDb' + :: ChainwebVersion + -> ChainId + -> RocksDb + -> IO SQLiteEnv + -> IO MemPoolAccess + -> PactServiceConfig + -> (IO (SQLiteEnv,PactQueue,TestBlockDb) -> TestTree) + -> TestTree +withPactTestBlockDb' version cid rdb sqlEnvIO mempoolIO pactConfig f = + withResource' (mkTestBlockDb version rdb) $ \bdbio -> + withResource (startPact bdbio) stopPact $ f . fmap (view _2) + where + startPact bdbio = do + reqQ <- newPactQueue 2000 + bdb <- bdbio + sqlEnv <- sqlEnvIO + mempool <- mempoolIO + bhdb <- getWebBlockHeaderDb (_bdbWebBlockHeaderDb bdb) cid + let pdb = _bdbPayloadDb bdb + a <- async $ runForever (\_ _ -> return ()) "Chainweb.Test.Pact.Utils.withPactTestBlockDb" $ + runPactService version cid logger reqQ mempool bhdb pdb sqlEnv pactConfig + return (a, (sqlEnv,reqQ,bdb)) + + stopPact (a, _) = cancel a + + -- Ideally, we should throw 'error' when the logger is invoked, because + -- error logs should not happen in production and should always be resolved. + -- Unfortunately, that's not yet always the case. So we just drop the + -- message. + -- + logger = genericLogger Error (\_ -> return ()) + +withSqliteDb :: () + => ChainId + -> IO FilePath + -> (IO SQLiteEnv -> TestTree) + -> TestTree +withSqliteDb cid iodir s = withResource start stop s + where + start = do + dir <- iodir + startSqliteDb cid logger dir False + + stop env = do + stopSqliteDb env + + -- Ideally, we should throw 'error' when the logger is invoked, because + -- error logs should not happen in production and should always be resolved. + -- Unfortunately, that's not yet always the case. So we just drop the + -- message. + -- + logger = genericLogger Error (\_ -> return ()) + -- | Single-chain Pact via service queue. withPactTestBlockDb :: ChainwebVersion @@ -812,12 +891,12 @@ withPactTestBlockDb -> RocksDb -> IO MemPoolAccess -> PactServiceConfig - -> (IO (PactQueue,TestBlockDb) -> TestTree) + -> (IO (SQLiteEnv,PactQueue,TestBlockDb) -> TestTree) -> TestTree withPactTestBlockDb version cid rdb mempoolIO pactConfig f = withTemporaryDir $ \iodir -> withResource' (mkTestBlockDb version rdb) $ \bdbio -> - withResource (startPact bdbio iodir) stopPact $ f . fmap (view _3) + withResource (startPact bdbio iodir) stopPact $ f . fmap (view _2) where startPact bdbio iodir = do reqQ <- newPactQueue 2000 @@ -829,9 +908,9 @@ withPactTestBlockDb version cid rdb mempoolIO pactConfig f = sqlEnv <- startSqliteDb cid logger dir False a <- async $ runForever (\_ _ -> return ()) "Chainweb.Test.Pact.Utils.withPactTestBlockDb" $ runPactService version cid logger reqQ mempool bhdb pdb sqlEnv pactConfig - return (a, sqlEnv, (reqQ,bdb)) + return (a, (sqlEnv,reqQ,bdb)) - stopPact (a, sqlEnv, _) = cancel a >> stopSqliteDb sqlEnv + stopPact (a, (sqlEnv, _, _)) = cancel a >> stopSqliteDb sqlEnv -- Ideally, we should throw 'error' when the logger is invoked, because -- error logs should not happen in production and should always be resolved. @@ -866,3 +945,43 @@ someBlockHeader v h = (!! (int h - 1)) makeLenses ''CmdBuilder makeLenses ''CmdSigner + +-- | Get all pact user tables. +-- +-- Note: This consumes a stream. If you are writing a test +-- with very large pact states (think: Gigabytes), use +-- the streaming version of this function from +-- 'Chainweb.Pact.Backend.PactState'. +getPactUserTables :: Database -> IO (Map Text [PactRow]) +getPactUserTables db = do + S.foldM_ + (\m tbl -> pure (M.insert tbl.name tbl.rows m)) + (pure M.empty) + pure + (PactState.getPactTables db) + +-- | Get active/latest pact state. +-- +-- Note: This consumes a stream. If you are writing a test +-- with very large pact states (think: Gigabytes), use +-- the streaming version of this function from +-- 'Chainweb.Pact.Backend.PactState'. +getLatestPactState :: Database -> IO (Map Text (Map ByteString ByteString)) +getLatestPactState db = do + S.foldM_ + (\m td -> pure (M.insert td.name td.rows m)) + (pure M.empty) + pure + (PactState.getLatestPactState db) + +-- | Compaction utility for testing. +-- Most of the time the flags will be ['C.NoVacuum', 'C.NoGrandHash'] +compact :: () + => LL.LogLevel + -> [C.CompactFlag] + -> SQLiteEnv + -> C.TargetBlockHeight + -> IO () +compact logLevel cFlags (SQLiteEnv db _) bh = do + C.withDefaultLogger logLevel $ \logger -> do + void $ C.compact bh logger db cFlags diff --git a/test/Chainweb/Test/Rosetta/RestAPI.hs b/test/Chainweb/Test/Rosetta/RestAPI.hs index 6bb470b4a3..f5adddb63c 100644 --- a/test/Chainweb/Test/Rosetta/RestAPI.hs +++ b/test/Chainweb/Test/Rosetta/RestAPI.hs @@ -76,7 +76,7 @@ import System.IO.Unsafe (unsafePerformIO) v :: ChainwebVersion v = fastForkingCpmTestVersion petersonChainGraph -nodes :: Natural +nodes :: Word nodes = 1 cid :: ChainId diff --git a/test/Chainweb/Test/Utils.hs b/test/Chainweb/Test/Utils.hs index 986bb02c10..36e6c21c39 100644 --- a/test/Chainweb/Test/Utils.hs +++ b/test/Chainweb/Test/Utils.hs @@ -1,5 +1,6 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiWayIf #-} {-# LANGUAGE NumericUnderscores #-} @@ -10,7 +11,6 @@ {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} -{-# LANGUAGE ViewPatterns #-} -- | -- Module: Chainweb.Test.Utils @@ -117,13 +117,14 @@ module Chainweb.Test.Utils , host , interface , testRetryPolicy +, withDbDirs ) where import Control.Concurrent import Control.Concurrent.Async import Control.Lens import Control.Monad -import Control.Monad.Catch (finally, bracket) +import Control.Monad.Catch (MonadCatch, catch, finally, bracket) import Control.Monad.IO.Class import Control.Monad.Trans.Resource import Control.Retry @@ -157,6 +158,7 @@ import Numeric.Natural import Servant.Client (BaseUrl(..), ClientEnv, Scheme(..), mkClientEnv, runClientM) +import System.Directory (removeDirectoryRecursive) import System.Environment (withArgs) import System.IO import System.IO.Temp @@ -897,6 +899,7 @@ matchTest pat = withArgs ["-p",pat] data ChainwebNetwork = ChainwebNetwork { _getClientEnv :: !ClientEnv , _getServiceClientEnv :: !ClientEnv + , _getNodeDbDirs :: ![(FilePath, FilePath)] } withNodes_ @@ -905,16 +908,17 @@ withNodes_ -> ChainwebVersion -> B.ByteString -> RocksDb - -> Natural + -> Word -> ResourceT IO ChainwebNetwork -withNodes_ logger v testLabel rdb n = - (uncurry ChainwebNetwork . snd) . snd <$> - allocate start (cancel . fst) +withNodes_ logger v testLabel rdb n = do + nodeDbDirs <- withDbDirs n + (_rkey, (_async, (p2p, service))) <- allocate (start nodeDbDirs) (cancel . fst) + pure (ChainwebNetwork p2p service nodeDbDirs) where - start :: IO (Async (), (ClientEnv, ClientEnv)) - start = do + start :: [(FilePath, FilePath)] -> IO (Async (), (ClientEnv, ClientEnv)) + start dbDirs = do peerInfoVar <- newEmptyMVar - a <- async $ runTestNodes testLabel rdb logger v n peerInfoVar + a <- async $ runTestNodes testLabel rdb logger v peerInfoVar dbDirs (i, servicePort) <- readMVar peerInfoVar cwEnv <- getClientEnv $ getCwBaseUrl Https $ _hostAddressPort $ _peerAddr i cwServiceEnv <- getClientEnv $ getCwBaseUrl Http servicePort @@ -932,7 +936,7 @@ withNodes :: ChainwebVersion -> B.ByteString -> RocksDb - -> Natural + -> Word -> ResourceT IO ChainwebNetwork withNodes = withNodes_ (genericLogger Error (error . T.unpack)) -- Test resources are part of test infrastructure and should never print @@ -943,7 +947,7 @@ withNodesAtLatestBehavior :: ChainwebVersion -> B.ByteString -> RocksDb - -> Natural + -> Word -> ResourceT IO ChainwebNetwork withNodesAtLatestBehavior v testLabel rdb n = do net <- withNodes v testLabel rdb n @@ -992,19 +996,19 @@ runTestNodes -> RocksDb -> logger -> ChainwebVersion - -> Natural -> MVar (PeerInfo, Port) + -> [(FilePath, FilePath)] + -- ^ A Map from Node Id to (Pact DB Dir, RocksDB Dir). + -- The index is just the position in the list. -> IO () -runTestNodes testLabel rdb logger ver n portMVar = - forConcurrently_ [0 .. int n - 1] $ \i -> do - threadDelay (1000 * int i) - let baseConf = config ver n - conf <- if - | i == 0 -> - return $ bootstrapConfig baseConf - | otherwise -> - setBootstrapPeerInfo <$> (fst <$> readMVar portMVar) <*> pure baseConf - node testLabel rdb logger portMVar conf i +runTestNodes testLabel rdb logger ver portMVar dbDirs = do + forConcurrently_ (zip [0 ..] dbDirs) $ \(nid, (pactDbDir, rocksDbDir)) -> do + threadDelay (1000 * int nid) + let baseConf = config ver (int (length dbDirs)) + conf <- if nid == 0 + then return $ bootstrapConfig baseConf + else setBootstrapPeerInfo <$> (fst <$> readMVar portMVar) <*> pure baseConf + node testLabel rdb logger portMVar conf pactDbDir rocksDbDir nid node :: Logger logger @@ -1013,28 +1017,29 @@ node -> logger -> MVar (PeerInfo, Port) -> ChainwebConfiguration - -> Int + -> FilePath + -- ^ pact db dir + -> FilePath + -- ^ rocksdb dir + -> Word -- ^ Unique Node Id. The node id 0 is used for the bootstrap node -> IO () -node testLabel rdb rawLogger peerInfoVar conf nid = do +node testLabel rdb rawLogger peerInfoVar conf pactDbDir rocksDbDir nid = do rocksDb <- testRocksDb (testLabel <> T.encodeUtf8 (toText nid)) rdb - withSystemTempDirectory "test-backupdir" $ \backupDir -> - withSystemTempDirectory "test-rocksdb" $ \dir -> - withChainweb conf logger rocksDb backupDir dir False $ \case - StartedChainweb cw -> do - - -- If this is the bootstrap node we extract the port number and publish via an MVar. - when (nid == 0) $ do - let bootStrapInfo = view (chainwebPeer . peerResPeer . peerInfo) cw - bootStrapPort = view (chainwebServiceSocket . _1) cw - putMVar peerInfoVar (bootStrapInfo, bootStrapPort) - - poisonDeadBeef cw - runChainweb cw `finally` do - logFunctionText logger Info "write sample data" - logFunctionText logger Info "shutdown node" - return () - Replayed _ _ -> error "node: should not be a replay" + withChainweb conf logger rocksDb pactDbDir rocksDbDir False $ \case + StartedChainweb cw -> do + -- If this is the bootstrap node we extract the port number and publish via an MVar. + when (nid == 0) $ do + let bootStrapInfo = view (chainwebPeer . peerResPeer . peerInfo) cw + bootStrapPort = view (chainwebServiceSocket . _1) cw + putMVar peerInfoVar (bootStrapInfo, bootStrapPort) + + poisonDeadBeef cw + runChainweb cw `finally` do + logFunctionText logger Info "write sample data" + logFunctionText logger Info "shutdown node" + return () + Replayed _ _ -> error "node: should not be a replay" where logger = addLabel ("node", sshow nid) rawLogger @@ -1043,6 +1048,31 @@ node testLabel rdb rawLogger peerInfoVar conf nid = do crs = map snd $ HashMap.toList $ view chainwebChains cw poison cr = mempoolAddToBadList (view chainResMempool cr) (V.singleton deadbeef) +withDbDirs :: Word -> ResourceT IO [(FilePath, FilePath)] +withDbDirs n = do + let create :: IO [(FilePath, FilePath)] + create = do + forM [0 .. n - 1] $ \nid -> do + targetDir1 <- getCanonicalTemporaryDirectory + targetDir2 <- getCanonicalTemporaryDirectory + + dir1 <- createTempDirectory targetDir1 ("pactdb-dir-" ++ show nid) + dir2 <- createTempDirectory targetDir2 ("rocksdb-dir-" ++ show nid) + + pure (dir1, dir2) + + let destroy :: [(FilePath, FilePath)] -> IO () + destroy m = flip foldMap m $ \(d1, d2) -> do + ignoringIOErrors $ do + removeDirectoryRecursive d1 + removeDirectoryRecursive d2 + + (_, m) <- allocate create destroy + pure m + where + ignoringIOErrors :: (MonadCatch m) => m () -> m () + ignoringIOErrors ioe = ioe `catch` (\(_ :: IOError) -> pure ()) + deadbeef :: TransactionHash deadbeef = TransactionHash "deadbeefdeadbeefdeadbeefdeadbeef" diff --git a/test/SlowTests.hs b/test/SlowTests.hs index 85bd784023..b92a229384 100644 --- a/test/SlowTests.hs +++ b/test/SlowTests.hs @@ -34,6 +34,7 @@ suite :: TestTree suite = testGroup "ChainwebSlowTests" [ Chainweb.Test.MultiNode.test loglevel (timedConsensusVersion petersonChainGraph twentyChainGraph) 10 30 , Chainweb.Test.MultiNode.replayTest loglevel (fastForkingCpmTestVersion pairChainGraph) 6 + , Chainweb.Test.MultiNode.compactAndResumeTest loglevel (fastForkingCpmTestVersion pairChainGraph) 6 , testGroup "Network.X05.SelfSigned.Test" [ Network.X509.SelfSigned.Test.tests ] diff --git a/tools/cwtool/CwTool.hs b/tools/cwtool/CwTool.hs index 2ef8c6a13b..c1d552e68a 100644 --- a/tools/cwtool/CwTool.hs +++ b/tools/cwtool/CwTool.hs @@ -9,6 +9,9 @@ import System.Environment import System.Exit import Text.Printf +import Chainweb.Pact.Backend.Compaction (main) +import Chainweb.Pact.Backend.PactState (pactDiffMain) + import qualified CheckpointerDBChecksum import qualified Ea import qualified EncodeDecodeB64Util @@ -99,6 +102,14 @@ topLevelCommands = "tx-sim" "Simulate tx execution against real pact dbs" TxSimulator.simulateMain + , CommandSpec + "compact" + "Compact pact database" + Chainweb.Pact.Backend.Compaction.main + , CommandSpec + "pact-diff" + "Diff the latest state of two pact databases" + Chainweb.Pact.Backend.PactState.pactDiffMain , CommandSpec "calculate-release" "Calculate next service date and block heights for upgrades" From 56b52150637088c5e8175cea1c474d119aaf0a0d Mon Sep 17 00:00:00 2001 From: chessai Date: Fri, 17 Nov 2023 22:07:45 -0600 Subject: [PATCH 2/8] only flush the logger handle if it's open. don't redundantly log the very first query (#1781) --- src/Chainweb/Pact/Backend/Compaction.hs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Chainweb/Pact/Backend/Compaction.hs b/src/Chainweb/Pact/Backend/Compaction.hs index 9df0dbf12a..1e4448b6cd 100644 --- a/src/Chainweb/Pact/Backend/Compaction.hs +++ b/src/Chainweb/Pact/Backend/Compaction.hs @@ -135,11 +135,14 @@ withPerChainFileLogger logDir chainId ll f = do done <- newMVar False void $ forkIO $ fix $ \go -> do doneYet <- readMVar done + let flush = do + w <- IO.hIsOpen h + when w (IO.hFlush h) when (not doneYet) $ do - IO.hFlush h + flush threadDelay 5_000_000 go - IO.hFlush h + flush withLogger defaultLoggerConfig b $ \l -> do let logger = setComponent "compaction" @@ -617,7 +620,7 @@ compactAll :: CompactConfig -> IO () compactAll CompactConfig{..} = do latestBlockHeightChain0 <- do let cid = unsafeChainId 0 - withDefaultLogger Debug $ \logger -> do + withDefaultLogger Error $ \logger -> do let resetDb = False withSqliteDb cid logger ccDbDir resetDb $ \(SQLiteEnv db _) -> do runCompactM (CompactEnv logger db []) getLatestBlockHeight From da03ee95ecd610a13646801d32695df865c4e75d Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 22 Nov 2023 21:28:36 +0100 Subject: [PATCH 3/8] Move the shift to using Musl back to block height 2_939_323 (again) (#1782) --- src/Chainweb/Version/Mainnet.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Chainweb/Version/Mainnet.hs b/src/Chainweb/Version/Mainnet.hs index 75494fb991..bd038a8d5c 100644 --- a/src/Chainweb/Version/Mainnet.hs +++ b/src/Chainweb/Version/Mainnet.hs @@ -134,7 +134,7 @@ mainnet = ChainwebVersion Chainweb213Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_447_315) -- 2022-02-26T00:00:00+00:00 Chainweb214Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_605_663) -- 2022-04-22T00:00:00+00:00 Chainweb215Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_766_630) -- 2022-06-17T00:00:00+00:00 - Pact44NewTrans -> AllChains (ForkAtBlockHeight $ BlockHeight 2_965_885) -- Todo: add date + Pact44NewTrans -> AllChains (ForkAtBlockHeight $ BlockHeight 2_939_323) -- Todo: add date Chainweb216Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_988_324) -- 2022-09-02T00:00:00+00:00 Chainweb217Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 3_250_348) -- 2022-12-02T00:00:00+00:00 Chainweb218Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 3_512_363) -- 2023-03-03 00:00:00+00:00 From 8d05f747416ce5fb82a93b9d41113984bb4987e2 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Thu, 23 Nov 2023 16:49:17 -0500 Subject: [PATCH 4/8] Webauthn jwk formats (#1779) * wip command verifier * Thread KeysetPublicKey through chainweb * Migrate to KeysetPublicKey in tests * Work the validateCommand function with chainid and version through more of chainweb. Next stop: execBlock * Update pact * Revert KeysetPublicKey * fix tests * Update git sha for pact * supply argument to enforceKeyFormats * Update pact * Fix tests * fix cwtool * Fork the valid webauthn signature provenance * WIP webauthn sig encoding fork tests * wip * One encoding per signature * Use assertCommand in tests * Add tests for invalid transactions, and for keyset prefixes * make post-fork test pass * wip remote test * RestAPI test * merge master * Fix type errors in benchmarks * Revert the changes to webauthn signature encoding * oops * wip * WebAuthn yes * tests for poseidon * Fixup poseidon tests * update cabal.project.freeze * Fix freeze file * fix tests * bump pact version for new primitive * Add a TODO for formalizing how applyLocal flags fork without preflight * bump pact version * revert spelling change on imports * Bring back IsWebAuthnPrefixLegal * Specify Version in PactSingleChainTests * Make Version optional in buildCwCmd * Fix gas logging * Revert "Make Version optional in buildCwCmd" This reverts commit 71a3d0524cf5745748ad7dbb5725c79116b191b4. * Fix tests * Delete commented line Co-authored-by: Jose * Amend test name Co-authored-by: Jose Cardona * Fix fork heights Co-authored-by: Jose * Delete unnecessary do and network validation Co-authored-by: Jose * Inline validPubKey Co-authored-by: Jose * Address feedback Co-authored-by: Jose --------- Co-authored-by: Greg Hale Co-authored-by: rsoeldner Co-authored-by: jmcardon Co-authored-by: Jose --- bench/Chainweb/Pact/Backend/ForkingBench.hs | 27 ++- cabal.project | 4 +- cabal.project.freeze | 146 ++---------- chainweb.cabal | 2 +- src/Chainweb/Miner/Config.hs | 1 - src/Chainweb/Pact/Backend/ChainwebPactDb.hs | 2 +- .../Pact/Backend/RelationalCheckpointer.hs | 2 +- src/Chainweb/Pact/PactService.hs | 4 +- src/Chainweb/Pact/PactService/ExecBlock.hs | 3 +- src/Chainweb/Pact/RestAPI/Server.hs | 63 +++-- src/Chainweb/Pact/TransactionExec.hs | 11 +- src/Chainweb/Pact/Utils.hs | 13 +- src/Chainweb/Pact/Validations.hs | 43 +++- src/Chainweb/Rosetta/RestAPI/Server.hs | 16 +- src/Chainweb/Rosetta/Utils.hs | 7 +- src/Chainweb/Transaction.hs | 12 +- src/Chainweb/Version.hs | 9 +- src/Chainweb/Version/Development.hs | 3 +- src/Chainweb/Version/Guards.hs | 25 +- src/Chainweb/Version/Mainnet.hs | 3 +- src/Chainweb/Version/Testnet.hs | 3 +- test/Chainweb/Test/Cut/TestBlockDb.hs | 2 +- test/Chainweb/Test/Pact/PactExec.hs | 64 ++--- test/Chainweb/Test/Pact/PactMultiChainTest.hs | 219 +++++++++++++++--- test/Chainweb/Test/Pact/PactReplay.hs | 8 +- .../Chainweb/Test/Pact/PactSingleChainTest.hs | 36 +-- test/Chainweb/Test/Pact/RemotePactTest.hs | 79 ++++--- test/Chainweb/Test/Pact/SPV.hs | 66 +++--- test/Chainweb/Test/Pact/TTL.hs | 4 +- test/Chainweb/Test/Pact/Utils.hs | 121 +++++++--- test/Chainweb/Test/Rosetta/RestAPI.hs | 23 +- test/Chainweb/Test/TestVersions.hs | 7 +- test/golden/create-accounts-expected.txt | 2 +- test/golden/create-table-expected.txt | 2 +- test/golden/new-block-0-expected.txt | 50 ++-- test/golden/transfer-accounts-expected.txt | 2 +- test/pact/contTXOUTNew.pact | 6 +- test/pact/continuation-gas-payer.pact | 2 +- test/pact/tfrTXOUTNew.pact | 4 +- tools/cwtool/TxSimulator.hs | 2 +- tools/ea/Ea.hs | 5 + 41 files changed, 673 insertions(+), 430 deletions(-) diff --git a/bench/Chainweb/Pact/Backend/ForkingBench.hs b/bench/Chainweb/Pact/Backend/ForkingBench.hs index df780f3292..21336fd60b 100644 --- a/bench/Chainweb/Pact/Backend/ForkingBench.hs +++ b/bench/Chainweb/Pact/Backend/ForkingBench.hs @@ -263,7 +263,7 @@ data Resources , blockHeaderDb :: !BlockHeaderDb , pactService :: !(Async (), PactQueue) , mainTrunkBlocks :: ![T3 ParentHeader BlockHeader PayloadWithOutputs] - , coinAccounts :: !(MVar (Map Account (NonEmpty Ed25519KeyPairCaps))) + , coinAccounts :: !(MVar (Map Account (NonEmpty (DynKeyPair, [SigCapability])))) , nonceCounter :: !(IORef Word64) , txPerBlock :: !(IORef Int) , sqlEnv :: !SQLiteEnv @@ -361,7 +361,7 @@ withResources rdb trunkLength logLevel compact f = C.envWithCleanup create destr -- | Mempool Access -- -testMemPoolAccess :: IORef Int -> MVar (Map Account (NonEmpty Ed25519KeyPairCaps)) -> IO MemPoolAccess +testMemPoolAccess :: IORef Int -> MVar (Map Account (NonEmpty (DynKeyPair, [SigCapability]))) -> IO MemPoolAccess testMemPoolAccess txsPerBlock accounts = do return $ mempty { mpaGetBlock = \bf validate bh hash header -> do @@ -402,7 +402,7 @@ testMemPoolAccess txsPerBlock accounts = do Right tx -> return tx return $! txs - mkTransferCaps :: ReceiverName -> Amount -> (Account, NonEmpty Ed25519KeyPairCaps) -> (Account, NonEmpty Ed25519KeyPairCaps) + mkTransferCaps :: ReceiverName -> Amount -> (Account, NonEmpty (DynKeyPair, [SigCapability])) -> (Account, NonEmpty (DynKeyPair, [SigCapability])) mkTransferCaps (ReceiverName (Account r)) (Amount m) (s@(Account ss),ks) = (s, (caps <$) <$> ks) where caps = [gas,tfr] @@ -431,7 +431,7 @@ createCoinAccount :: ChainwebVersion -> PublicMeta -> String - -> IO (NonEmpty Ed25519KeyPairCaps, Command Text) + -> IO (NonEmpty (DynKeyPair, [SigCapability]), Command Text) createCoinAccount v meta name = do sender00Keyset <- NEL.fromList <$> getKeyset "sender00" nameKeyset <- NEL.fromList <$> getKeyset name @@ -444,13 +444,14 @@ createCoinAccount v meta name = do isSenderAccount name' = elem name' (map getAccount coinAccountNames) - getKeyset :: String -> IO [Ed25519KeyPairCaps] + getKeyset :: String -> IO [(DynKeyPair, [SigCapability])] getKeyset s | isSenderAccount s = do keypair <- stockKey (T.pack s) mkKeyPairs [keypair] - | otherwise = (\k -> [(k, [])]) <$> genKeyPair + | otherwise = (\k -> [(DynEd25519KeyPair k, [])]) <$> generateEd25519KeyPair + attachCaps :: String -> String -> Decimal -> NonEmpty (DynKeyPair, [SigCapability]) -> NonEmpty (DynKeyPair, [SigCapability]) attachCaps s rcvr m ks = (caps <$) <$> ks where caps = [gas, tfr] @@ -474,7 +475,7 @@ stockKey s = do stockKeyFile :: ByteString stockKeyFile = $(embedFile "pact/genesis/devnet/keys.yaml") -createCoinAccounts :: ChainwebVersion -> PublicMeta -> IO (NonEmpty (Account, NonEmpty Ed25519KeyPairCaps, Command Text)) +createCoinAccounts :: ChainwebVersion -> PublicMeta -> IO (NonEmpty (Account, NonEmpty (DynKeyPair, [SigCapability]), Command Text)) createCoinAccounts v meta = traverse (go <*> createCoinAccount v meta) names where go a m = do @@ -484,12 +485,16 @@ createCoinAccounts v meta = traverse (go <*> createCoinAccount v meta) names names :: NonEmpty String names = NEL.map safeCapitalize . NEL.fromList $ Prelude.take 2 $ words "mary elizabeth patricia jennifer linda barbara margaret susan dorothy jessica james john robert michael william david richard joseph charles thomas" -formatB16PubKey :: Ed25519KeyPair -> Text -formatB16PubKey = toB16Text . getPublic +formatB16PubKey :: DynKeyPair -> Text +formatB16PubKey = \case + DynEd25519KeyPair kp -> toB16Text $ getPublic kp + DynWebAuthnKeyPair _ pub _ -> toB16Text $ exportWebAuthnPublicKey pub safeCapitalize :: String -> String safeCapitalize = maybe [] (uncurry (:) . bimap toUpper (Prelude.map toLower)) . Data.List.uncons + +-- TODO: Use the new `assertCommand` function. validateCommand :: Command Text -> Either String ChainwebTransaction validateCommand cmdText = case verifyCommand cmdBS of ProcSucc cmd -> Right (mkPayloadWithTextOld <$> cmd) @@ -501,7 +506,7 @@ validateCommand cmdText = case verifyCommand cmdBS of data TransferRequest = TransferRequest !SenderName !ReceiverName !Amount mkTransferRequest :: () - => M.Map Account (NonEmpty Ed25519KeyPairCaps) + => M.Map Account (NonEmpty (DynKeyPair, [SigCapability])) -> IO TransferRequest mkTransferRequest kacts = do (from, to) <- distinctAccounts (M.keys kacts) @@ -562,7 +567,7 @@ distinctAccounts xs = pick xs >>= go createTransfer :: () => ChainwebVersion -> PublicMeta - -> NEL.NonEmpty Ed25519KeyPairCaps + -> NEL.NonEmpty (DynKeyPair, [SigCapability]) -> TransferRequest -> IO (Command Text) createTransfer v meta ks request = diff --git a/cabal.project b/cabal.project index b358111d00..39a492abdd 100644 --- a/cabal.project +++ b/cabal.project @@ -63,8 +63,8 @@ package yet-another-logger source-repository-package type: git location: https://github.com/kadena-io/pact.git - tag: 9d8d0975d58fd811404130a57724bfb5a2f2af80 - --sha256: sha256-4oVJnV12kXraWRQjJJZv3wuoEGfGVFC1EIkzYJrUVs8= + tag: 79bf626042ad3dc9ed276e9b243be43584f20da2 + --sha256: sha256-ObE5aQjy/mX2PMEvtRnlJ1mG4E82PqZDDdzerd4I+HI= source-repository-package type: git diff --git a/cabal.project.freeze b/cabal.project.freeze index 631c45d997..d0ca41e727 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -6,121 +6,87 @@ constraints: any.Boolean ==0.2.4, any.Glob ==0.10.2, any.HUnit ==1.6.2.0, any.MemoTrie ==0.6.11, - MemoTrie -examples, any.NumInstances ==1.4, any.OneTuple ==0.4.1.1, any.Only ==0.1, any.QuickCheck ==2.14.3, - QuickCheck -old-random +templatehaskell, any.RSA ==2.4.1, any.SHA ==1.6.4.4, - SHA -exe, any.StateVar ==1.2.2, any.adjunctions ==4.4.2, - any.aeson ==2.2.1.0, - aeson +ordered-keymap, + any.aeson ==2.2.0.0, any.aeson-pretty ==0.8.10, - aeson-pretty -lib-only, any.ansi-terminal ==1.0, - ansi-terminal -example, any.ansi-terminal-types ==0.11.5, any.ap-normalize ==0.1.0.1, - ap-normalize -test-with-clang, any.appar ==0.1.8, any.array ==0.5.5.0, any.asn1-encoding ==0.9.6, any.asn1-parse ==0.9.5, any.asn1-types ==0.3.4, any.assoc ==1.1, - assoc +tagged, any.async ==2.2.4, - async -bench, any.atomic-primops ==0.8.4, - atomic-primops -debug, any.attoparsec ==0.14.4, - attoparsec -developer, - any.attoparsec-aeson ==2.2.0.1, any.authenticate-oauth ==1.7, any.auto-update ==0.1.6, any.base ==4.18.1.0, - any.base-compat ==0.13.1, - any.base-compat-batteries ==0.13.1, - any.base-orphans ==0.9.1, + any.base-compat ==0.13.0, + any.base-compat-batteries ==0.13.0, + any.base-orphans ==0.9.0, any.base-unicode-symbols ==0.2.4.2, - base-unicode-symbols +base-4-8 -old-base, any.base16-bytestring ==1.0.2.0, any.base64-bytestring ==1.2.1.0, any.base64-bytestring-kadena ==0.1, any.basement ==0.0.16, any.bifunctors ==5.6.1, - bifunctors +tagged, any.binary ==0.8.9.1, any.binary-orphans ==1.0.4.1, any.bitvec ==1.1.5.0, - bitvec +simd, any.blaze-builder ==0.4.2.3, any.blaze-html ==0.9.1.2, any.blaze-markup ==0.8.3.0, any.boring ==0.2.1, - boring +tagged, any.bound ==2.0.7, - bound +template-haskell, any.bsb-http-chunked ==0.0.0.4, any.byteorder ==1.0.4, any.bytes ==0.17.3, any.bytestring ==0.11.5.2, any.bytestring-builder ==0.10.8.2.0, - bytestring-builder +bytestring_has_builder, - any.cabal-doctest ==1.0.9, any.cache ==0.1.3.0, any.call-stack ==0.4.0, any.case-insensitive ==1.2.1.0, any.cassava ==0.5.3.0, - cassava -bytestring--lt-0_10_4, any.cborg ==0.2.9.0, - cborg +optimize-gmp, any.cereal ==0.5.8.3, - cereal -bytestring-builder, chainweb -debug -ed25519 -ghc-flags, any.chainweb-storage ==0.1.0.0, any.charset ==0.3.10, any.clock ==0.8.4, - clock -llvm, - any.cmdargs ==0.10.22, - cmdargs +quotation -testprog, any.code-page ==0.2.1, any.colour ==2.3.6, any.comonad ==5.0.8, - comonad +containers +distributive +indexed-traversable, any.conduit ==1.3.5, any.configuration-tools ==0.7.0, - configuration-tools -remote-configs, - any.constraints ==0.14, + any.constraints ==0.13.4, any.containers ==0.6.7, any.contiguous ==0.6.4.0, any.contravariant ==1.5.5, - contravariant +semigroups +statevar +tagged, any.cookie ==0.4.6, any.criterion ==1.6.3.0, - criterion -embed-data-files -fast, any.criterion-measurement ==0.2.1.0, - criterion-measurement -fast, any.crypto-api ==0.13.3, - crypto-api -all_cpolys, any.crypto-pubkey-types ==0.4.3, any.cryptohash-md5 ==0.11.101.0, any.cryptohash-sha1 ==0.11.101.0, any.crypton ==0.33, - crypton -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq +support_pclmuldq +support_rdrand -support_sse +use_target_attributes, any.crypton-connection ==0.3.1, any.crypton-x509 ==1.7.6, any.crypton-x509-store ==1.6.9, any.crypton-x509-system ==1.6.7, any.crypton-x509-validation ==1.6.12, any.cryptonite ==0.30, - cryptonite -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq -support_pclmuldq +support_rdrand -support_sse +use_target_attributes, any.cuckoo ==0.3.1, - cuckoo -mwc-random -pcg-random, any.data-bword ==0.1.0.2, any.data-default ==0.7.1.1, any.data-default-class ==0.1.2.0, @@ -134,19 +100,14 @@ constraints: any.Boolean ==0.2.4, any.deepseq ==1.4.8.1, any.dense-linear-algebra ==0.1.0.0, any.deriving-compat ==0.6.5, - deriving-compat +base-4-9 +new-functor-classes +template-haskell-2-11, any.digraph ==0.3.0, any.direct-sqlite ==2.3.28, - direct-sqlite +fulltextsearch +haveusleep +json1 -systemlib +urifilenames, any.directory ==1.3.8.1, any.distributive ==0.6.2.1, - distributive +semigroups +tagged, any.dlist ==1.0, - dlist -werror, any.easy-file ==0.2.5, any.enclosed-exceptions ==1.0.3, any.entropy ==0.4.1.10, - entropy -donotgetentropy, any.errors ==2.3.0, any.ethereum ==0.1.0.1, ethereum -ethhash -openssl-use-pkg-config, @@ -158,7 +119,6 @@ constraints: any.Boolean ==0.2.4, any.finite-typelits ==0.1.6.0, any.free ==5.2, any.generic-data ==1.1.0.0, - generic-data -enable-inspect, any.generically ==0.1.1, any.ghc-bignum ==1.3, any.ghc-boot-th ==9.6.3, @@ -168,100 +128,75 @@ constraints: any.Boolean ==0.2.4, any.groups ==0.5.3, any.growable-vector ==0.1, any.half ==0.3.1, - any.happy ==1.20.1.1, any.hashable ==1.4.3.0, - hashable +integer-gmp -random-initial-seed, any.hashes ==0.2.3, - hashes -benchmark-cryptonite -openssl-use-pkg-config -test-cryptonite +with-openssl, any.haskell-lexer ==1.1.1, any.heaps ==0.4, any.hourglass ==0.2.12, - any.hsc2hs ==0.68.10, - hsc2hs -in-ghc-tree, any.http-api-data ==0.6, - http-api-data -use-text-show, any.http-client ==0.7.14, - http-client +network-uri, any.http-client-tls ==0.3.6.3, any.http-date ==0.0.11, any.http-media ==0.8.1.1, any.http-types ==0.12.3, - any.http2 ==4.2.2, - http2 -devel -h2spec, + any.http2 ==4.2.0, any.indexed-list-literals ==0.2.1.3, - any.indexed-traversable ==0.1.3, + any.indexed-traversable ==0.1.2.1, any.indexed-traversable-instances ==0.1.1.2, - any.integer-conversion ==0.1.0.1, + any.integer-conversion ==0.1, any.integer-gmp ==1.1, any.integer-logarithms ==1.0.3.1, - integer-logarithms -check-bounds +integer-gmp, any.invariant ==0.6.2, any.iproute ==1.7.12, any.ixset-typed ==0.5, any.js-chart ==2.9.4.1, any.kan-extensions ==5.2.5, any.lens ==5.2.3, - lens -benchmark-uniplate -dump-splices +inlining -j +test-hunit +test-properties +test-templates +trustworthy, any.lens-aeson ==1.2.3, any.libBF ==0.6.6, - libBF -system-libbf, any.libyaml ==0.1.2, - libyaml -no-unicode -system-libyaml, any.lifted-base ==0.2.3.12, any.loglevel ==0.1.0.0, any.managed ==1.0.10, any.massiv ==1.0.4.0, - massiv -unsafe-checks, - any.math-functions ==0.3.4.3, - math-functions +system-erf +system-expm1, + any.math-functions ==0.3.4.2, any.megaparsec ==9.5.0, - megaparsec -dev, any.memory ==0.18.0, - memory +support_bytestring +support_deepseq, any.merkle-log ==0.2.0, any.microlens ==0.4.13.1, any.microstache ==1.0.2.3, - any.mime-types ==0.1.2.0, + any.mime-types ==0.1.1.0, any.mmorph ==1.2.0, any.mod ==0.2.0.1, - mod +semirings +vector, any.monad-control ==1.0.3.1, any.mono-traversable ==1.0.15.3, any.mtl ==2.3.1, any.mtl-compat ==0.2.2, - mtl-compat -two-point-one -two-point-two, any.mwc-probability ==2.3.1, any.mwc-random ==0.15.0.2, any.network ==3.1.4.0, - network -devel, - any.network-byte-order ==0.1.7, + any.network-byte-order ==0.1.6, any.network-info ==0.2.1, any.network-uri ==2.6.4.2, any.newtype-generics ==0.6.2, any.nothunks ==0.1.4, - nothunks +bytestring +text +vector, any.old-locale ==1.0.0.7, any.old-time ==1.1.0.3, any.optparse-applicative ==0.18.1.0, - optparse-applicative +process, any.pact ==4.9, pact -build-tool +cryptonite-ed25519 -tests-in-lib, any.pact-json ==0.1.0.0, any.pact-time ==0.2.0.2, - pact-time -with-time, any.parallel ==3.2.2.0, any.parsec ==3.1.16.1, any.parser-combinators ==1.3.0, - parser-combinators -dev, any.parsers ==0.12.11, - parsers +attoparsec +binary +parsec, + any.patience ==0.3, any.pem ==0.2.4, any.poly ==0.5.1.0, - poly +sparse, any.pretty ==1.1.3.6, any.pretty-show ==1.10, any.prettyprinter ==1.7.1, - prettyprinter -buildreadme +text, any.prettyprinter-ansi-terminal ==1.1.3, any.primitive ==0.8.0.0, any.primitive-unlifted ==2.1.0.0, @@ -269,20 +204,16 @@ constraints: any.Boolean ==0.2.4, any.profunctors ==5.6.2, any.psqueues ==0.2.7.3, any.pvar ==1.0.0.0, - any.quickcheck-instances ==0.3.30, - quickcheck-instances -bytestring-builder, + any.quickcheck-instances ==0.3.29.1, any.random ==1.2.1.1, any.recv ==0.1.0, any.reducers ==3.12.4, any.reflection ==2.1.7, - reflection -slow +template-haskell, any.regex-base ==0.94.0.2, any.regex-tdfa ==1.3.2.2, - regex-tdfa +doctest -force-o2, any.resource-pool ==0.4.0.0, any.resourcet ==1.3.0, any.retry ==0.9.3.1, - retry -lib-werror, any.rocksdb-haskell-kadena ==1.1.0, rocksdb-haskell-kadena -with-tbb, any.rosetta ==1.0.1, @@ -294,17 +225,11 @@ constraints: any.Boolean ==0.2.4, any.sbv ==9.2, any.scheduler ==2.0.0.1, any.scientific ==0.3.7.0, - scientific -bytestring-builder -integer-simple, any.semialign ==1.3, - semialign +semigroupoids, any.semigroupoids ==6.0.0.1, - semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, any.semigroups ==0.20, - semigroups +binary +bytestring -bytestring-builder +containers +deepseq +hashable +tagged +template-haskell +text +transformers +unordered-containers, any.semirings ==0.6, - semirings +containers +unordered-containers, any.serialise ==0.2.6.0, - serialise +newtime15, any.servant ==0.20.1, any.servant-client ==0.20, any.servant-client-core ==0.20, @@ -312,33 +237,26 @@ constraints: any.Boolean ==0.2.4, any.sha-validation ==0.1.0.0, any.show-combinators ==0.2.0.0, any.simple-sendfile ==0.2.32, - simple-sendfile +allow-bsd -fallback, any.singleton-bool ==0.1.7, any.socks ==0.6.1, any.some ==1.0.5, - some +newtype-unsafe, any.sop-core ==0.5.0.2, - any.split ==0.2.4, + any.split ==0.2.3.5, any.splitmix ==0.1.0.5, - splitmix -optimised-mixer, any.statistics ==0.16.2.1, any.stm ==2.5.1.0, any.stm-chans ==3.0.0.9, any.stopwatch ==0.1.0.6, - stopwatch -test_delay_upper_bound -test_threaded, any.streaming ==0.2.4.0, any.streaming-commons ==0.2.2.6, - streaming-commons -use-bytestring-builder, any.strict ==0.5, any.strict-concurrency ==0.2.4.3, any.string-conversions ==0.4.0.1, any.syb ==0.7.2.4, + any.system-cxx-std-lib ==1.0, any.tagged ==0.8.8, - tagged +deepseq +transformers, any.tasty ==1.5, - tasty +unix, any.tasty-golden ==2.3.5, - tasty-golden -build-example, any.tasty-hunit ==0.10.1, any.tasty-json ==0.1.0.0, any.tasty-quickcheck ==0.10.3, @@ -347,75 +265,55 @@ constraints: any.Boolean ==0.2.4, any.text ==2.0.2, any.text-iso8601 ==0.1, any.text-short ==0.1.5, - text-short -asserts, - any.th-abstraction ==0.6.0.0, + any.th-abstraction ==0.5.0.0, any.th-compat ==0.1.4, any.these ==1.2, any.time ==1.12.2, any.time-compat ==1.9.6.1, - time-compat -old-locale, any.time-locale-compat ==0.1.1.5, - time-locale-compat +old-locale, any.time-manager ==0.0.1, any.tls ==1.9.0, - tls +compat -hans +network, any.tls-session-manager ==0.0.4, any.token-bucket ==0.1.0.1, - token-bucket +use-cbits, any.transformers ==0.6.1.0, any.transformers-base ==0.4.6, - transformers-base +orphaninstances, any.transformers-compat ==0.7.2, - transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, any.trifecta ==2.1.3, any.type-equality ==1, - any.typed-process ==0.2.11.1, + any.typed-process ==0.2.11.0, any.uniplate ==1.6.13, any.unix ==2.8.1.0, any.unix-compat ==0.7, - unix-compat -old-time, any.unix-time ==0.4.11, any.unliftio ==0.2.25.0, any.unliftio-core ==0.2.1.0, any.unordered-containers ==0.2.19.1, - unordered-containers -debug, any.utf8-string ==1.0.2, any.uuid ==1.3.15, - any.uuid-types ==1.0.5.1, + any.uuid-types ==1.0.5, any.validation ==1.1.3, any.vault ==0.3.1.5, - vault +useghc, - any.vector ==0.13.1.0, - vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector ==0.13.0.0, any.vector-algorithms ==0.9.0.1, - vector-algorithms +bench +boundschecks -internalchecks -llvm +properties -unsafechecks, any.vector-binary-instances ==0.2.5.2, any.vector-sized ==1.5.0, any.vector-space ==0.16, any.vector-stream ==0.1.0.0, any.vector-th-unbox ==0.2.2, any.void ==0.7.3, - void -safe, any.wai ==3.2.3, any.wai-app-static ==3.1.8, - wai-app-static +crypton -print, any.wai-cors ==0.2.7, any.wai-extra ==3.1.13.0, - wai-extra -build-example, any.wai-logger ==2.4.0, any.wai-middleware-throttle ==0.3.0.1, any.wai-middleware-validation ==0.1.0.2, any.warp ==3.3.29, - warp +allow-sendfilefd -network-bytestring -warp-debug +x509, - any.warp-tls ==3.4.3, + any.warp-tls ==3.4.2, any.witherable ==0.4.2, any.word8 ==0.1.3, any.wreq ==0.5.4.2, - wreq -aws -developer +doctest -httpbin, any.yaml ==0.11.11.2, - yaml +no-examples +no-exe, any.yet-another-logger ==0.4.2, - yet-another-logger -tbmqueue, - any.zlib ==0.6.3.0, - zlib -bundled-c-zlib -non-blocking-ffi -pkg-config -index-state: hackage.haskell.org 2023-10-18T16:48:34Z + any.zlib ==0.6.3.0 +index-state: hackage.haskell.org 2023-11-08T16:26:43Z diff --git a/chainweb.cabal b/chainweb.cabal index fbbbc916ac..531e96c642 100644 --- a/chainweb.cabal +++ b/chainweb.cabal @@ -1,7 +1,7 @@ cabal-version: 3.8 name: chainweb -version: 2.21 +version: 2.22 synopsis: A Proof-of-Work Parallel-Chain Architecture for Massive Throughput description: A Proof-of-Work Parallel-Chain Architecture for Massive Throughput. homepage: https://github.com/kadena-io/chainweb diff --git a/src/Chainweb/Miner/Config.hs b/src/Chainweb/Miner/Config.hs index 520783c1a6..eb5ce112bb 100644 --- a/src/Chainweb/Miner/Config.hs +++ b/src/Chainweb/Miner/Config.hs @@ -259,4 +259,3 @@ defaultNodeMining = NodeMiningConfig invalidMiner :: Miner invalidMiner = Miner "" . MinerKeys $ mkKeySet [] "keys-all" - diff --git a/src/Chainweb/Pact/Backend/ChainwebPactDb.hs b/src/Chainweb/Pact/Backend/ChainwebPactDb.hs index 96f3e8680c..31964c2850 100644 --- a/src/Chainweb/Pact/Backend/ChainwebPactDb.hs +++ b/src/Chainweb/Pact/Backend/ChainwebPactDb.hs @@ -346,7 +346,7 @@ doKeys d = do collect pb `DL.append` maybe DL.empty collect mptx let !allKeys = fmap fromString - $ msort -- becomes available with pact420Upgrade + $ msort -- becomes available with Pact42Upgrade $ LHM.sort $ dbKeys ++ memKeys return allKeys diff --git a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs index ec8d7a17b4..939d2d94de 100644 --- a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs +++ b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs @@ -159,7 +159,7 @@ doRestore v cid dbenv (Just (bh, hash)) = runBlockEnv dbenv $ do where -- Module name fix follows the restore call to checkpointer. setModuleNameFix = bsModuleNameFix .= enableModuleNameFix v cid bh - setSortedKeys = bsSortedKeys .= pact420 v cid bh + setSortedKeys = bsSortedKeys .= pact42 v cid bh setLowerCaseTables = bsLowerCaseTables .= chainweb217Pact v cid bh doRestore _ _ dbenv Nothing = runBlockEnv dbenv $ do clearPendingTxState diff --git a/src/Chainweb/Pact/PactService.hs b/src/Chainweb/Pact/PactService.hs index 15d9c45578..d7952f2bb2 100644 --- a/src/Chainweb/Pact/PactService.hs +++ b/src/Chainweb/Pact/PactService.hs @@ -750,6 +750,8 @@ execLocal cwtx preflight sigVerify rdepth = pactLabel "execLocal" $ withDiscarde pure $ LocalResultWithWarns cr' warns' Left e -> pure $ MetadataValidationFailure e _ -> liftIO $ do + -- these flags are supposed to be basically the set of flags that we would enable at the maximum block height. + -- TODO: consider making this formal, using flagsFor, and just adding FlagAllowReadInLocal. let execConfig = P.mkExecutionConfig $ [ P.FlagAllowReadInLocal | _psAllowReadsInLocal ] ++ enablePactEvents' (ctxVersion ctx) (ctxChainId ctx) (ctxCurrentBlockHeight ctx) ++ @@ -935,5 +937,3 @@ getGasModel ctx pactLabel :: (Logger logger) => Text -> PactServiceM logger tbl x -> PactServiceM logger tbl x pactLabel lbl x = localLabel ("pact-request", lbl) x - - diff --git a/src/Chainweb/Pact/PactService/ExecBlock.hs b/src/Chainweb/Pact/PactService/ExecBlock.hs index 08fafb1091..dda9b42422 100644 --- a/src/Chainweb/Pact/PactService/ExecBlock.hs +++ b/src/Chainweb/Pact/PactService/ExecBlock.hs @@ -250,13 +250,14 @@ validateChainwebTxs logger v cid cp txValidationTime bh txs doBuyGas checkTxSigs :: ChainwebTransaction -> IO (Either InsertError ChainwebTransaction) checkTxSigs t - | assertValidateSigs validSchemes hsh signers sigs = pure $ Right t + | assertValidateSigs validSchemes webAuthnPrefixLegal hsh signers sigs = pure $ Right t | otherwise = return $ Left InsertErrorInvalidSigs where hsh = P._cmdHash t sigs = P._cmdSigs t signers = P._pSigners $ payloadObj $ P._cmdPayload t validSchemes = validPPKSchemes v cid bh + webAuthnPrefixLegal = isWebAuthnPrefixLegal v cid bh initTxList :: ValidateTxs initTxList = V.map Right txs diff --git a/src/Chainweb/Pact/RestAPI/Server.hs b/src/Chainweb/Pact/RestAPI/Server.hs index f50d014a57..603db5ed1f 100644 --- a/src/Chainweb/Pact/RestAPI/Server.hs +++ b/src/Chainweb/Pact/RestAPI/Server.hs @@ -36,7 +36,7 @@ import Control.Concurrent.STM (atomically, retry) import Control.Concurrent.STM.TVar import Control.DeepSeq import Control.Lens (set, view, preview) -import Control.Monad ((<$!>), forM, mzero, when, void) +import Control.Monad import Control.Monad.Catch hiding (Handler) import Control.Monad.Reader import Control.Monad.State.Strict @@ -45,7 +45,6 @@ import Control.Monad.Trans.Maybe import Data.Aeson as Aeson import Data.Bifunctor (second) -import Data.ByteString (ByteString) import qualified Data.ByteString.Lazy as BSL import qualified Data.ByteString.Lazy.Char8 as BSL8 import qualified Data.ByteString.Short as SB @@ -117,6 +116,8 @@ import Chainweb.Transaction import qualified Chainweb.TreeDB as TreeDB import Chainweb.Utils import Chainweb.Version +import Chainweb.Pact.Validations (assertCommand) +import Chainweb.Version.Guards (isWebAuthnPrefixLegal, pactParserVersion, validPPKSchemes) import Chainweb.WebPactExecutionService import Chainweb.Storage.Table @@ -185,12 +186,13 @@ pactServer d = logger = _pactServerDataLogger d pact = _pactServerDataPact d cdb = _pactServerDataCutDb d + v = _chainwebVersion cdb pactApiHandlers - = sendHandler logger mempool + = sendHandler logger v cid mempool :<|> pollHandler logger cdb cid pact mempool :<|> listenHandler logger cdb cid pact mempool - :<|> localHandler logger pact + :<|> localHandler logger v cid pact pactSpvHandler = spvHandler logger cdb cid pactSpv2Handler = spv2Handler logger cdb cid @@ -246,12 +248,14 @@ instance ToJSON PactCmdLog where sendHandler :: Logger logger => logger + -> ChainwebVersion + -> ChainId -> MempoolBackend ChainwebTransaction -> SubmitBatch -> Handler RequestKeys -sendHandler logger mempool (SubmitBatch cmds) = Handler $ do +sendHandler logger v cid mempool (SubmitBatch cmds) = Handler $ do liftIO $ logg Info (PactCmdLogSend cmds) - case traverse validateCommand cmds of + case traverse (validateCommand v cid) cmds of Right enriched -> do let txs = V.fromList $ NEL.toList enriched -- If any of the txs in the batch fail validation, we reject them all. @@ -364,6 +368,8 @@ listenHandler logger cdb cid pact mem (ListenerRequest key) = do localHandler :: Logger logger => logger + -> ChainwebVersion + -> ChainId -> PactExecutionService -> Maybe LocalPreflightSimulation -- ^ Preflight flag @@ -373,9 +379,9 @@ localHandler -- ^ Rewind depth -> Command Text -> Handler LocalResult -localHandler logger pact preflight sigVerify rewindDepth cmd = do +localHandler logger v cid pact preflight sigVerify rewindDepth cmd = do liftIO $ logg Info $ PactCmdLogLocal cmd - cmd' <- case doCommandValidation cmd of + cmd' <- case validatedCommand of Right c -> return c Left err -> throwError $ setErrText ("Validation failed: " <> T.pack err) err400 @@ -391,7 +397,7 @@ localHandler logger pact preflight sigVerify rewindDepth cmd = do where logg = logFunctionJson (setComponent "local-handler" logger) - doCommandValidation cmdText + validatedCommand | Just NoVerify <- sigVerify = do -- -- desnote(emily): This workflow is 'Pact.Types.Command.verifyCommand' @@ -400,15 +406,15 @@ localHandler logger pact preflight sigVerify rewindDepth cmd = do -- down in the 'execLocal' code, 'noSigVerify' triggers a nop on -- checking again if 'preflight' is set. -- - let cmdBS = encodeUtf8 <$> cmdText + let payloadBS = encodeUtf8 (_cmdPayload cmd) - void $ Pact.verifyHash @'Pact.Blake2b_256 (_cmdHash cmdBS) (_cmdPayload cmdBS) - decoded <- eitherDecodeStrict' $ _cmdPayload cmdBS - p <- traverse Pact.parsePact decoded + void $ Pact.verifyHash @'Pact.Blake2b_256 (_cmdHash cmd) payloadBS + decoded <- eitherDecodeStrict' payloadBS + payloadParsed <- traverse Pact.parsePact decoded - let cmd' = cmdBS { _cmdPayload = p } - pure $ mkPayloadWithText cmdBS <$> cmd' - | otherwise = validateCommand cmd + let cmd' = cmd { _cmdPayload = (payloadBS, payloadParsed) } + pure $ mkPayloadWithText cmd' + | otherwise = validateCommand v cid cmd -- -------------------------------------------------------------------------- -- -- Cross Chain SPV Handler @@ -679,14 +685,25 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do toPactTx :: Transaction -> Maybe (Command Text) toPactTx (Transaction b) = decodeStrict' b -validateCommand :: Command Text -> Either String ChainwebTransaction -validateCommand cmdText = case verifyCommand cmdBS of - ProcSucc cmd -> Right (mkPayloadWithText cmdBS <$> cmd) - ProcFail err -> Left err - where - cmdBS :: Command ByteString - cmdBS = encodeUtf8 <$> cmdText +-- TODO: all of the functions in this module can instead grab the current block height from consensus +-- and pass it here to get a better estimate of what behavior is correct. +validateCommand :: ChainwebVersion -> ChainId -> Command Text -> Either String ChainwebTransaction +validateCommand v cid (fmap encodeUtf8 -> cmdBs) = case parsedCmd of + Right (commandParsed :: ChainwebTransaction) -> + if assertCommand + commandParsed + (validPPKSchemes v cid bh) + (isWebAuthnPrefixLegal v cid bh) + then Right commandParsed + else Left "Command failed validation" + Left e -> Left $ "Pact parsing error: " ++ e + where + bh = maxBound :: BlockHeight + decodeAndParse bs = + traverse (parsePact (pactParserVersion v cid bh)) =<< Aeson.eitherDecodeStrict' bs + parsedCmd = mkPayloadWithText <$> + cmdPayload (\bs -> (bs,) <$> decodeAndParse bs) cmdBs -- | Validate the length of the request key's underlying hash. -- diff --git a/src/Chainweb/Pact/TransactionExec.hs b/src/Chainweb/Pact/TransactionExec.hs index cbc1dbec0d..fbe3646ec4 100644 --- a/src/Chainweb/Pact/TransactionExec.hs +++ b/src/Chainweb/Pact/TransactionExec.hs @@ -343,7 +343,7 @@ flagsFor :: ChainwebVersion -> V.ChainId -> BlockHeight -> S.Set ExecutionFlag flagsFor v cid bh = S.fromList $ concat [ enablePactEvents' v cid bh , enablePact40 v cid bh - , enablePact420 v cid bh + , enablePact42 v cid bh , enforceKeysetFormats' v cid bh , enablePactModuleMemcheck v cid bh , enablePact43 v cid bh @@ -356,6 +356,7 @@ flagsFor v cid bh = S.fromList $ concat , enablePact48 v cid bh , disableReturnRTC v cid bh , enablePact49 v cid bh + , enablePact410 v cid bh ] applyCoinbase @@ -382,6 +383,7 @@ applyCoinbase v logger dbEnv (Miner mid mks@(MinerKeys mk)) reward@(ParsedDecima | fork1_3InEffect || enablePC = do when chainweb213Pact' $ enforceKeyFormats (\k -> throwM $ CoinbaseFailure $ "Invalid miner key: " <> sshow k) + (validKeyFormats v (ctxChainId txCtx) (ctxCurrentBlockHeight txCtx)) mk let (cterm, cexec) = mkCoinbaseTerm mid mks reward interp = Interpreter $ \_ -> do put initState; fmap pure (eval cterm) @@ -751,8 +753,8 @@ enforceKeysetFormats' v cid bh = [FlagEnforceKeyFormats | enforceKeysetFormats v enablePact40 :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] enablePact40 v cid bh = [FlagDisablePact40 | not (pact4Coin3 v cid bh)] -enablePact420 :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] -enablePact420 v cid bh = [FlagDisablePact420 | not (pact420 v cid bh)] +enablePact42 :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] +enablePact42 v cid bh = [FlagDisablePact42 | not (pact42 v cid bh)] enablePactModuleMemcheck :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] enablePactModuleMemcheck v cid bh = [FlagDisableInlineMemCheck | not (chainweb213Pact v cid bh)] @@ -784,6 +786,9 @@ enablePact48 v cid bh = [FlagDisablePact48 | not (chainweb220Pact v cid bh)] enablePact49 :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] enablePact49 v cid bh = [FlagDisablePact49 | not (chainweb221Pact v cid bh)] +enablePact410 :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] +enablePact410 v cid bh = [FlagDisablePact410 | not (chainweb222Pact v cid bh)] + -- | Even though this is not forking, abstracting for future shutoffs disableReturnRTC :: ChainwebVersion -> V.ChainId -> BlockHeight -> [ExecutionFlag] disableReturnRTC _v _cid _bh = [FlagDisableRuntimeReturnTypeChecking] diff --git a/src/Chainweb/Pact/Utils.hs b/src/Chainweb/Pact/Utils.hs index 90930fc273..c42ad1e1e7 100644 --- a/src/Chainweb/Pact/Utils.hs +++ b/src/Chainweb/Pact/Utils.hs @@ -17,7 +17,6 @@ module Chainweb.Pact.Utils , toTxCreationTime -- * k:account helper functions - , validatePubKey , validateKAccount , extractPubKeyFromKAccount , generateKAccountFromPubKey @@ -38,7 +37,7 @@ import Pact.Parse import qualified Pact.Types.ChainId as P import qualified Pact.Types.Term as P import Pact.Types.ChainMeta -import Pact.Types.KeySet (validateKeyFormat) +import Pact.Types.KeySet (ed25519HexFormat) import qualified Pact.JSON.Encode as J @@ -64,15 +63,13 @@ toTxCreationTime (Time timespan) = TxCreationTime $ ParsedInteger $ fromIntegral $ timeSpanToSeconds timespan -validatePubKey :: P.PublicKeyText -> Bool -validatePubKey = validateKeyFormat validateKAccount :: T.Text -> Bool validateKAccount acctName = case T.take 2 acctName of "k:" -> let pubKey = P.PublicKeyText $ T.drop 2 acctName - in validateKeyFormat pubKey + in ed25519HexFormat pubKey _ -> False extractPubKeyFromKAccount :: T.Text -> Maybe P.PublicKeyText @@ -83,13 +80,15 @@ extractPubKeyFromKAccount kacct generateKAccountFromPubKey :: P.PublicKeyText -> Maybe T.Text generateKAccountFromPubKey pubKey - | validatePubKey pubKey = + | ed25519HexFormat pubKey = let pubKeyText = P._pubKey pubKey in Just $ "k:" <> pubKeyText | otherwise = Nothing + -- Warning: Only use if already certain that PublicKeyText -- is valid. +-- Note: We are assuming the k: account is ED25519. pubKeyToKAccountKeySet :: P.PublicKeyText -> P.KeySet pubKeyToKAccountKeySet pubKey = P.mkKeySet [pubKey] "keys-all" @@ -112,4 +111,4 @@ emptyPayload :: PayloadWithOutputs emptyPayload = newPayloadWithOutputs miner coinbase mempty where miner = MinerData $ J.encodeStrict noMiner - coinbase = noCoinbaseOutput \ No newline at end of file + coinbase = noCoinbaseOutput diff --git a/src/Chainweb/Pact/Validations.hs b/src/Chainweb/Pact/Validations.hs index d893867255..d8a8e7eea0 100644 --- a/src/Chainweb/Pact/Validations.hs +++ b/src/Chainweb/Pact/Validations.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE DataKinds #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -- | @@ -22,8 +23,10 @@ module Chainweb.Pact.Validations , assertNetworkId , assertSigSize , assertTxSize +, IsWebAuthnPrefixLegal(..) , assertValidateSigs , assertTxTimeRelativeToParent +, assertCommand -- * Defaults , defaultMaxCommandUserSigListSize , defaultMaxCoinDecimalPlaces @@ -35,8 +38,11 @@ import Control.Lens import Data.Decimal (decimalPlaces) import Data.Maybe (isJust, catMaybes, fromMaybe) +import Data.Either (isRight) import Data.List.NonEmpty (NonEmpty, nonEmpty) import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.ByteString.Short as SBS import Data.Word (Word8) -- internal modules @@ -47,15 +53,16 @@ import Chainweb.Pact.Types import Chainweb.Pact.Utils (fromPactChainId) import Chainweb.Pact.Service.Types import Chainweb.Time (Seconds(..), Time(..), secondsToTimeSpan, scaleTimeSpan, second, add) -import Chainweb.Transaction (cmdTimeToLive, cmdCreationTime) +import Chainweb.Transaction (cmdTimeToLive, cmdCreationTime, PayloadWithText, payloadBytes, payloadObj, IsWebAuthnPrefixLegal(..)) import Chainweb.Version -import Chainweb.Version.Guards (validPPKSchemes) +import Chainweb.Version.Guards (isWebAuthnPrefixLegal, validPPKSchemes) import qualified Pact.Types.Gas as P import qualified Pact.Types.Hash as P import qualified Pact.Types.ChainId as P import qualified Pact.Types.Command as P import qualified Pact.Types.ChainMeta as P +import qualified Pact.Types.KeySet as P import qualified Pact.Parse as P @@ -73,6 +80,7 @@ assertLocalMetadata cmd@(P.Command pay sigs hsh) txCtx sigVerify = do let bh = ctxCurrentBlockHeight txCtx let validSchemes = validPPKSchemes v cid bh + let webAuthnPrefixLegal = isWebAuthnPrefixLegal v cid bh let P.PublicMeta pcid _ gl gp _ _ = P._pMeta pay nid = P._pNetworkId pay @@ -85,7 +93,7 @@ assertLocalMetadata cmd@(P.Command pay sigs hsh) txCtx sigVerify = do , eUnless "Gas price decimal precision too high" $ assertGasPrice gp , eUnless "Network id mismatch" $ assertNetworkId v nid , eUnless "Signature list size too big" $ assertSigSize sigs - , eUnless "Invalid transaction signatures" $ sigValidate validSchemes signers + , eUnless "Invalid transaction signatures" $ sigValidate validSchemes webAuthnPrefixLegal signers , eUnless "Tx time outside of valid range" $ assertTxTimeRelativeToParent pct cmd ] @@ -93,9 +101,9 @@ assertLocalMetadata cmd@(P.Command pay sigs hsh) txCtx sigVerify = do Nothing -> Right () Just vs -> Left vs where - sigValidate validSchemes signers + sigValidate validSchemes webAuthnPrefixLegal signers | Just NoVerify <- sigVerify = True - | otherwise = assertValidateSigs validSchemes hsh signers sigs + | otherwise = assertValidateSigs validSchemes webAuthnPrefixLegal hsh signers sigs pct = ParentCreationTime . _blockCreationTime @@ -154,13 +162,19 @@ assertTxSize initialGas gasLimit = initialGas < fromIntegral gasLimit -- | Check and assert that signers and user signatures are valid for a given -- transaction hash. -- -assertValidateSigs :: [P.PPKScheme] -> P.PactHash -> [P.Signer] -> [P.UserSig] -> Bool -assertValidateSigs validSchemes hsh signers sigs +assertValidateSigs :: [P.PPKScheme] -> IsWebAuthnPrefixLegal -> P.PactHash -> [P.Signer] -> [P.UserSig] -> Bool +assertValidateSigs validSchemes webAuthnPrefixLegal hsh signers sigs | length signers /= length sigs = False | otherwise = and $ zipWith verifyUserSig sigs signers where verifyUserSig sig signer = - let sigScheme = fromMaybe P.ED25519 (P._siScheme signer) - in sigScheme `elem` validSchemes && P.verifyUserSig hsh sig signer + let + sigScheme = fromMaybe P.ED25519 (P._siScheme signer) + okScheme = sigScheme `elem` validSchemes + okPrefix = + webAuthnPrefixLegal == WebAuthnPrefixLegal || + not (P.webAuthnPrefix `T.isPrefixOf` P._siPubKey signer) + okSignature = isRight $ P.verifyUserSig hsh sig signer + in okScheme && okPrefix && okSignature -- prop_tx_ttl_newBlock/validateBlock -- @@ -186,6 +200,17 @@ assertTxTimeRelativeToParent (ParentCreationTime (BlockCreationTime txValidation P.TxCreationTime txOriginationTime = view cmdCreationTime tx lenientTxValidationTime = add (scaleTimeSpan defaultLenientTimeSlop second) txValidationTime +-- | Assert that the command hash matches its payload and +-- its signatures are valid, without parsing the payload. +assertCommand :: P.Command PayloadWithText -> [P.PPKScheme] -> IsWebAuthnPrefixLegal -> Bool +assertCommand (P.Command pwt sigs hsh) ppkSchemePassList webAuthnPrefixLegal = + isRight assertHash && + assertValidateSigs ppkSchemePassList webAuthnPrefixLegal hsh signers sigs + where + cmdBS = SBS.fromShort $ payloadBytes pwt + signers = P._pSigners (payloadObj pwt) + assertHash = P.verifyHash @'P.Blake2b_256 hsh cmdBS + -- -------------------------------------------------------------------- -- -- defaults diff --git a/src/Chainweb/Rosetta/RestAPI/Server.hs b/src/Chainweb/Rosetta/RestAPI/Server.hs index 52112d7e1b..44a19f9fbe 100644 --- a/src/Chainweb/Rosetta/RestAPI/Server.hs +++ b/src/Chainweb/Rosetta/RestAPI/Server.hs @@ -124,7 +124,7 @@ accountBalanceH -> AccountBalanceReq -> Handler AccountBalanceResp accountBalanceH _ _ _ (AccountBalanceReq _ (AccountId _ (Just _) _) _) = throwRosetta RosettaSubAcctUnsupported -accountBalanceH v cutDb pacts (AccountBalanceReq net (AccountId acct _ _) pbid) = do +accountBalanceH v cutDb pacts (AccountBalanceReq net (AccountId acct _ _) pbid) = runExceptT work >>= either throwRosetta pure where acctBalResp bid bal = AccountBalanceResp @@ -193,7 +193,7 @@ blockTransactionH -> [(ChainId, PactExecutionService)] -> BlockTransactionReq -> Handler BlockTransactionResp -blockTransactionH v cutDb ps pacts (BlockTransactionReq net bid t) = do +blockTransactionH v cutDb ps pacts (BlockTransactionReq net bid t) = runExceptT work >>= either throwRosetta pure where BlockId bheight bhash = bid @@ -243,7 +243,7 @@ constructionPreprocessH :: ChainwebVersion -> ConstructionPreprocessReq -> Handler ConstructionPreprocessResp -constructionPreprocessH v req = do +constructionPreprocessH v req = either throwRosettaError pure work where ConstructionPreprocessReq net ops someMeta someMaxFee someMult = req @@ -350,12 +350,12 @@ constructionParseH v (ConstructionParseReq net isSigned tx) = where work :: Either RosettaError ConstructionParseResp work = do - void $ annotate rosettaError' (validateNetwork v net) + cid <- annotate rosettaError' (validateNetwork v net) (EnrichedCommand cmd txInfo signAccts) <- note (rosettaError' RosettaUnparsableTx) $ textToEnrichedCommand tx - signers <- getRosettaSigners cmd signAccts + signers <- getRosettaSigners cid cmd signAccts let ops = txToOps txInfo pure $ ConstructionParseResp @@ -365,9 +365,9 @@ constructionParseH v (ConstructionParseReq net isSigned tx) = , _constructionParseResp_metadata = Nothing } - getRosettaSigners cmd expectedSignerAccts + getRosettaSigners cid cmd expectedSignerAccts | isSigned = do - _ <- toRosettaError RosettaInvalidTx $ validateCommand cmd + _ <- toRosettaError RosettaInvalidTx $ validateCommand v cid cmd pure expectedSignerAccts -- If transaction signatures successfully validates, -- it was signed correctly with all of the account public @@ -437,7 +437,7 @@ constructionSubmitH v ms (ConstructionSubmitReq net tx) = note (rosettaError' RosettaUnparsableTx) $ textToEnrichedCommand tx - case validateCommand cmd of + case validateCommand v cid cmd of Right validated -> do let txs = V.fromList [validated] -- If any of the txs in the batch fail validation, we reject them all. diff --git a/src/Chainweb/Rosetta/Utils.hs b/src/Chainweb/Rosetta/Utils.hs index f62369a809..38d16a86f1 100644 --- a/src/Chainweb/Rosetta/Utils.hs +++ b/src/Chainweb/Rosetta/Utils.hs @@ -719,7 +719,7 @@ createSigningPayloads (EnrichedCommand cmd _ _) = map f toRosettaSigType Nothing = Just RosettaEd25519 toRosettaSigType (Just P.ED25519) = Just RosettaEd25519 - toRosettaSigType (Just P.WebAuthn) = Nothing + toRosettaSigType (Just P.WebAuthn) = Nothing -- TODO: Linda Ortega (09/18/2023) -- Returning `Nothing` to discourage using WebAuthn for Rosetta. `sigToScheme` will eventually throw an error. -------------------------------------------------------------------------------- @@ -762,6 +762,8 @@ getCmdPayload (Command p _ _) = (decodeStrict' $! T.encodeUtf8 p) +-- TODO: This assumes Rosettas signatures are all Ed25519 signatures +-- (Not webauthn). matchSigs :: [RosettaSignature] -> [Signer] @@ -782,14 +784,13 @@ matchSigs sigs signers = do $ HM.lookup addr m sigAndAddr (RosettaSignature _ (RosettaPublicKey pk ct) sigTyp sig) = do - _ <- toRosettaError RosettaInvalidSignature $! P.parseB16TextOnly sig sigScheme <- sigToScheme sigTyp pkScheme <- getScheme ct when (sigScheme /= pkScheme) (Left $ stringRosettaError RosettaInvalidSignature $ "Expected the same Signature and PublicKey type for Signature=" ++ show sig) - let userSig = P.UserSig sig + let userSig = P.ED25519Sig sig addr <- toPactPubKeyAddr pk pure (addr, userSig) diff --git a/src/Chainweb/Transaction.hs b/src/Chainweb/Transaction.hs index 64c9ba345a..a84e350c44 100644 --- a/src/Chainweb/Transaction.hs +++ b/src/Chainweb/Transaction.hs @@ -12,6 +12,7 @@ module Chainweb.Transaction , HashableTrans(..) , PayloadWithText , PactParserVersion(..) + , IsWebAuthnPrefixLegal(..) , chainwebPayloadCodec , encodePayload , decodePayload @@ -66,9 +67,9 @@ payloadBytes = _payloadBytes payloadObj :: PayloadWithText -> Payload PublicMeta ParsedCode payloadObj = _payloadObj -mkPayloadWithText :: Command ByteString -> Payload PublicMeta ParsedCode -> PayloadWithText -mkPayloadWithText cmd p = PayloadWithText - { _payloadBytes = SB.toShort $ _cmdPayload cmd +mkPayloadWithText :: Command (ByteString, Payload PublicMeta ParsedCode) -> Command PayloadWithText +mkPayloadWithText = over cmdPayload $ \(bs, p) -> PayloadWithText + { _payloadBytes = SB.toShort bs , _payloadObj = p } @@ -85,6 +86,11 @@ data PactParserVersion | PactParserChainweb213 deriving (Eq, Ord, Bounded, Show, Enum) +data IsWebAuthnPrefixLegal + = WebAuthnPrefixIllegal + | WebAuthnPrefixLegal + deriving (Eq, Ord, Bounded, Show, Enum) + -- | Hashable newtype of ChainwebTransaction newtype HashableTrans a = HashableTrans { unHashable :: Command a } deriving (Eq, Functor, Ord) diff --git a/src/Chainweb/Version.hs b/src/Chainweb/Version.hs index 36c27c3300..ebef221327 100644 --- a/src/Chainweb/Version.hs +++ b/src/Chainweb/Version.hs @@ -179,7 +179,7 @@ data Fork | SPVBridge | Pact4Coin3 | EnforceKeysetFormats - | Pact420 + | Pact42 | CheckTxHash | Chainweb213Pact | Chainweb214Pact @@ -191,6 +191,7 @@ data Fork | Chainweb219Pact | Chainweb220Pact | Chainweb221Pact + | Chainweb222Pact -- always add new forks at the end, not in the middle of the constructors. deriving stock (Bounded, Generic, Eq, Enum, Ord, Show) deriving anyclass (NFData, Hashable) @@ -210,7 +211,7 @@ instance HasTextRepresentation Fork where toText SPVBridge = "spvBridge" toText Pact4Coin3 = "pact4Coin3" toText EnforceKeysetFormats = "enforceKeysetFormats" - toText Pact420 = "pact420" + toText Pact42 = "Pact42" toText CheckTxHash = "checkTxHash" toText Chainweb213Pact = "chainweb213Pact" toText Chainweb214Pact = "chainweb214Pact" @@ -222,6 +223,7 @@ instance HasTextRepresentation Fork where toText Chainweb219Pact = "chainweb219Pact" toText Chainweb220Pact = "chainweb220Pact" toText Chainweb221Pact = "chainweb221Pact" + toText Chainweb222Pact = "chainweb222Pact" fromText "slowEpoch" = return SlowEpoch fromText "vuln797Fix" = return Vuln797Fix @@ -237,7 +239,7 @@ instance HasTextRepresentation Fork where fromText "spvBridge" = return SPVBridge fromText "pact4Coin3" = return Pact4Coin3 fromText "enforceKeysetFormats" = return EnforceKeysetFormats - fromText "pact420" = return Pact420 + fromText "Pact42" = return Pact42 fromText "checkTxHash" = return CheckTxHash fromText "chainweb213Pact" = return Chainweb213Pact fromText "chainweb214Pact" = return Chainweb214Pact @@ -249,6 +251,7 @@ instance HasTextRepresentation Fork where fromText "chainweb219Pact" = return Chainweb219Pact fromText "chainweb220Pact" = return Chainweb220Pact fromText "chainweb221Pact" = return Chainweb221Pact + fromText "chainweb222Pact" = return Chainweb222Pact fromText t = throwM . TextFormatException $ "Unknown Chainweb fork: " <> t instance ToJSON Fork where diff --git a/src/Chainweb/Version/Development.hs b/src/Chainweb/Version/Development.hs index 2d3e3b2eeb..bcbea76a17 100644 --- a/src/Chainweb/Version/Development.hs +++ b/src/Chainweb/Version/Development.hs @@ -55,7 +55,7 @@ devnet = ChainwebVersion SPVBridge -> AllChains $ ForkAtBlockHeight $ BlockHeight 50 Pact4Coin3 -> AllChains $ ForkAtBlockHeight $ BlockHeight 80 EnforceKeysetFormats -> AllChains $ ForkAtBlockHeight $ BlockHeight 100 - Pact420 -> AllChains $ ForkAtBlockHeight $ BlockHeight 90 + Pact42 -> AllChains $ ForkAtBlockHeight $ BlockHeight 90 CheckTxHash -> AllChains $ ForkAtBlockHeight $ BlockHeight 110 Chainweb213Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 95 Chainweb214Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 115 @@ -67,6 +67,7 @@ devnet = ChainwebVersion Chainweb219Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 550 Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 560 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 580 + Chainweb222Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 590 , _versionUpgrades = foldr (chainZip HM.union) (AllChains mempty) [ forkUpgrades devnet diff --git a/src/Chainweb/Version/Guards.hs b/src/Chainweb/Version/Guards.hs index ac1280dc49..ec7d2d4a46 100644 --- a/src/Chainweb/Version/Guards.hs +++ b/src/Chainweb/Version/Guards.hs @@ -30,7 +30,7 @@ module Chainweb.Version.Guards , enablePactEvents , enableSPVBridge , pact4Coin3 - , pact420 + , pact42 , enforceKeysetFormats , doCheckTxHash , chainweb213Pact @@ -42,10 +42,13 @@ module Chainweb.Version.Guards , chainweb219Pact , chainweb220Pact , chainweb221Pact + , chainweb222Pact , pact44NewTrans , pactParserVersion , maxBlockGasLimit , validPPKSchemes + , isWebAuthnPrefixLegal + , validKeyFormats -- ** BlockHeader Validation Guards , slowEpochGuard @@ -57,6 +60,7 @@ module Chainweb.Version.Guards import Control.Lens import Numeric.Natural +import Pact.Types.KeySet (PublicKeyText, ed25519HexFormat, webAuthnFormat) import Pact.Types.Scheme (PPKScheme(ED25519, WebAuthn)) import Chainweb.BlockHeight @@ -208,8 +212,8 @@ pact44NewTrans = checkFork atOrAfter Pact44NewTrans pact4Coin3 :: ChainwebVersion -> ChainId -> BlockHeight -> Bool pact4Coin3 = checkFork after Pact4Coin3 -pact420 :: ChainwebVersion -> ChainId -> BlockHeight -> Bool -pact420 = checkFork atOrAfter Pact420 +pact42 :: ChainwebVersion -> ChainId -> BlockHeight -> Bool +pact42 = checkFork atOrAfter Pact42 chainweb213Pact :: ChainwebVersion -> ChainId -> BlockHeight -> Bool chainweb213Pact = checkFork atOrAfter Chainweb213Pact @@ -238,6 +242,9 @@ chainweb220Pact = checkFork atOrAfter Chainweb220Pact chainweb221Pact :: ChainwebVersion -> ChainId -> BlockHeight -> Bool chainweb221Pact = checkFork atOrAfter Chainweb221Pact +chainweb222Pact :: ChainwebVersion -> ChainId -> BlockHeight -> Bool +chainweb222Pact = checkFork atOrAfter Chainweb222Pact + pactParserVersion :: ChainwebVersion -> ChainId -> BlockHeight -> PactParserVersion pactParserVersion v cid bh | chainweb213Pact v cid bh = PactParserChainweb213 @@ -257,3 +264,15 @@ validPPKSchemes v cid bh = if chainweb221Pact v cid bh then [ED25519, WebAuthn] else [ED25519] + +isWebAuthnPrefixLegal :: ChainwebVersion -> ChainId -> BlockHeight -> IsWebAuthnPrefixLegal +isWebAuthnPrefixLegal v cid bh = + if chainweb222Pact v cid bh + then WebAuthnPrefixLegal + else WebAuthnPrefixIllegal + +validKeyFormats :: ChainwebVersion -> ChainId -> BlockHeight -> [PublicKeyText -> Bool] +validKeyFormats v cid bh = + if chainweb222Pact v cid bh + then [ed25519HexFormat, webAuthnFormat] + else [ed25519HexFormat] diff --git a/src/Chainweb/Version/Mainnet.hs b/src/Chainweb/Version/Mainnet.hs index bd038a8d5c..0148bc9275 100644 --- a/src/Chainweb/Version/Mainnet.hs +++ b/src/Chainweb/Version/Mainnet.hs @@ -129,7 +129,7 @@ mainnet = ChainwebVersion SPVBridge -> AllChains (ForkAtBlockHeight $ BlockHeight 1_275_000) Pact4Coin3 -> AllChains (ForkAtBlockHeight $ BlockHeight 1_722_500) -- 2021-06-19T03:34:05+00:00 EnforceKeysetFormats -> AllChains (ForkAtBlockHeight $ BlockHeight 2_162_000) -- 2022-01-17T17:51:12 - Pact420 -> AllChains (ForkAtBlockHeight $ BlockHeight 2_334_500) -- 2022-01-17T17:51:12+00:00 + Pact42 -> AllChains (ForkAtBlockHeight $ BlockHeight 2_334_500) -- 2022-01-17T17:51:12+00:00 CheckTxHash -> AllChains (ForkAtBlockHeight $ BlockHeight 2_349_800) -- 2022-01-23T02:53:38 Chainweb213Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_447_315) -- 2022-02-26T00:00:00+00:00 Chainweb214Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 2_605_663) -- 2022-04-22T00:00:00+00:00 @@ -141,6 +141,7 @@ mainnet = ChainwebVersion Chainweb219Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 3_774_423) -- 2023-06-02 00:00:00+00:00 Chainweb220Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 4_056_499) -- 2023-09-08 00:00:00+00:00 Chainweb221Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 4_177_889) -- 2023-10-20 00:00:00+00:00 + Chainweb222Pact -> AllChains ForkNever -- TODO: fill in , _versionGraphs = (to20ChainsMainnet, twentyChainGraph) `Above` diff --git a/src/Chainweb/Version/Testnet.hs b/src/Chainweb/Version/Testnet.hs index c6e198c052..576918df96 100644 --- a/src/Chainweb/Version/Testnet.hs +++ b/src/Chainweb/Version/Testnet.hs @@ -109,7 +109,7 @@ testnet = ChainwebVersion SPVBridge -> AllChains $ ForkAtBlockHeight $ BlockHeight 820_000 -- 2021-01-14T17:12:02 Pact4Coin3 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_261_000 -- 2021-06-17T15:54:14 EnforceKeysetFormats -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_701_000 -- 2021-11-18T17:54:36 - Pact420 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_862_000 -- 2021-06-19T03:34:05 + Pact42 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_862_000 -- 2021-06-19T03:34:05 CheckTxHash -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_889_000 -- 2022-01-24T04:19:24 Chainweb213Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 1_974_556 -- 2022-02-25 00:00:00 Chainweb214Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 2_134_331 -- 2022-04-21T12:00:00Z @@ -121,6 +121,7 @@ testnet = ChainwebVersion Chainweb219Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_299_753 -- 2023-06-01 12:00:00+00:00 Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_580_964 -- 2023-09-08 12:00:00+00:00 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_702_250 -- 2023-10-19 12:00:00+00:00 + Chainweb222Pact -> AllChains ForkNever -- TODO: fill in , _versionGraphs = (to20ChainsTestnet, twentyChainGraph) `Above` diff --git a/test/Chainweb/Test/Cut/TestBlockDb.hs b/test/Chainweb/Test/Cut/TestBlockDb.hs index cc8ec944e1..6209dd0b78 100644 --- a/test/Chainweb/Test/Cut/TestBlockDb.hs +++ b/test/Chainweb/Test/Cut/TestBlockDb.hs @@ -90,7 +90,7 @@ addTestBlockDb (TestBlockDb wdb pdb cmv) n gbt cid outs = do return False -- something went wrong - Left e -> throwM $ userError (show e) + Left e -> throwM $ userError ("addTestBlockDb: " <> show e) -- | Get header for chain on current cut. getParentTestBlockDb :: TestBlockDb -> ChainId -> IO BlockHeader diff --git a/test/Chainweb/Test/Pact/PactExec.hs b/test/Chainweb/Test/Pact/PactExec.hs index 75097651cc..b42a871e7e 100644 --- a/test/Chainweb/Test/Pact/PactExec.hs +++ b/test/Chainweb/Test/Pact/PactExec.hs @@ -221,8 +221,8 @@ testTfrNoGasFails = (V.singleton <$> tx, assertResultFail "Expected missing (GAS) failure" "Keyset failure") where - tx = buildCwCmd $ set cbSigners - [ mkSigner' sender00 + tx = buildCwCmd testVersion $ set cbSigners + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" "sender01" 1.0 ] ] $ mkCmd "testTfrNoGas" @@ -231,8 +231,8 @@ testTfrNoGasFails = testTfrGas :: TxsTest testTfrGas = (V.singleton <$> tx,test) where - tx = buildCwCmd $ set cbSigners - [ mkSigner' sender00 + tx = buildCwCmd testVersion $ set cbSigners + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" "sender01" 1.0 , mkGasCap ] @@ -253,8 +253,8 @@ testBadSenderFails = assertResultFail "Expected failure on bad sender" "row not found: some-unknown-sender") where - tx = buildCwCmd - $ set cbSigners [ mkSigner' sender00 [] ] + tx = buildCwCmd testVersion + $ set cbSigners [ mkEd25519Signer' sender00 [] ] $ set cbSender "some-unknown-sender" $ mkCmd "testBadSenderFails" $ mkExec' "(+ 1 2)" @@ -266,7 +266,7 @@ testGasPayer = (txs,checkResultSuccess test) impl <- loadGP forM [ impl, setupUser, fundGasAcct ] $ \rpc -> - buildCwCmd $ + buildCwCmd testVersion $ set cbSigners [s01] $ set cbSender "sender01" $ mkCmd "testGasPayer" rpc @@ -283,16 +283,16 @@ testGasPayer = (txs,checkResultSuccess test) fundGasAcct = mkExec' "(coin.transfer-create \"sender01\" \"gas-payer\" (gas-payer-v1-reference.create-gas-payer-guard) 100.0)" - s01 = mkSigner' sender01 + s01 = mkEd25519Signer' sender01 [ mkTransferCap "sender01" "gas-payer" 100.0 , mkGasCap , mkCapability "user.gas-payer-v1-reference" "FUND_USER" [] ] - runPaidTx = fmap V.singleton $ buildCwCmd $ + runPaidTx = fmap V.singleton $ buildCwCmd testVersion $ set cbSigners - [mkSigner' sender00 + [mkEd25519Signer' sender00 [mkCapability "user.gas-payer-v1-reference" "GAS_PAYER" [pString "sender00",pInteger 10_000,pDecimal 0.01]]] $ mkCmd "testGasPayer" $ @@ -327,20 +327,20 @@ testContinuationGasPayer = (txs,checkResultSuccess test) setupTest = fmap V.fromList $ do setupExprs' <- setupExprs - forM setupExprs' $ \se -> buildCwCmd $ + forM setupExprs' $ \se -> buildCwCmd testVersion $ set cbSigners - [ mkSigner' sender00 + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" "cont-gas-payer" 100.0 , mkGasCap ]] $ mkCmd "testContinuationGasPayer" $ mkExec' se - contPactId = "_sYA748a-Hsn_Qb3zsYTDu2H5JXgQcPr1dFkjMTgAm0" + contPactId = "9ylBanSjDGJJ6m0LgokZqb9P66P7JsQRWo9sYxqAjcQ" - runStepTwoWithGasPayer = fmap V.singleton $ buildCwCmd $ + runStepTwoWithGasPayer = fmap V.singleton $ buildCwCmd testVersion $ set cbSigners - [ mkSigner' sender01 + [ mkEd25519Signer' sender01 [ mkCapability "user.gas-payer-for-cont" "GAS_PAYER" [pString "sender01",pInteger 10_000,pDecimal 0.01] ]] $ @@ -348,8 +348,8 @@ testContinuationGasPayer = (txs,checkResultSuccess test) mkCmd "testContinuationGasPayer" $ mkCont $ mkContMsg (fromString contPactId) 1 - balanceCheck = fmap V.singleton $ buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + balanceCheck = fmap V.singleton $ buildCwCmd testVersion $ + set cbSigners [mkEd25519Signer' sender00 []] $ mkCmd "testContinuationGasPayer2" $ mkExec' "(coin.get-balance \"cont-gas-payer\")" @@ -384,26 +384,26 @@ testExecGasPayer = (txs,checkResultSuccess test) , "(coin.get-balance \"exec-gas-payer\")" ] setupTest = fmap V.fromList $ do setupExprs' <- setupExprs - forM setupExprs' $ \se -> buildCwCmd $ + forM setupExprs' $ \se -> buildCwCmd testVersion $ set cbSigners - [ mkSigner' sender00 + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" "exec-gas-payer" 100.0 , mkGasCap ]] $ mkCmd "testExecGasPayer" $ mkExec' se - runPaidTx = fmap V.singleton $ buildCwCmd $ + runPaidTx = fmap V.singleton $ buildCwCmd testVersion $ set cbSigners - [mkSigner' sender01 + [mkEd25519Signer' sender01 [mkCapability "user.gas-payer-for-exec" "GAS_PAYER" [pString "sender01",pInteger 10_000,pDecimal 0.01]]] $ set cbSender "exec-gas-payer" $ mkCmd "testExecGasPayer" $ mkExec' "(+ 1 2)" - balanceCheck = fmap V.singleton $ buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + balanceCheck = fmap V.singleton $ buildCwCmd testVersion $ + set cbSigners [mkEd25519Signer' sender00 []] $ mkCmd "testExecGasPayer" $ mkExec' "(coin.get-balance \"exec-gas-payer\")" @@ -420,14 +420,14 @@ testExecGasPayer = (txs,checkResultSuccess test) (pString "Write succeeded") checkPactResultSuccess "balCheck1" balCheck1 $ assertEqual "balCheck1" (pDecimal 100) checkPactResultSuccess "paidTx" paidTx $ assertEqual "paidTx" (pDecimal 3) - checkPactResultSuccess "balCheck2" balCheck2 $ assertEqual "balCheck2" (pDecimal 99.999_6) + checkPactResultSuccess "balCheck2" balCheck2 $ assertEqual "balCheck2" (pDecimal 99.999_5) test r = assertFailure $ "Expected 6 results, got: " ++ show r testFailureRedeem :: TxsTest testFailureRedeem = (txs,checkResultSuccess test) where - txs = fmap V.fromList $ forM exps $ \e -> buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + txs = fmap V.fromList $ forM exps $ \e -> buildCwCmd testVersion $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbGasPrice 0.01 $ set cbGasLimit 1000 $ mkCmd "testFailureRedeem" $ @@ -465,7 +465,7 @@ checkLocalSuccess test (Right cr) = test $ _crResult cr testAllowReadsLocalFails :: LocalTest testAllowReadsLocalFails = (tx,test) where - tx = buildCwCmd $ mkCmd "testAllowReadsLocalFails" $ + tx = buildCwCmd testVersion $ mkCmd "testAllowReadsLocalFails" $ mkExec' "(read coin.coin-table \"sender00\")" test = checkLocalSuccess $ checkPactResultFailure "testAllowReadsLocalFails" "Enforce non-upgradeability" @@ -473,7 +473,7 @@ testAllowReadsLocalFails = (tx,test) testAllowReadsLocalSuccess :: LocalTest testAllowReadsLocalSuccess = (tx,test) where - tx = buildCwCmd $ mkCmd "testAllowReadsLocalSuccess" $ + tx = buildCwCmd testVersion $ mkCmd "testAllowReadsLocalSuccess" $ mkExec' "(at 'balance (read coin.coin-table \"sender00\"))" test = checkLocalSuccess $ checkPactResultSuccessLocal "testAllowReadsLocalSuccess" $ @@ -503,8 +503,8 @@ execTest runPact request = _trEval request $ do where mkCmds cmdStrs = fmap V.fromList $ forM (zip cmdStrs [0..]) $ \(code,n :: Int) -> - buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + buildCwCmd testVersion $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbGasPrice 0.01 $ set cbTTL 1_000_000 $ mkCmd ("1" <> sshow n) $ @@ -592,8 +592,8 @@ fileCompareTxLogs label respIO = golden label $ do _showValidationFailure :: IO () _showValidationFailure = do - txs <- fmap V.singleton $ buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + txs <- fmap V.singleton $ buildCwCmd testVersion $ + set cbSigners [mkEd25519Signer' sender00 []] $ mkCmd "nonce" $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" let cr1 = CommandResult diff --git a/test/Chainweb/Test/Pact/PactMultiChainTest.hs b/test/Chainweb/Test/Pact/PactMultiChainTest.hs index df18fff525..3be41909d1 100644 --- a/test/Chainweb/Test/Pact/PactMultiChainTest.hs +++ b/test/Chainweb/Test/Pact/PactMultiChainTest.hs @@ -15,7 +15,7 @@ import Control.Lens hiding ((.=)) import Control.Monad import Control.Monad.Catch import Control.Monad.Reader -import Data.Aeson (object, (.=)) +import Data.Aeson (Value, object, (.=)) import Data.List(isPrefixOf) import qualified Data.ByteString.Base64.URL as B64U import qualified Data.HashMap.Strict as HM @@ -116,7 +116,7 @@ data PactTxTest = PactTxTest tests :: TestTree tests = testGroup testName [ test generousConfig freeGasModel "pact4coin3UpgradeTest" pact4coin3UpgradeTest - , test generousConfig freeGasModel "pact420UpgradeTest" pact420UpgradeTest + , test generousConfig freeGasModel "pact42UpgradeTest" pact42UpgradeTest , test generousConfig freeGasModel "minerKeysetTest" minerKeysetTest , test timeoutConfig freeGasModel "txTimeoutTest" txTimeoutTest , test generousConfig getGasModel "chainweb213Test" chainweb213Test @@ -130,6 +130,7 @@ tests = testGroup testName , test generousConfig getGasModel "pactLocalDepthTest" pactLocalDepthTest , test generousConfig getGasModel "pact48UpgradeTest" pact48UpgradeTest , test generousConfig getGasModel "pact49UpgradeTest" pact49UpgradeTest + , test generousConfig getGasModel "pact410UpgradeTest" pact410UpgradeTest ] where testName = "Chainweb.Test.Pact.PactMultiChainTest" @@ -292,7 +293,7 @@ pactLocalDepthTest = do res -> liftIO $ assertFailure $ "Expected LocalResultLegacy, but got: " ++ show res getSender00Balance = set cbGasLimit 700 $ mkCmd "nonce" $ mkExec' "(coin.get-balance \"sender00\")" buildCoinXfer code = buildBasic' - (set cbSigners [mkSigner' sender00 coinCaps] . set cbGasLimit 3000) + (set cbSigners [mkEd25519Signer' sender00 coinCaps] . set cbGasLimit 3000) $ mkExec' code where coinCaps = [ mkGasCap, mkTransferCap "sender00" "sender01" 1.0 ] @@ -309,9 +310,9 @@ pact45UpgradeTest = do [ PactTxTest (buildSimpleCmd "(enforce false 'hi)") $ assertTxFailure "Should fail with the error from the enforce" "hi" , PactTxTest (buildSimpleCmd "(enforce true (format \"{}-{}\" [12345, 657859]))") $ - assertTxGas "Enforce pre-fork evaluates the string with gas" 34 + assertTxGas "Enforce pre-fork evaluates the string with gas" 35 , PactTxTest (buildSimpleCmd "(enumerate 0 10) (str-to-list 'hi) (make-list 10 'hi)") $ - assertTxGas "List functions pre-fork gas" 19 + assertTxGas "List functions pre-fork gas" 20 , PactTxTest (buildBasicGas 70000 $ tblModule "Tbl") $ assertTxSuccess "mod53 table update succeeds" $ pString "TableCreated" @@ -323,9 +324,9 @@ pact45UpgradeTest = do [ PactTxTest (buildSimpleCmd "(+ 1 \'clearlyanerror)") $ assertTxFailure "Should replace tx error with empty error" "" , PactTxTest (buildSimpleCmd "(enforce true (format \"{}-{}\" [12345, 657859]))") $ - assertTxGas "Enforce post fork does not eval the string" (14 + coinTxBuyTransferGas) + assertTxGas "Enforce post fork does not eval the string" (15 + coinTxBuyTransferGas) , PactTxTest (buildSimpleCmd "(enumerate 0 10) (str-to-list 'hi) (make-list 10 'hi)") $ - assertTxGas "List functions post-fork change gas" (39 + coinTxBuyTransferGas) + assertTxGas "List functions post-fork change gas" (40 + coinTxBuyTransferGas) , PactTxTest (buildBasicGas 70000 $ tblModule "tBl") $ assertTxFailure "mod53 table update fails after fork" "" @@ -347,7 +348,7 @@ pact45UpgradeTest = do \ (deftable $TABLE$:{sch})) \ \ (create-table $TABLE$)" buildCoinXfer code = buildBasic' - (set cbSigners [mkSigner' sender00 coinCaps] . set cbGasLimit 3000) + (set cbSigners [mkEd25519Signer' sender00 coinCaps] . set cbGasLimit 3000) $ mkExec' code where coinCaps = [ mkGasCap, mkTransferCap "sender00" "sender01" 1.0 ] @@ -361,7 +362,7 @@ runLocal cid' cmd = runLocalWithDepth Nothing cid' cmd runLocalWithDepth :: Maybe RewindDepth -> ChainId -> CmdBuilder -> PactTestM (Either PactException LocalResult) runLocalWithDepth depth cid' cmd = do pact <- getPactService cid' - cwCmd <- buildCwCmd cmd + cwCmd <- buildCwCmd testVersion cmd liftIO $ _pactLocal pact Nothing Nothing depth cwCmd getPactService :: ChainId -> PactTestM PactExecutionService @@ -649,8 +650,8 @@ pact431UpgradeTest = do ]) -pact420UpgradeTest :: PactTestM () -pact420UpgradeTest = do +pact42UpgradeTest :: PactTestM () +pact42UpgradeTest = do -- run past genesis, upgrades runToHeight 3 @@ -735,7 +736,7 @@ chainweb216Test = do [ PactTxTest (buildSimpleCmd formatGas) $ assertTxGas "Pre-fork format gas" 21 , PactTxTest (buildSimpleCmd tryGas) $ - assertTxGas "Pre-fork try" 18 + assertTxGas "Pre-fork try" 19 , PactTxTest (buildSimpleCmd defineNonNamespacedPreFork) $ assertTxSuccess "Should pass when defining a non-namespaced keyset" @@ -751,7 +752,7 @@ chainweb216Test = do [ PactTxTest (buildSimpleCmd formatGas) $ assertTxGas "Post-fork format gas increase" 48 , PactTxTest (buildSimpleCmd tryGas) $ - assertTxGas "Post-fork try should charge a bit more gas" 19 + assertTxGas "Post-fork try should charge a bit more gas" 20 , PactTxTest (buildSimpleCmd defineNonNamespacedPostFork1) $ assertTxFailure "Should fail when defining a non-namespaced keyset post fork" @@ -1070,7 +1071,15 @@ pact48UpgradeTest = do pact49UpgradeTest :: PactTestM () pact49UpgradeTest = do - runToHeight 98 + runToHeight 97 + + -- Run block 98. + -- WebAuthn is not yet a valid PPK scheme, so this transaction + -- is not valid for insertion into the mempool. + expectInvalid + "WebAuthn should not yet be supported" + [ webAuthnSignedTransaction + ] -- run block 99 (before the pact-4.9 fork) runBlockTest @@ -1079,7 +1088,6 @@ pact49UpgradeTest = do "Non-canonical messages decode before pact-4.9" (pString "d") , PactTxTest base64DecodeBadPadding $ assertTxFailure "decoding illegally padded string" "Could not decode string: Base64URL decode failed: invalid padding near offset 16" - ] -- run block 100 (after the pact-4.9 fork) @@ -1087,12 +1095,118 @@ pact49UpgradeTest = do [ PactTxTest base64DecodeNonCanonical $ assertTxFailure "decoding non-canonical message" "Could not decode string: Could not base64-decode string" , PactTxTest base64DecodeBadPadding $ assertTxFailure "decoding illegally padded string" "Could not decode string: Could not base64-decode string" + + , PactTxTest webAuthnSignedTransaction $ + assertTxSuccess + "WebAuthn signatures should be valid now" + (pDecimal 3) + ] where + webAuthnSignedTransaction = buildBasicGasWebAuthnBareSigner 1000 $ mkExec' "(+ 1 2)" base64DecodeNonCanonical = buildBasicGas 10000 $ mkExec' "(base64-decode \"ZE==\")" base64DecodeBadPadding = buildBasicGas 10000 $ mkExec' "(base64-decode \"aGVsbG8gd29ybGQh%\")" +pact410UpgradeTest :: PactTestM () +pact410UpgradeTest = do + runToHeight 110 + + expectInvalid + "WebAuthn prefixed keys should not yet be supported in signatures" + [ prefixedSigned + ] + runBlockTest [ + PactTxTest poseidonTx $ + assertTxFailure "Should not resolve new pact native: poseidon-hash-hack-a-chain" + "Cannot resolve poseidon-hash-hack-a-chain" + ] + + runToHeight 120 + runBlockTest + [ PactTxTest prefixedSigned $ + assertTxSuccess + "Prefixed WebAuthn signers should be legal" + (pDecimal 1) + + , PactTxTest prefixedSignerPrefixedKey $ + assertTxSuccess + "WebAuthn prefixed keys should be enforceable with prefixed signers" + (pBool True) + + , PactTxTest bareSignerPrefixedKey $ + assertTxFailure + "WebAuthn prefixed keys should not be enforceable with bare signers" + "Keyset failure (keys-all): [WEBAUTHN...]" + + , PactTxTest definePrefixedKeySet $ + assertTxSuccess + "WebAuthn prefixed keys should be enforceable after defining them" + (pBool True) + + , PactTxTest prefixedSignerPrefixedKeyCreatePrincipal $ + assertTxSuccess + "WebAuthn prefixed keys in a keyset should be possible to make into w: principals" + (pString "w:XrscJ2X8aFxFF7oilzFyjQuA1mUN8jgwdxbAd8rt21M:keys-all") + + , PactTxTest prefixedSignerPrefixedKeyValidatePrincipal $ + assertTxSuccess + "WebAuthn prefixed keys in a keyset should be possible to make into *valid* w: principals" + (pBool True) + + , PactTxTest prefixedSignerBareKey $ + assertTxFailure + "WebAuthn bare keys should throw an error when read" + "Invalid keyset" + + , PactTxTest invalidPrefixedKey $ + assertTxFailure + "Invalid WebAuthn prefixed keys should throw an error when read" + "Invalid keyset" + + , PactTxTest poseidonTx $ + assertTxSuccess "Should resolve new pact native: poseidon-hash-hack-a-chain" + (pDecimal 18586133768512220936620570745912940619677854269274689475585506675881198879027) + ] + + where + poseidonTx = buildBasic $ mkExec' "(poseidon-hash-hack-a-chain 1)" + prefixedSigned = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec' "1" + + prefixedSignerBareKey = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec + "(enforce-keyset (read-keyset 'k))" + (mkKeyEnvData "a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + bareSignerPrefixedKey = buildBasicGasWebAuthnBareSigner 1000 $ mkExec + "(enforce-keyset (read-keyset 'k))" + (mkKeyEnvData "WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + prefixedSignerPrefixedKey = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec + "(enforce-keyset (read-keyset 'k))" + (mkKeyEnvData "WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + definePrefixedKeySet = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec + "(namespace 'free) (define-keyset \"free.edmund\" (read-keyset 'k)) (enforce-keyset \"free.edmund\")" + (mkKeyEnvData "WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + prefixedSignerPrefixedKeyCreatePrincipal = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec + "(create-principal (read-keyset 'k))" + (mkKeyEnvData "WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + prefixedSignerPrefixedKeyValidatePrincipal = buildBasicGasWebAuthnPrefixedSigner 1000 $ mkExec + "(let ((ks (read-keyset 'k))) (validate-principal ks (create-principal ks)))" + (mkKeyEnvData "WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf") + + -- This hardcoded public key is the same as the valid one above, except that the first + -- character is changed. CBOR parsing will fail. + invalidPrefixedKey = buildBasicGas 1000 $ mkExec + "(read-keyset 'k)" + (mkKeyEnvData "WEBAUTHN-a401010327200add79710303bf") + + mkKeyEnvData :: String -> Value + mkKeyEnvData key = object [ "k" .= [key] ] + + pact4coin3UpgradeTest :: PactTestM () pact4coin3UpgradeTest = do @@ -1145,7 +1259,7 @@ pact4coin3UpgradeTest = do "coin" v3Hash assertTxEvents "Events for tx1 @ block 22" [gasEv1,allocEv,allocTfr] cr , PactTxTest (buildXSend []) $ \cr -> do - gasEv2 <- mkTransferEvent "sender00" "NoMiner" 0.0014 "coin" v3Hash + gasEv2 <- mkTransferEvent "sender00" "NoMiner" 0.0015 "coin" v3Hash sendTfr <- mkTransferEvent "sender00" "" 0.0123 "coin" v3Hash yieldEv <- mkXYieldEvent "sender00" "sender00" 0.0123 sender00Ks "pact" v3Hash "0" "0" assertTxEvents "Events for tx2 @ block 22" [gasEv2,sendTfr, yieldEv] cr @@ -1205,8 +1319,8 @@ pact4coin3UpgradeTest = do ] buildReleaseCommand = buildBasic' - (set cbSigners [ mkSigner' sender00 [] - , mkSigner' allocation00KeyPair []]) + (set cbSigners [ mkEd25519Signer' sender00 [] + , mkEd25519Signer' allocation00KeyPair []]) $ mkExec' "(coin.release-allocation 'allocation00)" buildCont sendTx = do @@ -1232,16 +1346,18 @@ setPactMempool (PactMempool fs) = do mpaGetBlock = go mpsRef } where - go ref bf _ _ _ bh = do + go ref bf mempoolPreBlockCheck bHeight bHash blockHeader = do mps <- readIORef ref - let mi = MempoolInput bf bh + let mi = MempoolInput bf blockHeader runMps i = \case [] -> return mempty (mp:r) -> case _mempoolBlock mp mi of Just bs -> do writeIORef ref (take i mps ++ r) - fmap V.fromList $ forM bs $ \b -> - buildCwCmd $ _mempoolCmdBuilder b mi + cmds <- fmap V.fromList $ forM bs $ \b -> + buildCwCmd testVersion $ _mempoolCmdBuilder b mi + validationResults <- mempoolPreBlockCheck bHeight bHash cmds + return $ fmap fst $ V.filter snd (V.zip cmds validationResults) Nothing -> runMps (succ i) r runMps 0 mps @@ -1324,7 +1440,7 @@ assertTxFailure' msg needle tx = -- | Run a single mempool block on current chain with tests for each tx. -- Limitations: can only run a single-chain, single-refill test for -- a given cut height. -runBlockTest :: [PactTxTest] -> PactTestM () +runBlockTest :: HasCallStack => [PactTxTest] -> PactTestM () runBlockTest pts = do chid <- view menvChainId setPactMempool $ PactMempool [testsToBlock chid pts] @@ -1336,11 +1452,24 @@ testsToBlock :: ChainId -> [PactTxTest] -> MempoolBlock testsToBlock chid pts = blockForChain chid $ MempoolBlock $ \_ -> pure $ map _pttBuilder pts +-- | No tests in this list should even be submitted to the mempool, +-- they should be rejected early. +expectInvalid :: String -> [MempoolCmdBuilder] -> PactTestM () +expectInvalid msg pts = do + chid <- view menvChainId + setPactMempool $ PactMempool [blockForChain chid $ MempoolBlock $ \_ -> pure pts] + _ <- runCut' + rs <- txResults + liftIO $ assertEqual msg mempty rs + -- | Run tests on current cut and chain. -runBlockTests :: [PactTxTest] -> PactTestM () +runBlockTests :: HasCallStack => [PactTxTest] -> PactTestM () runBlockTests pts = do - txResults >>= zipWithM_ go pts . V.toList + rs <- txResults + liftIO $ assertEqual "Result length should equal transaction length" (length pts) (length rs) + zipWithM_ go pts (V.toList rs) where + go :: PactTxTest -> CommandResult Hash -> PactTestM () go (PactTxTest _ t) cr = liftIO $ t cr -- | Run cuts to block height. @@ -1354,7 +1483,7 @@ runToHeight bhi = do buildXSend :: [SigCapability] -> MempoolCmdBuilder buildXSend caps = MempoolCmdBuilder $ \(MempoolInput _ bh) -> - set cbSigners [mkSigner' sender00 caps] + set cbSigners [mkEd25519Signer' sender00 caps] $ setFromHeader bh $ mkCmd (sshow bh) $ mkExec @@ -1390,8 +1519,20 @@ buildXReceive buildXReceive (proof,pid) = buildBasic $ mkCont ((mkContMsg pid 1) { _cmProof = Just proof }) +signWebAuthn00Prefixed :: CmdBuilder -> CmdBuilder +signWebAuthn00Prefixed = + set cbSigners [mkWebAuthnSigner' sender02WebAuthnPrefixed [] + ,mkEd25519Signer' sender00 [] + ] + +signWebAuthn00 :: CmdBuilder -> CmdBuilder +signWebAuthn00 = + set cbSigners [mkWebAuthnSigner' sender02WebAuthn [] + ,mkEd25519Signer' sender00 [] + ] + signSender00 :: CmdBuilder -> CmdBuilder -signSender00 = set cbSigners [mkSigner' sender00 []] +signSender00 = set cbSigners [mkEd25519Signer' sender00 []] setFromHeader :: BlockHeader -> CmdBuilder -> CmdBuilder setFromHeader bh = @@ -1416,6 +1557,30 @@ buildBasic' f r = MempoolCmdBuilder $ \(MempoolInput _ bh) -> $ setFromHeader bh $ mkCmd (sshow bh) r +buildBasicWebAuthnBareSigner' + :: (CmdBuilder -> CmdBuilder) + -> PactRPC T.Text + -> MempoolCmdBuilder +buildBasicWebAuthnBareSigner' f r = MempoolCmdBuilder $ \(MempoolInput _ bh) -> + f $ signWebAuthn00 + $ setFromHeader bh + $ mkCmd (sshow bh) r + +buildBasicWebAuthnPrefixedSigner' + :: (CmdBuilder -> CmdBuilder) + -> PactRPC T.Text + -> MempoolCmdBuilder +buildBasicWebAuthnPrefixedSigner' f r = MempoolCmdBuilder $ \(MempoolInput _ bh) -> + f $ signWebAuthn00Prefixed + $ setFromHeader bh + $ mkCmd (sshow bh) r + +buildBasicGasWebAuthnPrefixedSigner :: GasLimit -> PactRPC T.Text -> MempoolCmdBuilder +buildBasicGasWebAuthnPrefixedSigner g = buildBasicWebAuthnPrefixedSigner' (set cbGasLimit g) + +buildBasicGasWebAuthnBareSigner :: GasLimit -> PactRPC T.Text -> MempoolCmdBuilder +buildBasicGasWebAuthnBareSigner g = buildBasicWebAuthnBareSigner' (set cbGasLimit g) + -- | Get output on latest cut for chain getPWO :: ChainId -> PactTestM (PayloadWithOutputs,BlockHeader) getPWO chid = do diff --git a/test/Chainweb/Test/Pact/PactReplay.hs b/test/Chainweb/Test/Pact/PactReplay.hs index c62c97d606..49cf6e4d30 100644 --- a/test/Chainweb/Test/Pact/PactReplay.hs +++ b/test/Chainweb/Test/Pact/PactReplay.hs @@ -110,8 +110,8 @@ testMemPoolAccess = mempty getTestBlock _ _ 1 _ = mempty getTestBlock txOrigTime validate bHeight hash = do let nonce = T.pack . show @(Time Micros) $ txOrigTime - tx <- buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + tx <- buildCwCmd testVer $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime (toTxCreationTime txOrigTime) $ mkCmd nonce $ mkExec' "1" @@ -137,8 +137,8 @@ dupegenMemPoolAccess = do if bHeight `elem` hs' then return mempty else do writeIORef hs (bHeight:hs') outtxs <- fmap V.singleton $ - buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + buildCwCmd testVer $ + set cbSigners [mkEd25519Signer' sender00 []] $ mkCmd "0" $ mkExec' "1" oks <- validate bHeight bHash outtxs diff --git a/test/Chainweb/Test/Pact/PactSingleChainTest.hs b/test/Chainweb/Test/Pact/PactSingleChainTest.hs index f8942d0edb..cf0db74d0a 100644 --- a/test/Chainweb/Test/Pact/PactSingleChainTest.hs +++ b/test/Chainweb/Test/Pact/PactSingleChainTest.hs @@ -136,7 +136,7 @@ tests rdb = testGroup testName testHistLookup2 = getHistoricalLookupNoTxs "randomAccount" (assertEqual "Return Nothing if key absent after a no txs block" Nothing) testHistLookup3 = getHistoricalLookupWithTxs "sender00" - (assertSender00Bal 9.999998051e7 "check latest entry for sender00 after block with txs") + (assertSender00Bal 9.999998042e7 "check latest entry for sender00 after block with txs") testWithConf' :: () => RocksDb @@ -316,8 +316,8 @@ pactStateSamePreAndPostCompaction rdb = setOneShotMempool cr.mempoolRef goldenMemPool let makeTx :: Int -> BlockHeader -> IO ChainwebTransaction - makeTx nth bh = buildCwCmd - $ set cbSigners [mkSigner' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] + makeTx nth bh = buildCwCmd testVersion + $ set cbSigners [mkEd25519Signer' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] $ setFromHeader bh $ mkCmd (sshow (nth, bh)) $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" @@ -388,8 +388,8 @@ compactionIsIdempotent rdb = setOneShotMempool cr.mempoolRef goldenMemPool let makeTx :: Int -> BlockHeader -> IO ChainwebTransaction - makeTx nth bh = buildCwCmd - $ set cbSigners [mkSigner' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] + makeTx nth bh = buildCwCmd testVersion + $ set cbSigners [mkEd25519Signer' sender00 [mkGasCap, mkTransferCap "sender00" "sender01" 1.0]] $ setFromHeader bh $ mkCmd (sshow (nth, bh)) $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" @@ -487,7 +487,7 @@ compactionUserTablesDropped rdb = , ")" , "(create-table " <> tblName <> ")" ] - buildCwCmd + buildCwCmd testVersion $ signSender00 $ set cbGasLimit gasLimit $ mkCmd ("createTable-" <> tblName <> "-" <> sshow n) @@ -654,7 +654,7 @@ newBlockRewindValidate mpRefIO reqIO = testCase "newBlockRewindValidate" $ do chainDataMemPool = mempty { mpaGetBlock = \_ _ _ _ bh -> do - fmap V.singleton $ buildCwCmd + fmap V.singleton $ buildCwCmd testVersion $ signSender00 $ setFromHeader bh $ mkCmd (sshow bh) @@ -662,7 +662,7 @@ newBlockRewindValidate mpRefIO reqIO = testCase "newBlockRewindValidate" $ do } signSender00 :: CmdBuilder -> CmdBuilder -signSender00 = set cbSigners [mkSigner' sender00 []] +signSender00 = set cbSigners [mkEd25519Signer' sender00 []] setFromHeader :: BlockHeader -> CmdBuilder -> CmdBuilder setFromHeader bh = @@ -681,7 +681,7 @@ blockGasLimitTest _ reqIO = testCase "blockGasLimitTest" $ do let useGas g = do - bigTx <- buildCwCmd $ set cbGasLimit g $ signSender00 $ mkCmd "cmd" $ mkExec' "TESTING" + bigTx <- buildCwCmd testVersion $ set cbGasLimit g $ signSender00 $ mkCmd "cmd" $ mkExec' "TESTING" let cr = CommandResult (RequestKey (Hash "0")) Nothing @@ -766,12 +766,12 @@ mempoolRefillTest mpRefIO reqIO = testCase "mempoolRefillTest" $ do i <- modifyMVar supply $ return . (succ &&& id) f i bh - goodTx i bh = buildCwCmd + goodTx i bh = buildCwCmd testVersion $ signSender00 $ mkCmd' bh (sshow (i,bh)) $ mkExec' "(+ 1 2)" - badTx i bh = buildCwCmd + badTx i bh = buildCwCmd testVersion $ signSender00 $ set cbSender "bad" $ mkCmd' bh (sshow (i,bh)) @@ -816,7 +816,7 @@ moduleNameMempool ns mn = mempty , ns <> "." <> mn <> ".G" ] fmap V.fromList $ forM (zip txs [0..]) $ \(code,n :: Int) -> - buildCwCmd $ + buildCwCmd testVersion $ signSender00 $ set cbCreationTime (toTxCreationTime $ _bct $ _blockCreationTime bh) $ mkCmd ("1" <> sshow n) $ @@ -846,7 +846,7 @@ mempoolCreationTimeTest mpRefIO reqIO = testCase "mempoolCreationTimeTest" $ do where - makeTx nonce t = buildCwCmd + makeTx nonce t = buildCwCmd testVersion $ signSender00 $ set cbChainId cid $ set cbCreationTime (toTxCreationTime t) @@ -871,19 +871,19 @@ preInsertCheckTimeoutTest _ reqIO = testCase "preInsertCheckTimeoutTest" $ do coinV4 <- T.readFile "pact/coin-contract/v4/coin-v4.pact" coinV5 <- T.readFile "pact/coin-contract/v5/coin-v5.pact" - txCoinV3 <- buildCwCmd + txCoinV3 <- buildCwCmd testVersion $ signSender00 $ set cbChainId cid $ mkCmd "tx-now-coinv3" $ mkExec' coinV3 - txCoinV4 <- buildCwCmd + txCoinV4 <- buildCwCmd testVersion $ signSender00 $ set cbChainId cid $ mkCmd "tx-now-coinv4" $ mkExec' coinV4 - txCoinV5 <- buildCwCmd + txCoinV5 <- buildCwCmd testVersion $ signSender00 $ set cbChainId cid $ mkCmd "tx-now-coinv5" @@ -897,7 +897,7 @@ badlistNewBlockTest mpRefIO reqIO = testCase "badlistNewBlockTest" $ do (_, reqQ, _) <- reqIO let hashToTxHashList = V.singleton . requestKeyToTransactionHash . RequestKey . toUntypedHash @'Blake2b_256 badHashRef <- newIORef $ hashToTxHashList initialHash - badTx <- buildCwCmd + badTx <- buildCwCmd testVersion $ signSender00 -- this should exceed the account balance $ set cbGasLimit 99999 @@ -966,7 +966,7 @@ goldenMemPool = mempty return outtxs mkTxs txs = fmap V.fromList $ forM (zip txs [0..]) $ \(code,n :: Int) -> - buildCwCmd $ + buildCwCmd testVersion $ signSender00 $ set cbGasPrice 0.01 $ set cbTTL 1_000_000 $ -- match old goldens diff --git a/test/Chainweb/Test/Pact/RemotePactTest.hs b/test/Chainweb/Test/Pact/RemotePactTest.hs index f6fa543e8c..10ed8ad83a 100644 --- a/test/Chainweb/Test/Pact/RemotePactTest.hs +++ b/test/Chainweb/Test/Pact/RemotePactTest.hs @@ -203,6 +203,7 @@ tests rdb = testGroup "Chainweb.Test.Pact.RemotePactTest" , after AllSucceed "poll returns correct results" $ testCaseSteps "poll correct results test" $ \step -> join $ pollingCorrectResults <$> iot <*> cenv <*> pure step + , after AllSucceed "webauthn signatures" $ testCase "webauthn sig" $ join $ webAuthnSignatureTest <$> iot <*> cenv ] ] @@ -247,16 +248,15 @@ txlogsCompactionTest :: Pact.TxCreationTime -> ClientEnv -> FilePath -> IO () txlogsCompactionTest t cenv pactDbDir = do let cmd :: Text -> Text -> CmdBuilder cmd nonce tx = do - set cbSigners [mkSigner' sender00 []] + set cbSigners [mkEd25519Signer' sender00 []] $ set cbTTL defaultMaxTTL $ set cbCreationTime t $ set cbChainId cid - $ set cbNetworkId (Just v) $ mkCmd nonce $ mkExec tx $ mkKeySetData "sender00" [sender00] - createTableTx <- buildTextCmd + createTableTx <- buildTextCmd v $ set cbGasLimit 300_000 $ cmd "create-table-persons" $ T.unlines @@ -328,7 +328,7 @@ txlogsCompactionTest t cenv pactDbDir = do -- Note there are two transactions that write to `persons`, which is -- why `numTxIds` = 2 (and not the number of rows). let gasLimit = 400_000 - buildTextCmd + buildTextCmd v $ set cbGasLimit gasLimit $ cmd ("test-txlogs-" <> sshow n) $ T.unlines @@ -344,7 +344,7 @@ txlogsCompactionTest t cenv pactDbDir = do createWriteTx n = do -- module = 60k, write = 100 let gasLimit = 70_000 - buildTextCmd + buildTextCmd v $ set cbGasLimit gasLimit $ cmd ("test-write-" <> sshow n) $ T.unlines @@ -430,19 +430,17 @@ localContTest t cenv step = do tx = "(namespace 'free)(module m G (defcap G () true) (defpact p () (step (yield { \"a\" : (+ 1 1) })) (step (resume { \"a\" := a } a))))(free.m.p)" firstStep = do - buildTextCmd - $ set cbSigners [mkSigner' sender00 []] + buildTextCmd v + $ set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ set cbGasLimit 70000 $ mkCmd "nonce-cont-1" $ mkExec' tx secondStep pid = do - buildTextCmd - $ set cbSigners [mkSigner' sender00 []] + buildTextCmd v + $ set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ mkCmd "nonce-cont-2" $ mkCont $ mkContMsg pid 1 @@ -472,10 +470,9 @@ pollingConfirmDepth t cenv step = do tx' = "43" firstStep transaction = do - buildTextCmd - $ set cbSigners [mkSigner' sender00 []] + buildTextCmd v + $ set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ set cbGasLimit 70000 $ mkCmd "nonce-cont-1" $ mkExec' transaction @@ -508,10 +505,9 @@ pollingCorrectResults t cenv step = do (tx, tx') = ("42", "43") stepTx transaction = do - buildTextCmd - $ set cbSigners [mkSigner' sender00 []] + buildTextCmd v + $ set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ set cbGasLimit 70_000 $ mkCmd "nonce-cont-2" $ mkExec' transaction @@ -552,7 +548,7 @@ localPreflightSimTest :: Pact.TxCreationTime -> ClientEnv -> (String -> IO ()) - localPreflightSimTest t cenv step = do mv <- newMVar (0 :: Int) sid <- mkChainId v maxBound 0 - let sigs = [mkSigner' sender00 []] + let sigs = [mkEd25519Signer' sender00 []] step "Execute preflight /local tx - preflight known /send success" let psid = Pact.ChainId $ chainIdToText sid @@ -582,21 +578,21 @@ localPreflightSimTest t cenv step = do step "Execute preflight /local tx - chain id mismatch" let fcid = unsafeChainId maxBound - cmd2 <- mkTx mv =<< mkCmdBuilder sigs v fcid 1000 gp + cmd2 <- mkTx v mv =<< mkCmdBuilder sigs fcid 1000 gp runClientFailureAssertion sid cenv cmd2 "Chain id mismatch" step "Execute preflight /local tx - tx gas limit too high" - cmd3 <- mkTx mv =<< mkCmdBuilder sigs v sid 100000000000000 gp + cmd3 <- mkTx v mv =<< mkCmdBuilder sigs sid 100000000000000 gp runClientFailureAssertion sid cenv cmd3 "Transaction Gas limit exceeds block gas limit" step "Execute preflight /local tx - tx gas price precision too high" - cmd4 <- mkTx mv =<< mkCmdBuilder sigs v sid 1000 0.00000000000000001 + cmd4 <- mkTx v mv =<< mkCmdBuilder sigs sid 1000 0.00000000000000001 runClientFailureAssertion sid cenv cmd4 "Gas price decimal precision too high" step "Execute preflight /local tx - network id mismatch" - cmd5 <- mkTx mv =<< mkCmdBuilder sigs Mainnet01 sid 1000 gp + cmd5 <- mkTx Mainnet01 mv =<< mkCmdBuilder sigs sid 1000 gp runClientFailureAssertion sid cenv cmd5 "Network id mismatch" step "Execute preflight /local tx - too many sigs" @@ -684,20 +680,19 @@ localPreflightSimTest t cenv step = do c <- Pact.mkExec code A.Null (pm t) kps (Just "fastfork-CPM-peterson") (Just nonce) pure (succ nn, c) - mkCmdBuilder sigs nid pcid limit price = do + mkCmdBuilder sigs pcid limit price = do pure $ set cbGasLimit limit $ set cbGasPrice price $ set cbChainId pcid $ set cbSigners sigs $ set cbCreationTime t - $ set cbNetworkId (Just nid) $ mkCmd "" $ mkExec' "(+ 1 2)" - mkTx mv tx = modifyMVar mv $ \nn -> do + mkTx v' mv tx = modifyMVar mv $ \nn -> do let n = "nonce-" <> sshow nn - tx' <- buildTextCmd $ set cbNonce n tx + tx' <- buildTextCmd v' $ set cbNonce n tx pure (succ nn, tx') pollingBadlistTest :: ClientEnv -> IO () @@ -1100,6 +1095,38 @@ allocationTest t cenv step = do , (FieldKey "guard", PGuard $ GKeySetRef (KeySetName "allocation02" Nothing)) ] +-- Test that transactions signed with (mock) WebAuthn keypairs are accepted +-- by the pact service. +webAuthnSignatureTest :: Pact.TxCreationTime -> ClientEnv -> IO () +webAuthnSignatureTest t cenv = do + + cmd1 <- buildTextCmd v + $ set cbSigners [mkWebAuthnSigner' sender02WebAuthn [], mkEd25519Signer' sender00 []] + $ set cbCreationTime t + $ set cbGasLimit 1000 + $ mkCmd "nonce-webauthn-1" + $ mkExec' "(concat [\"chainweb-\" \"node\"])" + + rks1 <- sending cid' cenv (SubmitBatch $ pure cmd1) + PollResponses _resp1 <- polling cid' cenv rks1 ExpectPactResult + + cmd2 <- buildTextCmd v + $ set cbSigners [mkWebAuthnSigner' sender02WebAuthn [], mkEd25519Signer' sender00 []] + $ set cbCreationTime t + $ set cbGasLimit 1000 + $ mkCmd "nonce-webauthn-2" + $ mkExec' "(concat [\"chainweb-\" \"node\"])" + + + rks2 <- sending cid' cenv (SubmitBatch $ pure cmd2) + PollResponses _resp2 <- polling cid' cenv rks2 ExpectPactResult + + return () + + where + cid' = unsafeChainId 0 + + -- -------------------------------------------------------------------------- -- -- Utils diff --git a/test/Chainweb/Test/Pact/SPV.hs b/test/Chainweb/Test/Pact/SPV.hs index d99bfbef07..4d2ffb04db 100644 --- a/test/Chainweb/Test/Pact/SPV.hs +++ b/test/Chainweb/Test/Pact/SPV.hs @@ -281,7 +281,7 @@ roundtrip' v sid0 tid0 burn create step = withTestBlockDb v $ \bdb -> do step "cut 1: burn" -- Creating the parent took at least 1 second. So 1s is fine as creation time let t1 = add second epoch - txGen1 <- burn t1 pidv sid tid + txGen1 <- burn v t1 pidv sid tid void $ swapMVar tg txGen1 co1 <- runCut' v bdb pact @@ -289,7 +289,7 @@ roundtrip' v sid0 tid0 burn create step = withTestBlockDb v $ \bdb -> do step "setup create txgen with cut 1" (BlockCreationTime t2) <- _blockCreationTime <$> getParentTestBlockDb bdb tid hi <- _blockHeight <$> getParentTestBlockDb bdb sid - txGen2 <- create t2 bdb pidv sid tid hi + txGen2 <- create v t2 bdb pidv sid tid hi -- cut 2: empty cut for diameter 1 step "cut 2: empty cut for diameter 1" @@ -349,10 +349,11 @@ type TransactionGenerator -> IO (Vector ChainwebTransaction) type BurnGenerator - = Time Micros -> MVar PactId -> Chainweb.ChainId -> Chainweb.ChainId -> IO TransactionGenerator + = ChainwebVersion -> Time Micros -> MVar PactId -> Chainweb.ChainId -> Chainweb.ChainId -> IO TransactionGenerator type CreatesGenerator - = Time Micros + = ChainwebVersion + -> Time Micros -> TestBlockDb -> MVar PactId -> Chainweb.ChainId @@ -363,7 +364,7 @@ type CreatesGenerator -- | Generate burn/create Pact Service commands on arbitrarily many chains -- burnGen :: BurnGenerator -burnGen time pidv sid tid = do +burnGen v time pidv sid tid = do ref0 <- newIORef False ref1 <- newIORef False return $ go ref0 ref1 @@ -376,8 +377,8 @@ burnGen time pidv sid tid = do readIORef ref1 >>= \case True -> return mempty False -> do - cmd <- buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + cmd <- buildCwCmd v $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime (toTxCreationTime time) $ set cbChainId sid $ mkCmd "0" $ @@ -414,7 +415,7 @@ burnGen time pidv sid tid = do -- | Generate arbitrary coin.transfer call. -- transferGen :: BurnGenerator -transferGen time pidv sid _tid = do +transferGen v time pidv sid _tid = do ref0 <- newIORef False ref1 <- newIORef False return $ go ref0 ref1 @@ -427,9 +428,9 @@ transferGen time pidv sid _tid = do readIORef ref1 >>= \case True -> return mempty False -> do - cmd <- buildCwCmd $ + cmd <- buildCwCmd v $ set cbSigners - [mkSigner' sender00 + [mkEd25519Signer' sender00 [mkTransferCap "sender00" "sender01" 1.0 ,mkGasCap]] $ set cbCreationTime (toTxCreationTime time) $ @@ -449,16 +450,17 @@ transferGen time pidv sid _tid = do tx1Code = "(coin.transfer 'sender00 'sender01 1.0)" createCont - :: ChainId - -> MVar PactId - -> Maybe ContProof - -> Time Micros - -> IO (Vector ChainwebTransaction) -createCont cid pidv proof time = do + :: ChainwebVersion + -> ChainId + -> MVar PactId + -> Maybe ContProof + -> Time Micros + -> IO (Vector ChainwebTransaction) +createCont v cid pidv proof time = do pid <- readMVar pidv fmap Vector.singleton $ - buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + buildCwCmd v $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime (toTxCreationTime time) $ set cbChainId cid $ mkCmd "1" $ @@ -469,7 +471,7 @@ createCont cid pidv proof time = do -- | Generate a tx to run 'verify-spv' tests. -- createVerify :: Bool -> Text -> Value -> CreatesGenerator -createVerify bridge code mdata time (TestBlockDb wdb pdb _c) _pidv sid tid bhe = do +createVerify bridge code mdata v time (TestBlockDb wdb pdb _c) _pidv sid tid bhe = do ref <- newIORef False return $ go ref where @@ -483,8 +485,8 @@ createVerify bridge code mdata time (TestBlockDb wdb pdb _c) _pidv sid tid bhe = [ ("proof", String $ encodeB64UrlNoPaddingText $ encodeToByteString pf) ] | otherwise = toJSON pf - cmd <- buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + cmd <- buildCwCmd v $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime (toTxCreationTime time) $ set cbChainId tid $ mkCmd "0" $ @@ -497,7 +499,7 @@ createVerify bridge code mdata time (TestBlockDb wdb pdb _c) _pidv sid tid bhe = -- | Generate a tx to run 'verify-spv' tests. -- createVerifyEth :: Text -> CreatesGenerator -createVerifyEth code time (TestBlockDb _wdb _pdb _c) _pidv _sid tid _bhe = do +createVerifyEth code v time (TestBlockDb _wdb _pdb _c) _pidv _sid tid _bhe = do ref <- newIORef False q <- encodeB64UrlNoPaddingText . putRlpByteString <$> receiptProofTest 2 return $ go q ref @@ -508,8 +510,8 @@ createVerifyEth code time (TestBlockDb _wdb _pdb _c) _pidv _sid tid _bhe = do True -> return mempty False -> do -- q <- toJSON <$> createTransactionOutputProof_ wdb pdb tid sid bhe 0 - cmd <- buildCwCmd $ - set cbSigners [mkSigner' sender00 []] $ + cmd <- buildCwCmd v $ + set cbSigners [mkEd25519Signer' sender00 []] $ set cbCreationTime (toTxCreationTime time) $ set cbChainId tid $ mkCmd "0" $ @@ -534,7 +536,7 @@ receiptProofTest i = do -- has already called the 'create-coin' half of the transaction, it will not do so again. -- createSuccess :: CreatesGenerator -createSuccess time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do +createSuccess v time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do ref <- newIORef False return $ go ref where @@ -545,13 +547,13 @@ createSuccess time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do False -> do q <- toJSON <$> createTransactionOutputProof_ wdb pdb tid sid bhe 0 let proof = Just . ContProof . B64U.encode . toStrict . Aeson.encode $ q - createCont tid pidv proof time + createCont v tid pidv proof time `finally` writeIORef ref True -- | Execute on the create-coin command on the wrong target chain -- createWrongTargetChain :: CreatesGenerator -createWrongTargetChain time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do +createWrongTargetChain v time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do ref <- newIORef False return $ go ref where @@ -564,13 +566,13 @@ createWrongTargetChain time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do let proof = Just . ContProof . B64U.encode . toStrict . Aeson.encode $ q - createCont sid pidv proof time + createCont v sid pidv proof time `finally` writeIORef ref True -- | Execute create-coin command with invalid proof -- createInvalidProof :: CreatesGenerator -createInvalidProof time _ pidv _ tid _ = do +createInvalidProof v time _ pidv _ tid _ = do ref <- newIORef False return $ go ref where @@ -579,14 +581,14 @@ createInvalidProof time _ pidv _ tid _ = do | otherwise = readIORef ref >>= \case True -> return mempty False -> - createCont tid pidv Nothing time + createCont v tid pidv Nothing time `finally` writeIORef ref True -- | Execute on the create-coin command on the correct target chain, with a proof -- pointing at the wrong target chain -- createProofBadTargetChain :: CreatesGenerator -createProofBadTargetChain time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do +createProofBadTargetChain v time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do ref <- newIORef False return $ go ref where @@ -600,5 +602,5 @@ createProofBadTargetChain time (TestBlockDb wdb pdb _c) pidv sid tid bhe = do let proof = Just . ContProof . B64U.encode . toStrict . Aeson.encode $ q - createCont sid pidv proof time + createCont v sid pidv proof time `finally` writeIORef ref True diff --git a/test/Chainweb/Test/Pact/TTL.hs b/test/Chainweb/Test/Pact/TTL.hs index 7a3cd167a7..015f505803 100644 --- a/test/Chainweb/Test/Pact/TTL.hs +++ b/test/Chainweb/Test/Pact/TTL.hs @@ -179,10 +179,10 @@ modAtTtl f (Seconds t) = mempty { mpaGetBlock = \_ validate bh hash ph -> do let txTime = toTxCreationTime $ f $ _bct $ _blockCreationTime ph tt = TTLSeconds (int t) - outtxs <- fmap V.singleton $ buildCwCmd + outtxs <- fmap V.singleton $ buildCwCmd testVer $ set cbCreationTime txTime $ set cbTTL tt - $ set cbSigners [mkSigner' sender00 []] + $ set cbSigners [mkEd25519Signer' sender00 []] $ mkCmd (sshow bh) $ mkExec' "1" diff --git a/test/Chainweb/Test/Pact/Utils.hs b/test/Chainweb/Test/Pact/Utils.hs index 0681b89717..3f4159fad3 100644 --- a/test/Chainweb/Test/Pact/Utils.hs +++ b/test/Chainweb/Test/Pact/Utils.hs @@ -31,6 +31,9 @@ module Chainweb.Test.Pact.Utils , sender00 , sender01 , sender00Ks +, sender02WebAuthn +, sender02WebAuthnPrefixed +, sender03WebAuthn , allocation00KeyPair , testKeyPairs , mkKeySetData @@ -65,13 +68,14 @@ module Chainweb.Test.Pact.Utils , mkCont , mkContMsg , ContMsg (..) -, mkSigner -, mkSigner' +, mkEd25519Signer +, mkEd25519Signer' +, mkWebAuthnSigner +, mkWebAuthnSigner' , CmdBuilder(..) , cbSigners , cbRPC , cbNonce -, cbNetworkId , cbChainId , cbSender , cbGasLimit @@ -170,6 +174,7 @@ import Pact.Types.Crypto import Pact.Types.Exp import Pact.Types.Gas import Pact.Types.Hash +import Pact.Types.KeySet import qualified Pact.Types.Logger as P import Pact.Types.Names import Pact.Types.PactValue @@ -198,6 +203,7 @@ import Chainweb.Pact.Backend.SQLite.DirectV2 import Chainweb.Pact.Backend.Types import Chainweb.Pact.Backend.Utils hiding (withSqliteDb) import Chainweb.Pact.PactService +import Chainweb.Pact.RestAPI.Server (validateCommand) import Chainweb.Pact.Service.PactQueue import Chainweb.Pact.Service.Types import Chainweb.Pact.Types @@ -226,9 +232,9 @@ type SimpleKeyPair = (Text,Text) -- | Legacy; better to use 'CmdSigner'/'CmdBuilder'. -- if caps are empty, gas cap is implicit. otherwise it must be included -testKeyPairs :: SimpleKeyPair -> Maybe [SigCapability] -> IO [Ed25519KeyPairCaps] +testKeyPairs :: SimpleKeyPair -> Maybe [SigCapability] -> IO [(DynKeyPair, [SigCapability])] testKeyPairs skp capsm = do - kp <- toApiKp $ mkSigner' skp (fromMaybe [] capsm) + kp <- toApiKp $ mkEd25519Signer' skp (fromMaybe [] capsm) mkKeyPairs [kp] testPactFilesDir :: FilePath @@ -242,6 +248,20 @@ sender01 :: SimpleKeyPair sender01 = ("6be2f485a7af75fedb4b7f153a903f7e6000ca4aa501179c91a2450b777bd2a7" ,"2beae45b29e850e6b1882ae245b0bab7d0689ebdd0cd777d4314d24d7024b4f7") +sender02WebAuthnPrefixed :: SimpleKeyPair +sender02WebAuthnPrefixed = + ("WEBAUTHN-a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf" + ,"fecd4feb1243d715d095e24713875ca76c476f8672ec487be8e3bc110dd329ab") + +sender02WebAuthn :: SimpleKeyPair +sender02WebAuthn = + ("a4010103272006215820c18831c6f15306d6271e154842906b68f26c1af79b132dde6f6add79710303bf" + ,"fecd4feb1243d715d095e24713875ca76c476f8672ec487be8e3bc110dd329ab") + +sender03WebAuthn :: SimpleKeyPair +sender03WebAuthn = + ("a4010103272006215820ad72392508272b4c45536976474cdd434e772bfd630738ee9aac7343e7222eb6" + ,"ebe7d1119a53863fa64be7347d82d9fcc9ebeb8cbbe480f5e8642c5c36831434") allocation00KeyPair :: SimpleKeyPair allocation00KeyPair = @@ -469,10 +489,11 @@ data CmdSigner = CmdSigner } deriving (Eq,Show,Ord,Generic) -- | Make ED25519 signer. -mkSigner :: Text -> Text -> [SigCapability] -> CmdSigner -mkSigner pubKey privKey caps = CmdSigner +mkEd25519Signer :: Text -> Text -> [SigCapability] -> CmdSigner +mkEd25519Signer pubKey privKey caps = CmdSigner { _csSigner = signer - , _csPrivKey = privKey } + , _csPrivKey = privKey + } where signer = Signer { _siScheme = Nothing @@ -480,15 +501,29 @@ mkSigner pubKey privKey caps = CmdSigner , _siAddress = Nothing , _siCapList = caps } -mkSigner' :: SimpleKeyPair -> [SigCapability] -> CmdSigner -mkSigner' (pub,priv) = mkSigner pub priv +mkEd25519Signer' :: SimpleKeyPair -> [SigCapability] -> CmdSigner +mkEd25519Signer' (pub,priv) = mkEd25519Signer pub priv + +mkWebAuthnSigner :: Text -> Text -> [SigCapability] -> CmdSigner +mkWebAuthnSigner pubKey privKey caps = CmdSigner + { _csSigner = signer + , _csPrivKey = privKey + } + where + signer = Signer + { _siScheme = Just WebAuthn + , _siPubKey = pubKey + , _siAddress = Nothing + , _siCapList = caps } + +mkWebAuthnSigner' :: SimpleKeyPair -> [SigCapability] -> CmdSigner +mkWebAuthnSigner' (pub, priv) caps = mkWebAuthnSigner pub priv caps -- | Chainweb-oriented command builder. data CmdBuilder = CmdBuilder { _cbSigners :: ![CmdSigner] , _cbRPC :: !(PactRPC Text) , _cbNonce :: !Text - , _cbNetworkId :: !(Maybe ChainwebVersion) , _cbChainId :: !ChainId , _cbSender :: !Text , _cbGasLimit :: !GasLimit @@ -522,7 +557,6 @@ defaultCmd = CmdBuilder { _cbSigners = [] , _cbRPC = mkExec' "1" , _cbNonce = "nonce" - , _cbNetworkId = Nothing , _cbChainId = unsafeChainId 0 , _cbSender = "sender00" , _cbGasLimit = 10_000 @@ -540,39 +574,61 @@ mkCmd nonce rpc = defaultCmd -- | Build parsed + verified Pact command -- -buildCwCmd :: (MonadThrow m, MonadIO m) => CmdBuilder -> m ChainwebTransaction -buildCwCmd cmd = buildRawCmd cmd >>= \c -> case verifyCommand c of - ProcSucc r -> return $ fmap (mkPayloadWithText c) r - ProcFail e -> throwM $ userError $ "buildCmd failed: " ++ e - - +-- TODO: Use the new `assertCommand` function. +buildCwCmd :: (MonadThrow m, MonadIO m) => ChainwebVersion -> CmdBuilder -> m ChainwebTransaction +buildCwCmd v cmd = buildRawCmd v cmd >>= \(c :: Command ByteString) -> + case validateCommand v (_cbChainId cmd) (T.decodeUtf8 <$> c) of + Left err -> throwM $ userError $ "buildCmd failed: " ++ err + Right cmd' -> return cmd' -- | Build unparsed, unverified command -- -buildTextCmd :: CmdBuilder -> IO (Command Text) -buildTextCmd = fmap (fmap T.decodeUtf8) . buildRawCmd +buildTextCmd :: ChainwebVersion -> CmdBuilder -> IO (Command Text) +buildTextCmd v = fmap (fmap T.decodeUtf8) . buildRawCmd v -- | Build a raw bytestring command -- -buildRawCmd :: (MonadThrow m, MonadIO m) => CmdBuilder -> m (Command ByteString) -buildRawCmd CmdBuilder{..} = do - akps <- mapM toApiKp _cbSigners - kps <- liftIO $ mkKeyPairs akps - liftIO $ mkCommand kps pm _cbNonce nid _cbRPC +buildRawCmd :: (MonadThrow m, MonadIO m) => ChainwebVersion -> CmdBuilder -> m (Command ByteString) +buildRawCmd v CmdBuilder{..} = do + kps <- liftIO $ traverse mkDynKeyPairs _cbSigners + cmd <- liftIO $ mkCommandWithDynKeys kps pm _cbNonce (Just nid) _cbRPC + pure cmd where - nid = fmap (P.NetworkId . sshow) _cbNetworkId + nid = P.NetworkId (sshow v) cid = fromString $ show (chainIdInt _cbChainId :: Int) pm = PublicMeta cid _cbSender _cbGasLimit _cbGasPrice _cbTTL _cbCreationTime dieL :: MonadThrow m => [Char] -> Either [Char] a -> m a dieL msg = either (\s -> throwM $ userError $ msg ++ ": " ++ s) return +mkDynKeyPairs :: MonadThrow m => CmdSigner -> m (DynKeyPair, [SigCapability]) +mkDynKeyPairs (CmdSigner Signer{..} privKey) = + case (fromMaybe ED25519 _siScheme, _siPubKey, privKey) of + (ED25519, pub, priv) -> do + pub' <- either diePubKey return $ parseEd25519PubKey =<< parseB16TextOnly pub + priv' <- either diePrivKey return $ parseEd25519SecretKey =<< parseB16TextOnly priv + return $ (DynEd25519KeyPair (pub', priv'), _siCapList) + + (WebAuthn, pub, priv) -> do + let (pubKeyStripped, wasPrefixed) = fromMaybe + (pub, WebAuthnPubKeyBare) + ((,WebAuthnPubKeyPrefixed) <$> T.stripPrefix webAuthnPrefix pub) + pubWebAuthn <- + either diePubKey return (parseWebAuthnPublicKey =<< parseB16TextOnly pubKeyStripped) + privWebAuthn <- + either diePrivKey return (parseWebAuthnPrivateKey =<< parseB16TextOnly priv) + return $ (DynWebAuthnKeyPair wasPrefixed pubWebAuthn privWebAuthn, _siCapList) + where + diePubKey str = error $ "pubkey: " <> str + diePrivKey str = error $ "privkey: " <> str + toApiKp :: MonadThrow m => CmdSigner -> m ApiKeyPair toApiKp (CmdSigner Signer{..} privKey) = do sk <- dieL "private key" $ parseB16TextOnly privKey pk <- dieL "public key" $ parseB16TextOnly _siPubKey - return $! - ApiKeyPair (PrivBS sk) (Just (PubBS pk)) _siAddress _siScheme (Just _siCapList) + let keyPair = ApiKeyPair (PrivBS sk) (Just (PubBS pk)) _siAddress _siScheme (Just _siCapList) + return $! keyPair + -- ----------------------------------------------------------------------- -- -- Service creation utilities @@ -644,7 +700,14 @@ testPactCtxSQLite logger v cid bhdb pdb sqlenv conf gasmodel = do , _psIsBatch = False , _psCheckpointerDepth = 0 , _psLogger = addLabel ("chain-id", chainIdToText cid) $ addLabel ("component", "pact") $ _cpLogger cp - , _psGasLogger = Nothing + , _psGasLogger = do + guard (_pactLogGas conf) + return + $ addLabel ("chain-id", chainIdToText cid) + $ addLabel ("component", "pact") + $ addLabel ("sub-component", "gas") + $ _cpLogger cp + , _psBlockGasLimit = _pactBlockGasLimit conf , _psChainId = cid } diff --git a/test/Chainweb/Test/Rosetta/RestAPI.hs b/test/Chainweb/Test/Rosetta/RestAPI.hs index f5adddb63c..fa6a836544 100644 --- a/test/Chainweb/Test/Rosetta/RestAPI.hs +++ b/test/Chainweb/Test/Rosetta/RestAPI.hs @@ -140,7 +140,7 @@ tests rdb = testGroup "Chainweb.Test.Rosetta.RestAPI" go , networkListTests , networkOptionsTests , networkStatusTests - , blockKAccountAfterPact420 + , blockKAccountAfterPact42 , constructionTransferTests , blockCoinV3RemediationTests ] @@ -177,8 +177,8 @@ accountBalanceTests tio envIo = -- TxLog parse error after fork to Pact 420. -- This assumes that this test occurs after the -- fork blockheight. -blockKAccountAfterPact420 :: RosettaTest -blockKAccountAfterPact420 tio envIo = +blockKAccountAfterPact42 :: RosettaTest +blockKAccountAfterPact42 tio envIo = testCaseSteps "Block k Account After Pact 420 Test" $ \step -> do cenv <- envIo rkmv <- newEmptyMVar @RequestKeys @@ -596,18 +596,19 @@ submitToConstructionAPI expectOps chainId' payer getKeys expectResult cenv step Right pk' <- pure $ P.parseB16TextOnly pk let akps = P.ApiKeyPair (P.PrivBS sk') (Just $ P.PubBS pk') Nothing Nothing Nothing - [(kp,_)] <- P.mkKeyPairs [akps] + [(DynEd25519KeyPair kp,_)] <- P.mkKeyPairs [akps] (Right (hsh :: P.PactHash)) <- pure $ fmap (P.fromUntypedHash . P.Hash . BS.toShort) (P.parseB16TextOnly $ _rosettaSigningPayload_hexBytes payload) - sig <- P.signHash hsh kp + let + sig = P.signHash hsh kp pure $! RosettaSignature { _rosettaSignature_signingPayload = payload , _rosettaSignature_publicKey = RosettaPublicKey pk CurveEdwards25519 , _rosettaSignature_signatureType = RosettaEd25519 - , _rosettaSignature_hexBytes = P._usSig sig + , _rosettaSignature_hexBytes = sig } acct n = AccountId n Nothing Nothing @@ -791,15 +792,14 @@ mkTransfer :: ChainId -> IO (Time Micros) -> IO SubmitBatch mkTransfer sid tio = do t <- toTxCreationTime <$> tio n <- readIORef nonceRef - c <- buildTextCmd + c <- buildTextCmd v $ set cbSigners - [ mkSigner' sender00 + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" "sender01" 1.0 , mkGasCap ] ] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ set cbChainId sid $ mkCmd ("nonce-transfer-" <> sshow t <> "-" <> sshow n) $ mkExec' "(coin.transfer \"sender00\" \"sender01\" 1.0)" @@ -812,14 +812,13 @@ mkKCoinAccount sid tio = do let kAcct = "k:" <> fst sender00 t <- toTxCreationTime <$> tio n <- readIORef nonceRef - c <- buildTextCmd + c <- buildTextCmd v $ set cbSigners - [ mkSigner' sender00 + [ mkEd25519Signer' sender00 [ mkTransferCap "sender00" kAcct 20.0 , mkGasCap ] ] $ set cbCreationTime t - $ set cbNetworkId (Just v) $ set cbChainId sid $ mkCmd ("nonce-transfer-" <> sshow t <> "-" <> sshow n) $ mkExec ("(coin.transfer-create \"sender00\" \"" <> kAcct <> "\" (read-keyset \"sender00\") 20.0)") diff --git a/test/Chainweb/Test/TestVersions.hs b/test/Chainweb/Test/TestVersions.hs index a79066ae13..1eeae049a7 100644 --- a/test/Chainweb/Test/TestVersions.hs +++ b/test/Chainweb/Test/TestVersions.hs @@ -126,7 +126,7 @@ fastForks = tabulateHashMap $ \case Chainweb213Pact -> AllChains ForkAtGenesis PactEvents -> AllChains ForkAtGenesis CoinV2 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1 - Pact420 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1 + Pact42 -> AllChains $ ForkAtBlockHeight $ BlockHeight 1 SkipTxTimingValidation -> AllChains $ ForkAtBlockHeight $ BlockHeight 2 ModuleNameFix -> AllChains $ ForkAtBlockHeight $ BlockHeight 2 ModuleNameFix2 -> AllChains $ ForkAtBlockHeight $ BlockHeight 2 @@ -139,6 +139,7 @@ fastForks = tabulateHashMap $ \case Chainweb219Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 27 Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 30 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 33 + Chainweb222Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 36 -- | A test version without Pact or PoW, with only one chain graph. barebonesTestVersion :: ChainGraph -> ChainwebVersion @@ -246,7 +247,7 @@ slowForkingCpmTestVersion g = buildTestVersion $ \v -> v SkipTxTimingValidation -> AllChains $ ForkAtBlockHeight (BlockHeight 2) ModuleNameFix -> AllChains $ ForkAtBlockHeight (BlockHeight 2) ModuleNameFix2 -> AllChains $ ForkAtBlockHeight (BlockHeight 2) - Pact420 -> AllChains $ ForkAtBlockHeight (BlockHeight 5) + Pact42 -> AllChains $ ForkAtBlockHeight (BlockHeight 5) CheckTxHash -> AllChains $ ForkAtBlockHeight (BlockHeight 7) EnforceKeysetFormats -> AllChains $ ForkAtBlockHeight (BlockHeight 10) PactEvents -> AllChains $ ForkAtBlockHeight (BlockHeight 10) @@ -260,6 +261,7 @@ slowForkingCpmTestVersion g = buildTestVersion $ \v -> v Chainweb219Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 71) Chainweb220Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 85) Chainweb221Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 100) + Chainweb222Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 115) -- | CPM version (see `cpmTestVersion`) with forks and upgrades quickly enabled. fastForkingCpmTestVersion :: ChainGraph -> ChainwebVersion @@ -275,4 +277,3 @@ noBridgeCpmTestVersion g = buildTestVersion $ \v -> v & cpmTestVersion g & versionName .~ ChainwebVersionName ("nobridge-CPM-" <> toText g) & versionForks .~ (fastForks & at SPVBridge ?~ AllChains ForkNever) - diff --git a/test/golden/create-accounts-expected.txt b/test/golden/create-accounts-expected.txt index 61fca2476c..28289208bb 100644 --- a/test/golden/create-accounts-expected.txt +++ b/test/golden/create-accounts-expected.txt @@ -3,4 +3,4 @@ - cmd: contents: (test1.create-global-accounts) tag: Code - output: G15srpFoJgfoVchKV2ilyjMGtscxfNFlxwV0tvUC6bE + output: PC05AVVNYfsSZFSUQOOIWp6jFe5eSDDeJs6QhyemyeA diff --git a/test/golden/create-table-expected.txt b/test/golden/create-table-expected.txt index 71b1f39b56..5bcc956d8d 100644 --- a/test/golden/create-table-expected.txt +++ b/test/golden/create-table-expected.txt @@ -3,4 +3,4 @@ - cmd: contents: (create-table test1.accounts) tag: Code - output: ExfKxPyfW_VnvUXfZva1taP6-woqfIYlNcnlFnKv5wQ + output: 2Ld8y7v3yS4WgJdp2jG4ayjrhnn0PJALeunzpdL3fe8 diff --git a/test/golden/new-block-0-expected.txt b/test/golden/new-block-0-expected.txt index 8da2011f16..82501ee2a2 100644 --- a/test/golden/new-block-0-expected.txt +++ b/test/golden/new-block-0-expected.txt @@ -1,30 +1,30 @@ results: coinbase: eyJnYXMiOjAsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6IldyaXRlIHN1Y2NlZWRlZCJ9LCJyZXFLZXkiOiJJbXRNTUVWWWEyc3lkMlJrVFZkeU1sRkJUVU5YUkRaelFqZHBNR3hPVTFZM2VGWm5XSGN5YWxBelNFMGkiLCJsb2dzIjoibFdaVWZjX0dKdGxUUy1LeDAtYm00aXRfSWE5bnNGZG5jZ2NwSE0td091OCIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjd9 minerData: eyJhY2NvdW50IjoiTm9NaW5lciIsInByZWRpY2F0ZSI6IjwiLCJwdWJsaWMta2V5cyI6W119 - outputsHash: 9ZzMlVJFvUJP7Z--MALWu9r8EKkuvvExTJUZyidk3yM - payloadHash: G3LQ7UEnUBMQBg2-mtICF7i2AKoAp_8tJZs7a8SG0lI + outputsHash: oh0ZsBr60JGxL41VpPKW2zL5RXztLbYvKsHtdO8AgB8 + payloadHash: EnDoblnPIEO9eDOjHz1P8nYdziWg91F6-dfibI69sQE transactions: - - - eyJoYXNoIjoiSWxGRDdhOVVZT0dXMUJHRUZjYWRwejVUVW1XLVc5ZkJ5NkZIVzYzaXEzayIsInNpZ3MiOlt7InNpZyI6ImYzZTc5Mjc4MGRmNTIwZmVkMjg5MDc3MjllMjI1NTA4NTk2MzJhMzNiMTQzMTM0MDlkOTkyZmRmMjJjNTZlZDEwMzdiZTcxNDYyNWRkMTZmYWI5MmVlNjZhZmQ1MjhiNWViZjEwYWNhZjY2ZjRlZGVmMWM2ZjZjY2Q4OTkxMjA0In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihkZWZpbmUta2V5c2V0ICd0ZXN0LWFkbWluIChyZWFkLWtleXNldCBcXFwidGVzdC1hZG1pbi1rZXlzZXRcXFwiKSlcXG5cXG4obmFtZXNwYWNlICdmcmVlKVxcblxcbihtb2R1bGUgdGVzdDEgJ3Rlc3QtYWRtaW5cXG5cXG4gIChkZWZzY2hlbWEgYWNjb3VudFxcbiAgICBiYWxhbmNlOmRlY2ltYWxcXG4gICAgYW1vdW50OmRlY2ltYWxcXG4gICAgZGF0YSlcXG5cXG4gIChkZWZ0YWJsZSBhY2NvdW50czp7YWNjb3VudH0pXFxuXFxuICAoZGVmdW4gY3JlYXRlLWFjY291bnQgKGlkIGluaXQtYmFsKVxcbiAgICAoaW5zZXJ0IGFjY291bnRzIGlkXFxuICAgICAgICAgeyBcXFwiYmFsYW5jZVxcXCI6IGluaXQtYmFsLCBcXFwiYW1vdW50XFxcIjogaW5pdC1iYWwsIFxcXCJkYXRhXFxcIjogXFxcIkNyZWF0ZWQgYWNjb3VudFxcXCIgfSkpXFxuXFxuICAoZGVmdW4gdHJhbnNmZXIgKHNyYyBkZXN0IGFtb3VudClcXG4gICAgXFxcInRyYW5zZmVyIEFNT1VOVCBmcm9tIFNSQyB0byBERVNUIGZvciB1bmVuY3J5cHRlZCBhY2NvdW50c1xcXCJcXG4gICAgKGRlYml0IHNyYyBhbW91bnQgeyBcXFwidHJhbnNmZXItdG9cXFwiOiBkZXN0IH0pXFxuICAgIChjcmVkaXQgZGVzdCBhbW91bnQgeyBcXFwidHJhbnNmZXItZnJvbVxcXCI6IHNyYyB9KSlcXG5cXG4gIChkZWZwYWN0IHBheW1lbnQgKHNyYy1lbnRpdHkgc3JjIGRlc3QtZW50aXR5IGRlc3QgYW1vdW50KVxcbiAgICBcXFwiVHdvLXBoYXNlIGNvbmZpZGVudGlhbCBwYXltZW50LCBzZW5kaW5nIG1vbmV5IGZyb20gU1JDIGF0IFNSQy1FTlRJVFkgdG8gREVTVCBhdCBERVNULUVOVElUWS5cXFwiXFxuXFxuICAgIChzdGVwLXdpdGgtcm9sbGJhY2tcXG4gICAgIHNyYy1lbnRpdHlcXG4gICAgIChsZXQgKChyZXN1bHQgKGRlYml0IHNyYyBhbW91bnQgeyBcXFwidHJhbnNmZXItdG9cXFwiOiBkZXN0LCBcXFwibWVzc2FnZVxcXCI6IFxcXCJTdGFydGluZyBwYWN0XFxcIiB9KSkpXFxuICAgICAgICh5aWVsZCB7IFxcXCJyZXN1bHRcXFwiOiByZXN1bHQsIFxcXCJhbW91bnRcXFwiOiBhbW91bnQsIFxcXCJ0eFxcXCI6IChwYWN0LWlkKSB9KSlcXG4gICAgIChjcmVkaXQgc3JjIGFtb3VudCB7IFxcXCJyb2xsYmFja1xcXCI6IChwYWN0LWlkKSB9KSlcXG5cXG4gICAgKHN0ZXBcXG4gICAgIGRlc3QtZW50aXR5XFxuICAgICAocmVzdW1lIHsgXFxcInJlc3VsdFxcXCI6PSByZXN1bHQsIFxcXCJhbW91bnRcXFwiOj0gZGViaXQtYW1vdW50IH1cXG4gICAgICAgKGNyZWRpdCBkZXN0IGRlYml0LWFtb3VudFxcbiAgICAgICAgICAgICAgIHsgXFxcInRyYW5zZmVyLWZyb21cXFwiOiBzcmMsIFxcXCJkZWJpdC1yZXN1bHRcXFwiOiByZXN1bHQsIFxcXCJ0eFxcXCI6IChwYWN0LWlkKSB9KSkpKVxcblxcbiAgKGRlZnVuIGRlYml0IChhY2N0IGFtb3VudCBkYXRhKVxcbiAgICBcXFwiRGViaXQgQUNDVCBmb3IgQU1PVU5ULCBlbmZvcmNpbmcgcG9zaXRpdmUgYW1vdW50IGFuZCBzdWZmaWNpZW50IGZ1bmRzLCBhbm5vdGF0aW5nIHdpdGggREFUQVxcXCJcXG4gICAgKGVuZm9yY2UtcG9zaXRpdmUgYW1vdW50KVxcbiAgICAod2l0aC1yZWFkIGFjY291bnRzIGFjY3QgeyBcXFwiYmFsYW5jZVxcXCI6PSBiYWxhbmNlIH1cXG4gICAgICAoY2hlY2stYmFsYW5jZSBiYWxhbmNlIGFtb3VudClcXG4gICAgICAodXBkYXRlIGFjY291bnRzIGFjY3RcXG4gICAgICAgICAgICB7IFxcXCJiYWxhbmNlXFxcIjogKC0gYmFsYW5jZSBhbW91bnQpLCBcXFwiYW1vdW50XFxcIjogKC0gYW1vdW50KVxcbiAgICAgICAgICAgICwgXFxcImRhdGFcXFwiOiBkYXRhIH0pKSlcXG5cXG4gIChkZWZ1biBjcmVkaXQgKGFjY3QgYW1vdW50IGRhdGEpXFxuICAgIFxcXCJDcmVkaXQgQUNDVCBmb3IgQU1PVU5ULCBlbmZvcmNpbmcgcG9zaXRpdmUgYW1vdW50XFxcIlxcbiAgICAoZW5mb3JjZS1wb3NpdGl2ZSBhbW91bnQpXFxuICAgICh3aXRoLXJlYWQgYWNjb3VudHMgYWNjdCB7IFxcXCJiYWxhbmNlXFxcIjo9IGJhbGFuY2UgfVxcbiAgICAgICh1cGRhdGUgYWNjb3VudHMgYWNjdFxcbiAgICAgICAgICAgIHsgXFxcImJhbGFuY2VcXFwiOiAoKyBiYWxhbmNlIGFtb3VudCksIFxcXCJhbW91bnRcXFwiOiBhbW91bnRcXG4gICAgICAgICAgICAsIFxcXCJkYXRhXFxcIjogZGF0YSB9KSkpXFxuXFxuXFxuICAoZGVmdW4gcmVhZC1hY2NvdW50IChpZClcXG4gICAgXFxcIlJlYWQgZGF0YSBmb3IgYWNjb3VudCBJRFxcXCJcXG4gICAgKCsgeyBcXFwiYWNjb3VudFxcXCI6IGlkIH0gKHJlYWQgYWNjb3VudHMgaWQpKSlcXG5cXG4gIChkZWZ1biBjaGVjay1iYWxhbmNlIChiYWxhbmNlIGFtb3VudClcXG4gICAgKGVuZm9yY2UgKDw9IGFtb3VudCBiYWxhbmNlKSBcXFwiSW5zdWZmaWNpZW50IGZ1bmRzXFxcIikpXFxuXFxuICAoZGVmdW4gZW5mb3JjZS1wb3NpdGl2ZSAoYW1vdW50KVxcbiAgICAoZW5mb3JjZSAoPj0gYW1vdW50IDAuMCkgXFxcImFtb3VudCBtdXN0IGJlIHBvc2l0aXZlXFxcIikpXFxuXFxuIChkZWZ1biByZWFkLWFsbCAoKVxcbiAgIChtYXAgKHJlYWQtYWNjb3VudCkgKGtleXMgYWNjb3VudHMpKSlcXG5cXG4gKGRlZnVuIHJlYWQtYWxsLWdsb2JhbCAoKVxcbiAgIChtYXAgKHJlYWQtYWNjb3VudCkgW1xcXCJBY2N0MVxcXCIgXFxcIkFjY3QyXFxcIl0pKVxcblxcbiAoZGVmdW4gY3JlYXRlLWdsb2JhbC1hY2NvdW50cyAoKVxcbiAgIChjcmVhdGUtYWNjb3VudCBcXFwiQWNjdDFcXFwiIDEwMDAwMDAuMClcXG4gICAoY3JlYXRlLWFjY291bnQgXFxcIkFjY3QyXFxcIiAwLjApXFxuICAgKHJlYWQtYWxsKSlcXG5cXG4gKGRlZnBhY3QgY3JlYXRlLXByaXZhdGUtYWNjb3VudHMgKClcXG4gICAoc3RlcCBcXFwiQWxpY2VcXFwiIChjcmVhdGUtYWNjb3VudCBcXFwiQVxcXCIgMTAwMC4wKSlcXG4gICAoc3RlcCBcXFwiQm9iXFxcIiAoY3JlYXRlLWFjY291bnQgXFxcIkJcXFwiIDEwMDAuMCkpXFxuICAgKHN0ZXAgXFxcIkNhcm9sXFxcIiAoY3JlYXRlLWFjY291bnQgXFxcIkNcXFwiIDEwMDAuMCkpXFxuICAgKHN0ZXAgXFxcIkRpbmVzaFxcXCIgKGNyZWF0ZS1hY2NvdW50IFxcXCJEXFxcIiAxMDAwLjApKSlcXG5cXG4pXFxuXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTBcIn0ifQ - - eyJnYXMiOjY0OSwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiTG9hZGVkIG1vZHVsZSBmcmVlLnRlc3QxLCBoYXNoIEI2WkpnZldENW9iUEdtWWNfaVJLaXJETVA2LVVHbTJwVVY3bXRIRC1Sc1EifSwicmVxS2V5IjoiSWxGRDdhOVVZT0dXMUJHRUZjYWRwejVUVW1XLVc5ZkJ5NkZIVzYzaXEzayIsImxvZ3MiOiJMRm51NzRDaE4wZS1IbGI3azlqZlAzcmpmcWh6WjFweUpibXpkX1lOMFdZIiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MTF9 - - - eyJoYXNoIjoidUFaMDFQQ21QWW43UTBrdGRMZ21abEZueVFUU2t3aXRXTmp0ZTNUQ3hOYyIsInNpZ3MiOlt7InNpZyI6IjU1Nzk3Zjk2YjEwYzY5ZjRhMGY2ZTg5ZDI4OGFjMTNlZDE1MDI4NWJkNDJmODNjZTMzNjE2M2IzYTllNmM3MjFiY2FlMGE2MzhjMTgwYTk0MTE2YjlhZDBkM2UwOWFlYmI3OGE2NjVhNTg2MzdhMmExZmE0OGU3NDZiYzU5NDA5In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihjcmVhdGUtdGFibGUgZnJlZS50ZXN0MS5hY2NvdW50cylcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxMVwifSJ9 - - eyJnYXMiOjI2NCwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiVGFibGVDcmVhdGVkIn0sInJlcUtleSI6InVBWjAxUENtUFluN1Ewa3RkTGdtWmxGbnlRVFNrd2l0V05qdGUzVEN4TmMiLCJsb2dzIjoianVuYlJrd3hEMVhiUHBvTnNCZlN3NEJGYUEtcW1mempEMmhfOHhQTmczbyIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjE0fQ - - - eyJoYXNoIjoiWjZxajdIZnBSN0pnZkJFZGhsYU9BSTdjdDd1ZFM2dE5jVUF0RFZMSHNJbyIsInNpZ3MiOlt7InNpZyI6ImM5YzExMGE2MGI4M2JmMmM1OTFiMTMxMWJjOTdjNjJiNzkxZTcwNDYzM2RhY2EyMGNhYmE2MTRhNGQwYThkYmFlODJlNmQxOWQ2NDZlMWJmNGNlMWUyMDZjMTM4MmM4NGExMGI5Y2NkYWY2MzAzOGU1ZWJmMDk3ZmE3YzZjMTAzIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihmcmVlLnRlc3QxLmNyZWF0ZS1nbG9iYWwtYWNjb3VudHMpXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTJcIn0ifQ - - eyJnYXMiOjU4MCwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjpbeyJhbW91bnQiOjEwMDAwMDAsImRhdGEiOiJDcmVhdGVkIGFjY291bnQiLCJiYWxhbmNlIjoxMDAwMDAwLCJhY2NvdW50IjoiQWNjdDEifSx7ImFtb3VudCI6MCwiZGF0YSI6IkNyZWF0ZWQgYWNjb3VudCIsImJhbGFuY2UiOjAsImFjY291bnQiOiJBY2N0MiJ9XX0sInJlcUtleSI6Ilo2cWo3SGZwUjdKZ2ZCRWRobGFPQUk3Y3Q3dWRTNnROY1VBdERWTEhzSW8iLCJsb2dzIjoia0VIU2ZyQXRLQV9vdi1xZDV4cmpSSTFtX2phenRkSF9JbGVJWG9QNW9wNCIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjE3fQ - - - eyJoYXNoIjoicFBrLWxFUE1jRnRDWXdXR2ZaejV2bGdzZ1ZYV3dYbXhmQVJ6bkFCbUVONCIsInNpZ3MiOlt7InNpZyI6IjM3YjI4MmY1YWQ0YWY0Yzk2YmE1NzA5YWEzOTI2NTU1YjU5MDEwYzczZmVmZTUwNzQyMGYyODVjZTI1MmQ5ZjM1YzQ5NjAxYTgwMDYxZGU2NzEzNTcxNzY2MTNmODIyODQxMDU0MjliZTVkOTEzNWMzODAxMzcyN2UyMmI3ZjA2In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihmcmVlLnRlc3QxLnRyYW5zZmVyIFxcXCJBY2N0MVxcXCIgXFxcIkFjY3QyXFxcIiAxLjAwKVwifX0sXCJzaWduZXJzXCI6W3tcInB1YktleVwiOlwiMzY4ODIwZjgwYzMyNGJiYzdjMmIwNjEwNjg4YTdkYTQzZTM5ZjkxZDExODczMjY3MWNkOWM3NTAwZmY0M2NjYVwifV0sXCJtZXRhXCI6e1wiY3JlYXRpb25UaW1lXCI6MCxcInR0bFwiOjEwMDAwMDAsXCJnYXNMaW1pdFwiOjEwMDAwLFwiY2hhaW5JZFwiOlwiMFwiLFwiZ2FzUHJpY2VcIjoxLjBlLTIsXCJzZW5kZXJcIjpcInNlbmRlcjAwXCJ9LFwibm9uY2VcIjpcIjEzXCJ9In0 - - eyJnYXMiOjQwNywicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiV3JpdGUgc3VjY2VlZGVkIn0sInJlcUtleSI6InBQay1sRVBNY0Z0Q1l3V0dmWno1dmxnc2dWWFd3WG14ZkFSem5BQm1FTjQiLCJsb2dzIjoib3hWd2tvU21xM2JZeVh5aHYwT0V3ZTN2c0tGbWlkRTBtTEotQ2FZWVhRayIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjIwfQ - - - eyJoYXNoIjoiX2RneUxfSGtlZGJFZjl5UTY5Wkhhc2g2M01JVUw2ZktibzBqeXJJRWhPYyIsInNpZ3MiOlt7InNpZyI6IjZhODRiMWRkNGFkNDE5ZGY3YThhODA4NGYwNWEyZDM5YzExNTBmMjI5ZTM3OTFlM2UwY2E0OTkyZDkwNGNhOGVlMTk4MTljNTE3YTVjNjM5M2E3ZmYyNWI1NjdiNjJjOTYwZTFiYmI4ZTE2YjZiMWU2YmRkNDU1Njk3ODc1MDA5In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAncHJldi1ibG9jay1oYXNoIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxNFwifSJ9 - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6ImdqTmVmdmM3STRpblNOOUxFZkVCN0c0OUlvWC0ycGxEQmVUbFpwU1czQTQifSwicmVxS2V5IjoiX2RneUxfSGtlZGJFZjl5UTY5Wkhhc2g2M01JVUw2ZktibzBqeXJJRWhPYyIsImxvZ3MiOiJUVVVlWFJmRFlnQlZTdjFZU0ktU3lfRVdtcHRaWXZqQlNmY1ZMc2ZzcVpVIiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MjN9 - - - eyJoYXNoIjoiTzhoWU1TVUhnSkNPRXFoTy0yRFRNRGJhTzBIZmx6VjM0UHRzM0dlUHBJUSIsInNpZ3MiOlt7InNpZyI6ImMxYjQ5OTQwNTQ0MjZmNzE2NWM2NmZjNjY0MzE0N2M1N2QxYTczYzM3NjEwMjU4OGU0M2QyMzA1ZjBhY2FhYTQyMDRkZGRmNGRhMWFiMzk0OTdkY2FiM2FjZWU1NmQ0NmQ3NzJkNDZkNjAyNmU5ZjQ4MGFmMWNiMWIyNWFmYzA3In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnYmxvY2stdGltZSAoY2hhaW4tZGF0YSkpXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTVcIn0ifQ - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6eyJ0aW1lIjoiMTk3MC0wMS0wMVQwMDowMDowMFoifX0sInJlcUtleSI6Ik84aFlNU1VIZ0pDT0VxaE8tMkRUTURiYU8wSGZselYzNFB0czNHZVBwSVEiLCJsb2dzIjoidTdKNDU3dGdzNDJ1dGhjOU1TbWtkaTkxT2tUb2FCNFpGei1QUWtSajJhdyIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjI2fQ - - - eyJoYXNoIjoiSGFsbTJLYUV5Qm5weXJqMkxHMmJQcVZDUmZBYk8yWHZzZWlnaURaZElLOCIsInNpZ3MiOlt7InNpZyI6ImI0MDNlZWI3ODI0NzA0OWU0NjEzNGQzYWQ0MWZlMTAwNWMxM2I0NzNhNjUzZGNmNTVlN2ZmMzIzOGU5N2VkYmRmZTMyZmFmODczODViYTc0MmNmMDFmYmFkMDU2ODEwNWVjNDhhMDk1MzIyNWJmMmY0ODJjNjhiMjkyZGNmOTAzIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnYmxvY2staGVpZ2h0IChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxNlwifSJ9 - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MX0sInJlcUtleSI6IkhhbG0yS2FFeUJucHlyajJMRzJiUHFWQ1JmQWJPMlh2c2VpZ2lEWmRJSzgiLCJsb2dzIjoiaEtSOEZUYkhSVnhxV092TTZWXzZ1dUdMSVA3T2loV25RR3dFSnV0REF0cyIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjI5fQ - - - eyJoYXNoIjoiRVI0TlZZaFp1Z3BJeXR6TFhQWjJUZ18wUGc5T3NvTEtva1RxN0R3WGpLdyIsInNpZ3MiOlt7InNpZyI6IjJlNDZjMWMzZDQ0YTQ3YzdiZDY2ZTM1MWNhZDBmY2JjYmMyMDk1ZWJhYTk5MzY4NjRmZmZhOTVlNTJhNjE5MDlkMTBlODY3Y2RhMjFmNTcyZjMxODhmYjNmNjUxYTAzMzY3NWM5ZTk2NDFhNDBlM2EwNGNlMmUyMDk4NjQ2NjAzIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnZ2FzLWxpbWl0IChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxN1wifSJ9 - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MTAwMDB9LCJyZXFLZXkiOiJFUjROVlloWnVncEl5dHpMWFBaMlRnXzBQZzlPc29MS29rVHE3RHdYakt3IiwibG9ncyI6IklwcDE1S0QtdlZXS3JwanVDM0NBWHk3cTZia1hudURIbF82TmFiaGVyd1EiLCJtZXRhRGF0YSI6bnVsbCwiY29udGludWF0aW9uIjpudWxsLCJ0eElkIjozMn0 - - - eyJoYXNoIjoia21ldS1LajJudVNMcHZzNmVsNVdiY0d6VnZ3ZjlaSjd2LTJCa2g4UmVlQSIsInNpZ3MiOlt7InNpZyI6IjIwNmZkMmMzYTBkMzBjMDhlN2FhY2U4ZGZmNmQ1MTIxYWZhMTA3NThhM2M3YTFhNDc2NTE1YmFlMjNiNmNiZmM5NjRiNjhkYmYxMzU5MzVhYWNiYzNlZmYzMzczZWE4YmUxMmI0OTdmOTU1NjU5MDQ3NDAxNTQ0ZTUyMDNjMjA4In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnZ2FzLXByaWNlIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxOFwifSJ9 - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MS4wZS0yfSwicmVxS2V5Ijoia21ldS1LajJudVNMcHZzNmVsNVdiY0d6VnZ3ZjlaSjd2LTJCa2g4UmVlQSIsImxvZ3MiOiJaUDhDRmdiOEZQUkQ0Q3E2MFV0aUp3aU9oaDduRFU3YXFBNnQtbDFFbS1NIiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MzV9 - - - eyJoYXNoIjoiU2REQTV6VXVPWEJOZEQ5cnpzYWEzU1NRTENUTTdVWWNPeWhoaWxGUlNURSIsInNpZ3MiOlt7InNpZyI6IjhiNGU0MjZiMDMwMGYwOTJmMWJhZDVkYTY2MWEzYjBmZWE0ZjUwNTAyZmI3ODZjOWQyOTZhNTgwZTRmOGViMTc1Njc5YjQ4NDQwMDAyNjQ2MTc2OWRjMzI4MzkxYTQzM2E5NjAxY2ZhMzk5ZDQyNmUxYTdhMmUwOWYxNzYyNzAxIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnY2hhaW4taWQgKGNoYWluLWRhdGEpKVwifX0sXCJzaWduZXJzXCI6W3tcInB1YktleVwiOlwiMzY4ODIwZjgwYzMyNGJiYzdjMmIwNjEwNjg4YTdkYTQzZTM5ZjkxZDExODczMjY3MWNkOWM3NTAwZmY0M2NjYVwifV0sXCJtZXRhXCI6e1wiY3JlYXRpb25UaW1lXCI6MCxcInR0bFwiOjEwMDAwMDAsXCJnYXNMaW1pdFwiOjEwMDAwLFwiY2hhaW5JZFwiOlwiMFwiLFwiZ2FzUHJpY2VcIjoxLjBlLTIsXCJzZW5kZXJcIjpcInNlbmRlcjAwXCJ9LFwibm9uY2VcIjpcIjE5XCJ9In0 - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6IjAifSwicmVxS2V5IjoiU2REQTV6VXVPWEJOZEQ5cnpzYWEzU1NRTENUTTdVWWNPeWhoaWxGUlNURSIsImxvZ3MiOiItNEMyR2Myc0hJWE9sREt0cFBZNGZpaEI0dndWc0NPYnpPc0JxOFAyNEQ0IiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6Mzh9 - - - eyJoYXNoIjoiV2dudUNnNkxfbDZsemJqV3RCZk1FdVB0dHlfdUdjTnJVb2w1SEdSRU9fbyIsInNpZ3MiOlt7InNpZyI6IjgxNzAxZTJiMDQ4MzdjMDk1NDY3NGNiNWM2YzcxMGJhZGZlNzA5MjJiYTM0YTEyNDVjMWZmZDJmY2Q2NTQ4NzExZjY2Y2FmNTc1Njk2ZjU2NmQzYWQ3ZGNlMzgxZGQxN2M1Njc0OWQwM2M3OWQ5NmI1OWIxNGQ3ZjAwMTE0NjBhIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpudWxsLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnc2VuZGVyIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxMTBcIn0ifQ - - eyJnYXMiOjcsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6InNlbmRlcjAwIn0sInJlcUtleSI6IldnbnVDZzZMX2w2bHpiald0QmZNRXVQdHR5X3VHY05yVW9sNUhHUkVPX28iLCJsb2dzIjoiVl96OG10RjdtMVpNRWJiQUNtZlFLbWFLejdhZF9qWjhYbXhaYV9KMTRWWSIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjQxfQ - transactionsHash: jI_aVcPfGdURmGMHOnzx9YdhzAQBvRZVLKQdpmd_9BQ + - - eyJoYXNoIjoiSzhiZVo0andPZlJWVXlnanBRbmI0cTg3YXYtdUpoNTNwRFFSWEVOZ3ZMOCIsInNpZ3MiOlt7InNpZyI6IjU2NzUxZjI3ZDQxM2I0MTMxMGRmYjdmM2NmZTYyN2UxZDgxOWZkMmEyNGE4OTkyZTQ2YmY2MTkzMzVmY2I4OWIwMzY1ZjY3MTU2ZGRlNzJlZGU1OTVjNTcyMDk4OGI3NjI5NGFkMjk3NGM3NWEwM2E3MDQ4NzQxN2VhZTc2MTAzIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihkZWZpbmUta2V5c2V0ICd0ZXN0LWFkbWluIChyZWFkLWtleXNldCBcXFwidGVzdC1hZG1pbi1rZXlzZXRcXFwiKSlcXG5cXG4obmFtZXNwYWNlICdmcmVlKVxcblxcbihtb2R1bGUgdGVzdDEgJ3Rlc3QtYWRtaW5cXG5cXG4gIChkZWZzY2hlbWEgYWNjb3VudFxcbiAgICBiYWxhbmNlOmRlY2ltYWxcXG4gICAgYW1vdW50OmRlY2ltYWxcXG4gICAgZGF0YSlcXG5cXG4gIChkZWZ0YWJsZSBhY2NvdW50czp7YWNjb3VudH0pXFxuXFxuICAoZGVmdW4gY3JlYXRlLWFjY291bnQgKGlkIGluaXQtYmFsKVxcbiAgICAoaW5zZXJ0IGFjY291bnRzIGlkXFxuICAgICAgICAgeyBcXFwiYmFsYW5jZVxcXCI6IGluaXQtYmFsLCBcXFwiYW1vdW50XFxcIjogaW5pdC1iYWwsIFxcXCJkYXRhXFxcIjogXFxcIkNyZWF0ZWQgYWNjb3VudFxcXCIgfSkpXFxuXFxuICAoZGVmdW4gdHJhbnNmZXIgKHNyYyBkZXN0IGFtb3VudClcXG4gICAgXFxcInRyYW5zZmVyIEFNT1VOVCBmcm9tIFNSQyB0byBERVNUIGZvciB1bmVuY3J5cHRlZCBhY2NvdW50c1xcXCJcXG4gICAgKGRlYml0IHNyYyBhbW91bnQgeyBcXFwidHJhbnNmZXItdG9cXFwiOiBkZXN0IH0pXFxuICAgIChjcmVkaXQgZGVzdCBhbW91bnQgeyBcXFwidHJhbnNmZXItZnJvbVxcXCI6IHNyYyB9KSlcXG5cXG4gIChkZWZwYWN0IHBheW1lbnQgKHNyYy1lbnRpdHkgc3JjIGRlc3QtZW50aXR5IGRlc3QgYW1vdW50KVxcbiAgICBcXFwiVHdvLXBoYXNlIGNvbmZpZGVudGlhbCBwYXltZW50LCBzZW5kaW5nIG1vbmV5IGZyb20gU1JDIGF0IFNSQy1FTlRJVFkgdG8gREVTVCBhdCBERVNULUVOVElUWS5cXFwiXFxuXFxuICAgIChzdGVwLXdpdGgtcm9sbGJhY2tcXG4gICAgIHNyYy1lbnRpdHlcXG4gICAgIChsZXQgKChyZXN1bHQgKGRlYml0IHNyYyBhbW91bnQgeyBcXFwidHJhbnNmZXItdG9cXFwiOiBkZXN0LCBcXFwibWVzc2FnZVxcXCI6IFxcXCJTdGFydGluZyBwYWN0XFxcIiB9KSkpXFxuICAgICAgICh5aWVsZCB7IFxcXCJyZXN1bHRcXFwiOiByZXN1bHQsIFxcXCJhbW91bnRcXFwiOiBhbW91bnQsIFxcXCJ0eFxcXCI6IChwYWN0LWlkKSB9KSlcXG4gICAgIChjcmVkaXQgc3JjIGFtb3VudCB7IFxcXCJyb2xsYmFja1xcXCI6IChwYWN0LWlkKSB9KSlcXG5cXG4gICAgKHN0ZXBcXG4gICAgIGRlc3QtZW50aXR5XFxuICAgICAocmVzdW1lIHsgXFxcInJlc3VsdFxcXCI6PSByZXN1bHQsIFxcXCJhbW91bnRcXFwiOj0gZGViaXQtYW1vdW50IH1cXG4gICAgICAgKGNyZWRpdCBkZXN0IGRlYml0LWFtb3VudFxcbiAgICAgICAgICAgICAgIHsgXFxcInRyYW5zZmVyLWZyb21cXFwiOiBzcmMsIFxcXCJkZWJpdC1yZXN1bHRcXFwiOiByZXN1bHQsIFxcXCJ0eFxcXCI6IChwYWN0LWlkKSB9KSkpKVxcblxcbiAgKGRlZnVuIGRlYml0IChhY2N0IGFtb3VudCBkYXRhKVxcbiAgICBcXFwiRGViaXQgQUNDVCBmb3IgQU1PVU5ULCBlbmZvcmNpbmcgcG9zaXRpdmUgYW1vdW50IGFuZCBzdWZmaWNpZW50IGZ1bmRzLCBhbm5vdGF0aW5nIHdpdGggREFUQVxcXCJcXG4gICAgKGVuZm9yY2UtcG9zaXRpdmUgYW1vdW50KVxcbiAgICAod2l0aC1yZWFkIGFjY291bnRzIGFjY3QgeyBcXFwiYmFsYW5jZVxcXCI6PSBiYWxhbmNlIH1cXG4gICAgICAoY2hlY2stYmFsYW5jZSBiYWxhbmNlIGFtb3VudClcXG4gICAgICAodXBkYXRlIGFjY291bnRzIGFjY3RcXG4gICAgICAgICAgICB7IFxcXCJiYWxhbmNlXFxcIjogKC0gYmFsYW5jZSBhbW91bnQpLCBcXFwiYW1vdW50XFxcIjogKC0gYW1vdW50KVxcbiAgICAgICAgICAgICwgXFxcImRhdGFcXFwiOiBkYXRhIH0pKSlcXG5cXG4gIChkZWZ1biBjcmVkaXQgKGFjY3QgYW1vdW50IGRhdGEpXFxuICAgIFxcXCJDcmVkaXQgQUNDVCBmb3IgQU1PVU5ULCBlbmZvcmNpbmcgcG9zaXRpdmUgYW1vdW50XFxcIlxcbiAgICAoZW5mb3JjZS1wb3NpdGl2ZSBhbW91bnQpXFxuICAgICh3aXRoLXJlYWQgYWNjb3VudHMgYWNjdCB7IFxcXCJiYWxhbmNlXFxcIjo9IGJhbGFuY2UgfVxcbiAgICAgICh1cGRhdGUgYWNjb3VudHMgYWNjdFxcbiAgICAgICAgICAgIHsgXFxcImJhbGFuY2VcXFwiOiAoKyBiYWxhbmNlIGFtb3VudCksIFxcXCJhbW91bnRcXFwiOiBhbW91bnRcXG4gICAgICAgICAgICAsIFxcXCJkYXRhXFxcIjogZGF0YSB9KSkpXFxuXFxuXFxuICAoZGVmdW4gcmVhZC1hY2NvdW50IChpZClcXG4gICAgXFxcIlJlYWQgZGF0YSBmb3IgYWNjb3VudCBJRFxcXCJcXG4gICAgKCsgeyBcXFwiYWNjb3VudFxcXCI6IGlkIH0gKHJlYWQgYWNjb3VudHMgaWQpKSlcXG5cXG4gIChkZWZ1biBjaGVjay1iYWxhbmNlIChiYWxhbmNlIGFtb3VudClcXG4gICAgKGVuZm9yY2UgKDw9IGFtb3VudCBiYWxhbmNlKSBcXFwiSW5zdWZmaWNpZW50IGZ1bmRzXFxcIikpXFxuXFxuICAoZGVmdW4gZW5mb3JjZS1wb3NpdGl2ZSAoYW1vdW50KVxcbiAgICAoZW5mb3JjZSAoPj0gYW1vdW50IDAuMCkgXFxcImFtb3VudCBtdXN0IGJlIHBvc2l0aXZlXFxcIikpXFxuXFxuIChkZWZ1biByZWFkLWFsbCAoKVxcbiAgIChtYXAgKHJlYWQtYWNjb3VudCkgKGtleXMgYWNjb3VudHMpKSlcXG5cXG4gKGRlZnVuIHJlYWQtYWxsLWdsb2JhbCAoKVxcbiAgIChtYXAgKHJlYWQtYWNjb3VudCkgW1xcXCJBY2N0MVxcXCIgXFxcIkFjY3QyXFxcIl0pKVxcblxcbiAoZGVmdW4gY3JlYXRlLWdsb2JhbC1hY2NvdW50cyAoKVxcbiAgIChjcmVhdGUtYWNjb3VudCBcXFwiQWNjdDFcXFwiIDEwMDAwMDAuMClcXG4gICAoY3JlYXRlLWFjY291bnQgXFxcIkFjY3QyXFxcIiAwLjApXFxuICAgKHJlYWQtYWxsKSlcXG5cXG4gKGRlZnBhY3QgY3JlYXRlLXByaXZhdGUtYWNjb3VudHMgKClcXG4gICAoc3RlcCBcXFwiQWxpY2VcXFwiIChjcmVhdGUtYWNjb3VudCBcXFwiQVxcXCIgMTAwMC4wKSlcXG4gICAoc3RlcCBcXFwiQm9iXFxcIiAoY3JlYXRlLWFjY291bnQgXFxcIkJcXFwiIDEwMDAuMCkpXFxuICAgKHN0ZXAgXFxcIkNhcm9sXFxcIiAoY3JlYXRlLWFjY291bnQgXFxcIkNcXFwiIDEwMDAuMCkpXFxuICAgKHN0ZXAgXFxcIkRpbmVzaFxcXCIgKGNyZWF0ZS1hY2NvdW50IFxcXCJEXFxcIiAxMDAwLjApKSlcXG5cXG4pXFxuXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTBcIn0ifQ + - eyJnYXMiOjY0OSwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiTG9hZGVkIG1vZHVsZSBmcmVlLnRlc3QxLCBoYXNoIEI2WkpnZldENW9iUEdtWWNfaVJLaXJETVA2LVVHbTJwVVY3bXRIRC1Sc1EifSwicmVxS2V5IjoiSzhiZVo0andPZlJWVXlnanBRbmI0cTg3YXYtdUpoNTNwRFFSWEVOZ3ZMOCIsImxvZ3MiOiJXc2hfempnODdSTUxBaXlkYTFmNFRsM2tCQnMwNm92RHNpS2xTRlNvMnVJIiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MTF9 + - - eyJoYXNoIjoicEVqZ1VLbGVSWHFxby13azNLTzBWeGNScVg5Vmtua0ZCYmhPQndfZUNzdyIsInNpZ3MiOlt7InNpZyI6Ijk4OGU3OThjZGU5MmFjYjJhZGM4YWJhOTZkNzc3MWI0ZDUxMmQ2OWM0ZTgxOGIwMTMxNzQ4Mzk0NzQ4YjhmMDRkM2Q1MjQyMTNhMjJiZGVlZDMzYmMzNTFlN2ZiMjc4ZGU0YTM1YzMzMjhkYTZhNDdkMGJjYTgyNTJlNDkxYjA5In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihjcmVhdGUtdGFibGUgZnJlZS50ZXN0MS5hY2NvdW50cylcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxMVwifSJ9 + - eyJnYXMiOjI2NSwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiVGFibGVDcmVhdGVkIn0sInJlcUtleSI6InBFamdVS2xlUlhxcW8td2szS08wVnhjUnFYOVZrbmtGQmJoT0J3X2VDc3ciLCJsb2dzIjoiUU9sempuT2Y1TDJ5dm1tNHk5NE5LU2VvSkVUaUVWZ281MGN6amhNMDVHTSIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjE0fQ + - - eyJoYXNoIjoidzJ4VEdJeTczeDhYTENHaFJXcnJ3WWV5TWRhTXVMLXM5alBPX3hTM3I4ayIsInNpZ3MiOlt7InNpZyI6ImYwNTA3N2RhNGU4NDM4NDBiMjA3OTI5MjFkMTMxYTg1NzgxZWU5ZmU1YzZiNTY5MDhhY2M3MDA0OGM2YmY0NWQxMzBlNTk2NTJhMmU2ZGM1MDg0ZjhlMTAzNjU4NGNmOGU5NGYwZjZjMTVlZGQ4YWYxZmM3ODc5ZWQ1N2JmZjA0In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihmcmVlLnRlc3QxLmNyZWF0ZS1nbG9iYWwtYWNjb3VudHMpXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTJcIn0ifQ + - eyJnYXMiOjU4MSwicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjpbeyJhbW91bnQiOjEwMDAwMDAsImRhdGEiOiJDcmVhdGVkIGFjY291bnQiLCJiYWxhbmNlIjoxMDAwMDAwLCJhY2NvdW50IjoiQWNjdDEifSx7ImFtb3VudCI6MCwiZGF0YSI6IkNyZWF0ZWQgYWNjb3VudCIsImJhbGFuY2UiOjAsImFjY291bnQiOiJBY2N0MiJ9XX0sInJlcUtleSI6IncyeFRHSXk3M3g4WExDR2hSV3Jyd1lleU1kYU11TC1zOWpQT194UzNyOGsiLCJsb2dzIjoiY0ZhTi1nbVpJbkpjVTRaekc1cGhuYm1GcWJlNS1fMzVvM1I3a3NXbFNOMCIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjE3fQ + - - eyJoYXNoIjoiX1JqeTFBbVY2cl9qSmhKM3h3TksxbjVDRHpVeEJ0MGRUOTAzLUg3SUpDMCIsInNpZ3MiOlt7InNpZyI6IjNlZGJiZTcwN2I2YTI4OGNlYmVkYTNlODIxMDQ3YjEyODc3MDgwMjM1MGQzMjE3Njg1N2JkN2Y0YmNiOTEwZjMxMjhiYzFkNGQyM2M1MjAwZDI0NDMxZDExMDM5ZWYyNDA4ZWM2ZmQxYTE4MTYxMmYwZTlhZjI0ZDcyMDUzYTBkIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihmcmVlLnRlc3QxLnRyYW5zZmVyIFxcXCJBY2N0MVxcXCIgXFxcIkFjY3QyXFxcIiAxLjAwKVwifX0sXCJzaWduZXJzXCI6W3tcInB1YktleVwiOlwiMzY4ODIwZjgwYzMyNGJiYzdjMmIwNjEwNjg4YTdkYTQzZTM5ZjkxZDExODczMjY3MWNkOWM3NTAwZmY0M2NjYVwifV0sXCJtZXRhXCI6e1wiY3JlYXRpb25UaW1lXCI6MCxcInR0bFwiOjEwMDAwMDAsXCJnYXNMaW1pdFwiOjEwMDAwLFwiY2hhaW5JZFwiOlwiMFwiLFwiZ2FzUHJpY2VcIjoxLjBlLTIsXCJzZW5kZXJcIjpcInNlbmRlcjAwXCJ9LFwibm9uY2VcIjpcIjEzXCJ9In0 + - eyJnYXMiOjQwNywicmVzdWx0Ijp7InN0YXR1cyI6InN1Y2Nlc3MiLCJkYXRhIjoiV3JpdGUgc3VjY2VlZGVkIn0sInJlcUtleSI6Il9SankxQW1WNnJfakpoSjN4d05LMW41Q0R6VXhCdDBkVDkwMy1IN0lKQzAiLCJsb2dzIjoiLVVKQktPTDFzTm5tTFBjVkt2b0ZxMUZ4R1R0WnZnZHhSd1NJTDdDSVNydyIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjIwfQ + - - eyJoYXNoIjoicHpKTkQ4TVMtVV9SN3Y3T0Qtd1psT1NOOUdBZDlaLVEtRmVpLWY4Q0g5VSIsInNpZ3MiOlt7InNpZyI6ImZiN2JhOTJlZDhjMGMwMWJiZDk5MzdlMjg2NzRkYzBiNzI3OGRlNjExNWMwYzY0MDc5NzkxZDkxMzQ5Mzk3ODQ4YmI2NDg2NjYwOTQ2MGUyYjE1ODMxMzZjMDc0YjdmNTJiOTAwMmVkYzVlNDUxNzkzOTZhNWU5ZjI0NTUwYTBmIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAncHJldi1ibG9jay1oYXNoIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxNFwifSJ9 + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6ImdqTmVmdmM3STRpblNOOUxFZkVCN0c0OUlvWC0ycGxEQmVUbFpwU1czQTQifSwicmVxS2V5IjoicHpKTkQ4TVMtVV9SN3Y3T0Qtd1psT1NOOUdBZDlaLVEtRmVpLWY4Q0g5VSIsImxvZ3MiOiJLbC1mN3AxMlc1R0hORmw1NVE5djRaWGhWWV85a3VscEdRRFhSQ1hLYkU0IiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MjN9 + - - eyJoYXNoIjoibkpyMUFFbDBjSHFyTkxCeUVXTEZuc1ZmV2dfNnZCT0tpS1cteWZxNW13byIsInNpZ3MiOlt7InNpZyI6IjdjODU4ZDZjMjFlNzI0YmY1NjNhMzQyOTBjYjRiN2Y4M2U0NjkyYmJkOGEwYzRiZDNiNzIzZWFiZmFjY2VlZTFjYmM3ZGVkNWE1N2YxMTUzMmE2MTI2NTI3M2UwZGViMGM3YmZlN2VmYmYyZWU4YWQyZjNhMzY4Zjg2NWEwMTA4In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnYmxvY2stdGltZSAoY2hhaW4tZGF0YSkpXCJ9fSxcInNpZ25lcnNcIjpbe1wicHViS2V5XCI6XCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJ9XSxcIm1ldGFcIjp7XCJjcmVhdGlvblRpbWVcIjowLFwidHRsXCI6MTAwMDAwMCxcImdhc0xpbWl0XCI6MTAwMDAsXCJjaGFpbklkXCI6XCIwXCIsXCJnYXNQcmljZVwiOjEuMGUtMixcInNlbmRlclwiOlwic2VuZGVyMDBcIn0sXCJub25jZVwiOlwiMTVcIn0ifQ + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6eyJ0aW1lIjoiMTk3MC0wMS0wMVQwMDowMDowMFoifX0sInJlcUtleSI6Im5KcjFBRWwwY0hxck5MQnlFV0xGbnNWZldnXzZ2Qk9LaUtXLXlmcTVtd28iLCJsb2dzIjoia0g5M2hlNTZsUFJ4YTZFVWlRNWdUanFnUngxeTU5NXdodXlqV2Nyd3BCbyIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjI2fQ + - - eyJoYXNoIjoieWdSODh2eHFnTGN2NWhodUpzcDNTMVFRbGZ1aS1DTWQ5WWRYYzA0Z1FkYyIsInNpZ3MiOlt7InNpZyI6Ijc4YTBmNjc1MzcyZDU5YjY1ZDdmZTZhNDgxZDI5MDIyYjU3ZTk0ODhhYTFlODBkZTE2OTE0YmM4OWE2YTk3ZThlYjZmYTRiYWI5ZDM0MTRmMGJiMzBmMzJkYjhhNDhiOGU3NGI4MDgzMTdiZjYwZDBkN2EyMGM1ZTEzZmY4NzBhIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnYmxvY2staGVpZ2h0IChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxNlwifSJ9 + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MX0sInJlcUtleSI6InlnUjg4dnhxZ0xjdjVoaHVKc3AzUzFRUWxmdWktQ01kOVlkWGMwNGdRZGMiLCJsb2dzIjoibzV5TWdNRDFOd0RpQXFFMVFDREg0Y29VUkpXcndPT3ZmbFh6TkY5UGV6NCIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjI5fQ + - - eyJoYXNoIjoiODJ4M085bGNEbHhVWUdhTkJYSy1iRm9kbHYtNlVTUVRqUVFRdFJibnFLVSIsInNpZ3MiOlt7InNpZyI6IjFmMGFkMjJmYjRhZmJhOTg4ZDNjYzI3ODFhZjJlNTE5NmIwZWEyZGFjMzU3YjA2OTc3NzI1YzQ5MTRlYjdiODdjN2EzMTk5MjU3NTk2YTVhNGFmOGUyMTg0YzMyYzQzZWQwYWEzMjVkYjU2YWFiZTkwM2VkMWY3ZmM1OGYwZTBiIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnZ2FzLWxpbWl0IChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxN1wifSJ9 + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MTAwMDB9LCJyZXFLZXkiOiI4MngzTzlsY0RseFVZR2FOQlhLLWJGb2Rsdi02VVNRVGpRUVF0UmJucUtVIiwibG9ncyI6IkRlZWxsZTNxVkhRbHJ0N3lBWkx4cEZqUWk0OVNVdm9tUkxGb0FTRmg4aUEiLCJtZXRhRGF0YSI6bnVsbCwiY29udGludWF0aW9uIjpudWxsLCJ0eElkIjozMn0 + - - eyJoYXNoIjoiSWtQejh4OXY3VS0wak51elhNcTNTOVpfVDRMNnFXYU1feVJxTzdRMV9ibyIsInNpZ3MiOlt7InNpZyI6ImE2ZjcwMDc1YzI2ZjMxNWRhYjJlM2FkMzM0ZDNjYTI1ODkwYzlmMWE1YmZkMmNmNTg0NDY1NzcxZmI3NWQwNDgwZmRhNDhhYmZmMWY0Yzg3ZWFhYmUzN2ZmYTEzMjg1NjllNDE5NmM3OTViY2VmN2JkNGYyYTZiNTI3MzBjMTA3In1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnZ2FzLXByaWNlIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxOFwifSJ9 + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6MS4wZS0yfSwicmVxS2V5IjoiSWtQejh4OXY3VS0wak51elhNcTNTOVpfVDRMNnFXYU1feVJxTzdRMV9ibyIsImxvZ3MiOiJ5bDh6SXRpaDU1NUJ0cTlWQ0Jvd1ZiWGNablBSRFJwc0NmMmVCNDdtZ2RZIiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6MzV9 + - - eyJoYXNoIjoiVVJxUzBBM0lqSnprX19fOWRBNzVUSTFGTVg3OEJWeVJXSW1xcjFTMTlKbyIsInNpZ3MiOlt7InNpZyI6IjcxZmQyYmY0MTg5Y2E0MzE4NjBlN2FjYTgxMzJjZTI4MmJmNDc3MTY2MTAwNTQ1OThlNDAzN2I5YmE1OTI3MzViODMxMWEwZTc0NDE5MDMyZTI5ZTUyMmRhMTllZmZiYjk3M2VmODUzOWJhNGRlMWQ2MDU4NmJlNDYxMjNhMDAxIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnY2hhaW4taWQgKGNoYWluLWRhdGEpKVwifX0sXCJzaWduZXJzXCI6W3tcInB1YktleVwiOlwiMzY4ODIwZjgwYzMyNGJiYzdjMmIwNjEwNjg4YTdkYTQzZTM5ZjkxZDExODczMjY3MWNkOWM3NTAwZmY0M2NjYVwifV0sXCJtZXRhXCI6e1wiY3JlYXRpb25UaW1lXCI6MCxcInR0bFwiOjEwMDAwMDAsXCJnYXNMaW1pdFwiOjEwMDAwLFwiY2hhaW5JZFwiOlwiMFwiLFwiZ2FzUHJpY2VcIjoxLjBlLTIsXCJzZW5kZXJcIjpcInNlbmRlcjAwXCJ9LFwibm9uY2VcIjpcIjE5XCJ9In0 + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6IjAifSwicmVxS2V5IjoiVVJxUzBBM0lqSnprX19fOWRBNzVUSTFGTVg3OEJWeVJXSW1xcjFTMTlKbyIsImxvZ3MiOiJVWTEwSjRzN3NiaDFWbTR3WmxaRHV4T1NBYWtfaG5YVWxrWWpXaHBTajB3IiwibWV0YURhdGEiOm51bGwsImNvbnRpbnVhdGlvbiI6bnVsbCwidHhJZCI6Mzh9 + - - eyJoYXNoIjoiZXNmMlN1UmlXbVdHa1NORW5zeGtMVkR4UEJVbVpmUWNMUk1WQ3k5RWZfcyIsInNpZ3MiOlt7InNpZyI6IjEwYjJiMTZhMDg3YjAwZmVjNTE1N2IwNTk3N2RiNDhmYWY5Mjg4MDJhZWIwYTQ0MzIxMjhmYjg0ZTlhYzBhY2U5Yjg5ZjM0MzJmZjBiZDIwZTU4MzY0MWExMmM4ZjdlNzA4YWU2MmU2ZGE3ZWZiMTM2NGFkNzU2OWEwZDA3MDBkIn1dLCJjbWQiOiJ7XCJuZXR3b3JrSWRcIjpcInNsb3dmb3JrLUNQTS1wZXRlcnNvblwiLFwicGF5bG9hZFwiOntcImV4ZWNcIjp7XCJkYXRhXCI6e1widGVzdC1hZG1pbi1rZXlzZXRcIjpbXCIzNjg4MjBmODBjMzI0YmJjN2MyYjA2MTA2ODhhN2RhNDNlMzlmOTFkMTE4NzMyNjcxY2Q5Yzc1MDBmZjQzY2NhXCJdfSxcImNvZGVcIjpcIihhdCAnc2VuZGVyIChjaGFpbi1kYXRhKSlcIn19LFwic2lnbmVyc1wiOlt7XCJwdWJLZXlcIjpcIjM2ODgyMGY4MGMzMjRiYmM3YzJiMDYxMDY4OGE3ZGE0M2UzOWY5MWQxMTg3MzI2NzFjZDljNzUwMGZmNDNjY2FcIn1dLFwibWV0YVwiOntcImNyZWF0aW9uVGltZVwiOjAsXCJ0dGxcIjoxMDAwMDAwLFwiZ2FzTGltaXRcIjoxMDAwMCxcImNoYWluSWRcIjpcIjBcIixcImdhc1ByaWNlXCI6MS4wZS0yLFwic2VuZGVyXCI6XCJzZW5kZXIwMFwifSxcIm5vbmNlXCI6XCIxMTBcIn0ifQ + - eyJnYXMiOjgsInJlc3VsdCI6eyJzdGF0dXMiOiJzdWNjZXNzIiwiZGF0YSI6InNlbmRlcjAwIn0sInJlcUtleSI6ImVzZjJTdVJpV21XR2tTTkVuc3hrTFZEeFBCVW1aZlFjTFJNVkN5OUVmX3MiLCJsb2dzIjoiVU95OWdjVjRJVjk4M1lNU3RGRWlFTGVIQ0ZEZEF0c3ZqY3lpMk5aeG53ayIsIm1ldGFEYXRhIjpudWxsLCJjb250aW51YXRpb24iOm51bGwsInR4SWQiOjQxfQ + transactionsHash: WdFstQYQSjfr0rwh9teCqZmmO23lOQhkpqBebSBtkYQ test-group: new-block diff --git a/test/golden/transfer-accounts-expected.txt b/test/golden/transfer-accounts-expected.txt index 3280359336..a60e081f97 100644 --- a/test/golden/transfer-accounts-expected.txt +++ b/test/golden/transfer-accounts-expected.txt @@ -3,4 +3,4 @@ - cmd: contents: (test1.transfer "Acct1" "Acct2" 1.00) tag: Code - output: noxWe0aOpl3J20QHMVJP8EvPGG8kftv63GeIjvJwrXQ + output: O4tU1VdTniyZH8CkVPKViCvHT6C_lke6lmObZeKjPL8 diff --git a/test/pact/contTXOUTNew.pact b/test/pact/contTXOUTNew.pact index 2c44fe5d6f..3527e074b0 100644 --- a/test/pact/contTXOUTNew.pact +++ b/test/pact/contTXOUTNew.pact @@ -7,7 +7,7 @@ , "name": "coin.transfer-crosschain" } , "executed": "" - , "pact-id": "rwFh_qoxumclpxZj6QIo5GOM0CmE5AztlKukhFCRTVA" + , "pact-id": "pHsX8RJkJNRlrGflD_vRiKOsjlLSVGdf1hYU_eI6fgU" , "step": 0 , "step-count": 2 , "step-has-rollback": false @@ -25,9 +25,9 @@ } , "events": [] , "gas": 6 - , "logs": "9JwtbUnJnZVC62lihy1YDA7Gmi2Mkopl62GkMGB3QZI" + , "logs": "c_Xbqb7BZetrHBtU7nGdZ4yoc1_Y1iE3CgsO03gOrbc" , "meta": {} - , "req-key": "rwFh_qoxumclpxZj6QIo5GOM0CmE5AztlKukhFCRTVA" + , "req-key": "pHsX8RJkJNRlrGflD_vRiKOsjlLSVGdf1hYU_eI6fgU" , "result": { "amount": 1.0 , "receiver": "sender01" diff --git a/test/pact/continuation-gas-payer.pact b/test/pact/continuation-gas-payer.pact index a927909ba9..c8e227ee6b 100644 --- a/test/pact/continuation-gas-payer.pact +++ b/test/pact/continuation-gas-payer.pact @@ -22,7 +22,7 @@ (enforce (= false (at "cont-has-proof" (read-msg))) "Not inside a cross-chain tx") (enforce (= 1 (at "cont-step" (read-msg))) "Inside 2nd step of continuation") (enforce (= {} (at "cont-user-data" (read-msg))) "Inside a continuation w/o user data") - (enforce (= "_sYA748a-Hsn_Qb3zsYTDu2H5JXgQcPr1dFkjMTgAm0" + (enforce (= "9ylBanSjDGJJ6m0LgokZqb9P66P7JsQRWo9sYxqAjcQ" (at "cont-pact-id" (read-msg))) "Inside a particular continuation") (compose-capability (ALLOW_GAS)) ) diff --git a/test/pact/tfrTXOUTNew.pact b/test/pact/tfrTXOUTNew.pact index 4b4cf61a69..e7e72b1e2d 100644 --- a/test/pact/tfrTXOUTNew.pact +++ b/test/pact/tfrTXOUTNew.pact @@ -10,9 +10,9 @@ } ] , "gas": 5 - , "logs": "0aw7N48LexZxuIDtmGf1-WLwmGndE3VkmFM50jxwFwo" + , "logs": "zF8hy3XGDeZR0sHrd5e-KehTlPUHguzwSbAQRMU8mcM" , "meta": {} - , "req-key": "0IEgLi4hZ7YMZ1yg1SOzfSHUEifH8gwTBdEC7MJ9BQQ" + , "req-key": "LCLlbFalZdnCr7ax2Cqntuj2PCUq0pVGhmgAE-oabKs" , "result": "Write succeeded" , "txid": "12" })) diff --git a/tools/cwtool/TxSimulator.hs b/tools/cwtool/TxSimulator.hs index b4f3822744..0a71654b5a 100644 --- a/tools/cwtool/TxSimulator.hs +++ b/tools/cwtool/TxSimulator.hs @@ -114,7 +114,7 @@ simulate sc@(SimConfig dbDir txIdx' _ _ cid ver gasLog doTypecheck) = do miner <- decodeStrictOrThrow $ _minerData md let Transaction tx = fst $ txs V.! txIdx cmdTx <- decodeStrictOrThrow tx - case validateCommand cmdTx of + case validateCommand ver cid cmdTx of Left _ -> error "bad cmd" Right cmdPwt -> do let cmd = payloadObj <$> cmdPwt diff --git a/tools/ea/Ea.hs b/tools/ea/Ea.hs index c0c77ccb1c..64943da134 100644 --- a/tools/ea/Ea.hs +++ b/tools/ea/Ea.hs @@ -182,6 +182,11 @@ mkChainwebTxs' :: [Command Text] -> IO [ChainwebTransaction] mkChainwebTxs' rawTxs = forM rawTxs $ \cmd -> do let cmdBS = fmap TE.encodeUtf8 cmd + -- TODO: Use the new `assertCommand` function. + -- We want to delete `verifyCommand` at some point. + -- It's not critical for Ea because WebAuthn signatures (which + -- the new `assertCommand` knows how to handle) are not present + -- in Genesis blocks. procCmd = verifyCommand cmdBS case procCmd of f@ProcFail{} -> fail (show f) From b0bbf8cf481f57fff0d1d05b7dcd5acfa1c7772f Mon Sep 17 00:00:00 2001 From: chessai Date: Fri, 24 Nov 2023 13:20:05 -0800 Subject: [PATCH 5/8] release 2.22 (#1783) * release 2.22 * respond to changelog review * Chainweb223Pact ForkNever --- CHANGELOG.md | 23 +++++++++++++++++++++ node/ChainwebNode.hs | 4 ++-- src/Chainweb/Version.hs | 4 +++- src/Chainweb/Version/Development.hs | 1 + src/Chainweb/Version/Guards.hs | 4 ++++ src/Chainweb/Version/Mainnet.hs | 3 ++- src/Chainweb/Version/Testnet.hs | 3 ++- test/Chainweb/Test/TestVersions.hs | 2 ++ tools/calculate-release/CalculateRelease.hs | 4 ++-- 9 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70f9a709cb..c208bd7a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # `chainweb-node` Changelog +## 2.22 (2023-11-24) +This version replaces all previous versions. Any prior version will stop working +on **2023-12-13T:00:00Z**. Node administrators must upgrade to this version before +that date. + +This version will expire on **2023-03-06T:00:00Z**. + +To upgrade, pull the latest docker image or download the binary and restart the node. + +Changes: + +* Node support for webauthn signers, scoped signatures, and webauthn keyset formats in Pact (#1779) +* Block endpoint added to Service API (#1720) +* Fix batch /polling so that it no longer omits results (#1775) +* Add block header to validation failure message (#1752) +* Halt block fill algorithm constructively if we exceeded the tx fetch limit (#1762) +* Be more careful not to write the results of invalid blocks to the pact state (#1740) +* Fix Mac M2 compatibility with older blocks (#1782) + +Internal Changes: +* Support aeson-2.2 (#1750) +* Fix benchmarks for block creation and validation (#1767) + ## 2.21 (2023-10-05) This version replaces all previous versions. Any prior version will stop working diff --git a/node/ChainwebNode.hs b/node/ChainwebNode.hs index f2cf011a92..1aeeafc540 100644 --- a/node/ChainwebNode.hs +++ b/node/ChainwebNode.hs @@ -526,10 +526,10 @@ pkgInfoScopes = -- -------------------------------------------------------------------------- -- -- main --- SERVICE DATE for version 2.21 +-- SERVICE DATE for version 2.22 -- serviceDate :: Maybe String -serviceDate = Just "2023-12-13T00:00:00Z" +serviceDate = Just "2024-03-06T00:00:00Z" mainInfo :: ProgramInfo ChainwebNodeConfiguration mainInfo = programInfoValidate diff --git a/src/Chainweb/Version.hs b/src/Chainweb/Version.hs index ebef221327..54b2938c5e 100644 --- a/src/Chainweb/Version.hs +++ b/src/Chainweb/Version.hs @@ -192,6 +192,7 @@ data Fork | Chainweb220Pact | Chainweb221Pact | Chainweb222Pact + | Chainweb223Pact -- always add new forks at the end, not in the middle of the constructors. deriving stock (Bounded, Generic, Eq, Enum, Ord, Show) deriving anyclass (NFData, Hashable) @@ -224,6 +225,7 @@ instance HasTextRepresentation Fork where toText Chainweb220Pact = "chainweb220Pact" toText Chainweb221Pact = "chainweb221Pact" toText Chainweb222Pact = "chainweb222Pact" + toText Chainweb223Pact = "chainweb223Pact" fromText "slowEpoch" = return SlowEpoch fromText "vuln797Fix" = return Vuln797Fix @@ -251,7 +253,7 @@ instance HasTextRepresentation Fork where fromText "chainweb219Pact" = return Chainweb219Pact fromText "chainweb220Pact" = return Chainweb220Pact fromText "chainweb221Pact" = return Chainweb221Pact - fromText "chainweb222Pact" = return Chainweb222Pact + fromText "chainweb223Pact" = return Chainweb223Pact fromText t = throwM . TextFormatException $ "Unknown Chainweb fork: " <> t instance ToJSON Fork where diff --git a/src/Chainweb/Version/Development.hs b/src/Chainweb/Version/Development.hs index bcbea76a17..66fffa15bb 100644 --- a/src/Chainweb/Version/Development.hs +++ b/src/Chainweb/Version/Development.hs @@ -68,6 +68,7 @@ devnet = ChainwebVersion Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 560 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 580 Chainweb222Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 590 + Chainweb223Pact -> AllChains ForkNever , _versionUpgrades = foldr (chainZip HM.union) (AllChains mempty) [ forkUpgrades devnet diff --git a/src/Chainweb/Version/Guards.hs b/src/Chainweb/Version/Guards.hs index ec7d2d4a46..54fc418ee7 100644 --- a/src/Chainweb/Version/Guards.hs +++ b/src/Chainweb/Version/Guards.hs @@ -43,6 +43,7 @@ module Chainweb.Version.Guards , chainweb220Pact , chainweb221Pact , chainweb222Pact + , chainweb223Pact , pact44NewTrans , pactParserVersion , maxBlockGasLimit @@ -245,6 +246,9 @@ chainweb221Pact = checkFork atOrAfter Chainweb221Pact chainweb222Pact :: ChainwebVersion -> ChainId -> BlockHeight -> Bool chainweb222Pact = checkFork atOrAfter Chainweb222Pact +chainweb223Pact :: ChainwebVersion -> ChainId -> BlockHeight -> Bool +chainweb223Pact = checkFork atOrAfter Chainweb223Pact + pactParserVersion :: ChainwebVersion -> ChainId -> BlockHeight -> PactParserVersion pactParserVersion v cid bh | chainweb213Pact v cid bh = PactParserChainweb213 diff --git a/src/Chainweb/Version/Mainnet.hs b/src/Chainweb/Version/Mainnet.hs index 0148bc9275..ed7971af02 100644 --- a/src/Chainweb/Version/Mainnet.hs +++ b/src/Chainweb/Version/Mainnet.hs @@ -141,7 +141,8 @@ mainnet = ChainwebVersion Chainweb219Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 3_774_423) -- 2023-06-02 00:00:00+00:00 Chainweb220Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 4_056_499) -- 2023-09-08 00:00:00+00:00 Chainweb221Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 4_177_889) -- 2023-10-20 00:00:00+00:00 - Chainweb222Pact -> AllChains ForkNever -- TODO: fill in + Chainweb222Pact -> AllChains (ForkAtBlockHeight $ BlockHeight 4_335_753) -- 2023-12-14 00:00:00+00:00 + Chainweb223Pact -> AllChains ForkNever , _versionGraphs = (to20ChainsMainnet, twentyChainGraph) `Above` diff --git a/src/Chainweb/Version/Testnet.hs b/src/Chainweb/Version/Testnet.hs index 576918df96..c49213a4ad 100644 --- a/src/Chainweb/Version/Testnet.hs +++ b/src/Chainweb/Version/Testnet.hs @@ -121,7 +121,8 @@ testnet = ChainwebVersion Chainweb219Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_299_753 -- 2023-06-01 12:00:00+00:00 Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_580_964 -- 2023-09-08 12:00:00+00:00 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_702_250 -- 2023-10-19 12:00:00+00:00 - Chainweb222Pact -> AllChains ForkNever -- TODO: fill in + Chainweb222Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 3_859_808 -- 2023-12-13 12:00:00+00:00 + Chainweb223Pact -> AllChains ForkNever , _versionGraphs = (to20ChainsTestnet, twentyChainGraph) `Above` diff --git a/test/Chainweb/Test/TestVersions.hs b/test/Chainweb/Test/TestVersions.hs index 1eeae049a7..86162ea145 100644 --- a/test/Chainweb/Test/TestVersions.hs +++ b/test/Chainweb/Test/TestVersions.hs @@ -140,6 +140,7 @@ fastForks = tabulateHashMap $ \case Chainweb220Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 30 Chainweb221Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 33 Chainweb222Pact -> AllChains $ ForkAtBlockHeight $ BlockHeight 36 + Chainweb223Pact -> AllChains ForkNever -- | A test version without Pact or PoW, with only one chain graph. barebonesTestVersion :: ChainGraph -> ChainwebVersion @@ -262,6 +263,7 @@ slowForkingCpmTestVersion g = buildTestVersion $ \v -> v Chainweb220Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 85) Chainweb221Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 100) Chainweb222Pact -> AllChains $ ForkAtBlockHeight (BlockHeight 115) + Chainweb223Pact -> AllChains ForkNever -- | CPM version (see `cpmTestVersion`) with forks and upgrades quickly enabled. fastForkingCpmTestVersion :: ChainGraph -> ChainwebVersion diff --git a/tools/calculate-release/CalculateRelease.hs b/tools/calculate-release/CalculateRelease.hs index 82d4b72054..bc28040ecb 100644 --- a/tools/calculate-release/CalculateRelease.hs +++ b/tools/calculate-release/CalculateRelease.hs @@ -48,8 +48,8 @@ main = do putStrLn $ "Mainnet fork height (at " <> show mainnetForkTime <> "): " <> show mainnetForkHeight putStrLn $ "Testnet fork height (at " <> show testnetForkTime <> "): " <> show testnetForkHeight - let nextServiceDateDay = addDays (8 * 7) serviceDateDay - putStrLn $ "Next service date (+8 weeks): " <> show nextServiceDateDay + let nextServiceDateDay = addDays (12 * 7) serviceDateDay + putStrLn $ "Next service date (+12 weeks): " <> show nextServiceDateDay where heightOfChain0 :: String -> IO BlockHeight heightOfChain0 cutUrl = From b2058802e00634a4c2f4b884503ee9c7bd27c376 Mon Sep 17 00:00:00 2001 From: chessai Date: Fri, 24 Nov 2023 14:35:22 -0800 Subject: [PATCH 6/8] update to pact410 (#1784) * update to pact410 * update changelog --- CHANGELOG.md | 4 +++- cabal.project | 4 ++-- cabal.project.freeze | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c208bd7a74..1d3f10827b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,9 @@ This version will expire on **2023-03-06T:00:00Z**. To upgrade, pull the latest docker image or download the binary and restart the node. Changes: - + +* Updated to Pact 4.10 (numerous, see [Pact + changelog](https://github.com/kadena-io/pact/releases/tag/v4.10)) * Node support for webauthn signers, scoped signatures, and webauthn keyset formats in Pact (#1779) * Block endpoint added to Service API (#1720) * Fix batch /polling so that it no longer omits results (#1775) diff --git a/cabal.project b/cabal.project index 39a492abdd..4a49e5b210 100644 --- a/cabal.project +++ b/cabal.project @@ -63,8 +63,8 @@ package yet-another-logger source-repository-package type: git location: https://github.com/kadena-io/pact.git - tag: 79bf626042ad3dc9ed276e9b243be43584f20da2 - --sha256: sha256-ObE5aQjy/mX2PMEvtRnlJ1mG4E82PqZDDdzerd4I+HI= + tag: 495c8738acaa0157958ab23a06cd94abbc99d2d5 + --sha256: 1dn322m5mx43bn3ki37zdk9daknnr6sz8y7d89si75m84rl0w27k source-repository-package type: git diff --git a/cabal.project.freeze b/cabal.project.freeze index d0ca41e727..9cd6569909 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -183,7 +183,7 @@ constraints: any.Boolean ==0.2.4, any.old-locale ==1.0.0.7, any.old-time ==1.1.0.3, any.optparse-applicative ==0.18.1.0, - any.pact ==4.9, + any.pact ==4.10, pact -build-tool +cryptonite-ed25519 -tests-in-lib, any.pact-json ==0.1.0.0, any.pact-time ==0.2.0.2, @@ -316,4 +316,4 @@ constraints: any.Boolean ==0.2.4, any.yaml ==0.11.11.2, any.yet-another-logger ==0.4.2, any.zlib ==0.6.3.0 -index-state: hackage.haskell.org 2023-11-08T16:26:43Z +index-state: hackage.haskell.org 2023-11-24T21:32:17Z From b88c2132eb95bf7f3910b7108b234319ea419db6 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 28 Nov 2023 13:46:51 -0600 Subject: [PATCH 7/8] poll endpoint: internal cleanups (#1786) * service: refactor internal /poll body a bit Summary: Just to make things more readable. Signed-off-by: Austin Seipp Change-Id: Icb9b8ac03473b6c637c1ec6dc88b5c15 * pact/restapi: migrate some use of `MaybeT` to `ExceptT` Summary: `MaybeT` is alluring and easy to use, but destroys debuggability in practice unless used with extreme care. Do not use it inside `internalPoll` in order to ease debugging; the hope is that this will make bugs like #1732 easier to find in the future. Signed-off-by: Austin Seipp Change-Id: I93f98d0213b48c9af809620b72d92d08 --------- Signed-off-by: Austin Seipp --- .../Pact/Backend/RelationalCheckpointer.hs | 44 +++++++++++-------- src/Chainweb/Pact/RestAPI/Server.hs | 31 +++++++------ 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs index 939d2d94de..4d722718de 100644 --- a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs +++ b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs @@ -319,41 +319,49 @@ doRegisterSuccessful dbenv (TypedHash hash) = doLookupSuccessful :: Db logger -> Maybe ConfirmationDepth -> V.Vector PactHash -> IO (HashMap.HashMap PactHash (T2 BlockHeight BlockHash)) doLookupSuccessful dbenv confDepth hashes = runBlockEnv dbenv $ do - withSavepoint DbTransaction $ do - r <- callDb "doLookupSuccessful" $ \db -> do - let - currentHeightQ = "SELECT blockheight FROM BlockHistory \ - \ ORDER BY blockheight DESC LIMIT 1" - + withSavepoint DbTransaction $ + fmap buildResultMap $ -- swizzle results of query into a HashMap + callDb "doLookupSuccessful" $ \db -> do -- if there is a confirmation depth, we get the current height and calculate -- the block height, to look for the transactions in range [0, current block height - confirmation depth] blockheight <- case confDepth of Nothing -> pure Nothing Just (ConfirmationDepth cd) -> do + let currentHeightQ = "SELECT blockheight FROM BlockHistory ORDER BY blockheight DESC LIMIT 1" currentHeight <- qry_ db currentHeightQ [RInt] case currentHeight of [[SInt bh]] -> pure $ Just (bh - fromIntegral cd) _ -> fail "impossible" let - blockheightval = maybe [] (\bh -> [SInt bh]) blockheight - qvals = [ SBlob (BS.fromShort hash) | (TypedHash hash) <- V.toList hashes ] ++ blockheightval + hss = V.toList hashes + params = Utf8 $ intercalate "," (map (const "?") hss) + qtext = "SELECT blockheight, hash, txhash FROM \ + \TransactionIndex INNER JOIN BlockHistory \ + \USING (blockheight) WHERE txhash IN (" <> params <> ")" + <> maybe "" (const " AND blockheight <= ?") confDepth + <> ";" + qvals + -- match query params above. first, hashes + = map (\(TypedHash h) -> SBlob $ BS.fromShort h) hss + -- then, the block depth, if any + ++ maybe [] (\bh -> [SInt bh]) blockheight qry db qtext qvals [RInt, RBlob, RBlob] >>= mapM go - return $ HashMap.fromList (map (\(T3 blockheight blockhash txhash) -> (txhash, T2 blockheight blockhash)) r) where - qtext = "SELECT blockheight, hash, txhash FROM \ - \TransactionIndex INNER JOIN BlockHistory \ - \USING (blockheight) WHERE txhash IN (" <> hashesParams <> ")" - <> maybe "" (const " AND blockheight <= ?") confDepth - <> ";" - hashesParams = Utf8 $ intercalate "," [ "?" | _ <- V.toList hashes] - - go :: [SType] -> IO (T3 BlockHeight BlockHash PactHash) + -- NOTE: it's useful to keep the types of 'go' and 'buildResultMap' in sync + -- for readability but also to ensure the compiler and reader infer the + -- right result types from the db query. + + buildResultMap :: [T3 PactHash BlockHeight BlockHash] -> HashMap.HashMap PactHash (T2 BlockHeight BlockHash) + buildResultMap xs = HashMap.fromList $ + map (\(T3 txhash blockheight blockhash) -> (txhash, T2 blockheight blockhash)) xs + + go :: [SType] -> IO (T3 PactHash BlockHeight BlockHash) go (SInt blockheight:SBlob blockhash:SBlob txhash:_) = do !blockhash' <- either fail return $ runGetEitherS decodeBlockHash blockhash let !txhash' = TypedHash $ BS.toShort txhash - return $! T3 (fromIntegral blockheight) blockhash' txhash' + return $! T3 txhash' (fromIntegral blockheight) blockhash' go _ = fail "impossible" doGetBlockHistory :: Db logger -> BlockHeader -> Domain RowKey RowData -> IO BlockTxHistory diff --git a/src/Chainweb/Pact/RestAPI/Server.hs b/src/Chainweb/Pact/RestAPI/Server.hs index 603db5ed1f..72fe83e1d5 100644 --- a/src/Chainweb/Pact/RestAPI/Server.hs +++ b/src/Chainweb/Pact/RestAPI/Server.hs @@ -40,8 +40,7 @@ import Control.Monad import Control.Monad.Catch hiding (Handler) import Control.Monad.Reader import Control.Monad.State.Strict -import Control.Monad.Trans.Except (ExceptT) -import Control.Monad.Trans.Maybe +import Control.Monad.Trans.Except (ExceptT, runExceptT, except) import Data.Aeson as Aeson import Data.Bifunctor (second) @@ -131,6 +130,7 @@ import Pact.Types.Hash (Hash(..)) import qualified Pact.Types.Hash as Pact import Pact.Types.PactError (PactError(..), PactErrorType(..)) import Pact.Types.Pretty (pretty) +import Control.Error (note, rights) -- -------------------------------------------------------------------------- -- @@ -619,10 +619,10 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do let results1 = V.map (\rk -> (rk, HM.lookup (Pact.fromUntypedHash $ unRequestKey rk) results0)) requestKeysV let (present0, missing) = V.unstablePartition (isJust . snd) results1 let present = V.map (second fromJuste) present0 - lookedUp <- catMaybes . V.toList <$> mapM lookup present badlisted <- V.toList <$> checkBadList (V.map fst missing) - let outputs = lookedUp ++ badlisted - return $! HM.fromList outputs + vs <- mapM lookup present + let good = rights $ V.toList vs + return $! HM.fromList (good ++ badlisted) where cid = _chainId bhdb !requestKeysV = V.fromList $ NEL.toList requestKeys0 @@ -630,28 +630,29 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do lookup :: (RequestKey, T2 BlockHeight BlockHash) - -> IO (Maybe (RequestKey, CommandResult Hash)) + -> IO (Either String (RequestKey, CommandResult Hash)) lookup (key, T2 _ ha) = fmap (key,) <$> lookupRequestKey key ha -- TODO: group by block for performance (not very important right now) - lookupRequestKey key bHash = runMaybeT $ do + lookupRequestKey key bHash = runExceptT $ do let keyHash = unRequestKey key let pactHash = Pact.fromUntypedHash keyHash let matchingHash = (== pactHash) . _cmdHash . fst blockHeader <- liftIO $ TreeDB.lookupM bhdb bHash let payloadHash = _blockPayloadHash blockHeader - (_payloadWithOutputsTransactions -> txsBs) <- MaybeT $ tableLookup pdb payloadHash + (_payloadWithOutputsTransactions -> txsBs) <- barf "tablelookupFailed" =<< liftIO (tableLookup pdb payloadHash) !txs <- mapM fromTx txsBs case find matchingHash txs of Just (_cmd, TransactionOutput output) -> do - out <- MaybeT $ return $! decodeStrict' output + out <- barf "decodeStrict' output" $! decodeStrict' output when (_crReqKey out /= key) $ fail "internal error: Transaction output doesn't match its hash!" enrichCR blockHeader out - Nothing -> mzero + Nothing -> throwError $ "Request key not found: " <> sshow keyHash + fromTx :: (Transaction, TransactionOutput) -> ExceptT String IO (Command Text, TransactionOutput) fromTx (!tx, !out) = do - !tx' <- MaybeT (return (toPactTx tx)) + !tx' <- except $ toPactTx tx return (tx', out) checkBadList :: Vector RequestKey -> IO (Vector (RequestKey, CommandResult Hash)) @@ -670,7 +671,7 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do !cr = CommandResult rk Nothing res 0 Nothing Nothing Nothing [] in (rk, cr) - enrichCR :: BlockHeader -> CommandResult Hash -> MaybeT IO (CommandResult Hash) + enrichCR :: BlockHeader -> CommandResult Hash -> ExceptT String IO (CommandResult Hash) enrichCR bh = return . set crMetaData (Just $ object [ "blockHeight" .= _blockHeight bh @@ -682,9 +683,11 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do -- -------------------------------------------------------------------------- -- -- Misc Utils -toPactTx :: Transaction -> Maybe (Command Text) -toPactTx (Transaction b) = decodeStrict' b +barf :: Monad m => e -> Maybe a -> ExceptT e m a +barf e = maybe (throwError e) return +toPactTx :: Transaction -> Either String (Command Text) +toPactTx (Transaction b) = note "toPactTx failure" (decodeStrict' b) -- TODO: all of the functions in this module can instead grab the current block height from consensus -- and pass it here to get a better estimate of what behavior is correct. From 6996c54f5627178de10c9177f49b081214715d0f Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 28 Nov 2023 15:24:18 -0600 Subject: [PATCH 8/8] backout of commit b88c2132eb95bf7f3910b7108b234319ea419db6 (#1787) --- .../Pact/Backend/RelationalCheckpointer.hs | 44 ++++++++----------- src/Chainweb/Pact/RestAPI/Server.hs | 31 ++++++------- 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs index 4d722718de..939d2d94de 100644 --- a/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs +++ b/src/Chainweb/Pact/Backend/RelationalCheckpointer.hs @@ -319,49 +319,41 @@ doRegisterSuccessful dbenv (TypedHash hash) = doLookupSuccessful :: Db logger -> Maybe ConfirmationDepth -> V.Vector PactHash -> IO (HashMap.HashMap PactHash (T2 BlockHeight BlockHash)) doLookupSuccessful dbenv confDepth hashes = runBlockEnv dbenv $ do - withSavepoint DbTransaction $ - fmap buildResultMap $ -- swizzle results of query into a HashMap - callDb "doLookupSuccessful" $ \db -> do + withSavepoint DbTransaction $ do + r <- callDb "doLookupSuccessful" $ \db -> do + let + currentHeightQ = "SELECT blockheight FROM BlockHistory \ + \ ORDER BY blockheight DESC LIMIT 1" + -- if there is a confirmation depth, we get the current height and calculate -- the block height, to look for the transactions in range [0, current block height - confirmation depth] blockheight <- case confDepth of Nothing -> pure Nothing Just (ConfirmationDepth cd) -> do - let currentHeightQ = "SELECT blockheight FROM BlockHistory ORDER BY blockheight DESC LIMIT 1" currentHeight <- qry_ db currentHeightQ [RInt] case currentHeight of [[SInt bh]] -> pure $ Just (bh - fromIntegral cd) _ -> fail "impossible" let - hss = V.toList hashes - params = Utf8 $ intercalate "," (map (const "?") hss) - qtext = "SELECT blockheight, hash, txhash FROM \ - \TransactionIndex INNER JOIN BlockHistory \ - \USING (blockheight) WHERE txhash IN (" <> params <> ")" - <> maybe "" (const " AND blockheight <= ?") confDepth - <> ";" - qvals - -- match query params above. first, hashes - = map (\(TypedHash h) -> SBlob $ BS.fromShort h) hss - -- then, the block depth, if any - ++ maybe [] (\bh -> [SInt bh]) blockheight + blockheightval = maybe [] (\bh -> [SInt bh]) blockheight + qvals = [ SBlob (BS.fromShort hash) | (TypedHash hash) <- V.toList hashes ] ++ blockheightval qry db qtext qvals [RInt, RBlob, RBlob] >>= mapM go + return $ HashMap.fromList (map (\(T3 blockheight blockhash txhash) -> (txhash, T2 blockheight blockhash)) r) where - -- NOTE: it's useful to keep the types of 'go' and 'buildResultMap' in sync - -- for readability but also to ensure the compiler and reader infer the - -- right result types from the db query. - - buildResultMap :: [T3 PactHash BlockHeight BlockHash] -> HashMap.HashMap PactHash (T2 BlockHeight BlockHash) - buildResultMap xs = HashMap.fromList $ - map (\(T3 txhash blockheight blockhash) -> (txhash, T2 blockheight blockhash)) xs - - go :: [SType] -> IO (T3 PactHash BlockHeight BlockHash) + qtext = "SELECT blockheight, hash, txhash FROM \ + \TransactionIndex INNER JOIN BlockHistory \ + \USING (blockheight) WHERE txhash IN (" <> hashesParams <> ")" + <> maybe "" (const " AND blockheight <= ?") confDepth + <> ";" + hashesParams = Utf8 $ intercalate "," [ "?" | _ <- V.toList hashes] + + go :: [SType] -> IO (T3 BlockHeight BlockHash PactHash) go (SInt blockheight:SBlob blockhash:SBlob txhash:_) = do !blockhash' <- either fail return $ runGetEitherS decodeBlockHash blockhash let !txhash' = TypedHash $ BS.toShort txhash - return $! T3 txhash' (fromIntegral blockheight) blockhash' + return $! T3 (fromIntegral blockheight) blockhash' txhash' go _ = fail "impossible" doGetBlockHistory :: Db logger -> BlockHeader -> Domain RowKey RowData -> IO BlockTxHistory diff --git a/src/Chainweb/Pact/RestAPI/Server.hs b/src/Chainweb/Pact/RestAPI/Server.hs index 72fe83e1d5..603db5ed1f 100644 --- a/src/Chainweb/Pact/RestAPI/Server.hs +++ b/src/Chainweb/Pact/RestAPI/Server.hs @@ -40,7 +40,8 @@ import Control.Monad import Control.Monad.Catch hiding (Handler) import Control.Monad.Reader import Control.Monad.State.Strict -import Control.Monad.Trans.Except (ExceptT, runExceptT, except) +import Control.Monad.Trans.Except (ExceptT) +import Control.Monad.Trans.Maybe import Data.Aeson as Aeson import Data.Bifunctor (second) @@ -130,7 +131,6 @@ import Pact.Types.Hash (Hash(..)) import qualified Pact.Types.Hash as Pact import Pact.Types.PactError (PactError(..), PactErrorType(..)) import Pact.Types.Pretty (pretty) -import Control.Error (note, rights) -- -------------------------------------------------------------------------- -- @@ -619,10 +619,10 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do let results1 = V.map (\rk -> (rk, HM.lookup (Pact.fromUntypedHash $ unRequestKey rk) results0)) requestKeysV let (present0, missing) = V.unstablePartition (isJust . snd) results1 let present = V.map (second fromJuste) present0 + lookedUp <- catMaybes . V.toList <$> mapM lookup present badlisted <- V.toList <$> checkBadList (V.map fst missing) - vs <- mapM lookup present - let good = rights $ V.toList vs - return $! HM.fromList (good ++ badlisted) + let outputs = lookedUp ++ badlisted + return $! HM.fromList outputs where cid = _chainId bhdb !requestKeysV = V.fromList $ NEL.toList requestKeys0 @@ -630,29 +630,28 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do lookup :: (RequestKey, T2 BlockHeight BlockHash) - -> IO (Either String (RequestKey, CommandResult Hash)) + -> IO (Maybe (RequestKey, CommandResult Hash)) lookup (key, T2 _ ha) = fmap (key,) <$> lookupRequestKey key ha -- TODO: group by block for performance (not very important right now) - lookupRequestKey key bHash = runExceptT $ do + lookupRequestKey key bHash = runMaybeT $ do let keyHash = unRequestKey key let pactHash = Pact.fromUntypedHash keyHash let matchingHash = (== pactHash) . _cmdHash . fst blockHeader <- liftIO $ TreeDB.lookupM bhdb bHash let payloadHash = _blockPayloadHash blockHeader - (_payloadWithOutputsTransactions -> txsBs) <- barf "tablelookupFailed" =<< liftIO (tableLookup pdb payloadHash) + (_payloadWithOutputsTransactions -> txsBs) <- MaybeT $ tableLookup pdb payloadHash !txs <- mapM fromTx txsBs case find matchingHash txs of Just (_cmd, TransactionOutput output) -> do - out <- barf "decodeStrict' output" $! decodeStrict' output + out <- MaybeT $ return $! decodeStrict' output when (_crReqKey out /= key) $ fail "internal error: Transaction output doesn't match its hash!" enrichCR blockHeader out - Nothing -> throwError $ "Request key not found: " <> sshow keyHash + Nothing -> mzero - fromTx :: (Transaction, TransactionOutput) -> ExceptT String IO (Command Text, TransactionOutput) fromTx (!tx, !out) = do - !tx' <- except $ toPactTx tx + !tx' <- MaybeT (return (toPactTx tx)) return (tx', out) checkBadList :: Vector RequestKey -> IO (Vector (RequestKey, CommandResult Hash)) @@ -671,7 +670,7 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do !cr = CommandResult rk Nothing res 0 Nothing Nothing Nothing [] in (rk, cr) - enrichCR :: BlockHeader -> CommandResult Hash -> ExceptT String IO (CommandResult Hash) + enrichCR :: BlockHeader -> CommandResult Hash -> MaybeT IO (CommandResult Hash) enrichCR bh = return . set crMetaData (Just $ object [ "blockHeight" .= _blockHeight bh @@ -683,11 +682,9 @@ internalPoll pdb bhdb mempool pactEx cut confDepth requestKeys0 = do -- -------------------------------------------------------------------------- -- -- Misc Utils -barf :: Monad m => e -> Maybe a -> ExceptT e m a -barf e = maybe (throwError e) return +toPactTx :: Transaction -> Maybe (Command Text) +toPactTx (Transaction b) = decodeStrict' b -toPactTx :: Transaction -> Either String (Command Text) -toPactTx (Transaction b) = note "toPactTx failure" (decodeStrict' b) -- TODO: all of the functions in this module can instead grab the current block height from consensus -- and pass it here to get a better estimate of what behavior is correct.