From 4f835aae98ae646c08c8804b6761d99fb8dc9a40 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Tue, 8 Aug 2023 15:59:57 -0400 Subject: [PATCH 1/6] state-proof-based catchup Add support for catchup to use state proofs to validate new blocks. The main changes are: - The catchup service now has a state proof fetcher, whose job is to retrieve state proofs for rounds beyond the current ledger state. These state proofs are stored in an sqlite DB, since there might be many state proofs that we need to fetch. - The catchup service exposes a way to set trusted "renaissance" parameters for authenticating the initial state proofs, for cases when we can't use state proofs from the genesis block (like the situation we have on mainnet now). - The BlockService HTTP interface adds support for retrieving a state proof, and for getting a light block header proof instead of a cert when retrieving a block. - The catchup service uses state proofs, if possible, to authenticate new blocks, in lieu of agreement certificates. The catchup service is backwards-compatible: if it requests a state proof from the BlockService, but receives an agreement certificate instead (e.g., because the BlockService has not been upgraded with the above changes), the catchup service will validate the certificate instead. - The config file has additional fields to optionally specify the renaissance catchup parameters, to allow catchup to start validating state proofs from some (trusted through out-of-band channels) block. It may be a good idea, for performance, to pre-compute the state proofs and distribute them in a single file, rather than asking many relays to find the state proofs on-demand. This is not done yet, because it largely depends on how we would want to distribute these bundled state proofs. It should be reasonably straightforward to feed a bundle of state proofs into the stateProofFetcher. --- catchup/catchpointService.go | 2 +- catchup/fetcher_test.go | 152 +++++++- catchup/pref_test.go | 2 +- catchup/service.go | 131 ++++++- catchup/service_test.go | 124 ++++-- catchup/stateproof.go | 464 +++++++++++++++++++++++ catchup/stateproof_test.go | 173 +++++++++ catchup/universalFetcher.go | 146 +++++-- catchup/universalFetcher_test.go | 36 +- components/mocks/mockNetwork.go | 4 + config/config.go | 5 + config/config_test.go | 8 + config/localTemplate.go | 52 ++- config/local_defaults.go | 7 +- daemon/algod/api/server/v2/handlers.go | 14 +- data/stateproofmsg/message.go | 11 +- data/stateproofmsg/msgp_gen.go | 23 +- installer/config.json.example | 6 +- node/follower_node.go | 12 +- node/node.go | 10 +- rpcs/blockService.go | 269 ++++++++++++- rpcs/ledgerService.go | 2 +- rpcs/msgp_gen.go | 62 ++- stateproof/stateproofMessageGenerator.go | 4 +- test/testdata/configs/config-v32.json | 138 +++++++ 25 files changed, 1668 insertions(+), 189 deletions(-) create mode 100644 catchup/stateproof.go create mode 100644 catchup/stateproof_test.go create mode 100644 test/testdata/configs/config-v32.json diff --git a/catchup/catchpointService.go b/catchup/catchpointService.go index 801901048b..80404d785e 100644 --- a/catchup/catchpointService.go +++ b/catchup/catchpointService.go @@ -674,7 +674,7 @@ func (cs *CatchpointCatchupService) fetchBlock(round basics.Round, retryCount ui return nil, time.Duration(0), psp, true, cs.abort(fmt.Errorf("fetchBlock: recurring non-HTTP peer was provided by the peer selector")) } fetcher := makeUniversalBlockFetcher(cs.log, cs.net, cs.config) - blk, _, downloadDuration, err = fetcher.fetchBlock(cs.ctx, round, httpPeer) + blk, _, _, downloadDuration, err = fetcher.fetchBlock(cs.ctx, round, httpPeer, false) if err != nil { if cs.ctx.Err() != nil { return nil, time.Duration(0), psp, true, cs.stopOrAbort() diff --git a/catchup/fetcher_test.go b/catchup/fetcher_test.go index 983de01475..1e73abd08a 100644 --- a/catchup/fetcher_test.go +++ b/catchup/fetcher_test.go @@ -31,6 +31,9 @@ import ( "github.com/algorand/go-algorand/components/mocks" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/crypto/merklesignature" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" @@ -38,18 +41,60 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/stateproof" ) -func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, next basics.Round, b bookkeeping.Block, err error) { +const ( + testLedgerKeyValidRounds = 10000 +) + +type testLedgerStateProofData struct { + Params config.ConsensusParams + User basics.Address + Secrets *merklesignature.Secrets + TotalWeight basics.MicroAlgos + Participants basics.ParticipantsArray + Tree *merklearray.Tree + TemplateBlock bookkeeping.Block +} + +func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, next basics.Round, b bookkeeping.Block, stateProofData *testLedgerStateProofData, err error) { var user basics.Address user[0] = 123 - proto := config.Consensus[protocol.ConsensusCurrentVersion] - genesis := make(map[basics.Address]basics.AccountData) - genesis[user] = basics.AccountData{ + ver := blk.CurrentProtocol + if ver == "" { + ver = protocol.ConsensusCurrentVersion + } + + proto := config.Consensus[ver] + + userData := basics.AccountData{ Status: basics.Offline, MicroAlgos: basics.MicroAlgos{Raw: proto.MinBalance * 2000000}, } + + if proto.StateProofInterval > 0 { + stateProofData = &testLedgerStateProofData{ + Params: proto, + User: user, + } + + stateProofData.Secrets, err = merklesignature.New(0, testLedgerKeyValidRounds, proto.StateProofInterval) + if err != nil { + t.Fatal("couldn't generate state proof keys", err) + return + } + + userData.StateProofID = stateProofData.Secrets.GetVerifier().Commitment + userData.VoteFirstValid = 0 + userData.VoteLastValid = testLedgerKeyValidRounds + userData.VoteKeyDilution = 1 + userData.Status = basics.Online + } + + genesis := make(map[basics.Address]basics.AccountData) + genesis[user] = userData genesis[sinkAddr] = basics.AccountData{ Status: basics.Offline, MicroAlgos: basics.MicroAlgos{Raw: proto.MinBalance * 2000000}, @@ -66,7 +111,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, cfg := config.GetDefaultLocal() cfg.Archival = true ledger, err = data.LoadLedger( - log, t.Name(), inMem, protocol.ConsensusCurrentVersion, genBal, "", genHash, + log, t.Name(), inMem, ver, genBal, "", genHash, nil, cfg, ) if err != nil { @@ -99,7 +144,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, b.RewardsLevel = prev.RewardsLevel b.BlockHeader.Round = next b.BlockHeader.GenesisHash = genHash - b.CurrentProtocol = protocol.ConsensusCurrentVersion + b.CurrentProtocol = ver txib, err := b.EncodeSignedTxn(signedtx, transactions.ApplyData{}) require.NoError(t, err) b.Payset = []transactions.SignedTxnInBlock{ @@ -107,15 +152,97 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, } b.TxnCommitments, err = b.PaysetCommit() require.NoError(t, err) + + if proto.StateProofInterval > 0 { + var p basics.Participant + p.Weight = userData.MicroAlgos.ToUint64() + p.PK.KeyLifetime = merklesignature.KeyLifetimeDefault + p.PK.Commitment = userData.StateProofID + + stateProofData.Participants = append(stateProofData.Participants, p) + stateProofData.TotalWeight = userData.MicroAlgos + stateProofData.Tree, err = merklearray.BuildVectorCommitmentTree(stateProofData.Participants, crypto.HashFactory{HashType: cryptostateproof.HashType}) + if err != nil { + t.Fatal("couldn't build state proof voters tree", err) + return + } + + b.StateProofTracking = map[protocol.StateProofType]bookkeeping.StateProofTrackingData{ + protocol.StateProofBasic: bookkeeping.StateProofTrackingData{ + StateProofVotersCommitment: stateProofData.Tree.Root(), + StateProofOnlineTotalWeight: stateProofData.TotalWeight, + StateProofNextRound: basics.Round(proto.StateProofInterval), + }, + } + } + require.NoError(t, ledger.AddBlock(b, agreement.Certificate{Round: next})) return } -func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, numBlocks int) { +func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, stateProofData *testLedgerStateProofData, numBlocks int) { var err error + origPayset := blk.Payset + nextStateProofTracking := blk.StateProofTracking + for i := 0; i < numBlocks; i++ { blk.BlockHeader.Round++ blk.BlockHeader.TimeStamp += int64(crypto.RandUint64() % 100 * 1000) + blk.Payset = origPayset + blk.StateProofTracking = nextStateProofTracking + + if stateProofData != nil && + (blk.BlockHeader.Round%basics.Round(stateProofData.Params.StateProofInterval)) == 0 && + blk.BlockHeader.Round > basics.Round(stateProofData.Params.StateProofInterval) { + proofrnd := blk.BlockHeader.Round.SubSaturate(basics.Round(stateProofData.Params.StateProofInterval)) + msg, err := stateproof.GenerateStateProofMessage(ledger, proofrnd) + require.NoError(t, err) + + provenWeight, overflowed := basics.Muldiv(stateProofData.TotalWeight.ToUint64(), uint64(stateProofData.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + msgHash := msg.Hash() + prover, err := cryptostateproof.MakeProver(msgHash, + uint64(proofrnd), + provenWeight, + stateProofData.Participants, + stateProofData.Tree, + stateProofData.Params.StateProofStrengthTarget) + require.NoError(t, err) + + sig, err := stateProofData.Secrets.GetSigner(uint64(proofrnd)).SignBytes(msgHash[:]) + require.NoError(t, err) + + err = prover.Add(0, sig) + require.NoError(t, err) + + require.True(t, prover.Ready()) + sp, err := prover.CreateProof() + require.NoError(t, err) + + var stxn transactions.SignedTxn + stxn.Txn.Type = protocol.StateProofTx + stxn.Txn.Sender = transactions.StateProofSender + stxn.Txn.FirstValid = blk.BlockHeader.Round + stxn.Txn.LastValid = blk.BlockHeader.Round + stxn.Txn.GenesisHash = blk.BlockHeader.GenesisHash + stxn.Txn.StateProofTxnFields.StateProofType = protocol.StateProofBasic + stxn.Txn.StateProofTxnFields.StateProof = *sp + stxn.Txn.StateProofTxnFields.Message = msg + + txib, err := blk.EncodeSignedTxn(stxn, transactions.ApplyData{}) + require.NoError(t, err) + blk.Payset = make([]transactions.SignedTxnInBlock, len(origPayset)+1) + copy(blk.Payset[:], origPayset[:]) + blk.Payset[len(origPayset)] = txib + + sptracking := blk.StateProofTracking[protocol.StateProofBasic] + sptracking.StateProofNextRound = blk.BlockHeader.Round + nextStateProofTracking = map[protocol.StateProofType]bookkeeping.StateProofTrackingData{ + protocol.StateProofBasic: sptracking, + } + } + blk.TxnCommitments, err = blk.PaysetCommit() require.NoError(t, err) @@ -126,6 +253,10 @@ func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, numBloc require.NoError(t, err) require.Equal(t, blk.BlockHeader, hdr) } + + blk.Payset = origPayset + blk.StateProofTracking = nextStateProofTracking + stateProofData.TemplateBlock = blk } type basicRPCNode struct { @@ -143,6 +274,13 @@ func (b *basicRPCNode) RegisterHTTPHandler(path string, handler http.Handler) { b.rmux.Handle(path, handler) } +func (b *basicRPCNode) RegisterHTTPHandlerFunc(path string, handler func(response http.ResponseWriter, request *http.Request)) { + if b.rmux == nil { + b.rmux = mux.NewRouter() + } + b.rmux.HandleFunc(path, handler) +} + func (b *basicRPCNode) RegisterHandlers(dispatch []network.TaggedMessageHandler) { } diff --git a/catchup/pref_test.go b/catchup/pref_test.go index 377575f23a..3d580ef74b 100644 --- a/catchup/pref_test.go +++ b/catchup/pref_test.go @@ -66,7 +66,7 @@ func BenchmarkServiceFetchBlocks(b *testing.B) { require.NoError(b, err) // Make Service - syncer := MakeService(logging.TestingLog(b), defaultConfig, net, local, new(mockedAuthenticator), nil, nil) + syncer := MakeService(logging.TestingLog(b), defaultConfig, net, local, new(mockedAuthenticator), nil, nil, nil) b.StartTimer() syncer.Start() for w := 0; w < 1000; w++ { diff --git a/catchup/service.go b/catchup/service.go index bc23b3d736..23db9fa903 100644 --- a/catchup/service.go +++ b/catchup/service.go @@ -30,11 +30,13 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/execpool" ) @@ -66,6 +68,7 @@ type Ledger interface { Validate(ctx context.Context, blk bookkeeping.Block, executionPool execpool.BacklogPool) (*ledgercore.ValidatedBlock, error) AddValidatedBlock(vb ledgercore.ValidatedBlock, cert agreement.Certificate) error WaitMem(r basics.Round) chan struct{} + GetStateProofVerificationContext(basics.Round) (*ledgercore.StateProofVerificationContext, error) } // Service represents the catchup service. Once started and until it is stopped, it ensures that the ledger is up to date with network. @@ -105,6 +108,54 @@ type Service struct { // unsupportedRoundMonitor goroutine, after detecting // an unsupported block. onceUnsupportedRound sync.Once + + // stateproofs contains validated state proof messages for + // future rounds. The round is the LastAttestedRound of + // the state proof message. + // + // To help with garbage-collecting state proofs for rounds + // that are already in the ledger, stateproofmin tracks the + // lowest-numbered round in stateproofs. stateproofmin=0 + // indicates that no state proofs have been fetched. + // Similarly, stateproofmax tracks the most recent available + // state proof. + // + // stateproofdb stores these state proofs to ensure progress + // if algod gets restarted halfway through a long catchup. + // + // stateproofproto is the protocol version that corresponds + // to the consensus parameters for state proofs, either from + // a ledger round where we saw state proofs enabled, or from + // a special renaissance block configuration setting (below). + // + // stateproofproto is set if stateproofmin>0. + // + // stateproofmu protects the stateproof state from concurrent + // access. + // + // stateproofwait tracks channels for waiting on state proofs + // to be fetched. If stateproofwait is nil, there is no active + // stateProofFetcher that can be waited for. + stateproofs map[basics.Round]stateProofInfo + stateproofmin basics.Round + stateproofmax basics.Round + stateproofdb *db.Accessor + stateproofproto protocol.ConsensusVersion + stateproofmu sync.Mutex + stateproofwait map[basics.Round]chan struct{} + + // renaissance specifies the parameters for a renaissance + // block from which we can start validating state proofs, + // in lieu of validating the entire sequence of blocks + // starting from the genesis block. + renaissance *StateProofVerificationContext +} + +// stateProofInfo is a validated state proof message for some round, +// along with the consensus protocol for that round. +type stateProofInfo struct { + message stateproofmsg.Message + proto protocol.ConsensusVersion } // A BlockAuthenticator authenticates blocks given a certificate. @@ -120,7 +171,7 @@ type BlockAuthenticator interface { } // MakeService creates a catchup service instance from its constituent components -func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate, blockValidationPool execpool.BacklogPool) (s *Service) { +func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate, blockValidationPool execpool.BacklogPool, spCatchupDB *db.Accessor) (s *Service) { s = &Service{} s.cfg = config @@ -133,6 +184,16 @@ func MakeService(log logging.Logger, config config.Local, net network.GossipNode s.deadlineTimeout = agreement.DeadlineTimeout() s.blockValidationPool = blockValidationPool s.syncNow = make(chan struct{}, 1) + s.stateproofs = make(map[basics.Round]stateProofInfo) + + if spCatchupDB != nil { + s.stateproofdb = spCatchupDB + + err := s.initStateProofs() + if err != nil { + s.log.Warnf("catchup.initStateProofs(): %v", err) + } + } return s } @@ -209,13 +270,15 @@ func (s *Service) SynchronizingTime() time.Duration { // errLedgerAlreadyHasBlock is returned by innerFetch in case the local ledger already has the requested block. var errLedgerAlreadyHasBlock = errors.New("ledger already has block") -// function scope to make a bunch of defer statements better -func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.Peer) (blk *bookkeeping.Block, cert *agreement.Certificate, ddur time.Duration, err error) { +// innerFetch retrieves a block with a certificate or state-proof-based +// light block header proof for round r from peer. proofOK specifies +// whether it's acceptable to fetch a proof instead of a certificate. +func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.Peer, proofOK bool) (blk *bookkeeping.Block, cert *agreement.Certificate, proof []byte, ddur time.Duration, err error) { ledgerWaitCh := s.ledger.WaitMem(r) select { case <-ledgerWaitCh: // if our ledger already have this block, no need to attempt to fetch it. - return nil, nil, time.Duration(0), errLedgerAlreadyHasBlock + return nil, nil, nil, time.Duration(0), errLedgerAlreadyHasBlock default: } @@ -229,7 +292,7 @@ func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.P cf() } }() - blk, cert, ddur, err = fetcher.fetchBlock(ctx, r, peer) + blk, cert, proof, ddur, err = fetcher.fetchBlock(ctx, r, peer, proofOK) // check to see if we aborted due to ledger. if err != nil { select { @@ -291,8 +354,23 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo } peer := psp.Peer + // Wait for a state proof to become available for this block. + // If no state proofs are available, this will return right away. + select { + case <-ctx.Done(): + s.log.Debugf("fetchAndWrite(%v): Aborted", r) + return false + case <-s.stateProofWait(r): + } + + spinfo := s.getStateProof(r) + // Try to fetch, timing out after retryInterval - block, cert, blockDownloadDuration, err := s.innerFetch(ctx, r, peer) + proofOK := true + if spinfo == nil || !config.Consensus[spinfo.proto].StateProofBlockHashInLightHeader { + proofOK = false + } + block, cert, proof, blockDownloadDuration, err := s.innerFetch(ctx, r, peer, proofOK) if err != nil { if err == errLedgerAlreadyHasBlock { @@ -314,11 +392,11 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo case <-lookbackComplete: } continue // retry the fetch - } else if block == nil || cert == nil { + } else if block == nil { // someone already wrote the block to the ledger, we should stop syncing return false } - s.log.Debugf("fetchAndWrite(%v): Got block and cert contents: %v %v", r, block, cert) + s.log.Debugf("fetchAndWrite(%v): Got block and cert/proof contents: %v %v %v", r, block, cert, proof) // Check that the block's contents match the block header (necessary with an untrusted block because b.Hash() only hashes the header) if s.cfg.CatchupVerifyPaysetHash() { @@ -335,21 +413,30 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo } } - // make sure that we have the lookBack block that's required for authenticating this block - select { - case <-ctx.Done(): - s.log.Debugf("fetchAndWrite(%v): Aborted while waiting for lookback block to ledger", r) - return false - case <-lookbackComplete: - } - - if s.cfg.CatchupVerifyCertificate() { - err = s.auth.Authenticate(block, cert) + if spinfo != nil && len(proof) > 0 { + err = verifyBlockStateProof(r, &spinfo.message, block, proof) if err != nil { - s.log.Warnf("fetchAndWrite(%v): cert did not authenticate block (attempt %d): %v", r, i, err) + s.log.Warnf("fetchAndWrite(%v): proof did not authenticate block (attempt %d): %v", r, i, err) peerSelector.rankPeer(psp, peerRankInvalidDownload) continue // retry the fetch } + } else { + // make sure that we have the lookBack block that's required for authenticating this block + select { + case <-ctx.Done(): + s.log.Debugf("fetchAndWrite(%v): Aborted while waiting for lookback block to ledger", r) + return false + case <-lookbackComplete: + } + + if s.cfg.CatchupVerifyCertificate() { + err = s.auth.Authenticate(block, cert) + if err != nil { + s.log.Warnf("fetchAndWrite(%v): cert did not authenticate block (attempt %d): %v", r, i, err) + peerSelector.rankPeer(psp, peerRankInvalidDownload) + continue // retry the fetch + } + } } peerRank := peerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration) @@ -453,6 +540,10 @@ func (s *Service) pipelinedFetch(seedLookback uint64) { ctx, cancelCtx := context.WithCancel(s.ctx) defer cancelCtx() + // Start the state proof fetcher, which will enable us to validate + // blocks using state proofs if available. + s.startStateProofFetcher(ctx) + // firstRound is the first round we're waiting to fetch. firstRound := s.ledger.NextRound() @@ -680,7 +771,7 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy peer := psp.Peer // Ask the fetcher to get the block somehow - block, fetchedCert, _, err := s.innerFetch(s.ctx, cert.Round, peer) + block, fetchedCert, _, _, err := s.innerFetch(s.ctx, cert.Round, peer, false) if err != nil { select { diff --git a/catchup/service_test.go b/catchup/service_test.go index 406f5ef819..3264ba8fea 100644 --- a/catchup/service_test.go +++ b/catchup/service_test.go @@ -137,12 +137,12 @@ func TestServiceFetchBlocksSameRange(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -157,7 +157,7 @@ func TestServiceFetchBlocksSameRange(t *testing.T) { net.addPeer(rootURL) // Make Service - syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) syncer.testStart() syncer.sync() @@ -188,12 +188,12 @@ func TestSyncRound(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -213,7 +213,7 @@ func TestSyncRound(t *testing.T) { // Make Service localCfg := config.GetDefaultLocal() - s := MakeService(logging.Base(), localCfg, net, local, auth, nil, nil) + s := MakeService(logging.Base(), localCfg, net, local, auth, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} s.deadlineTimeout = 2 * time.Second @@ -278,12 +278,12 @@ func TestPeriodicSync(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -302,7 +302,7 @@ func TestPeriodicSync(t *testing.T) { require.True(t, 0 == initialLocalRound) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, auth, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, auth, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} s.deadlineTimeout = 2 * time.Second @@ -344,12 +344,12 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { local.blocks = append(local.blocks, bookkeeping.Block{}) lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -364,7 +364,7 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) // Get last round @@ -378,9 +378,9 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { require.Equal(t, lastRoundAtStart+basics.Round(numBlocks), local.LastRound()) // Get the same block we wrote - block, _, _, err := makeUniversalBlockFetcher(logging.Base(), + block, _, _, _, err := makeUniversalBlockFetcher(logging.Base(), net, - defaultConfig).fetchBlock(context.Background(), lastRoundAtStart+1, net.peers[0]) + defaultConfig).fetchBlock(context.Background(), lastRoundAtStart+1, net.peers[0], false) require.NoError(t, err) @@ -408,12 +408,12 @@ func TestAbruptWrites(t *testing.T) { lastRound := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numberOfBlocks-1) + addBlocks(t, remote, blk, spdata, numberOfBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -428,7 +428,7 @@ func TestAbruptWrites(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) var wg sync.WaitGroup wg.Add(1) @@ -466,12 +466,12 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, int(numberOfBlocks)-1) + addBlocks(t, remote, blk, spdata, int(numberOfBlocks)-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -486,7 +486,7 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { net.addPeer(rootURL) // Make Service - syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) fetcher := makeUniversalBlockFetcher(logging.Base(), net, defaultConfig) // Start the service ( dummy ) @@ -500,7 +500,7 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { for i := basics.Round(1); i <= numberOfBlocks; i++ { // Get the same block we wrote - blk, _, _, err2 := fetcher.fetchBlock(context.Background(), i, net.GetPeers()[0]) + blk, _, _, _, err2 := fetcher.fetchBlock(context.Background(), i, net.GetPeers()[0], false) require.NoError(t, err2) // Check we wrote the correct block @@ -521,12 +521,12 @@ func TestServiceFetchBlocksMalformed(t *testing.T) { lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -541,7 +541,7 @@ func TestServiceFetchBlocksMalformed(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} // Start the service ( dummy ) @@ -670,7 +670,7 @@ func helperTestOnSwitchToUnSupportedProtocol( config.CatchupParallelBlocks = 2 block1 := mRemote.blocks[1] - remote, _, blk, err := buildTestLedger(t, block1) + remote, _, blk, spdata, err := buildTestLedger(t, block1) if err != nil { t.Fatal(err) return local, remote @@ -679,7 +679,7 @@ func helperTestOnSwitchToUnSupportedProtocol( blk.NextProtocolSwitchOn = mRemote.blocks[i+1].NextProtocolSwitchOn blk.NextProtocol = mRemote.blocks[i+1].NextProtocol // Adds blk.BlockHeader.Round + 1 - addBlocks(t, remote, blk, 1) + addBlocks(t, remote, blk, spdata, 1) blk.BlockHeader.Round++ } @@ -695,7 +695,7 @@ func helperTestOnSwitchToUnSupportedProtocol( net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), config, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), config, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) s.deadlineTimeout = 2 * time.Second s.Start() defer s.Stop() @@ -710,6 +710,7 @@ type mockedLedger struct { mu deadlock.Mutex blocks []bookkeeping.Block chans map[basics.Round]chan struct{} + certs map[basics.Round]agreement.Certificate } func (m *mockedLedger) NextRound() basics.Round { @@ -746,6 +747,12 @@ func (m *mockedLedger) AddBlock(blk bookkeeping.Block, cert agreement.Certificat delete(m.chans, r) } } + + if m.certs == nil { + m.certs = make(map[basics.Round]agreement.Certificate) + } + m.certs[blk.Round()] = cert + return nil } @@ -797,6 +804,15 @@ func (m *mockedLedger) Block(r basics.Round) (bookkeeping.Block, error) { return m.blocks[r], nil } +func (m *mockedLedger) BlockCert(r basics.Round) (bookkeeping.Block, agreement.Certificate, error) { + m.mu.Lock() + defer m.mu.Unlock() + if r > m.lastRound() { + return bookkeeping.Block{}, agreement.Certificate{}, errors.New("mockedLedger.Block: round too high") + } + return m.blocks[r], m.certs[r], nil +} + func (m *mockedLedger) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { blk, err := m.Block(r) return blk.BlockHeader, err @@ -830,6 +846,34 @@ func (m *mockedLedger) IsWritingCatchpointDataFile() bool { return false } +func (m *mockedLedger) GetStateProofVerificationContext(r basics.Round) (*ledgercore.StateProofVerificationContext, error) { + latest := m.LastRound() + latestHdr, err := m.BlockHdr(latest) + if err != nil { + return nil, err + } + + interval := basics.Round(config.Consensus[latestHdr.CurrentProtocol].StateProofInterval) + if interval == 0 { + return nil, errors.New("state proofs not supported") + } + + lastAttested := r.RoundUpToMultipleOf(interval) + votersRnd := lastAttested - interval + votersHdr, err := m.BlockHdr(votersRnd) + if err != nil { + return nil, err + } + + vc := &ledgercore.StateProofVerificationContext{ + LastAttestedRound: lastAttested, + VotersCommitment: votersHdr.StateProofTracking[protocol.StateProofBasic].StateProofVotersCommitment, + OnlineTotalWeight: votersHdr.StateProofTracking[protocol.StateProofBasic].StateProofOnlineTotalWeight, + Version: votersHdr.CurrentProtocol, + } + return vc, nil +} + func testingenvWithUpgrade( t testing.TB, numBlocks, @@ -885,13 +929,13 @@ func TestCatchupUnmatchedCertificate(t *testing.T) { local.blocks = append(local.blocks, bookkeeping.Block{}) lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } defer remote.Close() - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -906,7 +950,7 @@ func TestCatchupUnmatchedCertificate(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil, nil) s.testStart() for roundNumber := 2; roundNumber < 10; roundNumber += 3 { pc := &PendingUnmatchedCertificate{ @@ -931,7 +975,7 @@ func TestCreatePeerSelector(t *testing.T) { cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "someAddress" - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps := createPeerSelector(s.net, s.cfg, true) require.Equal(t, 5, len(ps.peerClasses)) require.Equal(t, peerRankInitialFirstPriority, ps.peerClasses[0].initialRank) @@ -949,7 +993,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress == ""; pipelineFetch = true; cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 4, len(ps.peerClasses)) require.Equal(t, peerRankInitialFirstPriority, ps.peerClasses[0].initialRank) @@ -965,7 +1009,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress != ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 5, len(ps.peerClasses)) @@ -984,7 +1028,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress == ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 4, len(ps.peerClasses)) @@ -1001,7 +1045,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress != ""; pipelineFetch = true cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 4, len(ps.peerClasses)) @@ -1018,7 +1062,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress == ""; pipelineFetch = true cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 3, len(ps.peerClasses)) @@ -1033,7 +1077,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress != ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 4, len(ps.peerClasses)) @@ -1050,7 +1094,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress == ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 3, len(ps.peerClasses)) @@ -1069,7 +1113,7 @@ func TestServiceStartStop(t *testing.T) { cfg := defaultConfig ledger := new(mockedLedger) ledger.blocks = append(ledger.blocks, bookkeeping.Block{}) - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) s.Start() s.Stop() @@ -1082,7 +1126,7 @@ func TestSynchronizingTime(t *testing.T) { cfg := defaultConfig ledger := new(mockedLedger) ledger.blocks = append(ledger.blocks, bookkeeping.Block{}) - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) require.Equal(t, time.Duration(0), s.SynchronizingTime()) atomic.StoreInt64(&s.syncStartNS, 1000000) diff --git a/catchup/stateproof.go b/catchup/stateproof.go new file mode 100644 index 0000000000..38ee2c76fd --- /dev/null +++ b/catchup/stateproof.go @@ -0,0 +1,464 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchup + +import ( + "context" + "database/sql" + "encoding/base64" + "fmt" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +// This file implements state-proof-based validation of new blocks, +// for catching up the state of a node with the rest of the network. + +// StateProofVerificationContext specifies the parameters needed to +// verify a state proof for the catchup code. +type StateProofVerificationContext struct { + // LastRound is the LastAttestedRound in the state proof message + // that we expect to verify with these parameters. + LastRound basics.Round + + // LnProvenWeight is passed to stateproof.MkVerifierWithLnProvenWeight. + LnProvenWeight uint64 + + // VotersCommitment is passed to stateproof.MkVerifierWithLnProvenWeight. + VotersCommitment crypto.GenericDigest + + // Proto specifies the protocol in which state proofs were enabled, + // used to determine StateProofStrengthTarget and StateProofInterval. + Proto protocol.ConsensusVersion +} + +func spSchemaUpgrade0(_ context.Context, tx *sql.Tx, _ bool) error { + const createProofsTable = `CREATE TABLE IF NOT EXISTS proofs ( + lastrnd integer, + proto text, + msg blob, + UNIQUE (lastrnd))` + + _, err := tx.Exec(createProofsTable) + return err +} + +func (s *Service) initStateProofs() error { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofdb == nil { + return nil + } + + migrations := []db.Migration{ + spSchemaUpgrade0, + } + + err := db.Initialize(*s.stateproofdb, migrations) + if err != nil { + return err + } + + stateproofs := make(map[basics.Round]stateProofInfo) + var stateproofmin basics.Round + var stateproofmax basics.Round + var stateproofproto protocol.ConsensusVersion + + err = s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + rows, err := tx.Query("SELECT proto, msg FROM proofs ORDER BY lastrnd") + if err != nil { + return err + } else { + defer rows.Close() + for rows.Next() { + var proto protocol.ConsensusVersion + var msgbuf []byte + err := rows.Scan(&proto, &msgbuf) + if err != nil { + s.log.Warnf("initStateProofs: cannot scan proof from db: %v", err) + continue + } + + var msg stateproofmsg.Message + err = protocol.Decode(msgbuf, &msg) + if err != nil { + s.log.Warnf("initStateProofs: cannot decode proof from db: %v", err) + continue + } + + stateproofs[msg.LastAttestedRound] = stateProofInfo{ + message: msg, + proto: proto, + } + stateproofmax = msg.LastAttestedRound + if stateproofmin == 0 { + stateproofmin = msg.LastAttestedRound + stateproofproto = proto + } + } + } + return nil + }) + if err != nil { + return err + } + + s.stateproofs = stateproofs + s.stateproofmin = stateproofmin + s.stateproofmax = stateproofmax + s.stateproofproto = stateproofproto + + return nil +} + +// addStateProof adds a verified state proof message. +func (s *Service) addStateProof(msg stateproofmsg.Message, proto protocol.ConsensusVersion) { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofdb != nil { + err := s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("INSERT INTO proofs (lastrnd, proto, msg) VALUES (?, ?, ?)", + msg.LastAttestedRound, proto, protocol.Encode(&msg)) + return err + }) + if err != nil { + s.log.Warnf("addStateProof: database error: %v", err) + } + } + + if s.stateproofmin == 0 { + s.stateproofmin = msg.LastAttestedRound + s.stateproofproto = proto + } + if msg.LastAttestedRound > s.stateproofmax { + s.stateproofmax = msg.LastAttestedRound + } + s.stateproofs[msg.LastAttestedRound] = stateProofInfo{ + message: msg, + proto: proto, + } + + for r := msg.FirstAttestedRound; r < msg.LastAttestedRound; r++ { + ch, ok := s.stateproofwait[r] + if ok { + close(ch) + delete(s.stateproofwait, r) + } + } +} + +// cleanupStateProofs removes state proofs that are for the latest +// round or earlier. +func (s *Service) cleanupStateProofs(latest basics.Round) { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofmin == 0 { + return + } + + if s.stateproofdb != nil { + err := s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("DELETE FROM proofs WHERE lastrnd<=?", latest) + return err + }) + if err != nil { + s.log.Warnf("cleanupStateProofs: database error: %v", err) + } + } + + for s.stateproofmin <= latest { + delete(s.stateproofs, s.stateproofmin) + s.stateproofmin += basics.Round(config.Consensus[s.stateproofproto].StateProofInterval) + } +} + +// nextStateProofVerifier() returns the latest state proof verification +// context that we have access to. This might be based on the latest block +// in the ledger, or based on the latest state proof (beyond the end of the +// ledger) that we have, or based on well-known "renaissance block" values. +// +// The return value might be nil if no verification context is available. +func (s *Service) nextStateProofVerifier() *StateProofVerificationContext { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + // As a baseline, use the renaissance verification context (if present). + res := s.renaissance + + // Check if we have a more recent verified state proof in memory. + lastProof, ok := s.stateproofs[s.stateproofmax] + if ok && (res == nil || lastProof.message.LastAttestedRound >= res.LastRound) { + res = &StateProofVerificationContext{ + LastRound: lastProof.message.LastAttestedRound + basics.Round(config.Consensus[lastProof.proto].StateProofInterval), + LnProvenWeight: lastProof.message.LnProvenWeight, + VotersCommitment: lastProof.message.VotersCommitment, + Proto: s.stateproofproto, + } + } + + // Check if the ledger has a more recent state proof verification context. + latest := s.ledger.LastRound() + + // If we don't know state proof parameters yet, check the ledger. + proto := s.stateproofproto + params, paramsOk := config.Consensus[proto] + if !paramsOk { + hdr, err := s.ledger.BlockHdr(latest) + if err != nil { + s.log.Warnf("nextStateProofVerifier: BlockHdr(%d): %v", latest, err) + } else { + proto = hdr.CurrentProtocol + params, paramsOk = config.Consensus[proto] + } + } + + if !paramsOk || params.StateProofInterval == 0 { + // Ledger's latest block does not support state proof. + // Return whatever verification context we've found so far. + return res + } + + // The next state proof verification context we should expect from + // the ledger is for StateProofInterval in the future from the most + // recent multiple of StateProofInterval. + nextLastRound := latest.RoundDownToMultipleOf(basics.Round(params.StateProofInterval)) + basics.Round(params.StateProofInterval) + if res != nil && nextLastRound <= res.LastRound { + // We already have a verification context that's no older. + return res + } + + vctx, err := s.ledger.GetStateProofVerificationContext(nextLastRound) + if err != nil { + s.log.Warnf("nextStateProofVerifier: GetStateProofVerificationContext(%d): %v", nextLastRound, err) + return res + } + + provenWeight, overflowed := basics.Muldiv(vctx.OnlineTotalWeight.ToUint64(), uint64(params.StateProofWeightThreshold), 1<<32) + if overflowed { + s.log.Warnf("nextStateProofVerifier: overflow computing provenWeight[%d]: %d * %d / (1<<32)", + nextLastRound, vctx.OnlineTotalWeight.ToUint64(), params.StateProofWeightThreshold) + return res + } + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + if err != nil { + s.log.Warnf("nextStateProofVerifier: LnIntApproximation(%d): %v", provenWeight, err) + return res + } + + return &StateProofVerificationContext{ + LastRound: nextLastRound, + LnProvenWeight: lnProvenWt, + VotersCommitment: vctx.VotersCommitment, + Proto: proto, + } +} + +func (s *Service) SetRenaissance(r StateProofVerificationContext) { + s.renaissance = &r +} + +func (s *Service) SetRenaissanceFromConfig(cfg config.Local) { + if cfg.RenaissanceCatchupRound == 0 { + return + } + + votersCommitment, err := base64.StdEncoding.DecodeString(cfg.RenaissanceCatchupVotersCommitment) + if err != nil { + s.log.Warnf("SetRenaissanceFromConfig: cannot decode voters commitment: %v", err) + return + } + + vc := StateProofVerificationContext{ + LastRound: basics.Round(cfg.RenaissanceCatchupRound), + LnProvenWeight: cfg.RenaissanceCatchupLnProvenWeight, + VotersCommitment: votersCommitment, + Proto: protocol.ConsensusVersion(cfg.RenaissanceCatchupProto), + } + + interval := basics.Round(config.Consensus[vc.Proto].StateProofInterval) + if interval == 0 { + s.log.Warnf("SetRenaissanceFromConfig: state proofs not enabled in specified proto %s", vc.Proto) + return + } + + if (vc.LastRound % interval) != 0 { + s.log.Warnf("SetRenaissanceFromConfig: round %d not multiple of state proof interval %d", vc.LastRound, interval) + return + } + + s.SetRenaissance(vc) +} + +func (s *Service) stateProofWaitEnable() { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + s.stateproofwait = make(map[basics.Round]chan struct{}) +} + +func (s *Service) stateProofWaitDisable() { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + for _, ch := range s.stateproofwait { + close(ch) + } + s.stateproofwait = nil +} + +func (s *Service) stateProofWait(r basics.Round) chan struct{} { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if r <= s.stateproofmax { + ch := make(chan struct{}) + close(ch) + return ch + } + + if s.stateproofwait == nil { + ch := make(chan struct{}) + close(ch) + return ch + } + + ch, ok := s.stateproofwait[r] + if !ok { + ch = make(chan struct{}) + s.stateproofwait[r] = ch + } + + return ch +} + +func (s *Service) getStateProof(r basics.Round) *stateProofInfo { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + interval := config.Consensus[s.stateproofproto].StateProofInterval + if interval == 0 { + return nil + } + + proofrnd := r.RoundUpToMultipleOf(basics.Round(interval)) + proofInfo, ok := s.stateproofs[proofrnd] + if !ok { + return nil + } + + return &proofInfo +} + +func (s *Service) startStateProofFetcher(ctx context.Context) { + s.stateProofWaitEnable() + s.workers.Add(1) + go s.stateProofFetcher(ctx) +} + +// stateProofFetcher repeatedly tries to fetch the next verifiable state proof, +// until cancelled or no more state proofs can be fetched. +// +// The caller must s.workers.Add(1) and s.stateProofWaitEnable() before spawning +// stateProofFetcher. +func (s *Service) stateProofFetcher(ctx context.Context) { + defer s.workers.Done() + defer s.stateProofWaitDisable() + + latest := s.ledger.LastRound() + s.cleanupStateProofs(latest) + + peerSelector := createPeerSelector(s.net, s.cfg, true) + + for { + vc := s.nextStateProofVerifier() + if vc == nil { + s.log.Debugf("catchup.stateProofFetcher: no verifier available") + return + } + + retry := 0 + for { + if retry >= catchupRetryLimit { + s.log.Debugf("catchup.stateProofFetcher: cannot fetch %d, giving up", vc.LastRound) + return + } + retry++ + + select { + case <-ctx.Done(): + s.log.Debugf("catchup.stateProofFetcher: aborted") + return + default: + } + + psp, err := peerSelector.getNextPeer() + if err != nil { + s.log.Warnf("catchup.stateProofFetcher: unable to getNextPeer: %v", err) + return + } + + fetcher := makeUniversalBlockFetcher(s.log, s.net, s.cfg) + pf, msg, _, err := fetcher.fetchStateProof(ctx, protocol.StateProofBasic, vc.LastRound, psp.Peer) + if err != nil { + s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: %v", vc.LastRound, retry, err) + peerSelector.rankPeer(psp, peerRankDownloadFailed) + continue + } + + verifier := stateproof.MkVerifierWithLnProvenWeight(vc.VotersCommitment, vc.LnProvenWeight, config.Consensus[vc.Proto].StateProofStrengthTarget) + err = verifier.Verify(uint64(vc.LastRound), msg.Hash(), &pf) + if err != nil { + s.log.Warnf("catchup.stateProofFetcher: cannot verify round %d: %v", vc.LastRound, err) + peerSelector.rankPeer(psp, peerRankInvalidDownload) + continue + } + + s.addStateProof(msg, vc.Proto) + break + } + } +} + +func verifyBlockStateProof(r basics.Round, spmsg *stateproofmsg.Message, block *bookkeeping.Block, proofData []byte) error { + l := block.ToLightBlockHeader() + + if !config.Consensus[block.CurrentProtocol].StateProofBlockHashInLightHeader { + return fmt.Errorf("block %d protocol %s does not authenticate block in light block header", r, block.CurrentProtocol) + } + + proof, err := merklearray.ProofDataToSingleLeafProof(crypto.Sha256.String(), proofData) + if err != nil { + return err + } + + elems := make(map[uint64]crypto.Hashable) + elems[uint64(r-spmsg.FirstAttestedRound)] = &l + + return merklearray.VerifyVectorCommitment(spmsg.BlockHeadersCommitment, elems, proof.ToProof()) +} diff --git a/catchup/stateproof_test.go b/catchup/stateproof_test.go new file mode 100644 index 0000000000..22bc7dff4d --- /dev/null +++ b/catchup/stateproof_test.go @@ -0,0 +1,173 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchup + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestServiceStateProofFetcherRenaissance(t *testing.T) { + partitiontest.PartitionTest(t) + + // Make Ledgers + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) + if err != nil { + t.Fatal(err) + return + } + addBlocks(t, remote, blk, spdata, 1000) + + local := new(mockedLedger) + local.blocks = append(local.blocks, bookkeeping.Block{}) + + // Create a network and block service + blockServiceConfig := config.GetDefaultLocal() + net := &httpTestPeerSource{} + ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID") + + nodeA := basicRPCNode{} + ls.RegisterHandlers(&nodeA) + nodeA.start() + defer nodeA.stop() + rootURL := nodeA.rootURL() + net.addPeer(rootURL) + + // Make Service + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) + syncer.testStart() + + provenWeight, overflowed := basics.Muldiv(spdata.TotalWeight.ToUint64(), uint64(spdata.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + require.NoError(t, err) + + syncer.SetRenaissance(StateProofVerificationContext{ + LastRound: 256, + LnProvenWeight: lnProvenWt, + VotersCommitment: spdata.Tree.Root(), + Proto: blk.CurrentProtocol, + }) + + ctx := context.Background() + syncer.startStateProofFetcher(ctx) + + ch := syncer.stateProofWait(500) + <-ch + + msg := syncer.getStateProof(500) + require.NotNil(t, msg) + + ch = syncer.stateProofWait(5000) + <-ch + + msg = syncer.getStateProof(5000) + require.Nil(t, msg) +} + +func TestServiceStateProofSync(t *testing.T) { + partitiontest.PartitionTest(t) + + // Make Ledgers + var blk0 bookkeeping.Block + blk0.CurrentProtocol = protocol.ConsensusFuture + remote, _, blk, spdata, err := buildTestLedger(t, blk0) + if err != nil { + t.Fatal(err) + return + } + addBlocks(t, remote, blk, spdata, 1000) + + local := new(mockedLedger) + local.blocks = append(local.blocks, bookkeeping.Block{}) + + // Create a network and block service + blockServiceConfig := config.GetDefaultLocal() + net := &httpTestPeerSource{} + ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID") + + nodeA := basicRPCNode{} + ls.RegisterHandlers(&nodeA) + nodeA.start() + defer nodeA.stop() + rootURL := nodeA.rootURL() + net.addPeer(rootURL) + + // Make Service + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) + syncer.testStart() + + provenWeight, overflowed := basics.Muldiv(spdata.TotalWeight.ToUint64(), uint64(spdata.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + require.NoError(t, err) + + syncer.SetRenaissance(StateProofVerificationContext{ + LastRound: 256, + LnProvenWeight: lnProvenWt, + VotersCommitment: spdata.Tree.Root(), + Proto: blk.CurrentProtocol, + }) + + syncer.sync() + + rr, lr := remote.LastRound(), local.LastRound() + require.Equal(t, rr, lr) + + // Block 500 should have been fetched using state proofs, which means + // we should have no cert in the local ledger. + _, cert, err := local.BlockCert(500) + require.NoError(t, err) + require.True(t, cert.MsgIsZero()) + + // Block 900 should have been fetched using certs, which means + // we should have a valid cert for it in the ledger. + _, cert, err = local.BlockCert(900) + require.NoError(t, err) + require.Equal(t, cert.Round, basics.Round(900)) + + // Now try to sync again, which should flush all of the state proofs already + // covered by the local ledger (which is ahead of the state proofs now). + syncer.sync() + + // Now extend the remote ledger, and make sure that the local ledger can sync + // using state proofs starting from the most recent ledger-generated state + // proof verification context. + addBlocks(t, remote, spdata.TemplateBlock, spdata, 1000) + syncer.sync() + + _, cert, err = local.BlockCert(1500) + require.NoError(t, err) + require.True(t, cert.MsgIsZero()) + + _, cert, err = local.BlockCert(1900) + require.NoError(t, err) + require.Equal(t, cert.Round, basics.Round(1900)) +} diff --git a/catchup/universalFetcher.go b/catchup/universalFetcher.go index c8dd8b9f9f..f78b6b681c 100644 --- a/catchup/universalFetcher.go +++ b/catchup/universalFetcher.go @@ -28,8 +28,11 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" + "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" @@ -52,8 +55,8 @@ func makeUniversalBlockFetcher(log logging.Logger, net network.GossipNode, confi } // fetchBlock returns a block from the peer. The peer can be either an http or ws peer. -func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Round, peer network.Peer) (blk *bookkeeping.Block, - cert *agreement.Certificate, downloadDuration time.Duration, err error) { +func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Round, peer network.Peer, proofOK bool) (blk *bookkeeping.Block, + cert *agreement.Certificate, proof []byte, downloadDuration time.Duration, err error) { var fetchedBuf []byte var address string @@ -65,7 +68,7 @@ func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Ro } fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } address = fetcherClient.address() } else if httpPeer, validHTTPPeer := peer.(network.HTTPPeer); validHTTPPeer { @@ -76,24 +79,24 @@ func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Ro client: httpPeer.GetHTTPClient(), log: uf.log, config: &uf.config} - fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round) + fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round, proofOK) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } address = fetcherClient.address() } else { - return nil, nil, time.Duration(0), fmt.Errorf("fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") + return nil, nil, nil, time.Duration(0), fmt.Errorf("fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") } downloadDuration = time.Now().Sub(blockDownloadStartTime) - block, cert, err := processBlockBytes(fetchedBuf, round, address) + block, cert, proof, err := processBlockBytes(fetchedBuf, round, address) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } uf.log.Debugf("fetchBlock: downloaded block %d in %d from %s", uint64(round), downloadDuration, address) - return block, cert, downloadDuration, err + return block, cert, proof, downloadDuration, err } -func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk *bookkeeping.Block, cert *agreement.Certificate, err error) { +func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk *bookkeeping.Block, cert *agreement.Certificate, proof []byte, err error) { var decodedEntry rpcs.EncodedBlockCert err = protocol.Decode(fetchedBuf, &decodedEntry) if err != nil { @@ -106,11 +109,56 @@ func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk return } - if decodedEntry.Certificate.Round != r { + if (decodedEntry.LightBlockHeaderProof == nil || decodedEntry.Certificate.Round != 0) && decodedEntry.Certificate.Round != r { err = makeErrWrongCertFromPeer(r, decodedEntry.Certificate.Round, peerAddr) return } - return &decodedEntry.Block, &decodedEntry.Certificate, nil + return &decodedEntry.Block, &decodedEntry.Certificate, decodedEntry.LightBlockHeaderProof, nil +} + +// fetchStateProof retrieves a state proof (and the corresponding message +// attested to by the state proof) for round. +// +// proofType specifies the expected state proof type. +func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType protocol.StateProofType, round basics.Round, peer network.Peer) (pf stateproof.StateProof, msg stateproofmsg.Message, downloadDuration time.Duration, err error) { + var fetchedBuf []byte + var address string + downloadStartTime := time.Now() + if httpPeer, validHTTPPeer := peer.(network.HTTPPeer); validHTTPPeer { + fetcherClient := &HTTPFetcher{ + peer: httpPeer, + rootURL: httpPeer.GetAddress(), + net: uf.net, + client: httpPeer.GetHTTPClient(), + log: uf.log, + config: &uf.config, + } + fetchedBuf, err = fetcherClient.getStateProofBytes(ctx, proofType, round) + if err != nil { + return pf, msg, 0, err + } + address = fetcherClient.address() + } else { + return pf, msg, 0, fmt.Errorf("fetchStateProof: UniversalFetcher only supports HTTPPeer") + } + downloadDuration = time.Now().Sub(downloadStartTime) + pf, msg, err = processStateProofBytes(fetchedBuf, round, address) + if err != nil { + return pf, msg, 0, err + } + uf.log.Debugf("fetchStateProof: downloaded proof for %d in %d from %s", uint64(round), downloadDuration, address) + return pf, msg, downloadDuration, err +} + +func processStateProofBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (pf stateproof.StateProof, msg stateproofmsg.Message, err error) { + var decodedEntry transactions.Transaction + err = protocol.Decode(fetchedBuf, &decodedEntry) + if err != nil { + err = fmt.Errorf("Cannot decode state proof for %d from %s: %v", r, peerAddr, err) + return + } + + return decodedEntry.StateProofTxnFields.StateProof, decodedEntry.StateProofTxnFields.Message, nil } // a stub fetcherClient to satisfy the NetworkFetcher interface @@ -209,18 +257,10 @@ type HTTPFetcher struct { config *config.Local } -// getBlockBytes gets a block. -// Core piece of FetcherClient interface -func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data []byte, err error) { - parsedURL, err := network.ParseHostOrURL(hf.rootURL) - if err != nil { - return nil, err - } - - parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net) - blockURL := parsedURL.String() - hf.log.Debugf("block GET %#v peer %#v %T", blockURL, hf.peer, hf.peer) - request, err := http.NewRequest("GET", blockURL, nil) +// getBytes requests a particular URL, expecting specific content types +func (hf *HTTPFetcher) getBytes(ctx context.Context, url string, expectedContentTypes map[string]struct{}) (data []byte, err error) { + hf.log.Debugf("block GET %#v peer %#v %T", url, hf.peer, hf.peer) + request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } @@ -230,7 +270,7 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data network.SetUserAgentHeader(request.Header) response, err := hf.client.Do(request) if err != nil { - hf.log.Debugf("GET %#v : %s", blockURL, err) + hf.log.Debugf("GET %#v : %s", url, err) return nil, err } @@ -242,11 +282,11 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return nil, errNoBlockForRound default: bodyBytes, err := rpcs.ResponseBytes(response, hf.log, fetcherMaxBlockBytes) - hf.log.Warnf("HTTPFetcher.getBlockBytes: response status code %d from '%s'. Response body '%s' ", response.StatusCode, blockURL, string(bodyBytes)) + hf.log.Warnf("HTTPFetcher.getBytes: response status code %d from '%s'. Response body '%s' ", response.StatusCode, url, string(bodyBytes)) if err == nil { - err = makeErrHTTPResponse(response.StatusCode, blockURL, fmt.Sprintf("Response body '%s'", string(bodyBytes))) + err = makeErrHTTPResponse(response.StatusCode, url, fmt.Sprintf("Response body '%s'", string(bodyBytes))) } else { - err = makeErrHTTPResponse(response.StatusCode, blockURL, err.Error()) + err = makeErrHTTPResponse(response.StatusCode, url, err.Error()) } return nil, err } @@ -261,11 +301,9 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return nil, err } - // TODO: Temporarily allow old and new content types so we have time for lazy upgrades - // Remove this 'old' string after next release. - const blockResponseContentTypeOld = "application/algorand-block-v1" - if contentTypes[0] != rpcs.BlockResponseContentType && contentTypes[0] != blockResponseContentTypeOld { - hf.log.Warnf("http block fetcher response has an invalid content type : %s", contentTypes[0]) + _, contentTypeOK := expectedContentTypes[contentTypes[0]] + if !contentTypeOK { + hf.log.Warnf("http fetcher response has an invalid content type : %s", contentTypes[0]) response.Body.Close() return nil, errHTTPResponseContentType{contentTypeCount: 1, contentType: contentTypes[0]} } @@ -273,6 +311,48 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return rpcs.ResponseBytes(response, hf.log, fetcherMaxBlockBytes) } +// getBlockBytes gets a block. +// Core piece of FetcherClient interface +func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round, proofOK bool) (data []byte, err error) { + parsedURL, err := network.ParseHostOrURL(hf.rootURL) + if err != nil { + return nil, err + } + + parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net) + if proofOK { + parsedURL.RawQuery = "stateproof=0" + } + blockURL := parsedURL.String() + + // TODO: Temporarily allow old and new content types so we have time for lazy upgrades + // Remove this 'old' string after next release. + const blockResponseContentTypeOld = "application/algorand-block-v1" + expectedContentTypes := map[string]struct{}{ + rpcs.BlockResponseContentType: struct{}{}, + blockResponseContentTypeOld: struct{}{}, + } + + return hf.getBytes(ctx, blockURL, expectedContentTypes) +} + +// getStateProofBytes gets a state proof. +func (hf *HTTPFetcher) getStateProofBytes(ctx context.Context, proofType protocol.StateProofType, r basics.Round) (data []byte, err error) { + parsedURL, err := network.ParseHostOrURL(hf.rootURL) + if err != nil { + return nil, err + } + + parsedURL.Path = rpcs.FormatStateProofQuery(uint64(r), proofType, parsedURL.Path, hf.net) + proofURL := parsedURL.String() + + expectedContentTypes := map[string]struct{}{ + rpcs.StateProofResponseContentType: struct{}{}, + } + + return hf.getBytes(ctx, proofURL, expectedContentTypes) +} + // Address is part of FetcherClient interface. // Returns the root URL of the connected peer. func (hf *HTTPFetcher) address() string { diff --git a/catchup/universalFetcher_test.go b/catchup/universalFetcher_test.go index 4414bb9c11..34a6eef81f 100644 --- a/catchup/universalFetcher_test.go +++ b/catchup/universalFetcher_test.go @@ -44,7 +44,7 @@ func TestUGetBlockWs(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, b, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, b, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -65,13 +65,13 @@ func TestUGetBlockWs(t *testing.T) { var cert *agreement.Certificate var duration time.Duration - block, cert, _, err = fetcher.fetchBlock(context.Background(), next, up) + block, cert, _, _, err = fetcher.fetchBlock(context.Background(), next, up, false) require.NoError(t, err) require.Equal(t, &b, block) require.GreaterOrEqual(t, int64(duration), int64(0)) - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next+1, up) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next+1, up, false) require.Error(t, err) require.Contains(t, err.Error(), "requested block is not available") @@ -86,7 +86,7 @@ func TestUGetBlockHTTP(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, b, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, b, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -111,13 +111,13 @@ func TestUGetBlockHTTP(t *testing.T) { var block *bookkeeping.Block var cert *agreement.Certificate var duration time.Duration - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next, net.GetPeers()[0]) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next, net.GetPeers()[0], false) require.NoError(t, err) require.Equal(t, &b, block) require.GreaterOrEqual(t, int64(duration), int64(0)) - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next+1, net.GetPeers()[0]) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next+1, net.GetPeers()[0], false) require.Error(t, errNoBlockForRound, err) require.Contains(t, err.Error(), "No block available for given round") @@ -132,7 +132,7 @@ func TestUGetBlockUnsupported(t *testing.T) { fetcher := universalBlockFetcher{} peer := "" - block, cert, duration, err := fetcher.fetchBlock(context.Background(), 1, peer) + block, cert, _, duration, err := fetcher.fetchBlock(context.Background(), 1, peer, false) require.Error(t, err) require.Contains(t, err.Error(), "fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") require.Nil(t, block) @@ -156,18 +156,18 @@ func TestProcessBlockBytesErrors(t *testing.T) { }) // Check for cert error - _, _, err := processBlockBytes(bc, 22, "test") + _, _, _, err := processBlockBytes(bc, 22, "test") var wcfpe errWrongCertFromPeer require.True(t, errors.As(err, &wcfpe)) // Check for round error - _, _, err = processBlockBytes(bc, 20, "test") + _, _, _, err = processBlockBytes(bc, 20, "test") var wbfpe errWrongBlockFromPeer require.True(t, errors.As(err, &wbfpe)) // Check for undecodable bc[11] = 0 - _, _, err = processBlockBytes(bc, 22, "test") + _, _, _, err = processBlockBytes(bc, 22, "test") var cdbe errCannotDecodeBlock require.True(t, errors.As(err, &cdbe)) } @@ -178,7 +178,7 @@ func TestRequestBlockBytesErrors(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, _, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, _, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -199,7 +199,7 @@ func TestRequestBlockBytesErrors(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) var wrfe errWsFetcherRequestFailed require.True(t, errors.As(err, &wrfe), "unexpected err: %w", wrfe) require.Equal(t, "context canceled", err.(errWsFetcherRequestFailed).cause) @@ -209,14 +209,14 @@ func TestRequestBlockBytesErrors(t *testing.T) { responseOverride := network.Response{Topics: network.Topics{network.MakeTopic(rpcs.BlockDataKey, make([]byte, 0))}} up = makeTestUnicastPeerWithResponseOverride(net, t, &responseOverride) - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) require.True(t, errors.As(err, &wrfe)) require.Equal(t, "Cert data not found", err.(errWsFetcherRequestFailed).cause) responseOverride = network.Response{Topics: network.Topics{network.MakeTopic(rpcs.CertDataKey, make([]byte, 0))}} up = makeTestUnicastPeerWithResponseOverride(net, t, &responseOverride) - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) require.True(t, errors.As(err, &wrfe)) require.Equal(t, "Block data not found", err.(errWsFetcherRequestFailed).cause) } @@ -259,26 +259,26 @@ func TestGetBlockBytesHTTPErrors(t *testing.T) { fetcher := makeUniversalBlockFetcher(logging.TestingLog(t), net, cfg) ls.status = http.StatusBadRequest - _, _, _, err := fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err := fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) var hre errHTTPResponse require.True(t, errors.As(err, &hre)) require.Equal(t, "Response body '\x00'", err.(errHTTPResponse).cause) ls.exceedLimit = true - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) require.True(t, errors.As(err, &hre)) require.Equal(t, "read limit exceeded", err.(errHTTPResponse).cause) ls.status = http.StatusOK ls.content = append(ls.content, "undefined") - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) var cte errHTTPResponseContentType require.True(t, errors.As(err, &cte)) require.Equal(t, "undefined", err.(errHTTPResponseContentType).contentType) ls.status = http.StatusOK ls.content = append(ls.content, "undefined2") - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) require.True(t, errors.As(err, &cte)) require.Equal(t, 2, err.(errHTTPResponseContentType).contentTypeCount) } diff --git a/components/mocks/mockNetwork.go b/components/mocks/mockNetwork.go index 8c8eb113f1..5d47e6726f 100644 --- a/components/mocks/mockNetwork.go +++ b/components/mocks/mockNetwork.go @@ -98,6 +98,10 @@ func (network *MockNetwork) ClearHandlers() { func (network *MockNetwork) RegisterHTTPHandler(path string, handler http.Handler) { } +// RegisterHTTPHandlerFunc - empty implementation +func (network *MockNetwork) RegisterHTTPHandlerFunc(path string, handler func(response http.ResponseWriter, request *http.Request)) { +} + // OnNetworkAdvance - empty implementation func (network *MockNetwork) OnNetworkAdvance() {} diff --git a/config/config.go b/config/config.go index fc2dd30050..6ee70213b2 100644 --- a/config/config.go +++ b/config/config.go @@ -72,6 +72,11 @@ const StateProofFileName = "stateproof.sqlite" // It is used for tracking participation key metadata. const ParticipationRegistryFilename = "partregistry.sqlite" +// StateProofCatchupFilename is the name of the database file that is used +// during catchup to temporarily store validated state proofs for blocks +// ahead of the ledger. +const StateProofCatchupFilename = "stateproofcatchup.sqlite" + // ConfigurableConsensusProtocolsFilename defines a set of consensus protocols that // are to be loaded from the data directory ( if present ), to override the // built-in supported consensus protocols. diff --git a/config/config_test.go b/config/config_test.go index c88b53834e..ac1041947e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -687,6 +687,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) { cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block") cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash") cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof") + cfg.StateproofCatchupDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof_catchup") cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") @@ -701,6 +702,8 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, testDirectory+"/custom_stateproof/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, testDirectory+"/custom_stateproof_catchup/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, testDirectory+"/custom_catchpoint/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) } @@ -722,6 +725,8 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, testDirectory+"/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, testDirectory+"/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, testDirectory+"/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) @@ -742,6 +747,8 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, cold+"/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) } @@ -758,6 +765,7 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) { cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block") cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash") cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof") + cfg.StateproofCatchupDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof_catchup") cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") // first try an error with an empty root dir diff --git a/config/localTemplate.go b/config/localTemplate.go index 4202c2648a..9d037e82b4 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -42,7 +42,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32"` // environmental (may be overridden) // When enabled, stores blocks indefinitely, otherwise, only the most recent blocks @@ -111,6 +111,10 @@ type Local struct { // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. // If not specified, the node will use the ColdDataDir. StateproofDir string `version[31]:""` + // StateproofCatchupDir is an optional directory to store stateproof catchup data. + // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. + // If not specified, the node will use the ColdDataDir. + StateproofCatchupDir string `version[31]:""` // CrashDBDir is an optional directory to store the crash database. // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. // If not specified, the node will use the ColdDataDir. @@ -581,6 +585,25 @@ type Local struct { // DisableAPIAuth turns off authentication for public (non-admin) API endpoints. DisableAPIAuth bool `version[30]:"false"` + + // RenaissanceCatchup* fields specify how to bootstrap authentication of + // state proofs without validating every block starting from genesis. + // + // RenaissanceCatchupRound is the next expected LastAttestedRound in the + // state proof we expect to verify using these renaissance parameters. + // If 0 (default), renaissance catchup parameters are disabled. + RenaissanceCatchupRound uint64 `version[32]:"0"` + + // RenaissanceCatchupLnProvenWeight is the corresponding field from the + // previous state proof message. + RenaissanceCatchupLnProvenWeight uint64 `version[32]:"0"` + + // RenaissanceCatchupVotersCommitment is the base64-encoded corresponding + // field from the previous state proof message. + RenaissanceCatchupVotersCommitment string `version[32]:""` + + // RenaissanceCatchupProto is the protocol of RenaissanceCatchupRound. + RenaissanceCatchupProto string `version[32]:""` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers @@ -708,14 +731,15 @@ func ensureAbsGenesisDir(path string, genesisID string) (string, error) { // ResolvedGenesisDirs is a collection of directories including Genesis ID // Subdirectories for execution of a node type ResolvedGenesisDirs struct { - RootGenesisDir string - HotGenesisDir string - ColdGenesisDir string - TrackerGenesisDir string - BlockGenesisDir string - CatchpointGenesisDir string - StateproofGenesisDir string - CrashGenesisDir string + RootGenesisDir string + HotGenesisDir string + ColdGenesisDir string + TrackerGenesisDir string + BlockGenesisDir string + CatchpointGenesisDir string + StateproofGenesisDir string + StateproofCatchupGenesisDir string + CrashGenesisDir string } // String returns the Genesis Directory values as a string @@ -728,6 +752,7 @@ func (rgd ResolvedGenesisDirs) String() string { ret += fmt.Sprintf("BlockGenesisDir: %s\n", rgd.BlockGenesisDir) ret += fmt.Sprintf("CatchpointGenesisDir: %s\n", rgd.CatchpointGenesisDir) ret += fmt.Sprintf("StateproofGenesisDir: %s\n", rgd.StateproofGenesisDir) + ret += fmt.Sprintf("StateproofCatchupGenesisDir: %s\n", rgd.StateproofCatchupGenesisDir) ret += fmt.Sprintf("CrashGenesisDir: %s\n", rgd.CrashGenesisDir) return ret } @@ -823,6 +848,15 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv } else { resolved.StateproofGenesisDir = resolved.ColdGenesisDir } + // if StateproofCatchupDir is not set, use ColdDataDir + if cfg.StateproofCatchupDir != "" { + resolved.StateproofCatchupGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofCatchupDir, genesisID) + if err != nil { + return ResolvedGenesisDirs{}, err + } + } else { + resolved.StateproofCatchupGenesisDir = resolved.ColdGenesisDir + } // if CrashDBDir is not set, use ColdDataDir if cfg.CrashDBDir != "" { resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID) diff --git a/config/local_defaults.go b/config/local_defaults.go index fd0aa20521..fd8a25f7f3 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 31, + Version: 32, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 15, @@ -131,6 +131,7 @@ var defaultLocal = Local{ RestWriteTimeoutSeconds: 120, RunHosted: false, StateproofDir: "", + StateproofCatchupDir: "", StorageEngine: "sqlite", SuggestedFeeBlockHistory: 3, SuggestedFeeSlidingWindowSize: 50, @@ -152,4 +153,8 @@ var defaultLocal = Local{ TxSyncTimeoutSeconds: 30, UseXForwardedForAddressField: "", VerifiedTranscationsCacheSize: 150000, + RenaissanceCatchupRound: 0, + RenaissanceCatchupLnProvenWeight: 0, + RenaissanceCatchupVotersCommitment: "", + RenaissanceCatchupProto: "", } diff --git a/daemon/algod/api/server/v2/handlers.go b/daemon/algod/api/server/v2/handlers.go index 7b190acd52..c330706ba2 100644 --- a/daemon/algod/api/server/v2/handlers.go +++ b/daemon/algod/api/server/v2/handlers.go @@ -213,8 +213,8 @@ func GetStateProofTransactionForRound(ctx context.Context, txnFetcher LedgerForA continue } - if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= uint64(round) && - uint64(round) <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { + if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= round && + round <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { return txn.Txn, nil } } @@ -1685,8 +1685,8 @@ func (v2 *Handlers) GetStateProof(ctx echo.Context, round uint64) error { response.Message.BlockHeadersCommitment = tx.Message.BlockHeadersCommitment response.Message.VotersCommitment = tx.Message.VotersCommitment response.Message.LnProvenWeight = tx.Message.LnProvenWeight - response.Message.FirstAttestedRound = tx.Message.FirstAttestedRound - response.Message.LastAttestedRound = tx.Message.LastAttestedRound + response.Message.FirstAttestedRound = uint64(tx.Message.FirstAttestedRound) + response.Message.LastAttestedRound = uint64(tx.Message.LastAttestedRound) return ctx.JSON(http.StatusOK, response) } @@ -1718,14 +1718,14 @@ func (v2 *Handlers) GetLightBlockHeaderProof(ctx echo.Context, round uint64) err lastAttestedRound := stateProof.Message.LastAttestedRound firstAttestedRound := stateProof.Message.FirstAttestedRound - stateProofInterval := lastAttestedRound - firstAttestedRound + 1 + stateProofInterval := uint64(lastAttestedRound - firstAttestedRound + 1) - lightHeaders, err := stateproof.FetchLightHeaders(ledger, stateProofInterval, basics.Round(lastAttestedRound)) + lightHeaders, err := stateproof.FetchLightHeaders(ledger, stateProofInterval, lastAttestedRound) if err != nil { return notFound(ctx, err, err.Error(), v2.Log) } - blockIndex := round - firstAttestedRound + blockIndex := round - uint64(firstAttestedRound) leafproof, err := stateproof.GenerateProofOfLightBlockHeaders(stateProofInterval, lightHeaders, blockIndex) if err != nil { return internalError(ctx, err, err.Error(), v2.Log) diff --git a/data/stateproofmsg/message.go b/data/stateproofmsg/message.go index 5f1c2e3433..4796f0acef 100644 --- a/data/stateproofmsg/message.go +++ b/data/stateproofmsg/message.go @@ -19,6 +19,7 @@ package stateproofmsg import ( "github.com/algorand/go-algorand/crypto" sp "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/protocol" ) @@ -29,11 +30,11 @@ import ( type Message struct { _struct struct{} `codec:",omitempty,omitemptyarray"` // BlockHeadersCommitment contains a commitment on all light block headers within a state proof interval. - BlockHeadersCommitment []byte `codec:"b,allocbound=crypto.Sha256Size"` - VotersCommitment []byte `codec:"v,allocbound=crypto.SumhashDigestSize"` - LnProvenWeight uint64 `codec:"P"` - FirstAttestedRound uint64 `codec:"f"` - LastAttestedRound uint64 `codec:"l"` + BlockHeadersCommitment []byte `codec:"b,allocbound=crypto.Sha256Size"` + VotersCommitment []byte `codec:"v,allocbound=crypto.SumhashDigestSize"` + LnProvenWeight uint64 `codec:"P"` + FirstAttestedRound basics.Round `codec:"f"` + LastAttestedRound basics.Round `codec:"l"` } // ToBeHashed returns the bytes of the message. diff --git a/data/stateproofmsg/msgp_gen.go b/data/stateproofmsg/msgp_gen.go index 02a17491bf..f59f844a32 100644 --- a/data/stateproofmsg/msgp_gen.go +++ b/data/stateproofmsg/msgp_gen.go @@ -6,6 +6,7 @@ import ( "github.com/algorand/msgp/msgp" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" ) // The following msgp objects are implemented in this file: @@ -33,11 +34,11 @@ func (z *Message) MarshalMsg(b []byte) (o []byte) { zb0001Len-- zb0001Mask |= 0x4 } - if (*z).FirstAttestedRound == 0 { + if (*z).FirstAttestedRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8 } - if (*z).LastAttestedRound == 0 { + if (*z).LastAttestedRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } @@ -61,12 +62,12 @@ func (z *Message) MarshalMsg(b []byte) (o []byte) { if (zb0001Mask & 0x8) == 0 { // if not empty // string "f" o = append(o, 0xa1, 0x66) - o = msgp.AppendUint64(o, (*z).FirstAttestedRound) + o = (*z).FirstAttestedRound.MarshalMsg(o) } if (zb0001Mask & 0x10) == 0 { // if not empty // string "l" o = append(o, 0xa1, 0x6c) - o = msgp.AppendUint64(o, (*z).LastAttestedRound) + o = (*z).LastAttestedRound.MarshalMsg(o) } if (zb0001Mask & 0x20) == 0 { // if not empty // string "v" @@ -141,7 +142,7 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).FirstAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).FirstAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FirstAttestedRound") return @@ -149,7 +150,7 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).LastAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).LastAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "LastAttestedRound") return @@ -217,13 +218,13 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "f": - (*z).FirstAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).FirstAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "FirstAttestedRound") return } case "l": - (*z).LastAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).LastAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "LastAttestedRound") return @@ -248,17 +249,17 @@ func (_ *Message) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *Message) Msgsize() (s int) { - s = 1 + 2 + msgp.BytesPrefixSize + len((*z).BlockHeadersCommitment) + 2 + msgp.BytesPrefixSize + len((*z).VotersCommitment) + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + s = 1 + 2 + msgp.BytesPrefixSize + len((*z).BlockHeadersCommitment) + 2 + msgp.BytesPrefixSize + len((*z).VotersCommitment) + 2 + msgp.Uint64Size + 2 + (*z).FirstAttestedRound.Msgsize() + 2 + (*z).LastAttestedRound.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *Message) MsgIsZero() bool { - return (len((*z).BlockHeadersCommitment) == 0) && (len((*z).VotersCommitment) == 0) && ((*z).LnProvenWeight == 0) && ((*z).FirstAttestedRound == 0) && ((*z).LastAttestedRound == 0) + return (len((*z).BlockHeadersCommitment) == 0) && (len((*z).VotersCommitment) == 0) && ((*z).LnProvenWeight == 0) && ((*z).FirstAttestedRound.MsgIsZero()) && ((*z).LastAttestedRound.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func MessageMaxSize() (s int) { - s = 1 + 2 + msgp.BytesPrefixSize + crypto.Sha256Size + 2 + msgp.BytesPrefixSize + crypto.SumhashDigestSize + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + s = 1 + 2 + msgp.BytesPrefixSize + crypto.Sha256Size + 2 + msgp.BytesPrefixSize + crypto.SumhashDigestSize + 2 + msgp.Uint64Size + 2 + basics.RoundMaxSize() + 2 + basics.RoundMaxSize() return } diff --git a/installer/config.json.example b/installer/config.json.example index 62cbe6427b..4c69bed1e5 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 31, + "Version": 32, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 15, @@ -103,6 +103,10 @@ "ProposalAssemblyTime": 500000000, "PublicAddress": "", "ReconnectTime": 60000000000, + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupLnProvenWeight": 0, + "RenaissanceCatchupVotersCommitment": "", + "RenaissanceCatchupProto": "", "ReservedFDs": 256, "RestConnectionsHardLimit": 2048, "RestConnectionsSoftLimit": 1024, diff --git a/node/follower_node.go b/node/follower_node.go index b7fb81065b..68dde144fd 100644 --- a/node/follower_node.go +++ b/node/follower_node.go @@ -20,6 +20,7 @@ package node import ( "context" "fmt" + "path/filepath" "time" "github.com/algorand/go-deadlock" @@ -40,6 +41,7 @@ import ( "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/execpool" ) @@ -126,10 +128,18 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo node, } + spCatchupFilename := filepath.Join(node.genesisDirs.StateproofCatchupGenesisDir, config.StateProofCatchupFilename) + spCatchupDB, err := db.MakeAccessor(spCatchupFilename, false, false) + if err != nil { + log.Errorf("Cannot create state-proof catchup DB (%s): %v", spCatchupFilename, err) + return nil, err + } + node.ledger.RegisterBlockListeners(blockListeners) node.blockService = rpcs.MakeBlockService(node.log, cfg, node.ledger, p2pNode, node.genesisID) node.catchupBlockAuth = blockAuthenticatorImpl{Ledger: node.ledger, AsyncVoteVerifier: agreement.MakeAsyncVoteVerifier(node.lowPriorityCryptoVerificationPool)} - node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, make(chan catchup.PendingUnmatchedCertificate), node.lowPriorityCryptoVerificationPool) + node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, make(chan catchup.PendingUnmatchedCertificate), node.lowPriorityCryptoVerificationPool, &spCatchupDB) + node.catchupService.SetRenaissanceFromConfig(cfg) // Initialize sync round to the latest db round + 1 so that nothing falls out of the cache on Start err = node.SetSyncRound(uint64(node.Ledger().LatestTrackerCommitted() + 1)) diff --git a/node/node.go b/node/node.go index 4c18ad1d51..2f2aa68d96 100644 --- a/node/node.go +++ b/node/node.go @@ -291,8 +291,16 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd return nil, err } + spCatchupFilename := filepath.Join(node.genesisDirs.StateproofCatchupGenesisDir, config.StateProofCatchupFilename) + spCatchupDB, err := db.MakeAccessor(spCatchupFilename, false, false) + if err != nil { + log.Errorf("Cannot create state-proof catchup DB (%s): %v", spCatchupFilename, err) + return nil, err + } + node.catchupBlockAuth = blockAuthenticatorImpl{Ledger: node.ledger, AsyncVoteVerifier: agreement.MakeAsyncVoteVerifier(node.lowPriorityCryptoVerificationPool)} - node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, agreementLedger.UnmatchedPendingCertificates, node.lowPriorityCryptoVerificationPool) + node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, agreementLedger.UnmatchedPendingCertificates, node.lowPriorityCryptoVerificationPool, &spCatchupDB) + node.catchupService.SetRenaissanceFromConfig(cfg) node.txPoolSyncerService = rpcs.MakeTxSyncer(node.transactionPool, node.net, node.txHandler.SolicitedTxHandler(), time.Duration(cfg.TxSyncIntervalSeconds)*time.Second, time.Duration(cfg.TxSyncTimeoutSeconds)*time.Second, cfg.TxSyncServeResponseSize) registry, err := ensureParticipationDB(node.genesisDirs.ColdGenesisDir, node.log) diff --git a/rpcs/blockService.go b/rpcs/blockService.go index b185f224e5..5d5e2bd20e 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -39,10 +39,12 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/stateproof" "github.com/algorand/go-algorand/util/metrics" ) @@ -54,10 +56,16 @@ const blockResponseRetryAfter = "3" const blockServerMaxBodyLength = 512 // we don't really pass meaningful content here, so 512 bytes should be a safe limit const blockServerCatchupRequestBufferSize = 10 +// StateProofResponseContentType is the HTTP Content-Type header for a raw state proof transaction +const StateProofResponseContentType = "application/x-algorand-stateproof-v1" + // BlockServiceBlockPath is the path to register BlockService as a handler for when using gorilla/mux // e.g. .HandleFunc(BlockServiceBlockPath, ls.ServeBlockPath) const BlockServiceBlockPath = "/v{version:[0-9.]+}/{genesisID}/block/{round:[0-9a-z]+}" +// BlockServiceStateProofPath is the path to register BlockService's ServeStateProofPath handler +const BlockServiceStateProofPath = "/v{version:[0-9.]+}/{genesisID}/stateproof/type{type:[0-9.]+}/{round:[0-9a-z]+}" + // Constant strings used as keys for topics const ( RoundKey = "roundKey" // Block round-number topic-key in the request @@ -87,6 +95,10 @@ var httpBlockMessagesDroppedCounter = metrics.MakeCounter( // LedgerForBlockService describes the Ledger methods used by BlockService. type LedgerForBlockService interface { EncodedBlockCert(rnd basics.Round) (blk []byte, cert []byte, err error) + BlockHdr(rnd basics.Round) (bookkeeping.BlockHeader, error) + Block(rnd basics.Round) (bookkeeping.Block, error) + Latest() basics.Round + AddressTxns(id basics.Address, r basics.Round) ([]transactions.SignedTxnWithAD, error) } // BlockService represents the Block RPC API @@ -108,12 +120,17 @@ type BlockService struct { memoryCap uint64 } -// EncodedBlockCert defines how GetBlockBytes encodes a block and its certificate +// EncodedBlockCert defines how GetBlockBytes and GetBlockStateProofBytes +// encodes a block and its certificate or light header proof. It is +// compatible with encoding of PreEncodedBlockCert, but currently we use +// two different structs, because we don't store pre-msgpack'ed light +// header proofs. type EncodedBlockCert struct { - _struct struct{} `codec:""` + _struct struct{} `codec:",omitempty,omitemptyarray"` - Block bookkeeping.Block `codec:"block"` - Certificate agreement.Certificate `codec:"cert"` + Block bookkeeping.Block `codec:"block,omitempty"` + Certificate agreement.Certificate `codec:"cert,omitempty"` + LightBlockHeaderProof []byte `codec:"proof,omitempty"` } // PreEncodedBlockCert defines how GetBlockBytes encodes a block and its certificate, @@ -159,6 +176,7 @@ type HTTPRegistrar interface { // RegisterHandlers registers the request handlers for BlockService's paths with the registrar. func (bs *BlockService) RegisterHandlers(registrar HTTPRegistrar) { registrar.RegisterHTTPHandlerFunc(BlockServiceBlockPath, bs.ServeBlockPath) + registrar.RegisterHTTPHandlerFunc(BlockServiceStateProofPath, bs.ServeStateProofPath) } // Start listening to catchup requests over ws @@ -185,8 +203,118 @@ func (bs *BlockService) Stop() { bs.closeWaitGroup.Wait() } +// ServeStateProofPath returns state proofs. +// It expects to be invoked via: +// +// /v{version}/{genesisID}/stateproof/type{type}/{round} +func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, request *http.Request) { + pathVars := mux.Vars(request) + versionStr := pathVars["version"] + roundStr := pathVars["round"] + genesisID := pathVars["genesisID"] + typeStr := pathVars["type"] + if versionStr != "1" { + bs.log.Debug("http stateproof bad version", versionStr) + response.WriteHeader(http.StatusBadRequest) + return + } + if genesisID != bs.genesisID { + bs.log.Debugf("http stateproof bad genesisID mine=%#v theirs=%#v", bs.genesisID, genesisID) + response.WriteHeader(http.StatusBadRequest) + return + } + if typeStr != "0" { // StateProofBasic + bs.log.Debugf("http stateproof bad type", typeStr) + response.WriteHeader(http.StatusBadRequest) + return + } + uround, err := strconv.ParseUint(roundStr, 36, 64) + if err != nil { + bs.log.Debug("http stateproof round parse fail", roundStr, err) + response.WriteHeader(http.StatusBadRequest) + return + } + round := basics.Round(uround) + + latestRound := bs.ledger.Latest() + ctx := request.Context() + hdr, err := bs.ledger.BlockHdr(round) + if err != nil { + bs.log.Debug("http stateproof cannot get blockhdr", round, err) + switch err.(type) { + case ledgercore.ErrNoEntry: + goto notfound + default: + response.WriteHeader(http.StatusInternalServerError) + return + } + } + + if config.Consensus[hdr.CurrentProtocol].StateProofInterval == 0 { + bs.log.Debug("http stateproof not enabled", round) + response.WriteHeader(http.StatusBadRequest) + return + } + + // As an optimization to prevent expensive searches for state + // proofs that don't exist yet, don't bother searching if we + // are looking for a state proof for a round that's within + // StateProofInterval of latest. + if round + basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) >= latestRound { + goto notfound + } + + for i := round + 1; i <= latestRound; i++ { + select { + case <-ctx.Done(): + return + default: + } + + txns, err := bs.ledger.AddressTxns(transactions.StateProofSender, i) + if err != nil { + bs.log.Debug("http stateproof address txns error", err) + response.WriteHeader(http.StatusInternalServerError) + return + } + for _, txn := range txns { + if txn.Txn.Type != protocol.StateProofTx { + continue + } + + if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= round && round <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { + encodedStateProof := protocol.Encode(&txn.Txn) + response.Header().Set("Content-Type", StateProofResponseContentType) + response.Header().Set("Content-Length", strconv.Itoa(len(encodedStateProof))) + response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) + response.WriteHeader(http.StatusOK) + _, err = response.Write(encodedStateProof) + if err != nil { + bs.log.Warn("http stateproof write failed ", err) + } + return + } + } + } + +notfound: + ok := bs.redirectRequest(response, request, bs.formatStateProofQuery(uint64(round))) + if !ok { + response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) + response.WriteHeader(http.StatusNotFound) + } +} + // ServeBlockPath returns blocks -// Either /v{version}/{genesisID}/block/{round} or ?b={round}&v={version} +// It expects to be invoked via several possible paths: +// +// /v{version}/{genesisID}/block/{round} +// ?b={round}&v={version} +// +// It optionally takes a ?stateproof={n} argument, where n is the type of +// state proof for which we should return a light block header proof. In +// the absence of this argument, we will return an agreement certificate. +// // Uses gorilla/mux for path argument parsing. func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *http.Request) { pathVars := mux.Vars(request) @@ -211,15 +339,17 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht response.WriteHeader(http.StatusBadRequest) return } + + request.Body = http.MaxBytesReader(response, request.Body, blockServerMaxBodyLength) + err := request.ParseForm() + if err != nil { + bs.log.Debug("http block parse form err", err) + response.WriteHeader(http.StatusBadRequest) + return + } + if (!hasVersionStr) || (!hasRoundStr) { // try query arg ?b={round} - request.Body = http.MaxBytesReader(response, request.Body, blockServerMaxBodyLength) - err := request.ParseForm() - if err != nil { - bs.log.Debug("http block parse form err", err) - response.WriteHeader(http.StatusBadRequest) - return - } roundStrs, ok := request.Form["b"] if !ok || len(roundStrs) != 1 { bs.log.Debug("http block bad block id form arg") @@ -248,12 +378,31 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht response.WriteHeader(http.StatusBadRequest) return } - encodedBlockCert, err := bs.rawBlockBytes(basics.Round(round)) + + sendStateProof := false + stateProof, hasStateProof := request.Form["stateproof"] + if hasStateProof { + if len(stateProof) != 1 || stateProof[0] != "0" { + bs.log.Debug("http block stateproof version %v unsupported", stateProof) + response.WriteHeader(http.StatusBadRequest) + return + } + + sendStateProof = true + } + + var encodedBlock []byte + if sendStateProof { + encodedBlock, err = bs.rawBlockStateProofBytes(basics.Round(round)) + } else { + encodedBlock, err = bs.rawBlockBytes(basics.Round(round)) + } + if err != nil { switch err.(type) { case ledgercore.ErrNoEntry: // entry cound not be found. - ok := bs.redirectRequest(round, response, request) + ok := bs.redirectRequest(response, request, bs.formatBlockQuery(round, sendStateProof)) if !ok { response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) response.WriteHeader(http.StatusNotFound) @@ -261,7 +410,7 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht return case errMemoryAtCapacity: // memory used by HTTP block requests is over the cap - ok := bs.redirectRequest(round, response, request) + ok := bs.redirectRequest(response, request, bs.formatBlockQuery(round, sendStateProof)) if !ok { response.Header().Set("Retry-After", blockResponseRetryAfter) response.WriteHeader(http.StatusServiceUnavailable) @@ -278,16 +427,16 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht } response.Header().Set("Content-Type", BlockResponseContentType) - response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlockCert))) + response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlock))) response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) response.WriteHeader(http.StatusOK) - _, err = response.Write(encodedBlockCert) + _, err = response.Write(encodedBlock) if err != nil { bs.log.Warn("http block write failed ", err) } bs.mu.Lock() defer bs.mu.Unlock() - bs.memoryUsed = bs.memoryUsed - uint64(len(encodedBlockCert)) + bs.memoryUsed = bs.memoryUsed - uint64(len(encodedBlock)) } func (bs *BlockService) processIncomingMessage(msg network.IncomingMessage) (n network.OutgoingMessage) { @@ -392,7 +541,7 @@ func (bs *BlockService) handleCatchupReq(ctx context.Context, reqMsg network.Inc // redirectRequest redirects the request to the next round robin fallback endpoing if available, otherwise, // if EnableBlockServiceFallbackToArchiver is enabled, redirects to a random archiver. -func (bs *BlockService) redirectRequest(round uint64, response http.ResponseWriter, request *http.Request) (ok bool) { +func (bs *BlockService) redirectRequest(response http.ResponseWriter, request *http.Request, pathsuffix string) (ok bool) { peerAddress := bs.getNextCustomFallbackEndpoint() if peerAddress == "" && bs.enableArchiverFallback { peerAddress = bs.getRandomArchiver() @@ -406,12 +555,25 @@ func (bs *BlockService) redirectRequest(round uint64, response http.ResponseWrit bs.log.Debugf("redirectRequest: %s", err.Error()) return false } - parsedURL.Path = strings.Replace(FormatBlockQuery(round, parsedURL.Path, bs.net), "{genesisID}", bs.genesisID, 1) + parsedURL.Path = path.Join(parsedURL.Path, pathsuffix) http.Redirect(response, request, parsedURL.String(), http.StatusTemporaryRedirect) bs.log.Debugf("redirectRequest: redirected block request to %s", parsedURL.String()) return true } +func (bs *BlockService) formatBlockQuery(round uint64, sendStateProof bool) string { + stateProofArg := "" + if sendStateProof { + stateProofArg = "?stateproof=0" + } + + return fmt.Sprintf("/v1/%s/block/%s%s", bs.genesisID, strconv.FormatUint(uint64(round), 36), stateProofArg) +} + +func (bs *BlockService) formatStateProofQuery(round uint64) string { + return fmt.Sprintf("/v1/%s/stateproof/type0/%s", bs.genesisID, strconv.FormatUint(uint64(round), 36)) +} + // getNextCustomFallbackEndpoint returns the next custorm fallback endpoint in RR ordering func (bs *BlockService) getNextCustomFallbackEndpoint() (endpointAddress string) { if len(bs.fallbackEndpoints.endpoints) == 0 { @@ -464,6 +626,29 @@ func (bs *BlockService) rawBlockBytes(round basics.Round) ([]byte, error) { return data, err } +// rawBlockStateProofBytes returns the block and light header proof for a given round, while taking the lock +// to ensure the block service is currently active. +func (bs *BlockService) rawBlockStateProofBytes(round basics.Round) ([]byte, error) { + bs.mu.Lock() + defer bs.mu.Unlock() + select { + case _, ok := <-bs.stop: + if !ok { + // service is closed. + return nil, errBlockServiceClosed + } + default: + } + if bs.memoryUsed > bs.memoryCap { + return nil, errMemoryAtCapacity{used: bs.memoryUsed, capacity: bs.memoryCap} + } + data, err := RawBlockStateProofBytes(bs.ledger, round) + if err == nil { + bs.memoryUsed = bs.memoryUsed + uint64(len(data)) + } + return data, err +} + func topicBlockBytes(log logging.Logger, dataLedger LedgerForBlockService, round basics.Round, requestType string) (network.Topics, uint64) { blk, cert, err := dataLedger.EncodedBlockCert(round) if err != nil { @@ -506,11 +691,55 @@ func RawBlockBytes(l LedgerForBlockService, round basics.Round) ([]byte, error) }), nil } +// RawBlockStateProofBytes return the msgpack bytes for a block and light header proof +func RawBlockStateProofBytes(l LedgerForBlockService, round basics.Round) ([]byte, error) { + blk, err := l.Block(round) + if err != nil { + return nil, err + } + + stateProofInterval := basics.Round(config.Consensus[blk.CurrentProtocol].StateProofInterval) + if stateProofInterval == 0 { + return nil, fmt.Errorf("state proofs not supported in block %d", round) + } + + lastAttestedRound := round.RoundUpToMultipleOf(stateProofInterval) + firstAttestedRound := lastAttestedRound - stateProofInterval + 1 + + latest := l.Latest() + if lastAttestedRound > latest { + return nil, fmt.Errorf("light block header proof not available for block %d yet, latest is %d", lastAttestedRound, latest) + } + + lightHeaders, err := stateproof.FetchLightHeaders(l, uint64(stateProofInterval), lastAttestedRound) + if err != nil { + return nil, err + } + + blockIndex := uint64(round - firstAttestedRound) + leafproof, err := stateproof.GenerateProofOfLightBlockHeaders(uint64(stateProofInterval), lightHeaders, blockIndex) + if err != nil { + return nil, err + } + + r := EncodedBlockCert{ + Block: blk, + LightBlockHeaderProof: leafproof.GetConcatenatedProof(), + } + + return protocol.Encode(&r), nil +} + // FormatBlockQuery formats a block request query for the given network and round number func FormatBlockQuery(round uint64, parsedURL string, net network.GossipNode) string { return net.SubstituteGenesisID(path.Join(parsedURL, "/v1/{genesisID}/block/"+strconv.FormatUint(uint64(round), 36))) } +// FormatStateProofQuery formats a state proof request for the given network, proof type, and round number +func FormatStateProofQuery(round uint64, proofType protocol.StateProofType, parsedURL string, net network.GossipNode) string { + return net.SubstituteGenesisID(path.Join(parsedURL, "/v1/{genesisID}/stateproof/type"+strconv.FormatUint(uint64(proofType), 10)+"/"+strconv.FormatUint(uint64(round), 36))) +} + func makeFallbackEndpoints(log logging.Logger, customFallbackEndpoints string) (fe fallbackEndpoints) { if customFallbackEndpoints == "" { return diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index 8abf87e3ba..3e21bdfa12 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -101,7 +101,7 @@ func (ls *LedgerService) Stop() { } } -// ServerHTTP returns ledgers for a particular round +// ServeHTTP returns ledgers for a particular round // Either /v{version}/{genesisID}/ledger/{round} or ?r={round}&v={version} // Uses gorilla/mux for path argument parsing. func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.Request) { diff --git a/rpcs/msgp_gen.go b/rpcs/msgp_gen.go index 5f8af433f7..a94a2a15da 100644 --- a/rpcs/msgp_gen.go +++ b/rpcs/msgp_gen.go @@ -23,13 +23,40 @@ import ( // MarshalMsg implements msgp.Marshaler func (z *EncodedBlockCert) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "block" - o = append(o, 0x82, 0xa5, 0x62, 0x6c, 0x6f, 0x63, 0x6b) - o = (*z).Block.MarshalMsg(o) - // string "cert" - o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) - o = (*z).Certificate.MarshalMsg(o) + // omitempty: check for empty values + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 4 bits */ + if (*z).Block.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Certificate.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if len((*z).LightBlockHeaderProof) == 0 { + zb0001Len-- + zb0001Mask |= 0x8 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "block" + o = append(o, 0xa5, 0x62, 0x6c, 0x6f, 0x63, 0x6b) + o = (*z).Block.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "cert" + o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) + o = (*z).Certificate.MarshalMsg(o) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "proof" + o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x6f, 0x66) + o = msgp.AppendBytes(o, (*z).LightBlockHeaderProof) + } + } return } @@ -67,6 +94,14 @@ func (z *EncodedBlockCert) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + (*z).LightBlockHeaderProof, bts, err = msgp.ReadBytesBytes(bts, (*z).LightBlockHeaderProof) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "LightBlockHeaderProof") + return + } + } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -102,6 +137,12 @@ func (z *EncodedBlockCert) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Certificate") return } + case "proof": + (*z).LightBlockHeaderProof, bts, err = msgp.ReadBytesBytes(bts, (*z).LightBlockHeaderProof) + if err != nil { + err = msgp.WrapError(err, "LightBlockHeaderProof") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -122,17 +163,18 @@ func (_ *EncodedBlockCert) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *EncodedBlockCert) Msgsize() (s int) { - s = 1 + 6 + (*z).Block.Msgsize() + 5 + (*z).Certificate.Msgsize() + s = 1 + 6 + (*z).Block.Msgsize() + 5 + (*z).Certificate.Msgsize() + 6 + msgp.BytesPrefixSize + len((*z).LightBlockHeaderProof) return } // MsgIsZero returns whether this is a zero value func (z *EncodedBlockCert) MsgIsZero() bool { - return ((*z).Block.MsgIsZero()) && ((*z).Certificate.MsgIsZero()) + return ((*z).Block.MsgIsZero()) && ((*z).Certificate.MsgIsZero()) && (len((*z).LightBlockHeaderProof) == 0) } // MaxSize returns a maximum valid message size for this message type func EncodedBlockCertMaxSize() (s int) { - s = 1 + 6 + bookkeeping.BlockMaxSize() + 5 + agreement.CertificateMaxSize() + s = 1 + 6 + bookkeeping.BlockMaxSize() + 5 + agreement.CertificateMaxSize() + 6 + panic("Unable to determine max size: Byteslice type z.LightBlockHeaderProof is unbounded") return } diff --git a/stateproof/stateproofMessageGenerator.go b/stateproof/stateproofMessageGenerator.go index c51d767195..1437697103 100644 --- a/stateproof/stateproofMessageGenerator.go +++ b/stateproof/stateproofMessageGenerator.go @@ -59,7 +59,7 @@ func GenerateStateProofMessage(l BlockHeaderFetcher, round basics.Round) (statep } proto := config.Consensus[latestRoundHeader.CurrentProtocol] - votersRound := uint64(round.SubSaturate(basics.Round(proto.StateProofInterval))) + votersRound := round.SubSaturate(basics.Round(proto.StateProofInterval)) commitment, err := createHeaderCommitment(l, &proto, &latestRoundHeader) if err != nil { return stateproofmsg.Message{}, err @@ -75,7 +75,7 @@ func GenerateStateProofMessage(l BlockHeaderFetcher, round basics.Round) (statep VotersCommitment: latestRoundHeader.StateProofTracking[protocol.StateProofBasic].StateProofVotersCommitment, LnProvenWeight: lnProvenWeight, FirstAttestedRound: votersRound + 1, - LastAttestedRound: uint64(latestRoundHeader.Round), + LastAttestedRound: latestRoundHeader.Round, }, nil } diff --git a/test/testdata/configs/config-v32.json b/test/testdata/configs/config-v32.json new file mode 100644 index 0000000000..2051ba6801 --- /dev/null +++ b/test/testdata/configs/config-v32.json @@ -0,0 +1,138 @@ +{ + "Version": 32, + "AccountUpdatesStatsInterval": 5000000000, + "AccountsRebuildSynchronousMode": 1, + "AgreementIncomingBundlesQueueLength": 15, + "AgreementIncomingProposalsQueueLength": 50, + "AgreementIncomingVotesQueueLength": 20000, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BlockDBDir": "", + "BlockServiceCustomFallbackEndpoints": "", + "BlockServiceMemCap": 500000000, + "BroadcastConnectionsLimit": -1, + "CadaverDirectory": "", + "CadaverSizeTarget": 0, + "CatchpointDir": "", + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupBlockValidateMode": 0, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ColdDataDir": "", + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "CrashDBDir": "", + "DNSBootstrapID": ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DeadlockDetectionThreshold": 30, + "DisableAPIAuth": false, + "DisableLedgerLRUCache": false, + "DisableLocalhostConnectionRateLimit": true, + "DisableNetworking": false, + "DisableOutgoingConnectionThrottling": false, + "EnableAccountUpdatesStats": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableBlockServiceFallbackToArchiver": true, + "EnableCatchupFromArchiveServers": false, + "EnableDeveloperAPI": false, + "EnableExperimentalAPI": false, + "EnableFollowMode": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnableP2P": false, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableRuntimeMetrics": false, + "EnableTopAccountsReporting": false, + "EnableTxBacklogRateLimiting": true, + "EnableTxnEvalTracer": false, + "EnableUsageLog": false, + "EnableVerbosedTransactionSyncLogging": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceFetchTransactions": false, + "ForceRelayMessages": false, + "GossipFanout": 4, + "HeartbeatUpdateInterval": 600, + "HotDataDir": "", + "IncomingConnectionsLimit": 2400, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "LedgerSynchronousMode": 2, + "LogArchiveDir": "", + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogFileDir": "", + "LogSizeLimit": 1073741824, + "MaxAPIBoxPerApplication": 100000, + "MaxAPIResourcesPerAccount": 100000, + "MaxAcctLookback": 4, + "MaxCatchpointDownloadDuration": 43200000000000, + "MaxConnectionsPerIP": 15, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkMessageTraceServer": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "P2PPersistPeerID": false, + "P2PPrivateKeyLocation": "", + "ParticipationKeysRefreshInterval": 60000000000, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "ProposalAssemblyTime": 500000000, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestConnectionsHardLimit": 2048, + "RestConnectionsSoftLimit": 1024, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "StateproofDir": "", + "StorageEngine": "sqlite", + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TrackerDBDir": "", + "TransactionSyncDataExchangeRate": 0, + "TransactionSyncSignificantMessageThreshold": 0, + "TxBacklogReservedCapacityPerPeer": 20, + "TxBacklogServiceRateWindowSeconds": 10, + "TxBacklogSize": 26000, + "TxIncomingFilterMaxSize": 500000, + "TxIncomingFilteringFlags": 1, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 75000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "", + "VerifiedTranscationsCacheSize": 150000, + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupLnProvenWeight": 0, + "RenaissanceCatchupVotersCommitment": "", + "RenaissanceCatchupProto": "" +} From ca71bf2179bbd7090cd9b2f6cd5b0e1e5e85b139 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 30 Aug 2023 22:39:22 -0400 Subject: [PATCH 2/6] make codegen checks happy --- catchup/service.go | 4 +++- config/local_defaults.go | 10 +++++----- installer/config.json.example | 5 +++-- test/testdata/configs/config-v32.json | 11 ++++++----- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/catchup/service.go b/catchup/service.go index 23db9fa903..a1545fdae4 100644 --- a/catchup/service.go +++ b/catchup/service.go @@ -25,6 +25,8 @@ import ( "sync/atomic" "time" + "github.com/algorand/go-deadlock" + "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -141,7 +143,7 @@ type Service struct { stateproofmax basics.Round stateproofdb *db.Accessor stateproofproto protocol.ConsensusVersion - stateproofmu sync.Mutex + stateproofmu deadlock.Mutex stateproofwait map[basics.Round]chan struct{} // renaissance specifies the parameters for a renaissance diff --git a/config/local_defaults.go b/config/local_defaults.go index fd8a25f7f3..08431f2013 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -124,14 +124,18 @@ var defaultLocal = Local{ ProposalAssemblyTime: 500000000, PublicAddress: "", ReconnectTime: 60000000000, + RenaissanceCatchupLnProvenWeight: 0, + RenaissanceCatchupProto: "", + RenaissanceCatchupRound: 0, + RenaissanceCatchupVotersCommitment: "", ReservedFDs: 256, RestConnectionsHardLimit: 2048, RestConnectionsSoftLimit: 1024, RestReadTimeoutSeconds: 15, RestWriteTimeoutSeconds: 120, RunHosted: false, - StateproofDir: "", StateproofCatchupDir: "", + StateproofDir: "", StorageEngine: "sqlite", SuggestedFeeBlockHistory: 3, SuggestedFeeSlidingWindowSize: 50, @@ -153,8 +157,4 @@ var defaultLocal = Local{ TxSyncTimeoutSeconds: 30, UseXForwardedForAddressField: "", VerifiedTranscationsCacheSize: 150000, - RenaissanceCatchupRound: 0, - RenaissanceCatchupLnProvenWeight: 0, - RenaissanceCatchupVotersCommitment: "", - RenaissanceCatchupProto: "", } diff --git a/installer/config.json.example b/installer/config.json.example index 4c69bed1e5..47300a4332 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -103,16 +103,17 @@ "ProposalAssemblyTime": 500000000, "PublicAddress": "", "ReconnectTime": 60000000000, - "RenaissanceCatchupRound": 0, "RenaissanceCatchupLnProvenWeight": 0, - "RenaissanceCatchupVotersCommitment": "", "RenaissanceCatchupProto": "", + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupVotersCommitment": "", "ReservedFDs": 256, "RestConnectionsHardLimit": 2048, "RestConnectionsSoftLimit": 1024, "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, + "StateproofCatchupDir": "", "StateproofDir": "", "StorageEngine": "sqlite", "SuggestedFeeBlockHistory": 3, diff --git a/test/testdata/configs/config-v32.json b/test/testdata/configs/config-v32.json index 2051ba6801..47300a4332 100644 --- a/test/testdata/configs/config-v32.json +++ b/test/testdata/configs/config-v32.json @@ -103,12 +103,17 @@ "ProposalAssemblyTime": 500000000, "PublicAddress": "", "ReconnectTime": 60000000000, + "RenaissanceCatchupLnProvenWeight": 0, + "RenaissanceCatchupProto": "", + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupVotersCommitment": "", "ReservedFDs": 256, "RestConnectionsHardLimit": 2048, "RestConnectionsSoftLimit": 1024, "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, + "StateproofCatchupDir": "", "StateproofDir": "", "StorageEngine": "sqlite", "SuggestedFeeBlockHistory": 3, @@ -130,9 +135,5 @@ "TxSyncServeResponseSize": 1000000, "TxSyncTimeoutSeconds": 30, "UseXForwardedForAddressField": "", - "VerifiedTranscationsCacheSize": 150000, - "RenaissanceCatchupRound": 0, - "RenaissanceCatchupLnProvenWeight": 0, - "RenaissanceCatchupVotersCommitment": "", - "RenaissanceCatchupProto": "" + "VerifiedTranscationsCacheSize": 150000 } From 4cbc453aa6a60db87336f38499b8e734e3cc4098 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 30 Aug 2023 22:49:35 -0400 Subject: [PATCH 3/6] make go vet happy --- daemon/algod/api/server/v2/test/handlers_test.go | 4 ++-- ledger/apply/stateproof_test.go | 12 ++++++------ ledger/eval/cow_test.go | 6 +++--- stateproof/stateproofMessageGenerator_test.go | 12 ++++++------ stateproof/worker_test.go | 6 +++--- test/e2e-go/features/stateproofs/stateproofs_test.go | 10 +++++----- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/daemon/algod/api/server/v2/test/handlers_test.go b/daemon/algod/api/server/v2/test/handlers_test.go index 48f89538bb..6de13b26cd 100644 --- a/daemon/algod/api/server/v2/test/handlers_test.go +++ b/daemon/algod/api/server/v2/test/handlers_test.go @@ -1872,8 +1872,8 @@ func addStateProof(blk bookkeeping.Block) bookkeeping.Block { StateProofType: 0, Message: stateproofmsg.Message{ BlockHeadersCommitment: []byte{0x0, 0x1, 0x2}, - FirstAttestedRound: stateProofRound + 1, - LastAttestedRound: stateProofRound + stateProofInterval, + FirstAttestedRound: basics.Round(stateProofRound + 1), + LastAttestedRound: basics.Round(stateProofRound + stateProofInterval), }, }, }, diff --git a/ledger/apply/stateproof_test.go b/ledger/apply/stateproof_test.go index 155a4eef5d..efc877d97c 100644 --- a/ledger/apply/stateproof_test.go +++ b/ledger/apply/stateproof_test.go @@ -106,7 +106,7 @@ func TestApplyStateProofV34(t *testing.T) { stateProofTx.StateProofType = protocol.StateProofBasic // stateproof txn doesn't confirm the next state proof round. expected is in the past validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(8) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -114,7 +114,7 @@ func TestApplyStateProofV34(t *testing.T) { // stateproof txn doesn't confirm the next state proof round. expected is in the future validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(32) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -152,7 +152,7 @@ func TestApplyStateProofV34(t *testing.T) { spHdr.Round = 15 blocks[spHdr.Round] = spHdr - stateProofTx.Message.LastAttestedRound = uint64(spHdr.Round) + stateProofTx.Message.LastAttestedRound = spHdr.Round applier.SetStateProofNextRound(15) blockErr[13] = noBlockErr err = StateProof(stateProofTx, atRound, applier, validate) @@ -179,7 +179,7 @@ func TestApplyStateProofV34(t *testing.T) { atRoundBlock.CurrentProtocol = version blocks[atRound] = atRoundBlock - stateProofTx.Message.LastAttestedRound = 2 * config.Consensus[version].StateProofInterval + stateProofTx.Message.LastAttestedRound = 2 * basics.Round(config.Consensus[version].StateProofInterval) stateProofTx.StateProof.SignedWeight = 100 applier.SetStateProofNextRound(basics.Round(2 * config.Consensus[version].StateProofInterval)) @@ -220,7 +220,7 @@ func TestApplyStateProof(t *testing.T) { stateProofTx.StateProofType = protocol.StateProofBasic // stateproof txn doesn't confirm the next state proof round. expected is in the past validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(8) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -228,7 +228,7 @@ func TestApplyStateProof(t *testing.T) { // stateproof txn doesn't confirm the next state proof round. expected is in the future validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(32) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) diff --git a/ledger/eval/cow_test.go b/ledger/eval/cow_test.go index 225b037997..c1d2d03d25 100644 --- a/ledger/eval/cow_test.go +++ b/ledger/eval/cow_test.go @@ -307,20 +307,20 @@ func TestCowStateProof(t *testing.T) { c0.SetStateProofNextRound(firstStateproof) stateproofTxn := transactions.StateProofTxnFields{ StateProofType: protocol.StateProofBasic, - Message: stateproofmsg.Message{LastAttestedRound: uint64(firstStateproof) + version.StateProofInterval}, + Message: stateproofmsg.Message{LastAttestedRound: firstStateproof + basics.Round(version.StateProofInterval)}, } // can not apply state proof for 3*version.StateProofInterval when we expect 2*version.StateProofInterval err := apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.ErrorIs(err, apply.ErrExpectedDifferentStateProofRound) - stateproofTxn.Message.LastAttestedRound = uint64(firstStateproof) + stateproofTxn.Message.LastAttestedRound = firstStateproof err = apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.NoError(err) a.Equal(3*basics.Round(version.StateProofInterval), c0.GetStateProofNextRound()) // try to apply the next stateproof 3*version.StateProofInterval - stateproofTxn.Message.LastAttestedRound = 3 * version.StateProofInterval + stateproofTxn.Message.LastAttestedRound = basics.Round(3 * version.StateProofInterval) err = apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.NoError(err) a.Equal(4*basics.Round(version.StateProofInterval), c0.GetStateProofNextRound()) diff --git a/stateproof/stateproofMessageGenerator_test.go b/stateproof/stateproofMessageGenerator_test.go index a990143b05..1c1e5d10d3 100644 --- a/stateproof/stateproofMessageGenerator_test.go +++ b/stateproof/stateproofMessageGenerator_test.go @@ -76,20 +76,20 @@ func TestStateProofMessage(t *testing.T) { if !lastMessage.MsgIsZero() { verifier := stateproof.MkVerifierWithLnProvenWeight(lastMessage.VotersCommitment, lastMessage.LnProvenWeight, proto.StateProofStrengthTarget) - err := verifier.Verify(tx.Txn.Message.LastAttestedRound, tx.Txn.Message.Hash(), &tx.Txn.StateProof) + err := verifier.Verify(uint64(tx.Txn.Message.LastAttestedRound), tx.Txn.Message.Hash(), &tx.Txn.StateProof) a.NoError(err) } // since a state proof txn was created, we update the header with the next state proof round // i.e network has accepted the state proof. - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastMessage = tx.Txn.Message } } func verifySha256BlockHeadersCommitments(a *require.Assertions, message stateproofmsg.Message, blocks map[basics.Round]bookkeeping.BlockHeader) { blkHdrArr := make(lightBlockHeaders, message.LastAttestedRound-message.FirstAttestedRound+1) - for i := uint64(0); i < message.LastAttestedRound-message.FirstAttestedRound+1; i++ { - hdr := blocks[basics.Round(message.FirstAttestedRound+i)] + for i := uint64(0); i < uint64(message.LastAttestedRound-message.FirstAttestedRound+1); i++ { + hdr := blocks[message.FirstAttestedRound+basics.Round(i)] blkHdrArr[i] = hdr.ToLightBlockHeader() } @@ -217,7 +217,7 @@ func TestGenerateBlockProof(t *testing.T) { verifyLightBlockHeaderProof(&tx, &proto, headers, a) - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastAttestedRound = basics.Round(tx.Txn.Message.LastAttestedRound) } } @@ -225,7 +225,7 @@ func TestGenerateBlockProof(t *testing.T) { func verifyLightBlockHeaderProof(tx *transactions.SignedTxn, proto *config.ConsensusParams, headers []bookkeeping.LightBlockHeader, a *require.Assertions) { // attempting to get block proof for every block in the interval for j := tx.Txn.Message.FirstAttestedRound; j < tx.Txn.Message.LastAttestedRound; j++ { - headerIndex := j - tx.Txn.Message.FirstAttestedRound + headerIndex := uint64(j - tx.Txn.Message.FirstAttestedRound) proof, err := GenerateProofOfLightBlockHeaders(proto.StateProofInterval, headers, headerIndex) a.NoError(err) a.NotNil(proof) diff --git a/stateproof/worker_test.go b/stateproof/worker_test.go index 68a19dcd21..0f9efbad55 100644 --- a/stateproof/worker_test.go +++ b/stateproof/worker_test.go @@ -742,7 +742,7 @@ func TestWorkerRestart(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] s.advanceRoundsWithoutStateProof(t, 1) - lastRound := uint64(0) + lastRound := basics.Round(0) for i := 0; i < expectedStateProofs; i++ { s.advanceRoundsWithoutStateProof(t, proto.StateProofInterval/2-1) w.Stop() @@ -763,10 +763,10 @@ func TestWorkerRestart(t *testing.T) { // since a state proof txn was created, we update the header with the next state proof round // i.e network has accepted the state proof. - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastRound = tx.Txn.Message.LastAttestedRound } - a.Equal(uint64(expectedStateProofs+1), lastRound/proto.StateProofInterval) + a.Equal(uint64(expectedStateProofs+1), uint64(lastRound)/proto.StateProofInterval) } func TestWorkerHandleSig(t *testing.T) { diff --git a/test/e2e-go/features/stateproofs/stateproofs_test.go b/test/e2e-go/features/stateproofs/stateproofs_test.go index bba0838c21..7c5ec26990 100644 --- a/test/e2e-go/features/stateproofs/stateproofs_test.go +++ b/test/e2e-go/features/stateproofs/stateproofs_test.go @@ -358,10 +358,10 @@ func TestStateProofMessageCommitmentVerification(t *testing.T) { t.Logf("found first stateproof, attesting to rounds %d - %d. Verifying.\n", stateProofMessage.FirstAttestedRound, stateProofMessage.LastAttestedRound) for rnd := stateProofMessage.FirstAttestedRound; rnd <= stateProofMessage.LastAttestedRound; rnd++ { - proofResp, singleLeafProof, err := fixture.LightBlockHeaderProof(rnd) + proofResp, singleLeafProof, err := fixture.LightBlockHeaderProof(uint64(rnd)) r.NoError(err) - blk, err := libgoalClient.BookkeepingBlock(rnd) + blk, err := libgoalClient.BookkeepingBlock(uint64(rnd)) r.NoError(err) lightBlockHeader := blk.ToLightBlockHeader() @@ -410,8 +410,8 @@ func getStateProofByLastRound(r *require.Assertions, fixture *fixtures.RestClien BlockHeadersCommitment: res.Message.BlockHeadersCommitment, VotersCommitment: res.Message.VotersCommitment, LnProvenWeight: res.Message.LnProvenWeight, - FirstAttestedRound: res.Message.FirstAttestedRound, - LastAttestedRound: res.Message.LastAttestedRound, + FirstAttestedRound: basics.Round(res.Message.FirstAttestedRound), + LastAttestedRound: basics.Round(res.Message.LastAttestedRound), } return stateProof, msg } @@ -1284,7 +1284,7 @@ func TestStateProofCheckTotalStake(t *testing.T) { stateProof, stateProofMsg := getStateProofByLastRound(r, &fixture, nextStateProofRound) - accountSnapshot := accountSnapshotAtRound[stateProofMsg.LastAttestedRound-consensusParams.StateProofInterval-consensusParams.StateProofVotersLookback] + accountSnapshot := accountSnapshotAtRound[uint64(stateProofMsg.LastAttestedRound)-consensusParams.StateProofInterval-consensusParams.StateProofVotersLookback] // once the state proof is accepted we want to make sure that the weight for _, v := range stateProof.Reveals { From 604626931a0a2d274fc06bf588950d3efbb493e9 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 30 Aug 2023 22:50:28 -0400 Subject: [PATCH 4/6] go fmt --- rpcs/blockService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpcs/blockService.go b/rpcs/blockService.go index 5d5e2bd20e..12905c8cf8 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -260,7 +260,7 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques // proofs that don't exist yet, don't bother searching if we // are looking for a state proof for a round that's within // StateProofInterval of latest. - if round + basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) >= latestRound { + if round+basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) >= latestRound { goto notfound } From d4bf217012acf8ad61f4e242a2894ffc9d405790 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Wed, 30 Aug 2023 22:56:12 -0400 Subject: [PATCH 5/6] try to make lint happy --- catchup/fetcher_test.go | 2 +- catchup/stateproof.go | 57 +++++++++++++++++++------------------ catchup/universalFetcher.go | 10 +++---- rpcs/blockService.go | 4 +-- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/catchup/fetcher_test.go b/catchup/fetcher_test.go index 1e73abd08a..176d36615a 100644 --- a/catchup/fetcher_test.go +++ b/catchup/fetcher_test.go @@ -168,7 +168,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, } b.StateProofTracking = map[protocol.StateProofType]bookkeeping.StateProofTrackingData{ - protocol.StateProofBasic: bookkeeping.StateProofTrackingData{ + protocol.StateProofBasic: { StateProofVotersCommitment: stateProofData.Tree.Root(), StateProofOnlineTotalWeight: stateProofData.TotalWeight, StateProofNextRound: basics.Round(proto.StateProofInterval), diff --git a/catchup/stateproof.go b/catchup/stateproof.go index 38ee2c76fd..882fd88186 100644 --- a/catchup/stateproof.go +++ b/catchup/stateproof.go @@ -91,33 +91,33 @@ func (s *Service) initStateProofs() error { rows, err := tx.Query("SELECT proto, msg FROM proofs ORDER BY lastrnd") if err != nil { return err - } else { - defer rows.Close() - for rows.Next() { - var proto protocol.ConsensusVersion - var msgbuf []byte - err := rows.Scan(&proto, &msgbuf) - if err != nil { - s.log.Warnf("initStateProofs: cannot scan proof from db: %v", err) - continue - } - - var msg stateproofmsg.Message - err = protocol.Decode(msgbuf, &msg) - if err != nil { - s.log.Warnf("initStateProofs: cannot decode proof from db: %v", err) - continue - } - - stateproofs[msg.LastAttestedRound] = stateProofInfo{ - message: msg, - proto: proto, - } - stateproofmax = msg.LastAttestedRound - if stateproofmin == 0 { - stateproofmin = msg.LastAttestedRound - stateproofproto = proto - } + } + + defer rows.Close() + for rows.Next() { + var proto protocol.ConsensusVersion + var msgbuf []byte + err := rows.Scan(&proto, &msgbuf) + if err != nil { + s.log.Warnf("initStateProofs: cannot scan proof from db: %v", err) + continue + } + + var msg stateproofmsg.Message + err = protocol.Decode(msgbuf, &msg) + if err != nil { + s.log.Warnf("initStateProofs: cannot decode proof from db: %v", err) + continue + } + + stateproofs[msg.LastAttestedRound] = stateProofInfo{ + message: msg, + proto: proto, + } + stateproofmax = msg.LastAttestedRound + if stateproofmin == 0 { + stateproofmin = msg.LastAttestedRound + stateproofproto = proto } } return nil @@ -279,10 +279,13 @@ func (s *Service) nextStateProofVerifier() *StateProofVerificationContext { } } +// SetRenaissance sets the "renaissance" parameters for validating state proofs. func (s *Service) SetRenaissance(r StateProofVerificationContext) { s.renaissance = &r } +// SetRenaissanceFromConfig sets the "renaissance" parameters for validating state +// proofs based on the settings in the specified cfg. func (s *Service) SetRenaissanceFromConfig(cfg config.Local) { if cfg.RenaissanceCatchupRound == 0 { return diff --git a/catchup/universalFetcher.go b/catchup/universalFetcher.go index f78b6b681c..a0b2a68f6d 100644 --- a/catchup/universalFetcher.go +++ b/catchup/universalFetcher.go @@ -87,7 +87,7 @@ func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Ro } else { return nil, nil, nil, time.Duration(0), fmt.Errorf("fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") } - downloadDuration = time.Now().Sub(blockDownloadStartTime) + downloadDuration = time.Since(blockDownloadStartTime) block, cert, proof, err := processBlockBytes(fetchedBuf, round, address) if err != nil { return nil, nil, nil, time.Duration(0), err @@ -141,7 +141,7 @@ func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType } else { return pf, msg, 0, fmt.Errorf("fetchStateProof: UniversalFetcher only supports HTTPPeer") } - downloadDuration = time.Now().Sub(downloadStartTime) + downloadDuration = time.Since(downloadStartTime) pf, msg, err = processStateProofBytes(fetchedBuf, round, address) if err != nil { return pf, msg, 0, err @@ -329,8 +329,8 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round, proofO // Remove this 'old' string after next release. const blockResponseContentTypeOld = "application/algorand-block-v1" expectedContentTypes := map[string]struct{}{ - rpcs.BlockResponseContentType: struct{}{}, - blockResponseContentTypeOld: struct{}{}, + rpcs.BlockResponseContentType: {}, + blockResponseContentTypeOld: {}, } return hf.getBytes(ctx, blockURL, expectedContentTypes) @@ -347,7 +347,7 @@ func (hf *HTTPFetcher) getStateProofBytes(ctx context.Context, proofType protoco proofURL := parsedURL.String() expectedContentTypes := map[string]struct{}{ - rpcs.StateProofResponseContentType: struct{}{}, + rpcs.StateProofResponseContentType: {}, } return hf.getBytes(ctx, proofURL, expectedContentTypes) diff --git a/rpcs/blockService.go b/rpcs/blockService.go index 12905c8cf8..c7532c5d1b 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -224,7 +224,7 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques return } if typeStr != "0" { // StateProofBasic - bs.log.Debugf("http stateproof bad type", typeStr) + bs.log.Debugf("http stateproof bad type %s", typeStr) response.WriteHeader(http.StatusBadRequest) return } @@ -383,7 +383,7 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht stateProof, hasStateProof := request.Form["stateproof"] if hasStateProof { if len(stateProof) != 1 || stateProof[0] != "0" { - bs.log.Debug("http block stateproof version %v unsupported", stateProof) + bs.log.Debugf("http block stateproof version %v unsupported", stateProof) response.WriteHeader(http.StatusBadRequest) return } From 87cfe4fb21209ca818d574d3cfa8aecd8bf97c15 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Thu, 31 Aug 2023 09:19:42 -0400 Subject: [PATCH 6/6] add support for state proof bundles --- catchup/stateproof.go | 72 +++--- catchup/universalFetcher.go | 21 +- rpcs/blockService.go | 74 ++++-- rpcs/msgp_gen.go | 471 ++++++++++++++++++++++++++++++++++++ rpcs/msgp_gen_test.go | 120 +++++++++ 5 files changed, 700 insertions(+), 58 deletions(-) diff --git a/catchup/stateproof.go b/catchup/stateproof.go index 882fd88186..e819470cd2 100644 --- a/catchup/stateproof.go +++ b/catchup/stateproof.go @@ -397,6 +397,7 @@ func (s *Service) stateProofFetcher(ctx context.Context) { s.cleanupStateProofs(latest) peerSelector := createPeerSelector(s.net, s.cfg, true) + retry := 0 for { vc := s.nextStateProofVerifier() @@ -405,45 +406,60 @@ func (s *Service) stateProofFetcher(ctx context.Context) { return } - retry := 0 - for { - if retry >= catchupRetryLimit { - s.log.Debugf("catchup.stateProofFetcher: cannot fetch %d, giving up", vc.LastRound) - return - } - retry++ + if retry >= catchupRetryLimit { + s.log.Debugf("catchup.stateProofFetcher: cannot fetch %d, giving up", vc.LastRound) + return + } + retry++ - select { - case <-ctx.Done(): - s.log.Debugf("catchup.stateProofFetcher: aborted") - return - default: - } + select { + case <-ctx.Done(): + s.log.Debugf("catchup.stateProofFetcher: aborted") + return + default: + } - psp, err := peerSelector.getNextPeer() - if err != nil { - s.log.Warnf("catchup.stateProofFetcher: unable to getNextPeer: %v", err) - return - } + psp, err := peerSelector.getNextPeer() + if err != nil { + s.log.Warnf("catchup.stateProofFetcher: unable to getNextPeer: %v", err) + return + } - fetcher := makeUniversalBlockFetcher(s.log, s.net, s.cfg) - pf, msg, _, err := fetcher.fetchStateProof(ctx, protocol.StateProofBasic, vc.LastRound, psp.Peer) - if err != nil { - s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: %v", vc.LastRound, retry, err) - peerSelector.rankPeer(psp, peerRankDownloadFailed) - continue + fetcher := makeUniversalBlockFetcher(s.log, s.net, s.cfg) + proofs, _, err := fetcher.fetchStateProof(ctx, protocol.StateProofBasic, vc.LastRound, psp.Peer) + if err != nil { + s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: %v", vc.LastRound, retry, err) + peerSelector.rankPeer(psp, peerRankDownloadFailed) + continue + } + + if len(proofs.Proofs) == 0 { + s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: no proofs returned", vc.LastRound, retry) + peerSelector.rankPeer(psp, peerRankDownloadFailed) + continue + } + + for idx, pf := range proofs.Proofs { + if idx > 0 { + // This is an extra state proof returned optimistically by the server. + // We need to get the corresponding verification context. + vc = s.nextStateProofVerifier() + if vc == nil { + break + } } verifier := stateproof.MkVerifierWithLnProvenWeight(vc.VotersCommitment, vc.LnProvenWeight, config.Consensus[vc.Proto].StateProofStrengthTarget) - err = verifier.Verify(uint64(vc.LastRound), msg.Hash(), &pf) + err = verifier.Verify(uint64(vc.LastRound), pf.Message.Hash(), &pf.StateProof) if err != nil { s.log.Warnf("catchup.stateProofFetcher: cannot verify round %d: %v", vc.LastRound, err) peerSelector.rankPeer(psp, peerRankInvalidDownload) - continue + break } - s.addStateProof(msg, vc.Proto) - break + s.log.Debugf("catchup.stateProofFetcher: validated proof for %d", vc.LastRound) + s.addStateProof(pf.Message, vc.Proto) + retry = 0 } } } diff --git a/catchup/universalFetcher.go b/catchup/universalFetcher.go index a0b2a68f6d..f157728ecd 100644 --- a/catchup/universalFetcher.go +++ b/catchup/universalFetcher.go @@ -28,11 +28,8 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" - "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/data/stateproofmsg" - "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" @@ -120,7 +117,7 @@ func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk // attested to by the state proof) for round. // // proofType specifies the expected state proof type. -func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType protocol.StateProofType, round basics.Round, peer network.Peer) (pf stateproof.StateProof, msg stateproofmsg.Message, downloadDuration time.Duration, err error) { +func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType protocol.StateProofType, round basics.Round, peer network.Peer) (proofs rpcs.StateProofResponse, downloadDuration time.Duration, err error) { var fetchedBuf []byte var address string downloadStartTime := time.Now() @@ -135,30 +132,30 @@ func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType } fetchedBuf, err = fetcherClient.getStateProofBytes(ctx, proofType, round) if err != nil { - return pf, msg, 0, err + return proofs, 0, err } address = fetcherClient.address() } else { - return pf, msg, 0, fmt.Errorf("fetchStateProof: UniversalFetcher only supports HTTPPeer") + return proofs, 0, fmt.Errorf("fetchStateProof: UniversalFetcher only supports HTTPPeer") } downloadDuration = time.Since(downloadStartTime) - pf, msg, err = processStateProofBytes(fetchedBuf, round, address) + proofs, err = processStateProofBytes(fetchedBuf, round, address) if err != nil { - return pf, msg, 0, err + return proofs, 0, err } uf.log.Debugf("fetchStateProof: downloaded proof for %d in %d from %s", uint64(round), downloadDuration, address) - return pf, msg, downloadDuration, err + return proofs, downloadDuration, err } -func processStateProofBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (pf stateproof.StateProof, msg stateproofmsg.Message, err error) { - var decodedEntry transactions.Transaction +func processStateProofBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (proofs rpcs.StateProofResponse, err error) { + var decodedEntry rpcs.StateProofResponse err = protocol.Decode(fetchedBuf, &decodedEntry) if err != nil { err = fmt.Errorf("Cannot decode state proof for %d from %s: %v", r, peerAddr, err) return } - return decodedEntry.StateProofTxnFields.StateProof, decodedEntry.StateProofTxnFields.Message, nil + return decodedEntry, nil } // a stub fetcherClient to satisfy the NetworkFetcher interface diff --git a/rpcs/blockService.go b/rpcs/blockService.go index c7532c5d1b..7e791c0ad0 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -37,8 +37,10 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" @@ -55,6 +57,7 @@ const blockResponseMissingBlockCacheControl = "public, max-age=1, must-revalidat const blockResponseRetryAfter = "3" // retry after 3 seconds const blockServerMaxBodyLength = 512 // we don't really pass meaningful content here, so 512 bytes should be a safe limit const blockServerCatchupRequestBufferSize = 10 +const stateProofMaxCount = 128 // StateProofResponseContentType is the HTTP Content-Type header for a raw state proof transaction const StateProofResponseContentType = "application/x-algorand-stateproof-v1" @@ -203,7 +206,24 @@ func (bs *BlockService) Stop() { bs.closeWaitGroup.Wait() } -// ServeStateProofPath returns state proofs. +// OneStateProof is used to encode one state proof in the response +// to a state proof fetch request. +type OneStateProof struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + StateProof cryptostateproof.StateProof `codec:"sp"` + Message stateproofmsg.Message `codec:"spmsg"` +} + +// StateProofResponse is used to encode the response to a state proof +// fetch request, consisting of one or more state proofs. +type StateProofResponse struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Proofs []OneStateProof `codec:"p,allocbound=stateProofMaxCount"` +} + +// ServeStateProofPath returns state proofs, starting with the specified round. // It expects to be invoked via: // // /v{version}/{genesisID}/stateproof/type{type}/{round} @@ -236,6 +256,8 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques } round := basics.Round(uround) + var res StateProofResponse + latestRound := bs.ledger.Latest() ctx := request.Context() hdr, err := bs.ledger.BlockHdr(round) @@ -243,7 +265,8 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques bs.log.Debug("http stateproof cannot get blockhdr", round, err) switch err.(type) { case ledgercore.ErrNoEntry: - goto notfound + // Send a 404 response, since res.Proofs is empty. + goto done default: response.WriteHeader(http.StatusInternalServerError) return @@ -261,7 +284,8 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques // are looking for a state proof for a round that's within // StateProofInterval of latest. if round+basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) >= latestRound { - goto notfound + // Send a 404 response, since res.Proofs is empty. + goto done } for i := round + 1; i <= latestRound; i++ { @@ -271,6 +295,10 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques default: } + if len(res.Proofs) >= stateProofMaxCount { + break + } + txns, err := bs.ledger.AddressTxns(transactions.StateProofSender, i) if err != nil { bs.log.Debug("http stateproof address txns error", err) @@ -283,25 +311,35 @@ func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, reques } if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= round && round <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { - encodedStateProof := protocol.Encode(&txn.Txn) - response.Header().Set("Content-Type", StateProofResponseContentType) - response.Header().Set("Content-Length", strconv.Itoa(len(encodedStateProof))) - response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) - response.WriteHeader(http.StatusOK) - _, err = response.Write(encodedStateProof) - if err != nil { - bs.log.Warn("http stateproof write failed ", err) - } - return + res.Proofs = append(res.Proofs, OneStateProof{ + StateProof: txn.Txn.StateProofTxnFields.StateProof, + Message: txn.Txn.StateProofTxnFields.Message, + }) + + // Keep looking for more state proofs, since the caller will + // likely want a sequence of them until the latest round. + round = round + basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) } } } -notfound: - ok := bs.redirectRequest(response, request, bs.formatStateProofQuery(uint64(round))) - if !ok { - response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) - response.WriteHeader(http.StatusNotFound) +done: + if len(res.Proofs) == 0 { + ok := bs.redirectRequest(response, request, bs.formatStateProofQuery(uint64(round))) + if !ok { + response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) + response.WriteHeader(http.StatusNotFound) + } + } else { + encodedResponse := protocol.Encode(&res) + response.Header().Set("Content-Type", StateProofResponseContentType) + response.Header().Set("Content-Length", strconv.Itoa(len(encodedResponse))) + response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) + response.WriteHeader(http.StatusOK) + _, err = response.Write(encodedResponse) + if err != nil { + bs.log.Warn("http stateproof write failed ", err) + } } } diff --git a/rpcs/msgp_gen.go b/rpcs/msgp_gen.go index a94a2a15da..2ece973295 100644 --- a/rpcs/msgp_gen.go +++ b/rpcs/msgp_gen.go @@ -6,7 +6,9 @@ import ( "github.com/algorand/msgp/msgp" "github.com/algorand/go-algorand/agreement" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" ) // The following msgp objects are implemented in this file: @@ -19,6 +21,24 @@ import ( // |-----> (*) MsgIsZero // |-----> EncodedBlockCertMaxSize() // +// OneStateProof +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> OneStateProofMaxSize() +// +// StateProofResponse +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> StateProofResponseMaxSize() +// // MarshalMsg implements msgp.Marshaler func (z *EncodedBlockCert) MarshalMsg(b []byte) (o []byte) { @@ -178,3 +198,454 @@ func EncodedBlockCertMaxSize() (s int) { panic("Unable to determine max size: Byteslice type z.LightBlockHeaderProof is unbounded") return } + +// MarshalMsg implements msgp.Marshaler +func (z *OneStateProof) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).StateProof.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Message.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = (*z).StateProof.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "spmsg" + o = append(o, 0xa5, 0x73, 0x70, 0x6d, 0x73, 0x67) + o = (*z).Message.MarshalMsg(o) + } + } + return +} + +func (_ *OneStateProof) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*OneStateProof) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *OneStateProof) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProof") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Message") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = OneStateProof{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "sp": + bts, err = (*z).StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *OneStateProof) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*OneStateProof) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *OneStateProof) Msgsize() (s int) { + s = 1 + 3 + (*z).StateProof.Msgsize() + 6 + (*z).Message.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *OneStateProof) MsgIsZero() bool { + return ((*z).StateProof.MsgIsZero()) && ((*z).Message.MsgIsZero()) +} + +// MaxSize returns a maximum valid message size for this message type +func OneStateProofMaxSize() (s int) { + s = 1 + 3 + cryptostateproof.StateProofMaxSize() + 6 + stateproofmsg.MessageMaxSize() + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *StateProofResponse) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0002Len := uint32(1) + var zb0002Mask uint8 /* 2 bits */ + if len((*z).Proofs) == 0 { + zb0002Len-- + zb0002Mask |= 0x2 + } + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if zb0002Len != 0 { + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + if (*z).Proofs == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).Proofs))) + } + for zb0001 := range (*z).Proofs { + // omitempty: check for empty values + zb0003Len := uint32(2) + var zb0003Mask uint8 /* 3 bits */ + if (*z).Proofs[zb0001].StateProof.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x2 + } + if (*z).Proofs[zb0001].Message.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x4 + } + // variable map header, size zb0003Len + o = append(o, 0x80|uint8(zb0003Len)) + if (zb0003Mask & 0x2) == 0 { // if not empty + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = (*z).Proofs[zb0001].StateProof.MarshalMsg(o) + } + if (zb0003Mask & 0x4) == 0 { // if not empty + // string "spmsg" + o = append(o, 0xa5, 0x73, 0x70, 0x6d, 0x73, 0x67) + o = (*z).Proofs[zb0001].Message.MarshalMsg(o) + } + } + } + } + return +} + +func (_ *StateProofResponse) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*StateProofResponse) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *StateProofResponse) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 > 0 { + zb0002-- + var zb0004 int + var zb0005 bool + zb0004, zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs") + return + } + if zb0004 > stateProofMaxCount { + err = msgp.ErrOverflow(uint64(zb0004), uint64(stateProofMaxCount)) + err = msgp.WrapError(err, "struct-from-array", "Proofs") + return + } + if zb0005 { + (*z).Proofs = nil + } else if (*z).Proofs != nil && cap((*z).Proofs) >= zb0004 { + (*z).Proofs = ((*z).Proofs)[:zb0004] + } else { + (*z).Proofs = make([]OneStateProof, zb0004) + } + for zb0001 := range (*z).Proofs { + var zb0006 int + var zb0007 bool + zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array", "StateProof") + return + } + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array", "Message") + return + } + } + if zb0006 > 0 { + err = msgp.ErrTooManyArrayFields(zb0006) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + if zb0007 { + (*z).Proofs[zb0001] = OneStateProof{} + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + switch string(field) { + case "sp": + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + } + } + } + } + } + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = StateProofResponse{} + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "p": + var zb0008 int + var zb0009 bool + zb0008, zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs") + return + } + if zb0008 > stateProofMaxCount { + err = msgp.ErrOverflow(uint64(zb0008), uint64(stateProofMaxCount)) + err = msgp.WrapError(err, "Proofs") + return + } + if zb0009 { + (*z).Proofs = nil + } else if (*z).Proofs != nil && cap((*z).Proofs) >= zb0008 { + (*z).Proofs = ((*z).Proofs)[:zb0008] + } else { + (*z).Proofs = make([]OneStateProof, zb0008) + } + for zb0001 := range (*z).Proofs { + var zb0010 int + var zb0011 bool + zb0010, zb0011, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0010, zb0011, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array", "StateProof") + return + } + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array", "Message") + return + } + } + if zb0010 > 0 { + err = msgp.ErrTooManyArrayFields(zb0010) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + if zb0011 { + (*z).Proofs[zb0001] = OneStateProof{} + } + for zb0010 > 0 { + zb0010-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + switch string(field) { + case "sp": + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + } + } + } + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *StateProofResponse) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*StateProofResponse) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *StateProofResponse) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for zb0001 := range (*z).Proofs { + s += 1 + 3 + (*z).Proofs[zb0001].StateProof.Msgsize() + 6 + (*z).Proofs[zb0001].Message.Msgsize() + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z *StateProofResponse) MsgIsZero() bool { + return (len((*z).Proofs) == 0) +} + +// MaxSize returns a maximum valid message size for this message type +func StateProofResponseMaxSize() (s int) { + s = 1 + 2 + // Calculating size of slice: z.Proofs + s += msgp.ArrayHeaderSize + ((stateProofMaxCount) * (OneStateProofMaxSize())) + return +} diff --git a/rpcs/msgp_gen_test.go b/rpcs/msgp_gen_test.go index d88b73039b..30771c759e 100644 --- a/rpcs/msgp_gen_test.go +++ b/rpcs/msgp_gen_test.go @@ -73,3 +73,123 @@ func BenchmarkUnmarshalEncodedBlockCert(b *testing.B) { } } } + +func TestMarshalUnmarshalOneStateProof(t *testing.T) { + partitiontest.PartitionTest(t) + v := OneStateProof{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingOneStateProof(t *testing.T) { + protocol.RunEncodingTest(t, &OneStateProof{}) +} + +func BenchmarkMarshalMsgOneStateProof(b *testing.B) { + v := OneStateProof{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgOneStateProof(b *testing.B) { + v := OneStateProof{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalOneStateProof(b *testing.B) { + v := OneStateProof{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalStateProofResponse(t *testing.T) { + partitiontest.PartitionTest(t) + v := StateProofResponse{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingStateProofResponse(t *testing.T) { + protocol.RunEncodingTest(t, &StateProofResponse{}) +} + +func BenchmarkMarshalMsgStateProofResponse(b *testing.B) { + v := StateProofResponse{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgStateProofResponse(b *testing.B) { + v := StateProofResponse{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalStateProofResponse(b *testing.B) { + v := StateProofResponse{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +}