Pre-share nonces (#218)

* Pre-share nonces

* lint

* next cosigner retry

* improve

* fix test

* lint

* noplogger

* Add fallback to old nonces fetch mechanism

* lint

* use new raft get leader ID

* lint

* lint

* add back leader election timeout metric. Use existing grpc clients for ping

* error types

* Expired nonce pruning

* should use nonceExpiration

* make local cosigner nonce cache expiration twice the leader nonce cache

cmd/horcrux/cmd/leader_election.go

@@ -83,7 +83,7 @@ horcrux elect 2 # elect specific leader`,
 				return err
 			}
 
-			fmt.Printf("Leader election successful. New leader: %s\n", res.Leader)
+			fmt.Printf("Leader election successful. New leader: %d\n", res.Leader)
 
 			return nil
 		},
@@ -173,7 +173,7 @@ func getLeaderCmd() *cobra.Command {
 				return err
 			}
 
-			fmt.Printf("Current leader: %s\n", res.Leader)
+			fmt.Printf("Current leader: %d\n", res.Leader)
 
 			return nil
 		},

cmd/horcrux/cmd/start.go

@@ -48,7 +48,7 @@ func startCmd() *cobra.Command {
 
 			switch config.Config.SignMode {
 			case signer.SignModeThreshold:
-				services, val, err = NewThresholdValidator(logger)
+				services, val, err = NewThresholdValidator(cmd.Context(), logger)
 				if err != nil {
 					return err
 				}

cmd/horcrux/cmd/threshold.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -14,6 +15,7 @@ import (
 const maxWaitForSameBlockAttempts = 3
 
 func NewThresholdValidator(
+	ctx context.Context,
 	logger cometlog.Logger,
 ) ([]cometservice.Service, *signer.ThresholdValidator, error) {
 	if err := config.Config.ValidateThresholdModeConfig(); err != nil {
@@ -39,9 +41,13 @@ func NewThresholdValidator(
 
 	for _, c := range thresholdCfg.Cosigners {
 		if c.ShardID != security.GetID() {
+			rc, err := signer.NewRemoteCosigner(c.ShardID, c.P2PAddr)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to initialize remote cosigner: %w", err)
+			}
 			remoteCosigners = append(
 				remoteCosigners,
-				signer.NewRemoteCosigner(c.ShardID, c.P2PAddr),
+				rc,
 			)
 		} else {
 			p2pListen = c.P2PAddr
@@ -92,5 +98,9 @@ func NewThresholdValidator(
 
 	raftStore.SetThresholdValidator(val)
 
+	if err := val.Start(ctx); err != nil {
+		return nil, nil, fmt.Errorf("failed to start threshold validator: %w", err)
+	}
+
 	return services, val, nil
 }

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/cosmos/gogoproto v1.4.10
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/gogo/protobuf v1.3.2
+	github.com/google/uuid v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/raft v1.5.0
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2

go.sum

@@ -285,6 +285,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=

proto/strangelove/horcrux/cosigner.proto

@@ -9,6 +9,7 @@ service Cosigner {
 	rpc GetNonces (GetNoncesRequest) returns (GetNoncesResponse) {}
 	rpc TransferLeadership (TransferLeadershipRequest) returns (TransferLeadershipResponse) {}
 	rpc GetLeader (GetLeaderRequest) returns (GetLeaderResponse) {}
+	rpc Ping(PingRequest) returns (PingResponse) {}
 }
 
 message Block {
@@ -37,6 +38,11 @@ message Nonce {
 	bytes signature = 5;
 }
 
+message UUIDNonce {
+	bytes uuid = 1;
+	repeated Nonce nonces = 2;
+}
+
 message HRST {
 	int64 height = 1;
 	int64 round = 2;
@@ -45,10 +51,11 @@ message HRST {
 }
 
 message SetNoncesAndSignRequest {
-	repeated Nonce nonces = 1;
-	HRST hrst = 2;
-	bytes signBytes = 3;
-	string chainID = 4;
+	bytes uuid = 1;
+	repeated Nonce nonces = 2;
+	HRST hrst = 3;
+	bytes signBytes = 4;
+	string chainID = 5;
 }
 
 message SetNoncesAndSignResponse {
@@ -58,12 +65,11 @@ message SetNoncesAndSignResponse {
 }
 
 message GetNoncesRequest {
-	HRST hrst = 1;
-	string chainID = 2;
+	repeated bytes uuids = 1;
 }
 
 message GetNoncesResponse {
-	repeated Nonce nonces = 1;
+	repeated UUIDNonce nonces = 1;
 }
 
 message TransferLeadershipRequest {
@@ -78,5 +84,8 @@ message TransferLeadershipResponse {
 message GetLeaderRequest {}
 
 message GetLeaderResponse {
-	string leader = 1;
+	int32 leader = 1;
 }
+
+message PingRequest {}
+message PingResponse {}

signer/cosigner.go

@@ -1,9 +1,11 @@
 package signer
 
 import (
+	"context"
 	"time"
 
 	cometcrypto "github.com/cometbft/cometbft/crypto"
+	"github.com/google/uuid"
 	"github.com/strangelove-ventures/horcrux/signer/proto"
 )
 
@@ -23,17 +25,29 @@ type Cosigner interface {
 	VerifySignature(chainID string, payload, signature []byte) bool
 
 	// Get nonces for all cosigner shards
-	GetNonces(chainID string, hrst HRSTKey) (*CosignerNoncesResponse, error)
+	GetNonces(ctx context.Context, uuids []uuid.UUID) (CosignerUUIDNoncesMultiple, error)
 
 	// Sign the requested bytes
-	SetNoncesAndSign(req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error)
+	SetNoncesAndSign(ctx context.Context, req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error)
+}
+
+type Cosigners []Cosigner
+
+func (cosigners Cosigners) GetByID(id int) Cosigner {
+	for _, cosigner := range cosigners {
+		if cosigner.GetID() == id {
+			return cosigner
+		}
+	}
+	return nil
 }
 
 // CosignerSignRequest is sent to a co-signer to obtain their signature for the SignBytes
 // The SignBytes should be a serialized block
 type CosignerSignRequest struct {
 	ChainID   string
 	SignBytes []byte
+	UUID      uuid.UUID
 }
 
 type CosignerSignResponse struct {
@@ -87,18 +101,6 @@ func CosignerNoncesFromProto(secretParts []*proto.Nonce) []CosignerNonce {
 	return out
 }
 
-type CosignerSetNonceRequest struct {
-	ChainID   string
-	SourceID  int
-	PubKey    []byte
-	Share     []byte
-	Signature []byte
-	Height    int64
-	Round     int64
-	Step      int8
-	Timestamp time.Time
-}
-
 type CosignerSignBlockRequest struct {
 	ChainID string
 	Block   *Block
@@ -107,14 +109,37 @@ type CosignerSignBlockRequest struct {
 type CosignerSignBlockResponse struct {
 	Signature []byte
 }
+type CosignerUUIDNonces struct {
+	UUID   uuid.UUID
+	Nonces CosignerNonces
+}
+
+func (n *CosignerUUIDNonces) For(id int) *CosignerUUIDNonces {
+	res := &CosignerUUIDNonces{UUID: n.UUID}
+	for _, nonce := range n.Nonces {
+		if nonce.DestinationID == id {
+			res.Nonces = append(res.Nonces, nonce)
+		}
+	}
+	return res
+}
 
-type CosignerNoncesResponse struct {
-	Nonces []CosignerNonce
+type CosignerUUIDNoncesMultiple []*CosignerUUIDNonces
+
+func (n CosignerUUIDNoncesMultiple) toProto() []*proto.UUIDNonce {
+	out := make([]*proto.UUIDNonce, len(n))
+	for i, nonces := range n {
+		out[i] = &proto.UUIDNonce{
+			Uuid:   nonces.UUID[:],
+			Nonces: nonces.Nonces.toProto(),
+		}
+	}
+	return out
 }
 
 type CosignerSetNoncesAndSignRequest struct {
 	ChainID   string
-	Nonces    []CosignerNonce
+	Nonces    *CosignerUUIDNonces
 	HRST      HRSTKey
 	SignBytes []byte
 }

signer/cosigner_grpc_server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/google/uuid"
 	"github.com/hashicorp/raft"
 	"github.com/strangelove-ventures/horcrux/signer/proto"
 )
@@ -30,10 +31,10 @@ func NewCosignerGRPCServer(
 }
 
 func (rpc *CosignerGRPCServer) SignBlock(
-	_ context.Context,
+	ctx context.Context,
 	req *proto.SignBlockRequest,
 ) (*proto.SignBlockResponse, error) {
-	res, _, err := rpc.thresholdValidator.SignBlock(req.ChainID, BlockFromProto(req.Block))
+	res, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
 	if err != nil {
 		return nil, err
 	}
@@ -43,12 +44,15 @@ func (rpc *CosignerGRPCServer) SignBlock(
 }
 
 func (rpc *CosignerGRPCServer) SetNoncesAndSign(
-	_ context.Context,
+	ctx context.Context,
 	req *proto.SetNoncesAndSignRequest,
 ) (*proto.SetNoncesAndSignResponse, error) {
-	res, err := rpc.cosigner.SetNoncesAndSign(CosignerSetNoncesAndSignRequest{
-		ChainID:   req.ChainID,
-		Nonces:    CosignerNoncesFromProto(req.GetNonces()),
+	res, err := rpc.cosigner.SetNoncesAndSign(ctx, CosignerSetNoncesAndSignRequest{
+		ChainID: req.ChainID,
+		Nonces: &CosignerUUIDNonces{
+			UUID:   uuid.UUID(req.Uuid),
+			Nonces: CosignerNoncesFromProto(req.GetNonces()),
+		},
 		HRST:      HRSTKeyFromProto(req.GetHrst()),
 		SignBytes: req.GetSignBytes(),
 	})
@@ -78,18 +82,23 @@ func (rpc *CosignerGRPCServer) SetNoncesAndSign(
 }
 
 func (rpc *CosignerGRPCServer) GetNonces(
-	_ context.Context,
+	ctx context.Context,
 	req *proto.GetNoncesRequest,
 ) (*proto.GetNoncesResponse, error) {
+	uuids := make([]uuid.UUID, len(req.Uuids))
+	for i, uuidBytes := range req.Uuids {
+		uuids[i] = uuid.UUID(uuidBytes)
+	}
 	res, err := rpc.cosigner.GetNonces(
-		req.ChainID,
-		HRSTKeyFromProto(req.GetHrst()),
+		ctx,
+		uuids,
 	)
 	if err != nil {
 		return nil, err
 	}
+
 	return &proto.GetNoncesResponse{
-		Nonces: CosignerNonces(res.Nonces).toProto(),
+		Nonces: res.toProto(),
 	}, nil
 }
 
@@ -122,5 +131,9 @@ func (rpc *CosignerGRPCServer) GetLeader(
 	*proto.GetLeaderRequest,
 ) (*proto.GetLeaderResponse, error) {
 	leader := rpc.raftStore.GetLeader()
-	return &proto.GetLeaderResponse{Leader: string(leader)}, nil
+	return &proto.GetLeaderResponse{Leader: int32(leader)}, nil
+}
+
+func (rpc *CosignerGRPCServer) Ping(context.Context, *proto.PingRequest) (*proto.PingResponse, error) {
+	return &proto.PingResponse{}, nil
 }