diff --git a/cmd/horcrux/cmd/threshold.go b/cmd/horcrux/cmd/threshold.go index 743a9e35..c1e035be 100644 --- a/cmd/horcrux/cmd/threshold.go +++ b/cmd/horcrux/cmd/threshold.go @@ -41,9 +41,13 @@ func NewThresholdValidator( for _, c := range thresholdCfg.Cosigners { if c.ShardID != security.GetID() { + rc, err := signer.NewRemoteCosigner(c.ShardID, c.P2PAddr) + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize remote cosigner: %w", err) + } remoteCosigners = append( remoteCosigners, - signer.NewRemoteCosigner(c.ShardID, c.P2PAddr), + rc, ) } else { p2pListen = c.P2PAddr diff --git a/proto/strangelove/horcrux/cosigner.proto b/proto/strangelove/horcrux/cosigner.proto index 788f71dc..52475b85 100644 --- a/proto/strangelove/horcrux/cosigner.proto +++ b/proto/strangelove/horcrux/cosigner.proto @@ -84,7 +84,7 @@ message TransferLeadershipResponse { message GetLeaderRequest {} message GetLeaderResponse { - string leader = 1; + int32 leader = 1; } message PingRequest {} diff --git a/signer/cosigner.go b/signer/cosigner.go index 97a3c82c..bf89a568 100644 --- a/signer/cosigner.go +++ b/signer/cosigner.go @@ -31,6 +31,17 @@ type Cosigner interface { SetNoncesAndSign(ctx context.Context, req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error) } +type Cosigners []Cosigner + +func (cosigners Cosigners) GetByID(id int) Cosigner { + for _, cosigner := range cosigners { + if cosigner.GetID() == id { + return cosigner + } + } + return nil +} + // CosignerSignRequest is sent to a co-signer to obtain their signature for the SignBytes // The SignBytes should be a serialized block type CosignerSignRequest struct { diff --git a/signer/cosigner_grpc_server.go b/signer/cosigner_grpc_server.go index 617334c5..89805290 100644 --- a/signer/cosigner_grpc_server.go +++ b/signer/cosigner_grpc_server.go @@ -131,7 +131,7 @@ func (rpc *CosignerGRPCServer) GetLeader( *proto.GetLeaderRequest, ) (*proto.GetLeaderResponse, error) { leader := rpc.raftStore.GetLeader() - return &proto.GetLeaderResponse{Leader: string(leader)}, nil + return &proto.GetLeaderResponse{Leader: int32(leader)}, nil } func (rpc *CosignerGRPCServer) Ping(context.Context, *proto.PingRequest) (*proto.PingResponse, error) { diff --git a/signer/leader.go b/signer/leader.go index 3f796a2d..beafb3ee 100644 --- a/signer/leader.go +++ b/signer/leader.go @@ -5,9 +5,9 @@ type Leader interface { // IsLeader returns true if the cosigner is the leader. IsLeader() bool - // SignBlock asks the leader to manage the signing of a block. - SignBlock(CosignerSignBlockRequest) (*CosignerSignBlockResponse, error) - // ShareSigned shares the last signed state with the other cosigners. ShareSigned(lss ChainSignStateConsensus) error + + // Get current leader + GetLeader() int } diff --git a/signer/leader_mock.go b/signer/leader_mock.go index b211a3d7..bd1c7c09 100644 --- a/signer/leader_mock.go +++ b/signer/leader_mock.go @@ -1,10 +1,7 @@ package signer import ( - "context" - "errors" "sync" - "time" ) var _ Leader = (*MockLeader)(nil) @@ -28,36 +25,8 @@ func (m *MockLeader) SetLeader(tv *ThresholdValidator) { m.leader = tv } -func (m *MockLeader) SignBlock(req CosignerSignBlockRequest) (*CosignerSignBlockResponse, error) { - var l *ThresholdValidator - for i := 0; i < 30; i++ { - m.mu.Lock() - l = m.leader - m.mu.Unlock() - if l != nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - if l == nil { - return nil, errors.New("timed out waiting for leader election to complete") - } - - block := Block{ - Height: req.Block.Height, - Round: req.Block.Round, - Step: req.Block.Step, - SignBytes: req.Block.SignBytes, - Timestamp: req.Block.Timestamp, - } - res, _, err := l.Sign(context.TODO(), req.ChainID, block) - if err != nil { - return nil, err - } - return &CosignerSignBlockResponse{ - Signature: res, - }, nil +func (m *MockLeader) GetLeader() int { + return m.id } func (m *MockLeader) ShareSigned(_ ChainSignStateConsensus) error { diff --git a/signer/proto/cosigner.pb.go b/signer/proto/cosigner.pb.go index 9fa3333a..ff9bf4d6 100644 --- a/signer/proto/cosigner.pb.go +++ b/signer/proto/cosigner.pb.go @@ -760,7 +760,7 @@ func (m *GetLeaderRequest) XXX_DiscardUnknown() { var xxx_messageInfo_GetLeaderRequest proto.InternalMessageInfo type GetLeaderResponse struct { - Leader string `protobuf:"bytes,1,opt,name=leader,proto3" json:"leader,omitempty"` + Leader int32 `protobuf:"varint,1,opt,name=leader,proto3" json:"leader,omitempty"` } func (m *GetLeaderResponse) Reset() { *m = GetLeaderResponse{} } @@ -796,11 +796,11 @@ func (m *GetLeaderResponse) XXX_DiscardUnknown() { var xxx_messageInfo_GetLeaderResponse proto.InternalMessageInfo -func (m *GetLeaderResponse) GetLeader() string { +func (m *GetLeaderResponse) GetLeader() int32 { if m != nil { return m.Leader } - return "" + return 0 } type PingRequest struct { @@ -900,53 +900,53 @@ func init() { var fileDescriptor_b7a1f695b94b848a = []byte{ // 744 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x51, 0x4f, 0xd3, 0x50, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdf, 0x4f, 0xd3, 0x50, 0x14, 0x5e, 0xb7, 0x76, 0xb2, 0x33, 0x30, 0x70, 0x25, 0x58, 0x1a, 0xb3, 0xcc, 0x1b, 0x35, 0x4b, 0x94, 0xcd, 0x4c, 0xa3, 0xcf, 0x20, 0x89, 0x12, 0x14, 0x49, 0x07, 0x2f, 0x86, 0x90, 0x74, 0xdd, - 0x65, 0x6d, 0x1c, 0xed, 0xb8, 0xf7, 0x16, 0xe1, 0x07, 0xf8, 0xee, 0x8b, 0xff, 0x89, 0x47, 0x1e, - 0x7d, 0xd3, 0xc0, 0x1f, 0x31, 0xf7, 0xf6, 0xb6, 0xac, 0xa5, 0x03, 0x1e, 0x78, 0x5a, 0xcf, 0xe9, - 0x77, 0xee, 0xf9, 0xbe, 0x6f, 0x5f, 0x9b, 0x02, 0x66, 0x9c, 0x3a, 0xc1, 0x90, 0x8c, 0xc2, 0x63, + 0x65, 0x6d, 0x1c, 0xed, 0xb8, 0xf7, 0x16, 0xe1, 0x0f, 0xf0, 0xdd, 0x17, 0xff, 0x27, 0x1e, 0x79, + 0xf4, 0x4d, 0x03, 0xff, 0x88, 0xb9, 0xb7, 0xb7, 0x65, 0x2d, 0x1d, 0xf0, 0xc0, 0xd3, 0x7a, 0x4e, + 0xcf, 0x8f, 0xef, 0xfb, 0xf6, 0xdd, 0x9b, 0x02, 0x66, 0x9c, 0x3a, 0xc1, 0x90, 0x8c, 0xc2, 0x63, 0xd2, 0xf1, 0x42, 0xea, 0xd2, 0xe8, 0xa4, 0xe3, 0x86, 0xcc, 0x1f, 0x06, 0x84, 0xb6, 0xc7, 0x34, - 0xe4, 0x21, 0x7a, 0x34, 0x81, 0x69, 0x2b, 0x0c, 0xfe, 0xa9, 0x81, 0xb1, 0x36, 0x0a, 0xdd, 0xef, - 0x68, 0x09, 0xaa, 0x1e, 0xf1, 0x87, 0x1e, 0x37, 0xb5, 0xa6, 0xd6, 0xaa, 0xd8, 0xaa, 0x42, 0x8b, - 0x60, 0xd0, 0x30, 0x0a, 0x06, 0x66, 0x59, 0xb6, 0xe3, 0x02, 0x21, 0xd0, 0x19, 0x27, 0x63, 0xb3, - 0xd2, 0xd4, 0x5a, 0x86, 0x2d, 0xaf, 0xd1, 0x13, 0xa8, 0x89, 0x85, 0x6b, 0xa7, 0x9c, 0x30, 0x53, - 0x6f, 0x6a, 0xad, 0x59, 0xfb, 0xaa, 0x21, 0xee, 0x72, 0xff, 0x90, 0x30, 0xee, 0x1c, 0x8e, 0x4d, - 0x43, 0x9e, 0x75, 0xd5, 0xc0, 0xfb, 0x30, 0xdf, 0x13, 0x50, 0x41, 0xc5, 0x26, 0x47, 0x11, 0x61, - 0x1c, 0x99, 0xf0, 0xc0, 0xf5, 0x1c, 0x3f, 0xd8, 0x58, 0x97, 0x94, 0x6a, 0x76, 0x52, 0xa2, 0xd7, - 0x60, 0xf4, 0x05, 0x52, 0x72, 0xaa, 0x77, 0xad, 0x76, 0x81, 0xb4, 0x76, 0x7c, 0x56, 0x0c, 0xc4, - 0x5f, 0x61, 0x61, 0xe2, 0x7c, 0x36, 0x0e, 0x03, 0x46, 0x12, 0xc2, 0x0e, 0x8f, 0x28, 0x91, 0x2b, - 0x14, 0x61, 0xd9, 0xc8, 0x12, 0x2e, 0xe7, 0x09, 0xff, 0xd6, 0xc0, 0xd8, 0x0a, 0x03, 0x97, 0x20, - 0x0b, 0x66, 0x58, 0x18, 0x51, 0x97, 0x28, 0x9e, 0x86, 0x9d, 0xd6, 0xe8, 0x19, 0xcc, 0x0d, 0x08, - 0xe3, 0x7e, 0xe0, 0x70, 0x3f, 0x14, 0x42, 0xca, 0x12, 0x90, 0x6d, 0x0a, 0xeb, 0xc7, 0x51, 0x7f, - 0x93, 0x9c, 0x4a, 0x3b, 0x67, 0x6d, 0x55, 0x09, 0xeb, 0x99, 0xe7, 0x50, 0xa2, 0xcc, 0x8c, 0x8b, - 0x2c, 0x6b, 0x23, 0xc7, 0x1a, 0xf7, 0xa0, 0xb6, 0xbb, 0xbb, 0xb1, 0x1e, 0x53, 0x43, 0xa0, 0x47, - 0x91, 0x3f, 0x50, 0xda, 0xe4, 0x35, 0xea, 0x42, 0x35, 0x10, 0x37, 0x99, 0x59, 0x6e, 0x56, 0xa6, - 0x9a, 0x27, 0xe7, 0x6d, 0x85, 0xc4, 0x07, 0xa0, 0x7f, 0xb2, 0x7b, 0x3b, 0xf7, 0x93, 0x91, 0x2b, - 0x53, 0xf5, 0xbc, 0xa9, 0x67, 0x1a, 0x3c, 0xee, 0x11, 0x2e, 0x97, 0xb3, 0xd5, 0x60, 0x20, 0xfe, - 0xb2, 0x24, 0x0d, 0xf7, 0xa4, 0x05, 0xad, 0x80, 0xee, 0x51, 0xc6, 0x25, 0xab, 0x7a, 0x77, 0xb9, - 0x70, 0x42, 0x88, 0xb5, 0x25, 0xec, 0x96, 0x50, 0x4f, 0x44, 0xd4, 0xc8, 0x44, 0x14, 0x9f, 0x80, - 0x79, 0x5d, 0x89, 0xca, 0x5d, 0x13, 0xea, 0x92, 0xcc, 0x76, 0xd4, 0x1f, 0xf9, 0xae, 0x52, 0x34, - 0xd9, 0xba, 0x39, 0x7b, 0xd9, 0x04, 0x54, 0xf2, 0x09, 0x68, 0xc1, 0xfc, 0xc7, 0x64, 0x73, 0x62, - 0xde, 0x22, 0x18, 0xc2, 0x30, 0x66, 0x6a, 0xcd, 0x8a, 0x48, 0x92, 0x2c, 0xf0, 0x26, 0x2c, 0x4c, - 0x20, 0x15, 0xb9, 0x77, 0xa9, 0xa7, 0x9a, 0xf4, 0xb4, 0x51, 0xe8, 0x50, 0x9a, 0xb1, 0x34, 0x23, - 0xef, 0x61, 0x79, 0x87, 0x3a, 0x01, 0x3b, 0x20, 0xf4, 0x33, 0x71, 0x06, 0x84, 0x32, 0xcf, 0x1f, - 0x27, 0xfb, 0x2d, 0x98, 0x19, 0xc9, 0x66, 0xfa, 0x2c, 0xa7, 0x35, 0xde, 0x07, 0xab, 0x68, 0x50, - 0xd1, 0xb9, 0x61, 0x52, 0x3c, 0x5d, 0xf1, 0xf5, 0xea, 0x60, 0x40, 0x09, 0x63, 0xd2, 0xa9, 0x9a, - 0x9d, 0x6d, 0x62, 0x24, 0xfd, 0x88, 0x8f, 0x56, 0x7c, 0xf0, 0x4b, 0xa9, 0x3c, 0xe9, 0xa9, 0x55, - 0x4b, 0x50, 0x8d, 0x27, 0xd5, 0x22, 0x55, 0xe1, 0x39, 0xa8, 0x6f, 0xfb, 0xc1, 0x30, 0x99, 0x7d, - 0x08, 0xb3, 0x71, 0x19, 0x8f, 0x75, 0xff, 0xea, 0x30, 0xf3, 0x41, 0xbd, 0x6a, 0xd1, 0x1e, 0xd4, - 0xd2, 0xf7, 0x0c, 0x7a, 0x5e, 0x68, 0x5d, 0xfe, 0x3d, 0x67, 0xbd, 0xb8, 0x0d, 0x16, 0x2f, 0xc2, - 0x25, 0x74, 0x04, 0xf3, 0xf9, 0x50, 0xa1, 0x57, 0xc5, 0xd3, 0xc5, 0x4f, 0x91, 0xb5, 0x72, 0x47, - 0x74, 0xba, 0x72, 0x0f, 0x6a, 0x69, 0x46, 0xa6, 0x08, 0xca, 0xa7, 0x6d, 0x8a, 0xa0, 0x6b, 0x51, - 0xc3, 0x25, 0xf4, 0x03, 0xd0, 0xf5, 0xff, 0x1e, 0xb5, 0x0b, 0xe7, 0xa7, 0xa6, 0xcb, 0xea, 0xdc, - 0x19, 0x9f, 0x93, 0x15, 0xdf, 0x9a, 0x2e, 0x2b, 0x13, 0x9a, 0xe9, 0xb2, 0xb2, 0x39, 0xc2, 0x25, - 0xf4, 0x05, 0x74, 0x11, 0x11, 0xd4, 0x2c, 0x9c, 0x98, 0x08, 0x93, 0xf5, 0xf4, 0x06, 0x44, 0x72, - 0xdc, 0xda, 0xd6, 0xd9, 0x45, 0x43, 0x3b, 0xbf, 0x68, 0x68, 0xff, 0x2e, 0x1a, 0xda, 0xaf, 0xcb, - 0x46, 0xe9, 0xfc, 0xb2, 0x51, 0xfa, 0x73, 0xd9, 0x28, 0x7d, 0x7b, 0x3b, 0xf4, 0xb9, 0x17, 0xf5, - 0xdb, 0x6e, 0x78, 0xd8, 0x99, 0x38, 0x68, 0xe5, 0x98, 0x04, 0xe2, 0x5d, 0xc0, 0xd2, 0x6f, 0x81, - 0x38, 0x9e, 0x1d, 0xf9, 0x25, 0xd0, 0xaf, 0xca, 0x9f, 0x37, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, - 0x9f, 0xb6, 0xf6, 0x12, 0x36, 0x08, 0x00, 0x00, + 0xe4, 0x21, 0x7a, 0x34, 0x51, 0xd3, 0x56, 0x35, 0xf8, 0xa7, 0x06, 0xc6, 0xda, 0x28, 0x74, 0xbf, + 0xa3, 0x25, 0xa8, 0x7a, 0xc4, 0x1f, 0x7a, 0xdc, 0xd4, 0x9a, 0x5a, 0xab, 0x62, 0xab, 0x08, 0x2d, + 0x82, 0x41, 0xc3, 0x28, 0x18, 0x98, 0x65, 0x99, 0x8e, 0x03, 0x84, 0x40, 0x67, 0x9c, 0x8c, 0xcd, + 0x4a, 0x53, 0x6b, 0x19, 0xb6, 0x7c, 0x46, 0x4f, 0xa0, 0x26, 0x16, 0xae, 0x9d, 0x72, 0xc2, 0x4c, + 0xbd, 0xa9, 0xb5, 0x66, 0xed, 0xab, 0x84, 0x78, 0xcb, 0xfd, 0x43, 0xc2, 0xb8, 0x73, 0x38, 0x36, + 0x0d, 0x39, 0xeb, 0x2a, 0x81, 0xf7, 0x61, 0xbe, 0x27, 0x4a, 0x05, 0x14, 0x9b, 0x1c, 0x45, 0x84, + 0x71, 0x64, 0xc2, 0x03, 0xd7, 0x73, 0xfc, 0x60, 0x63, 0x5d, 0x42, 0xaa, 0xd9, 0x49, 0x88, 0x5e, + 0x83, 0xd1, 0x17, 0x95, 0x12, 0x53, 0xbd, 0x6b, 0xb5, 0x0b, 0xa8, 0xb5, 0xe3, 0x59, 0x71, 0x21, + 0xfe, 0x0a, 0x0b, 0x13, 0xf3, 0xd9, 0x38, 0x0c, 0x18, 0x49, 0x00, 0x3b, 0x3c, 0xa2, 0x44, 0xae, + 0x50, 0x80, 0x65, 0x22, 0x0b, 0xb8, 0x9c, 0x07, 0xfc, 0x5b, 0x03, 0x63, 0x2b, 0x0c, 0x5c, 0x82, + 0x2c, 0x98, 0x61, 0x61, 0x44, 0x5d, 0xa2, 0x70, 0x1a, 0x76, 0x1a, 0xa3, 0x67, 0x30, 0x37, 0x20, + 0x8c, 0xfb, 0x81, 0xc3, 0xfd, 0x50, 0x10, 0x29, 0xcb, 0x82, 0x6c, 0x52, 0x48, 0x3f, 0x8e, 0xfa, + 0x9b, 0xe4, 0x54, 0xca, 0x39, 0x6b, 0xab, 0x48, 0x48, 0xcf, 0x3c, 0x87, 0x12, 0x25, 0x66, 0x1c, + 0x64, 0x51, 0x1b, 0x39, 0xd4, 0xb8, 0x07, 0xb5, 0xdd, 0xdd, 0x8d, 0xf5, 0x18, 0x1a, 0x02, 0x3d, + 0x8a, 0xfc, 0x81, 0xe2, 0x26, 0x9f, 0x51, 0x17, 0xaa, 0x81, 0x78, 0xc9, 0xcc, 0x72, 0xb3, 0x32, + 0x55, 0x3c, 0xd9, 0x6f, 0xab, 0x4a, 0x7c, 0x00, 0xfa, 0x27, 0xbb, 0xb7, 0x73, 0x3f, 0x1e, 0xb9, + 0x12, 0x55, 0xcf, 0x8b, 0x7a, 0xa6, 0xc1, 0xe3, 0x1e, 0xe1, 0x72, 0x39, 0x5b, 0x0d, 0x06, 0xe2, + 0x2f, 0x4b, 0xdc, 0x70, 0x4f, 0x5c, 0xd0, 0x0a, 0xe8, 0x1e, 0x65, 0x5c, 0xa2, 0xaa, 0x77, 0x97, + 0x0b, 0x3b, 0x04, 0x59, 0x5b, 0x96, 0xdd, 0x62, 0xea, 0x09, 0x8b, 0x1a, 0x19, 0x8b, 0xe2, 0x13, + 0x30, 0xaf, 0x33, 0x51, 0xbe, 0x6b, 0x42, 0x5d, 0x82, 0xd9, 0x8e, 0xfa, 0x23, 0xdf, 0x55, 0x8c, + 0x26, 0x53, 0x37, 0x7b, 0x2f, 0xeb, 0x80, 0x4a, 0xde, 0x01, 0x2d, 0x98, 0xff, 0x98, 0x6c, 0x4e, + 0xc4, 0x5b, 0x04, 0x43, 0x08, 0xc6, 0x4c, 0xad, 0x59, 0x11, 0x4e, 0x92, 0x01, 0xde, 0x84, 0x85, + 0x89, 0x4a, 0x05, 0xee, 0x5d, 0xaa, 0xa9, 0x26, 0x35, 0x6d, 0x14, 0x2a, 0x94, 0x7a, 0x2c, 0xf5, + 0xc8, 0x7b, 0x58, 0xde, 0xa1, 0x4e, 0xc0, 0x0e, 0x08, 0xfd, 0x4c, 0x9c, 0x01, 0xa1, 0xcc, 0xf3, + 0xc7, 0xc9, 0x7e, 0x0b, 0x66, 0x46, 0x32, 0x99, 0x9e, 0xe5, 0x34, 0xc6, 0xfb, 0x60, 0x15, 0x35, + 0x2a, 0x38, 0x37, 0x74, 0x8a, 0xd3, 0x15, 0x3f, 0xaf, 0x0e, 0x06, 0x94, 0x30, 0x26, 0x95, 0xaa, + 0xd9, 0xd9, 0x24, 0x46, 0x52, 0x8f, 0x78, 0xb4, 0xc2, 0x83, 0x5f, 0x4a, 0xe6, 0x49, 0x4e, 0xad, + 0x5a, 0x82, 0x6a, 0xdc, 0xa9, 0x8e, 0xb1, 0x8a, 0xf0, 0x1c, 0xd4, 0xb7, 0xfd, 0x60, 0x98, 0xf4, + 0x3e, 0x84, 0xd9, 0x38, 0x8c, 0xdb, 0xba, 0x7f, 0x75, 0x98, 0xf9, 0xa0, 0xae, 0x5a, 0xb4, 0x07, + 0xb5, 0xf4, 0x9e, 0x41, 0xcf, 0x0b, 0xa5, 0xcb, 0xdf, 0x73, 0xd6, 0x8b, 0xdb, 0xca, 0xe2, 0x45, + 0xb8, 0x84, 0x8e, 0x60, 0x3e, 0x6f, 0x2a, 0xf4, 0xaa, 0xb8, 0xbb, 0xf8, 0x14, 0x59, 0x2b, 0x77, + 0xac, 0x4e, 0x57, 0xee, 0x41, 0x2d, 0xf5, 0xc8, 0x14, 0x42, 0x79, 0xb7, 0x4d, 0x21, 0x74, 0xcd, + 0x6a, 0xb8, 0x84, 0x7e, 0x00, 0xba, 0xfe, 0xdf, 0xa3, 0x76, 0x61, 0xff, 0x54, 0x77, 0x59, 0x9d, + 0x3b, 0xd7, 0xe7, 0x68, 0xc5, 0xaf, 0xa6, 0xd3, 0xca, 0x98, 0x66, 0x3a, 0xad, 0xac, 0x8f, 0x70, + 0x09, 0x7d, 0x01, 0x5d, 0x58, 0x04, 0x35, 0x0b, 0x3b, 0x26, 0xcc, 0x64, 0x3d, 0xbd, 0xa1, 0x22, + 0x19, 0xb7, 0xb6, 0x75, 0x76, 0xd1, 0xd0, 0xce, 0x2f, 0x1a, 0xda, 0xbf, 0x8b, 0x86, 0xf6, 0xeb, + 0xb2, 0x51, 0x3a, 0xbf, 0x6c, 0x94, 0xfe, 0x5c, 0x36, 0x4a, 0xdf, 0xde, 0x0e, 0x7d, 0xee, 0x45, + 0xfd, 0xb6, 0x1b, 0x1e, 0x76, 0x26, 0x06, 0xad, 0x1c, 0x93, 0x40, 0xdc, 0x05, 0x2c, 0xfd, 0x16, + 0x88, 0xed, 0xd9, 0x91, 0x5f, 0x02, 0xfd, 0xaa, 0xfc, 0x79, 0xf3, 0x3f, 0x00, 0x00, 0xff, 0xff, + 0x97, 0xd9, 0x62, 0x0b, 0x36, 0x08, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1768,12 +1768,10 @@ func (m *GetLeaderResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Leader) > 0 { - i -= len(m.Leader) - copy(dAtA[i:], m.Leader) - i = encodeVarintCosigner(dAtA, i, uint64(len(m.Leader))) + if m.Leader != 0 { + i = encodeVarintCosigner(dAtA, i, uint64(m.Leader)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -2086,9 +2084,8 @@ func (m *GetLeaderResponse) Size() (n int) { } var l int _ = l - l = len(m.Leader) - if l > 0 { - n += 1 + l + sovCosigner(uint64(l)) + if m.Leader != 0 { + n += 1 + sovCosigner(uint64(m.Leader)) } return n } @@ -3731,10 +3728,10 @@ func (m *GetLeaderResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { + if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Leader", wireType) } - var stringLen uint64 + m.Leader = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowCosigner @@ -3744,24 +3741,11 @@ func (m *GetLeaderResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + m.Leader |= int32(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthCosigner - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthCosigner - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Leader = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipCosigner(dAtA[iNdEx:]) diff --git a/signer/raft_events.go b/signer/raft_events.go index ac6ed566..e3c70cd6 100644 --- a/signer/raft_events.go +++ b/signer/raft_events.go @@ -2,12 +2,6 @@ package signer import ( "encoding/json" - "errors" - "time" - - "github.com/strangelove-ventures/horcrux/signer/proto" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) const ( @@ -46,43 +40,3 @@ func (f *fsm) handleLSSEvent(value string) { _ = f.thresholdValidator.SaveLastSignedState(lss.ChainID, lss.SignStateConsensus) _ = f.cosigner.SaveLastSignedState(lss.ChainID, lss.SignStateConsensus) } - -func (s *RaftStore) getLeaderGRPCClient() (proto.CosignerClient, *grpc.ClientConn, error) { - var leader string - for i := 0; i < 30; i++ { - leader = string(s.GetLeader()) - if leader != "" { - break - } - time.Sleep(100 * time.Millisecond) - } - if leader == "" { - totalRaftLeaderElectiontimeout.Inc() - return nil, nil, errors.New("timed out waiting for leader election to complete") - } - conn, err := grpc.Dial(leader, grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - return nil, nil, err - } - return proto.NewCosignerClient(conn), conn, nil -} - -func (s *RaftStore) SignBlock(req CosignerSignBlockRequest) (*CosignerSignBlockResponse, error) { - client, conn, err := s.getLeaderGRPCClient() - if err != nil { - return nil, err - } - defer conn.Close() - context, cancelFunc := getContext() - defer cancelFunc() - res, err := client.SignBlock(context, &proto.SignBlockRequest{ - ChainID: req.ChainID, - Block: req.Block.ToProto(), - }) - if err != nil { - return nil, err - } - return &CosignerSignBlockResponse{ - Signature: res.GetSignature(), - }, nil -} diff --git a/signer/raft_store.go b/signer/raft_store.go index e89e7a5c..ba0b5a50 100644 --- a/signer/raft_store.go +++ b/signer/raft_store.go @@ -15,6 +15,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "sync" "time" @@ -287,11 +288,19 @@ func (s *RaftStore) IsLeader() bool { return s.raft.State() == raft.Leader } -func (s *RaftStore) GetLeader() raft.ServerAddress { +func (s *RaftStore) GetLeader() int { if s == nil || s.raft == nil { - return "" + return -1 } - return s.raft.Leader() + _, leaderID := s.raft.LeaderWithID() + if leaderID == "" { + return -1 + } + id, err := strconv.Atoi(string(leaderID)) + if err != nil { + return -1 + } + return id } func (s *RaftStore) ShareSigned(lss ChainSignStateConsensus) error { diff --git a/signer/remote_cosigner.go b/signer/remote_cosigner.go index 3a1f26da..d506f2bf 100644 --- a/signer/remote_cosigner.go +++ b/signer/remote_cosigner.go @@ -19,26 +19,30 @@ var _ Cosigner = &RemoteCosigner{} type RemoteCosigner struct { id int address string + + client proto.CosignerClient } // NewRemoteCosigner returns a newly initialized RemoteCosigner -func NewRemoteCosigner(id int, address string) *RemoteCosigner { +func NewRemoteCosigner(id int, address string) (*RemoteCosigner, error) { + client, err := getGRPCClient(address) + if err != nil { + return nil, err + } cosigner := &RemoteCosigner{ id: id, address: address, + client: client, } - return cosigner + + return cosigner, nil } const ( rpcTimeout = 4 * time.Second ) -func getContext() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), rpcTimeout) -} - // GetID returns the ID of the remote cosigner // Implements the cosigner interface func (cosigner *RemoteCosigner) GetID() int { @@ -63,19 +67,19 @@ func (cosigner *RemoteCosigner) VerifySignature(_ string, _, _ []byte) bool { return false } -func (cosigner *RemoteCosigner) getGRPCClient() (proto.CosignerClient, *grpc.ClientConn, error) { +func getGRPCClient(address string) (proto.CosignerClient, error) { var grpcAddress string - url, err := url.Parse(cosigner.address) + url, err := url.Parse(address) if err != nil { - grpcAddress = cosigner.address + grpcAddress = address } else { grpcAddress = url.Host } conn, err := grpc.Dial(grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { - return nil, nil, err + return nil, err } - return proto.NewCosignerClient(conn), conn, nil + return proto.NewCosignerClient(conn), nil } // Implements the cosigner interface @@ -83,17 +87,12 @@ func (cosigner *RemoteCosigner) GetNonces( ctx context.Context, uuids []uuid.UUID, ) (CosignerUUIDNoncesMultiple, error) { - client, conn, err := cosigner.getGRPCClient() - if err != nil { - return nil, err - } - defer conn.Close() us := make([][]byte, len(uuids)) for i, u := range uuids { us[i] = make([]byte, 16) copy(us[i], u[:]) } - res, err := client.GetNonces(ctx, &proto.GetNoncesRequest{ + res, err := cosigner.client.GetNonces(ctx, &proto.GetNoncesRequest{ Uuids: us, }) if err != nil { @@ -113,13 +112,7 @@ func (cosigner *RemoteCosigner) GetNonces( func (cosigner *RemoteCosigner) SetNoncesAndSign( ctx context.Context, req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error) { - client, conn, err := cosigner.getGRPCClient() - if err != nil { - return nil, err - } - defer conn.Close() - - res, err := client.SetNoncesAndSign(ctx, &proto.SetNoncesAndSignRequest{ + res, err := cosigner.client.SetNoncesAndSign(ctx, &proto.SetNoncesAndSignRequest{ Uuid: req.Nonces.UUID[:], ChainID: req.ChainID, Nonces: req.Nonces.Nonces.toProto(), @@ -135,3 +128,19 @@ func (cosigner *RemoteCosigner) SetNoncesAndSign( Signature: res.GetSignature(), }, nil } + +func (cosigner *RemoteCosigner) Sign( + ctx context.Context, + req CosignerSignBlockRequest, +) (*CosignerSignBlockResponse, error) { + res, err := cosigner.client.SignBlock(ctx, &proto.SignBlockRequest{ + ChainID: req.ChainID, + Block: req.Block.ToProto(), + }) + if err != nil { + return nil, err + } + return &CosignerSignBlockResponse{ + Signature: res.GetSignature(), + }, nil +} diff --git a/signer/threshold_validator.go b/signer/threshold_validator.go index 4d445161..2df08f7a 100644 --- a/signer/threshold_validator.go +++ b/signer/threshold_validator.go @@ -32,7 +32,7 @@ type ThresholdValidator struct { myCosigner *LocalCosigner // peer cosigners - peerCosigners []Cosigner + peerCosigners Cosigners leader Leader @@ -545,6 +545,59 @@ func (pv *ThresholdValidator) waitForPeerNonces( mu.Unlock() } +func (pv *ThresholdValidator) proxyIfNecessary(ctx context.Context, chainID string, block Block) (bool, []byte, time.Time, error) { + height, round, step, stamp := block.Height, block.Round, block.Step, block.Timestamp + + if pv.leader.IsLeader() { + return false, nil, time.Time{}, nil + } + + leader := pv.leader.GetLeader() + + // TODO is there a better way than to poll during leader election? + for i := 0; i < 500 && leader == -1; i++ { + time.Sleep(10 * time.Millisecond) + leader = pv.leader.GetLeader() + } + + if leader == -1 { + return true, nil, stamp, fmt.Errorf("timed out waiting for raft leader") + } + + if leader == pv.myCosigner.GetID() { + return false, nil, time.Time{}, nil + } + + pv.logger.Debug("I am not the leader. Proxying request to the leader", + "chain_id", chainID, + "height", height, + "round", round, + "step", step, + ) + totalNotRaftLeader.Inc() + + cosignerLeader := pv.peerCosigners.GetByID(leader) + if cosignerLeader == nil { + return true, nil, stamp, fmt.Errorf("failed to find cosigner with id %d", leader) + } + + signRes, err := cosignerLeader.(*RemoteCosigner).Sign(ctx, CosignerSignBlockRequest{ + ChainID: chainID, + Block: &block, + }) + if err != nil { + if _, ok := err.(*cometrpcjsontypes.RPCError); ok { + rpcErrUnwrapped := err.(*cometrpcjsontypes.RPCError).Data + // Need to return BeyondBlockError after proxy since the error type will be lost over RPC + if len(rpcErrUnwrapped) > 33 && rpcErrUnwrapped[:33] == "Progress already started on block" { + return true, nil, stamp, &BeyondBlockError{msg: rpcErrUnwrapped} + } + } + return true, nil, stamp, err + } + return true, signRes.Signature, stamp, nil +} + func (pv *ThresholdValidator) Sign(ctx context.Context, chainID string, block Block) ([]byte, time.Time, error) { height, round, step, stamp, signBytes := block.Height, block.Round, block.Step, block.Timestamp, block.SignBytes @@ -554,29 +607,9 @@ func (pv *ThresholdValidator) Sign(ctx context.Context, chainID string, block Bl // Only the leader can execute this function. Followers can handle the requests, // but they just need to proxy the request to the raft leader - if !pv.leader.IsLeader() { - pv.logger.Debug("I am not the leader. Proxying request to the leader", - "chain_id", chainID, - "height", height, - "round", round, - "step", step, - ) - totalNotRaftLeader.Inc() - signRes, err := pv.leader.SignBlock(CosignerSignBlockRequest{ - ChainID: chainID, - Block: &block, - }) - if err != nil { - if _, ok := err.(*cometrpcjsontypes.RPCError); ok { - rpcErrUnwrapped := err.(*cometrpcjsontypes.RPCError).Data - // Need to return BeyondBlockError after proxy since the error type will be lost over RPC - if len(rpcErrUnwrapped) > 33 && rpcErrUnwrapped[:33] == "Progress already started on block" { - return nil, stamp, &BeyondBlockError{msg: rpcErrUnwrapped} - } - } - return nil, stamp, err - } - return signRes.Signature, stamp, nil + isProxied, proxySig, proxyStamp, err := pv.proxyIfNecessary(ctx, chainID, block) + if isProxied { + return proxySig, proxyStamp, err } totalRaftLeader.Inc() @@ -799,5 +832,14 @@ func (pv *ThresholdValidator) Sign(ctx context.Context, chainID string, block Bl timeSignBlockSec := timeSignBlock.Seconds() timedSignBlockLag.Observe(timeSignBlockSec) + pv.logger.Info( + "Signed", + "chain_id", chainID, + "height", height, + "round", round, + "type", signType(step), + "duration_ms", timeSignBlock.Round(time.Millisecond), + ) + return signature, stamp, nil } diff --git a/test/horcrux_test.go b/test/horcrux_test.go index 3feab1f2..c7fc77c1 100644 --- a/test/horcrux_test.go +++ b/test/horcrux_test.go @@ -138,12 +138,11 @@ func TestUpgradeValidatorToHorcrux(t *testing.T) { } // TestDownedSigners2of3 tests taking down 2 nodes at a time in the 2/3 threshold horcrux cluster for a period of time. - func TestDownedSigners2of3(t *testing.T) { ctx := context.Background() const ( - totalValidators = 2 + totalValidators = 4 totalSigners = 3 threshold = 2 totalSentries = 3 @@ -186,7 +185,7 @@ func TestDownedSigners3of5(t *testing.T) { ctx := context.Background() const ( - totalValidators = 2 + totalValidators = 4 totalSigners = 5 threshold = 3 totalSentries = 3 @@ -273,7 +272,7 @@ func TestLeaderElection2of3(t *testing.T) { for _, s := range cosigners { s := s eg.Go(func() error { - return pollForLeader(ctx, t, s, cosigner.Name()+":"+signerPort) + return pollForLeader(ctx, t, s, cosigner.Index+1) }) } if err := eg.Wait(); err == nil { @@ -282,7 +281,7 @@ func TestLeaderElection2of3(t *testing.T) { // electing a specific leader can fail, but this is okay as long as all nodes agree on one leader. // will retry electing the specific leader in the next iteration. - var commonLeader string + var commonLeader int for i, s := range cosigners { leader, err := getLeader(ctx, s) require.NoErrorf(t, err, "failed to get leader from signer: %s", s.Name()) diff --git a/test/validator.go b/test/validator.go index 9334d1bb..45ee25eb 100644 --- a/test/validator.go +++ b/test/validator.go @@ -121,9 +121,9 @@ func startChains( } // modifyGenesisStrictUptime modifies the genesis file to have a strict uptime slashing window. -// 10 block window, 90% signed blocks required, so more than 1 missed block in 10 blocks will slash and jail the validator. +// 10 block window, 80% signed blocks required, so more than 2 missed blocks in 10 blocks will slash and jail the validator. func modifyGenesisStrictUptime(cc ibc.ChainConfig, b []byte) ([]byte, error) { - return modifyGenesisSlashingUptime(10, 0.9)(cc, b) + return modifyGenesisSlashingUptime(10, 0.8)(cc, b) } // modifyGenesisSlashingUptime modifies the genesis slashing period parameters. @@ -244,7 +244,7 @@ func requireHealthyValidator(t *testing.T, referenceNode *cosmos.ChainNode, vali require.False(t, signingInfo.Tombstoned) require.Equal(t, time.Unix(0, 0).UTC(), signingInfo.JailedUntil) - require.Zero(t, signingInfo.MissedBlocksCounter) + require.LessOrEqual(t, signingInfo.MissedBlocksCounter, int64(1)) } // transferLeadership elects a new raft leader. @@ -254,7 +254,7 @@ func transferLeadership(ctx context.Context, cosigner *cosmos.SidecarProcess) er } // pollForLeader polls for the given cosigner to become the leader. -func pollForLeader(ctx context.Context, t *testing.T, cosigner *cosmos.SidecarProcess, expectedLeader string) error { +func pollForLeader(ctx context.Context, t *testing.T, cosigner *cosmos.SidecarProcess, expectedLeader int) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -265,7 +265,7 @@ func pollForLeader(ctx context.Context, t *testing.T, cosigner *cosmos.SidecarPr select { case <-ticker.C: leader, err := getLeader(ctx, cosigner) - t.Logf("{%s} => current leader: {%s}, expected leader: {%s}", cosigner.Name(), leader, expectedLeader) + t.Logf("{%s} => current leader: {%d}, expected leader: {%d}", cosigner.Name(), leader, expectedLeader) if err != nil { return fmt.Errorf("failed to get leader from cosigner: %s - %w", cosigner.Name(), err) } @@ -279,10 +279,10 @@ func pollForLeader(ctx context.Context, t *testing.T, cosigner *cosmos.SidecarPr } // getLeader returns the current raft leader. -func getLeader(ctx context.Context, cosigner *cosmos.SidecarProcess) (string, error) { +func getLeader(ctx context.Context, cosigner *cosmos.SidecarProcess) (int, error) { ports, err := cosigner.GetHostPorts(ctx, signerPortDocker) if err != nil { - return "", err + return -1, err } grpcAddress := ports[0] conn, err := grpc.Dial(grpcAddress, @@ -290,7 +290,7 @@ func getLeader(ctx context.Context, cosigner *cosmos.SidecarProcess) (string, er grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), ) if err != nil { - return "", fmt.Errorf("dialing failed: %w", err) + return -1, fmt.Errorf("dialing failed: %w", err) } defer conn.Close() @@ -301,7 +301,7 @@ func getLeader(ctx context.Context, cosigner *cosmos.SidecarProcess) (string, er res, err := grpcClient.GetLeader(ctx, &proto.GetLeaderRequest{}) if err != nil { - return "", err + return -1, err } - return res.GetLeader(), nil + return int(res.GetLeader()), nil }