Skip to content

Commit

Permalink
Merge pull request etcd-io#10727 from jingyih/learner_part2
Browse files Browse the repository at this point in the history
*: support raft learner in etcd - part 2
  • Loading branch information
xiang90 authored May 15, 2019
2 parents 42acdfc + 23f1d02 commit d4cdbb1
Show file tree
Hide file tree
Showing 25 changed files with 728 additions and 67 deletions.
2 changes: 2 additions & 0 deletions .words
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,5 @@ PermitWithoutStream
__lostleader
ErrConnClosing
unfreed
grpcAddr
clientURLs
9 changes: 6 additions & 3 deletions clientv3/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package clientv3

import (
"context"
"errors"

pb "go.etcd.io/etcd/v3/etcdserver/etcdserverpb"
"go.etcd.io/etcd/v3/pkg/types"
Expand Down Expand Up @@ -133,6 +132,10 @@ func (c *cluster) MemberList(ctx context.Context) (*MemberListResponse, error) {
}

func (c *cluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) {
// TODO: implement
return nil, errors.New("not implemented")
r := &pb.MemberPromoteRequest{ID: id}
resp, err := c.remote.MemberPromote(ctx, r, c.callOpts...)
if err != nil {
return nil, toErr(ctx, err)
}
return (*MemberPromoteResponse)(resp), nil
}
57 changes: 44 additions & 13 deletions clientv3/integration/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package integration

import (
"context"
"fmt"
"reflect"
"strings"
"testing"
Expand Down Expand Up @@ -204,27 +203,59 @@ func TestMemberAddForLearner(t *testing.T) {
t.Errorf("Added a member as learner, got resp.Member.IsLearner = %v", resp.Member.IsLearner)
}

numOfLearners, err := getNumberOfLearners(clus)
if err != nil {
t.Fatalf("failed to get the number of learners in cluster: %v", err)
numberOfLearners := 0
for _, m := range resp.Members {
if m.IsLearner {
numberOfLearners++
}
}
if numOfLearners != 1 {
t.Errorf("Added 1 learner node to cluster, got %d", numOfLearners)
if numberOfLearners != 1 {
t.Errorf("Added 1 learner node to cluster, got %d", numberOfLearners)
}
}

// getNumberOfLearners return the number of learner nodes in cluster using MemberList API
func getNumberOfLearners(clus *integration.ClusterV3) (int, error) {
cli := clus.RandClient()
resp, err := cli.MemberList(context.Background())
func TestMemberPromoteForLearner(t *testing.T) {
// TODO test not ready learner promotion.
defer testutil.AfterTest(t)

clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
// TODO change the random client to client that talk to leader directly.
capi := clus.RandClient()

urls := []string{"http://127.0.0.1:1234"}
memberAddResp, err := capi.MemberAddAsLearner(context.Background(), urls)
if err != nil {
return 0, fmt.Errorf("failed to list member %v", err)
t.Fatalf("failed to add member %v", err)
}

if !memberAddResp.Member.IsLearner {
t.Fatalf("Added a member as learner, got resp.Member.IsLearner = %v", memberAddResp.Member.IsLearner)
}
learnerID := memberAddResp.Member.ID

numberOfLearners := 0
for _, m := range resp.Members {
for _, m := range memberAddResp.Members {
if m.IsLearner {
numberOfLearners++
}
}
if numberOfLearners != 1 {
t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners)
}

memberPromoteResp, err := capi.MemberPromote(context.Background(), learnerID)
if err != nil {
t.Fatalf("failed to promote member: %v", err)
}

numberOfLearners = 0
for _, m := range memberPromoteResp.Members {
if m.IsLearner {
numberOfLearners++
}
}
return numberOfLearners, nil
if numberOfLearners != 0 {
t.Errorf("learner promoted, expect 0 learner, got %d", numberOfLearners)
}
}
80 changes: 80 additions & 0 deletions clientv3/integration/kv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -971,3 +971,83 @@ func TestKVLargeRequests(t *testing.T) {
clus.Terminate(t)
}
}

// TestKVForLearner ensures learner member only accepts serializable read request.
func TestKVForLearner(t *testing.T) {
defer testutil.AfterTest(t)

clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)

// we have to add and launch learner member after initial cluster was created, because
// bootstrapping a cluster with learner member is not supported.
clus.AddAndLaunchLearnerMember(t)

learners, err := clus.GetLearnerMembers()
if err != nil {
t.Fatalf("failed to get the learner members in cluster: %v", err)
}
if len(learners) != 1 {
t.Fatalf("added 1 learner to cluster, got %d", len(learners))
}

if len(clus.Members) != 4 {
t.Fatalf("expecting 4 members in cluster after adding the learner member, got %d", len(clus.Members))
}
// note:
// 1. clus.Members[3] is the newly added learner member, which was appended to clus.Members
// 2. we are using member's grpcAddr instead of clientURLs as the endpoint for clientv3.Config,
// because the implementation of integration test has diverged from embed/etcd.go.
learnerEp := clus.Members[3].GRPCAddr()
cfg := clientv3.Config{
Endpoints: []string{learnerEp},
DialTimeout: 5 * time.Second,
DialOptions: []grpc.DialOption{grpc.WithBlock()},
}
// this client only has endpoint of the learner member
cli, err := clientv3.New(cfg)
if err != nil {
t.Fatalf("failed to create clientv3: %v", err)
}
defer cli.Close()

// TODO: expose servers's ReadyNotify() in test and use it instead.
// waiting for learner member to catch up applying the config change entries in raft log.
time.Sleep(3 * time.Second)

tests := []struct {
op clientv3.Op
wErr bool
}{
{
op: clientv3.OpGet("foo", clientv3.WithSerializable()),
wErr: false,
},
{
op: clientv3.OpGet("foo"),
wErr: true,
},
{
op: clientv3.OpPut("foo", "bar"),
wErr: true,
},
{
op: clientv3.OpDelete("foo"),
wErr: true,
},
{
op: clientv3.OpTxn([]clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("foo"), "=", 0)}, nil, nil),
wErr: true,
},
}

for idx, test := range tests {
_, err := cli.Do(context.TODO(), test.op)
if err != nil && !test.wErr {
t.Errorf("%d: expect no error, got %v", idx, err)
}
if err == nil && test.wErr {
t.Errorf("%d: expect error, got nil", idx)
}
}
}
2 changes: 1 addition & 1 deletion etcdctl/ctlv3/command/ep_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func newEpStatusCommand() *cobra.Command {
Use: "status",
Short: "Prints out the status of endpoints specified in `--endpoints` flag",
Long: `When --write-out is set to simple, this command prints out comma-separated status lists for each endpoint.
The items in the lists are endpoint, ID, version, db size, is leader, raft term, raft index.
The items in the lists are endpoint, ID, version, db size, is leader, is learner, raft term, raft index, raft applied index, errors.
`,
Run: epStatusCommandFunc,
}
Expand Down
35 changes: 35 additions & 0 deletions etcdctl/ctlv3/command/member_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func NewMemberCommand() *cobra.Command {
mc.AddCommand(NewMemberRemoveCommand())
mc.AddCommand(NewMemberUpdateCommand())
mc.AddCommand(NewMemberListCommand())
mc.AddCommand(NewMemberPromoteCommand())

return mc
}
Expand Down Expand Up @@ -100,6 +101,20 @@ The items in the lists are ID, Status, Name, Peer Addrs, Client Addrs, Is Learne
return cc
}

// NewMemberPromoteCommand returns the cobra command for "member promote".
func NewMemberPromoteCommand() *cobra.Command {
cc := &cobra.Command{
Use: "promote <memberID>",
Short: "Promotes a non-voting member in the cluster",
Long: `Promotes a non-voting learner member to a voting one in the cluster.
`,

Run: memberPromoteCommandFunc,
}

return cc
}

// memberAddCommandFunc executes the "member add" command.
func memberAddCommandFunc(cmd *cobra.Command, args []string) {
if len(args) < 1 {
Expand Down Expand Up @@ -238,3 +253,23 @@ func memberListCommandFunc(cmd *cobra.Command, args []string) {

display.MemberList(*resp)
}

// memberPromoteCommandFunc executes the "member promote" command.
func memberPromoteCommandFunc(cmd *cobra.Command, args []string) {
if len(args) != 1 {
ExitWithError(ExitBadArgs, fmt.Errorf("member ID is not provided"))
}

id, err := strconv.ParseUint(args[0], 16, 64)
if err != nil {
ExitWithError(ExitBadArgs, fmt.Errorf("bad member ID arg (%v), expecting ID in Hex", err))
}

ctx, cancel := commandCtx(cmd)
resp, err := mustClientFromCmd(cmd).MemberPromote(ctx, id)
cancel()
if err != nil {
ExitWithError(ExitError, err)
}
display.MemberPromote(id, *resp)
}
5 changes: 4 additions & 1 deletion etcdctl/ctlv3/command/printer.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type printer interface {
MemberAdd(v3.MemberAddResponse)
MemberRemove(id uint64, r v3.MemberRemoveResponse)
MemberUpdate(id uint64, r v3.MemberUpdateResponse)
MemberPromote(id uint64, r v3.MemberPromoteResponse)
MemberList(v3.MemberListResponse)

EndpointHealth([]epHealth)
Expand Down Expand Up @@ -194,14 +195,16 @@ func makeEndpointHealthTable(healthList []epHealth) (hdr []string, rows [][]stri
}

func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]string) {
hdr = []string{"endpoint", "ID", "version", "db size", "is leader", "raft term", "raft index", "raft applied index", "errors"}
hdr = []string{"endpoint", "ID", "version", "db size", "is leader", "is learner", "raft term",
"raft index", "raft applied index", "errors"}
for _, status := range statusList {
rows = append(rows, []string{
status.Ep,
fmt.Sprintf("%x", status.Resp.Header.MemberId),
status.Resp.Version,
humanize.Bytes(uint64(status.Resp.DbSize)),
fmt.Sprint(status.Resp.Leader == status.Resp.Header.MemberId),
fmt.Sprint(status.Resp.IsLearner),
fmt.Sprint(status.Resp.RaftTerm),
fmt.Sprint(status.Resp.RaftIndex),
fmt.Sprint(status.Resp.RaftAppliedIndex),
Expand Down
1 change: 1 addition & 0 deletions etcdctl/ctlv3/command/printer_fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ func (p *fieldsPrinter) EndpointStatus(eps []epStatus) {
fmt.Printf("\"Version\" : %q\n", ep.Resp.Version)
fmt.Println(`"DBSize" :`, ep.Resp.DbSize)
fmt.Println(`"Leader" :`, ep.Resp.Leader)
fmt.Println(`"IsLearner" :`, ep.Resp.IsLearner)
fmt.Println(`"RaftIndex" :`, ep.Resp.RaftIndex)
fmt.Println(`"RaftTerm" :`, ep.Resp.RaftTerm)
fmt.Println(`"RaftAppliedIndex" :`, ep.Resp.RaftAppliedIndex)
Expand Down
4 changes: 4 additions & 0 deletions etcdctl/ctlv3/command/printer_simple.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ func (s *simplePrinter) MemberUpdate(id uint64, r v3.MemberUpdateResponse) {
fmt.Printf("Member %16x updated in cluster %16x\n", id, r.Header.ClusterId)
}

func (s *simplePrinter) MemberPromote(id uint64, r v3.MemberPromoteResponse) {
fmt.Printf("Member %16x promoted in cluster %16x\n", id, r.Header.ClusterId)
}

func (s *simplePrinter) MemberList(resp v3.MemberListResponse) {
_, rows := makeMemberListTable(resp)
for _, row := range rows {
Expand Down
Loading

0 comments on commit d4cdbb1

Please sign in to comment.