From 2189d6937b146509e333ba5ff6f68fce4d8b1868 Mon Sep 17 00:00:00 2001 From: xiaojingchen Date: Fri, 12 Oct 2018 16:31:31 +0800 Subject: [PATCH] tidb graceful upgrade (#112) * tidb graceful upgrade --- pkg/apis/pingcap.com/v1alpha1/types.go | 9 +- pkg/controller/tidb_control.go | 52 +++++- .../tidbcluster/tidb_cluster_controller.go | 6 +- pkg/manager/member/tidb_member_manager.go | 7 +- pkg/manager/member/tidb_upgrader.go | 51 +++++- pkg/manager/member/tidb_upgrader_test.go | 169 +++++++++++++++--- pkg/manager/member/utils.go | 4 + 7 files changed, 255 insertions(+), 43 deletions(-) diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index cb8f6678146..9ec6abb9de8 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -197,10 +197,11 @@ type PDFailureMember struct { // TiDBStatus is TiDB status type TiDBStatus struct { - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Members map[string]TiDBMember `json:"members,omitempty"` - FailureMembers map[string]TiDBFailureMember `json:"failureMembers,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Members map[string]TiDBMember `json:"members,omitempty"` + FailureMembers map[string]TiDBFailureMember `json:"failureMembers,omitempty"` + ResignDDLOwnerRetryCount int32 `json:"resignDDLOwnerRetryCount,omitempty"` } // TiDBMember is TiDB member diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 61046f645c3..15b9dd0f62e 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -21,10 +21,17 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" ) +const ( + // NotDDLOwnerError is the error message which was returned when the tidb node is not a ddl owner + NotDDLOwnerError = "This node is not a ddl owner, can't be resigned." +) + // TiDBControlInterface is the interface that knows how to manage tidb peers type TiDBControlInterface interface { // GetHealth returns tidb's health info GetHealth(tc *v1alpha1.TidbCluster) map[string]bool + // ResignDDLOwner resigns the ddl owner of tidb, if the tidb node is not a ddl owner returns (true,nil),else returns (false,err) + ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) } // defaultTiDBControl is default implementation of TiDBControlInterface. @@ -45,7 +52,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo result := map[string]bool{} for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) - url := fmt.Sprintf("http://%s.%s-tidb-peer.%s:10080/status", hostName, tcName, ns) + url := fmt.Sprintf("http://%s.%s.%s:10080/status", hostName, TiDBPeerMemberName(tcName), ns) _, err := tdc.getBodyOK(url) if err != nil { result[hostName] = false @@ -56,6 +63,31 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo return result } +func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) { + tcName := tc.GetName() + ns := tc.GetNamespace() + + hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) + url := fmt.Sprintf("http://%s.%s.%s:10080/ddl/owner/resign", hostName, TiDBPeerMemberName(tcName), ns) + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return false, err + } + res, err := tdc.httpClient.Do(req) + if err != nil { + return false, err + } + defer DeferClose(res.Body, &err) + if res.StatusCode == http.StatusOK { + return false, nil + } + err2 := readErrorBody(res.Body) + if err2.Error() == NotDDLOwnerError { + return true, nil + } + return false, err2 +} + func (tdc *defaultTiDBControl) getBodyOK(apiURL string) ([]byte, error) { res, err := tdc.httpClient.Get(apiURL) if err != nil { @@ -76,7 +108,9 @@ func (tdc *defaultTiDBControl) getBodyOK(apiURL string) ([]byte, error) { // FakeTiDBControl is a fake implementation of TiDBControlInterface. type FakeTiDBControl struct { - healthInfo map[string]bool + healthInfo map[string]bool + resignDDLOwnerError error + notDDLOwner bool } // NewFakeTiDBControl returns a FakeTiDBControl instance @@ -89,6 +123,20 @@ func (ftd *FakeTiDBControl) SetHealth(healthInfo map[string]bool) { ftd.healthInfo = healthInfo } +// NotDDLOwner sets whether the tidb is the ddl owner +func (ftd *FakeTiDBControl) NotDDLOwner(notDDLOwner bool) { + ftd.notDDLOwner = notDDLOwner +} + +// SetResignDDLOwner sets error of resign ddl owner for FakeTiDBControl +func (ftd *FakeTiDBControl) SetResignDDLOwnerError(err error) { + ftd.resignDDLOwnerError = err +} + func (ftd *FakeTiDBControl) GetHealth(_ *v1alpha1.TidbCluster) map[string]bool { return ftd.healthInfo } + +func (ftd *FakeTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) { + return ftd.notDDLOwner, ftd.resignDDLOwnerError +} diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index b9d6ea2f2ae..bd426dd4472 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -100,11 +100,11 @@ func NewController( pdScaler := mm.NewPDScaler(pdControl, pvcInformer.Lister(), pvcControl) tikvScaler := mm.NewTiKVScaler(pdControl, pvcInformer.Lister(), pvcControl) pdFailover := mm.NewPDFailover(cli, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister()) - pdUpgrader := mm.NewPDUpgrader(pdControl, podControl, podInformer.Lister()) tikvFailover := mm.NewTiKVFailover(pdControl) - tikvUpgrader := mm.NewTiKVUpgrader(pdControl, podControl, podInformer.Lister()) - tidbUpgrader := mm.NewTiDBUpgrader() tidbFailover := mm.NewTiDBFailover(tidbFailoverPeriod) + pdUpgrader := mm.NewPDUpgrader(pdControl, podControl, podInformer.Lister()) + tikvUpgrader := mm.NewTiKVUpgrader(pdControl, podControl, podInformer.Lister()) + tidbUpgrader := mm.NewTiDBUpgrader(tidbControl) tcc := &Controller{ kubeClient: kubeCli, diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 0fde8ea7d06..f39d1b82958 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -135,7 +135,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid return err } - if !templateEqual(newTiDBSet.Spec.Template, oldTiDBSet.Spec.Template) { + if !templateEqual(newTiDBSet.Spec.Template, oldTiDBSet.Spec.Template) || tc.Status.TiDB.Phase == v1alpha1.UpgradePhase { if err := tmm.tidbUpgrader.Upgrade(tc, oldTiDBSet, newTiDBSet); err != nil { return err } @@ -155,6 +155,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid set := *oldTiDBSet set.Spec.Template = newTiDBSet.Spec.Template *set.Spec.Replicas = *newTiDBSet.Spec.Replicas + set.Spec.UpdateStrategy = newTiDBSet.Spec.UpdateStrategy err := SetLastAppliedConfigAnnotation(&set) if err != nil { return err @@ -322,7 +323,9 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }, ServiceName: controller.TiDBPeerMemberName(tcName), PodManagementPolicy: apps.ParallelPodManagement, - UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &apps.RollingUpdateStatefulSetStrategy{Partition: func() *int32 { r := tc.TiDBRealReplicas(); return &r }()}, + }, }, } return tidbSet diff --git a/pkg/manager/member/tidb_upgrader.go b/pkg/manager/member/tidb_upgrader.go index d2f51132b77..5d35517348c 100644 --- a/pkg/manager/member/tidb_upgrader.go +++ b/pkg/manager/member/tidb_upgrader.go @@ -15,26 +15,67 @@ package member import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" apps "k8s.io/api/apps/v1beta1" ) -type tidbUpgrader struct{} +const ( + // MaxResignDDLOwnerCount is the max regign DDL owner count + MaxResignDDLOwnerCount = 3 +) + +type tidbUpgrader struct { + tidbControl controller.TiDBControlInterface +} // NewTiDBUpgrader returns a tidb Upgrader -func NewTiDBUpgrader() Upgrader { - return &tidbUpgrader{} +func NewTiDBUpgrader(tidbControl controller.TiDBControlInterface) Upgrader { + return &tidbUpgrader{tidbControl: tidbControl} } func (tdu *tidbUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + if tc.Status.PD.Phase == v1alpha1.UpgradePhase || tc.Status.TiKV.Phase == v1alpha1.UpgradePhase { _, podSpec, err := GetLastAppliedConfig(oldSet) if err != nil { return err } newSet.Spec.Template.Spec = *podSpec - } else { - tc.Status.TiDB.Phase = v1alpha1.UpgradePhase + return nil } + + tc.Status.TiDB.Phase = v1alpha1.UpgradePhase + setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition) + + if tc.Status.TiDB.StatefulSet.CurrentReplicas == 0 { + return controller.RequeueErrorf("tidbcluster: [%s/%s]'s tidb doesn't have old version pod to upgrade", ns, tcName) + } + + if !tc.TiDBAllPodsStarted() { + return controller.RequeueErrorf("tidbcluster: [%s/%s]'s tidb pods are not all created", ns, tcName) + } + + for i := tc.Status.TiDB.StatefulSet.Replicas; i > tc.Status.TiDB.StatefulSet.CurrentReplicas; i-- { + if member, exist := tc.Status.TiDB.Members[tidbPodName(tcName, i-1)]; !exist || !member.Health { + return controller.RequeueErrorf("tidbcluster: [%s/%s]'s tidb upgraded pods are not all ready", ns, tcName) + } + } + + upgradeOrdinal := tc.Status.TiDB.StatefulSet.CurrentReplicas - 1 + if tc.Spec.TiDB.Replicas > 1 { + if member, exist := tc.Status.TiDB.Members[tidbPodName(tcName, upgradeOrdinal)]; exist && member.Health { + hasResign, err := tdu.tidbControl.ResignDDLOwner(tc, upgradeOrdinal) + if (!hasResign || err != nil) && tc.Status.TiDB.ResignDDLOwnerRetryCount < MaxResignDDLOwnerCount { + tc.Status.TiDB.ResignDDLOwnerRetryCount++ + return err + } + } + } + + tc.Status.TiDB.ResignDDLOwnerRetryCount = 0 + setUpgradePartition(newSet, upgradeOrdinal) return nil } diff --git a/pkg/manager/member/tidb_upgrader_test.go b/pkg/manager/member/tidb_upgrader_test.go index fef290712c6..31737be194a 100644 --- a/pkg/manager/member/tidb_upgrader_test.go +++ b/pkg/manager/member/tidb_upgrader_test.go @@ -14,10 +14,12 @@ package member import ( + "fmt" "testing" . "github.com/onsi/gomega" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" apps "k8s.io/api/apps/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,24 +31,25 @@ func TestTiDBUpgrader_Upgrade(t *testing.T) { type testcase struct { name string - pdUpgrading bool - tikvUpgrading bool + changeFn func(*v1alpha1.TidbCluster) getLastAppliedConfigErr bool + resignDDLOwnerError bool + errorExpect bool + expectFn func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) } testFn := func(test *testcase, t *testing.T) { t.Log(test.name) - upgrader := newTiDBUpgrader() - tc := newTidbClusterForTiDBUpgrader() - if test.pdUpgrading { - tc.Status.PD.Phase = v1alpha1.UpgradePhase + upgrader, tidbControl := newTiDBUpgrader() + if test.resignDDLOwnerError { + tidbControl.SetResignDDLOwnerError(fmt.Errorf("resign DDL owner failed")) + tidbControl.NotDDLOwner(false) } else { - tc.Status.PD.Phase = v1alpha1.NormalPhase + tidbControl.NotDDLOwner(true) } - if test.tikvUpgrading { - tc.Status.TiKV.Phase = v1alpha1.UpgradePhase - } else { - tc.Status.TiKV.Phase = v1alpha1.NormalPhase + tc := newTidbClusterForTiDBUpgrader() + if test.changeFn != nil { + test.changeFn(tc) } oldSet := newStatefulSetForTiDBUpgrader() newSet := oldSet.DeepCopy() @@ -58,26 +61,105 @@ func TestTiDBUpgrader_Upgrade(t *testing.T) { newSet.Spec.Template.Spec.Containers[0].Image = "tidb-test-images:v2" err := upgrader.Upgrade(tc, oldSet, newSet) - if test.getLastAppliedConfigErr { + if test.errorExpect { g.Expect(err).To(HaveOccurred()) - g.Expect(tc.Status.TiDB.Phase).NotTo(Equal(v1alpha1.UpgradePhase)) - return - } - - g.Expect(err).NotTo(HaveOccurred()) - if test.pdUpgrading || test.tikvUpgrading { - g.Expect(newSet.Spec.Template.Spec).To(Equal(oldSet.Spec.Template.Spec)) - g.Expect(tc.Status.TiDB.Phase).NotTo(Equal(v1alpha1.UpgradePhase)) } else { - g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.UpgradePhase)) + g.Expect(err).NotTo(HaveOccurred()) } + test.expectFn(g, tc, newSet) } tests := []*testcase{ - {name: "normal", pdUpgrading: false, tikvUpgrading: false, getLastAppliedConfigErr: false}, - {name: "pd is upgrading", pdUpgrading: true, tikvUpgrading: false, getLastAppliedConfigErr: false}, - {name: "tikv is upgrading", pdUpgrading: false, tikvUpgrading: true, getLastAppliedConfigErr: false}, - {name: "get apply config error", pdUpgrading: true, tikvUpgrading: false, getLastAppliedConfigErr: true}, + { + name: "normal", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.NormalPhase + }, + getLastAppliedConfigErr: false, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(0); return &i }()))) + }, + }, + { + name: "pd is upgrading", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.UpgradePhase + tc.Status.TiKV.Phase = v1alpha1.NormalPhase + }, + getLastAppliedConfigErr: false, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(1); return &i }()))) + }, + }, + { + name: "tikv is upgrading", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.UpgradePhase + }, + getLastAppliedConfigErr: false, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(1); return &i }()))) + }, + }, + { + name: "get apply config error", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.UpgradePhase + }, + getLastAppliedConfigErr: true, + errorExpect: true, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(1); return &i }()))) + }, + }, + { + name: "upgraded pods are not ready", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.NormalPhase + tc.Status.TiDB.Members["upgrader-tidb-1"] = v1alpha1.TiDBMember{ + Name: "upgrader-tidb-1", + Health: false, + } + }, + getLastAppliedConfigErr: false, + errorExpect: true, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(1); return &i }()))) + }, + }, + { + name: "resign DDL owner error", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.NormalPhase + }, + getLastAppliedConfigErr: false, + resignDDLOwnerError: true, + errorExpect: true, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(1); return &i }()))) + g.Expect(tc.Status.TiDB.ResignDDLOwnerRetryCount).To(Equal(int32(1))) + }, + }, + { + name: "resign DDL owner error count larger than MaxResignDDLOwnerCount", + changeFn: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Phase = v1alpha1.NormalPhase + tc.Status.TiKV.Phase = v1alpha1.NormalPhase + tc.Status.TiDB.ResignDDLOwnerRetryCount = 4 + }, + getLastAppliedConfigErr: false, + resignDDLOwnerError: true, + errorExpect: false, + expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) { + g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal((func() *int32 { i := int32(0); return &i }()))) + g.Expect(tc.Status.TiDB.ResignDDLOwnerRetryCount).To(Equal(int32(0))) + }, + }, } for _, test := range tests { @@ -86,8 +168,9 @@ func TestTiDBUpgrader_Upgrade(t *testing.T) { } -func newTiDBUpgrader() Upgrader { - return &tidbUpgrader{} +func newTiDBUpgrader() (Upgrader, *controller.FakeTiDBControl) { + tidbControl := controller.NewFakeTiDBControl() + return &tidbUpgrader{tidbControl: tidbControl}, tidbControl } func newStatefulSetForTiDBUpgrader() *apps.StatefulSet { @@ -108,6 +191,19 @@ func newStatefulSetForTiDBUpgrader() *apps.StatefulSet { }, }, }, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &apps.RollingUpdateStatefulSetStrategy{ + Partition: int32Pointer(1), + }, + }, + }, + Status: apps.StatefulSetStatus{ + CurrentRevision: "1", + UpdateRevision: "2", + ReadyReplicas: 2, + Replicas: 2, + CurrentReplicas: 1, + UpdatedReplicas: 1, }, } } @@ -146,5 +242,24 @@ func newTidbClusterForTiDBUpgrader() *v1alpha1.TidbCluster { StorageClassName: "my-storage-class", }, }, + Status: v1alpha1.TidbClusterStatus{ + TiDB: v1alpha1.TiDBStatus{ + StatefulSet: &apps.StatefulSetStatus{ + CurrentReplicas: 1, + UpdatedReplicas: 1, + Replicas: 2, + }, + Members: map[string]v1alpha1.TiDBMember{ + "upgrader-tidb-0": { + Name: "upgrader-tidb-0", + Health: true, + }, + "upgrader-tidb-1": { + Name: "upgrader-tidb-1", + Health: true, + }, + }, + }, + }, } } diff --git a/pkg/manager/member/utils.go b/pkg/manager/member/utils.go index 9d59c7cebd8..d4f97a6ebbd 100644 --- a/pkg/manager/member/utils.go +++ b/pkg/manager/member/utils.go @@ -205,3 +205,7 @@ func tikvPodName(tcName string, ordinal int32) string { func pdPodName(tcName string, ordinal int32) string { return fmt.Sprintf("%s-%d", controller.PDMemberName(tcName), ordinal) } + +func tidbPodName(tcName string, ordinal int32) string { + return fmt.Sprintf("%s-%d", controller.TiDBMemberName(tcName), ordinal) +}