fix(status): update cvr status on graceful corresponding pool-manager…

… termination (#349)

- cvr status will be updated to offline on graceful shutdown/terminate
  of pool-manager pod/container.
- Add the missing vendor lib dependencies

Signed-off-by: prateekpandey14 <[email protected]>

changelogs/unreleased/349-prateekpandey14

@@ -0,0 +1 @@
+fix(status): update cvr status on graceful corresponding pool-manager termination

go.mod

@@ -3,6 +3,7 @@ module github.com/openebs/cstor-operators
 go 1.13
 
 require (
+	github.com/cespare/xxhash v1.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/hashicorp/go-version v1.2.1

go.sum

@@ -34,11 +34,15 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -199,6 +203,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

pkg/controllers/replica-controller/controller.go

@@ -147,7 +147,7 @@ func NewCStorVolumeReplicaController(
 				//  for every restart of controller container
 				// this informer handler will get add event
 				// for each cvr resource present in k8s
-				if IsEmptyStatus(cvrObj) {
+				if IsEmptyStatus(cvrObj) || IsInitStatus(cvrObj) {
 					cvrObj.Status.Phase = apis.CVRStatusInit
 				} else {
 					cvrObj.Status.Phase = apis.CVRStatusRecreate

pkg/controllers/replica-controller/handler.go

@@ -705,3 +705,30 @@ func (c *CStorVolumeReplicaController) reconcileVersion(cvr *apis.CStorVolumeRep
 	}
 	return cvr, nil
 }
+
+// markCVRStatusToOffline will fetch all the CVR resources present
+// in etcd and mark it them CVR.Status to Offline
+func (c *CStorVolumeReplicaController) markCVRStatusToOffline() {
+	cspiuuidLabel := "cstorpoolinstance.openebs.io/uid" + "=" + os.Getenv(string(common.OpenEBSIOCSPIID))
+	// list the cvr matching with the cstorpoolinstance uuid label
+	cvrList, err := c.clientset.CstorV1().CStorVolumeReplicas("").List(context.TODO(), metav1.ListOptions{LabelSelector: cspiuuidLabel})
+	if err != nil {
+		klog.Errorf("failed to fetch CVR list, error: %v", err)
+		return
+	}
+	for _, cvr := range cvrList.Items {
+		cvr := cvr // pin it
+		// If pool-manager container restarts or pod is deleted
+		// before even creating a zfs datasets, then we can skip those cvrs
+		if cvr.Status.Phase == "" || cvr.Status.Phase == apis.CVRStatusInit {
+			continue
+		}
+		cvr.Status.Phase = apis.CVRStatusOffline
+		_, err = c.clientset.CstorV1().CStorVolumeReplicas(cvr.Namespace).Update(context.TODO(), &cvr, metav1.UpdateOptions{})
+		if err != nil {
+			klog.Errorf("failed to update CVR: %s status to %s", cvr.Name, apis.CVRStatusOffline)
+			continue
+		}
+		klog.Infof("status marked %s for CVR: %s", apis.CVRStatusOffline, cvr.Name)
+	}
+}

pkg/controllers/replica-controller/runner.go

@@ -53,6 +53,15 @@ func (c *CStorVolumeReplicaController) Run(threadiness int, stopCh <-chan struct
 
 	klog.Info("Started CStorVolumeReplica workers")
 	<-stopCh
+
+	klog.Info("changing the CVR state to offline before shutting down")
+	// Changing the state of corresponding CVR to Offline before shutting down.
+	// Similar as when pod is running and if you stopped kubelet it will make
+	// pod status unknown.
+	// NOTE: CVR status will be updated to OFFLINE even cStor pool-manager
+	// container alone get restarted but in this case pool will be still in
+	// running state. This inconsistency will be resolved in subsequent reconciliations
+	c.markCVRStatusToOffline()
 	klog.Info("Shutting down CStorVolumeReplica workers")
 
 	return nil