leader: check the node status for the leader-election (operator-frame…

…work#24)

leader/leader.go

@@ -164,6 +164,7 @@ func Become(ctx context.Context, lockName string, opts ...Option) error {
 				log.Info("Leader lock configmap owner reference must be a pod.", "OwnerReference", existingOwners[0])
 			default:
 				leaderPod := &corev1.Pod{}
+				leaderNode := &corev1.Node{}
 				key = crclient.ObjectKey{Namespace: ns, Name: existingOwners[0].Name}
 				err = config.Client.Get(ctx, key, leaderPod)
 				switch {
@@ -179,6 +180,24 @@ func Become(ctx context.Context, lockName string, opts ...Option) error {
 					if err != nil {
 						log.Error(err, "Leader pod could not be deleted.")
 					}
+				case getNode(ctx, config.Client, leaderPod.Spec.NodeName, leaderNode) == nil && isNotReadyNode(*leaderNode):
+					log.Info("the status of the node where operator pod with leader lock was running has been 'notReady'")
+					log.Info("Deleting the leader.")
+
+					//Mark the termainating status to the leader
+					err := config.Client.Delete(ctx, leaderPod)
+					if err != nil {
+						log.Error(err, "Leader pod could not be deleted.")
+					} else {
+						err := config.Client.Delete(ctx, existing)
+						switch {
+						case err == nil:
+						case apierrors.IsNotFound(err):
+							log.Info("cm has been deleted by prior operator.")
+						case err != nil:
+							return err
+						}
+					}
 
 				default:
 					log.Info("Not the leader. Waiting.")
@@ -267,3 +286,22 @@ func getPod(ctx context.Context, client crclient.Client, ns string) (*corev1.Pod
 
 	return pod, nil
 }
+
+func getNode(ctx context.Context, client crclient.Client, nodeName string, node *corev1.Node) error {
+	key := crclient.ObjectKey{Namespace: "", Name: nodeName}
+	err := client.Get(ctx, key, node)
+	if err != nil {
+		log.Error(err, "Failed to get Node", "Node.Name", nodeName)
+		return err
+	}
+	return nil
+}
+
+func isNotReadyNode(node corev1.Node) bool {
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == corev1.NodeReady && condition.Status != corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}

leader/leader_test.go

@@ -209,4 +209,67 @@ var _ = Describe("Leader election", func() {
 			Expect(pod.TypeMeta.Kind).To(Equal("Pod"))
 		})
 	})
+
+	Describe("getNode", func() {
+		var (
+			client crclient.Client
+		)
+		BeforeEach(func() {
+			client = fake.NewFakeClient(
+				&corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "mynode",
+					},
+				},
+			)
+		})
+		It("should return an error if no node is found", func() {
+			node := corev1.Node{}
+			err := getNode(context.TODO(), client, "", &node)
+			Expect(err).ShouldNot(BeNil())
+		})
+		It("should return the node with the given name", func() {
+			node := corev1.Node{}
+			err := getNode(context.TODO(), client, "mynode", &node)
+			Expect(err).Should(BeNil())
+			Expect(node.TypeMeta.APIVersion).To(Equal("v1"))
+			Expect(node.TypeMeta.Kind).To(Equal("Node"))
+		})
+	})
+
+	Describe("isNotReadyNode", func() {
+		var (
+			leaderNode *corev1.Node
+		)
+		BeforeEach(func() {
+			leaderNode = &corev1.Node{
+				Status: corev1.NodeStatus{
+					Conditions: make([]corev1.NodeCondition, 1),
+				},
+			}
+		})
+		It("should return false with an empty status", func() {
+			Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
+		})
+		It("should return false if type is incorrect", func() {
+			leaderNode.Status.Conditions[0].Type = corev1.NodeMemoryPressure
+			leaderNode.Status.Conditions[0].Status = corev1.ConditionFalse
+			Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
+		})
+		It("should return false if NodeReady's status is true", func() {
+			leaderNode.Status.Conditions[0].Type = corev1.NodeReady
+			leaderNode.Status.Conditions[0].Status = corev1.ConditionTrue
+			Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
+		})
+		It("should return true when Type is set and Status is set to false", func() {
+			leaderNode.Status.Conditions[0].Type = corev1.NodeReady
+			leaderNode.Status.Conditions[0].Status = corev1.ConditionFalse
+			Expect(isNotReadyNode(*leaderNode)).To(Equal(true))
+		})
+		It("should return true when Type is set and Status is set to unknown", func() {
+			leaderNode.Status.Conditions[0].Type = corev1.NodeReady
+			leaderNode.Status.Conditions[0].Status = corev1.ConditionUnknown
+			Expect(isNotReadyNode(*leaderNode)).To(Equal(true))
+		})
+	})
 })