Skip to content

Commit

Permalink
Merge pull request #319 from hardikdr/bugfix-secret-mclass
Browse files Browse the repository at this point in the history 
Bugfix: Improve the management of finalizers and re-enqueue-mechanism for MachineClasses and Secrets
  • Loading branch information
@prashanth26
prashanth26 authored Aug 21, 2019
2 parents 3c54de2 + 292ea21 commit 373b416
Show file tree
Hide file tree
Showing 7 changed files with 235 additions and 89 deletions.
47 changes: 34 additions & 13 deletions pkg/controller/alicloudmachineclass.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ limitations under the License.
package controller

import (
"time"

"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
Expand Down Expand Up @@ -107,6 +109,12 @@ func (c *controller) reconcileClusterAlicloudMachineClassKey(key string) error {
}

func (c *controller) reconcileClusterAlicloudMachineClass(class *v1alpha1.AlicloudMachineClass) error {
glog.V(4).Info("Start Reconciling alicloudmachineclass: ", class.Name)
defer func() {
c.enqueueAlicloudMachineClassAfter(class, 10*time.Minute)
glog.V(4).Info("Stop Reconciling alicloudmachineclass: ", class.Name)
}()

internalClass := &machine.AlicloudMachineClass{}
err := c.internalExternalScheme.Convert(class, internalClass, nil)
if err != nil {
Expand All @@ -121,7 +129,10 @@ func (c *controller) reconcileClusterAlicloudMachineClass(class *v1alpha1.Aliclo

// Manipulate finalizers
if class.DeletionTimestamp == nil {
c.addAlicloudMachineClassFinalizers(class)
err = c.addAlicloudMachineClassFinalizers(class)
if err != nil {
return err
}
}

machines, err := c.findMachinesForClass(AlicloudMachineClassKind, class.Name)
Expand All @@ -143,11 +154,10 @@ func (c *controller) reconcileClusterAlicloudMachineClass(class *v1alpha1.Aliclo
return err
}
if len(machineDeployments) == 0 && len(machineSets) == 0 && len(machines) == 0 {
c.deleteAlicloudMachineClassFinalizers(class)
return nil
return c.deleteAlicloudMachineClassFinalizers(class)
}

glog.V(4).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
glog.V(3).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
return nil
}

Expand All @@ -162,37 +172,48 @@ func (c *controller) reconcileClusterAlicloudMachineClass(class *v1alpha1.Aliclo
Manipulate Finalizers
*/

func (c *controller) addAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass) {
func (c *controller) addAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) {
finalizers.Insert(DeleteFinalizerName)
c.updateAlicloudMachineClassFinalizers(clone, finalizers.List())
return c.updateAlicloudMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) deleteAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass) {
func (c *controller) deleteAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) {
finalizers.Delete(DeleteFinalizerName)
c.updateAlicloudMachineClassFinalizers(clone, finalizers.List())
return c.updateAlicloudMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) updateAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass, finalizers []string) {
func (c *controller) updateAlicloudMachineClassFinalizers(class *v1alpha1.AlicloudMachineClass, finalizers []string) error {
// Get the latest version of the class so that we can avoid conflicts
class, err := c.controlMachineClient.AlicloudMachineClasses(class.Namespace).Get(class.Name, metav1.GetOptions{})
if err != nil {
return
return err
}

clone := class.DeepCopy()
clone.Finalizers = finalizers
_, err = c.controlMachineClient.AlicloudMachineClasses(class.Namespace).Update(clone)
if err != nil {
// Keep retrying until update goes through
glog.Warning("Updated failed, retrying: ", err)
c.updateAlicloudMachineClassFinalizers(class, finalizers)
glog.Warning("Updating AlicloudMachineClass failed, retrying. ", class.Name, err)
return err
}
glog.V(3).Infof("Successfully added/removed finalizer on the alicloudmachineclass %q", class.Name)
return err
}

func (c *controller) enqueueAlicloudMachineClassAfter(obj interface{}, after time.Duration) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
return
}
c.alicloudMachineClassQueue.AddAfter(key, after)
}
47 changes: 34 additions & 13 deletions pkg/controller/awsmachineclass.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ limitations under the License.
package controller

import (
"time"

"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
Expand Down Expand Up @@ -107,6 +109,12 @@ func (c *controller) reconcileClusterAWSMachineClassKey(key string) error {
}

func (c *controller) reconcileClusterAWSMachineClass(class *v1alpha1.AWSMachineClass) error {
glog.V(4).Info("Start Reconciling awsmachineclass: ", class.Name)
defer func() {
c.enqueueAwsMachineClassAfter(class, 10*time.Minute)
glog.V(4).Info("Stop Reconciling awsmachineclass: ", class.Name)
}()

internalClass := &machine.AWSMachineClass{}
err := c.internalExternalScheme.Convert(class, internalClass, nil)
if err != nil {
Expand All @@ -121,7 +129,10 @@ func (c *controller) reconcileClusterAWSMachineClass(class *v1alpha1.AWSMachineC

// Manipulate finalizers
if class.DeletionTimestamp == nil {
c.addAWSMachineClassFinalizers(class)
err = c.addAWSMachineClassFinalizers(class)
if err != nil {
return err
}
}

machines, err := c.findMachinesForClass(AWSMachineClassKind, class.Name)
Expand All @@ -143,11 +154,10 @@ func (c *controller) reconcileClusterAWSMachineClass(class *v1alpha1.AWSMachineC
return err
}
if len(machineDeployments) == 0 && len(machineSets) == 0 && len(machines) == 0 {
c.deleteAWSMachineClassFinalizers(class)
return nil
return c.deleteAWSMachineClassFinalizers(class)
}

glog.V(4).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
glog.V(3).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
return nil
}

Expand All @@ -162,37 +172,48 @@ func (c *controller) reconcileClusterAWSMachineClass(class *v1alpha1.AWSMachineC
Manipulate Finalizers
*/

func (c *controller) addAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass) {
func (c *controller) addAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) {
finalizers.Insert(DeleteFinalizerName)
c.updateAWSMachineClassFinalizers(clone, finalizers.List())
return c.updateAWSMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) deleteAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass) {
func (c *controller) deleteAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) {
finalizers.Delete(DeleteFinalizerName)
c.updateAWSMachineClassFinalizers(clone, finalizers.List())
return c.updateAWSMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) updateAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass, finalizers []string) {
func (c *controller) updateAWSMachineClassFinalizers(class *v1alpha1.AWSMachineClass, finalizers []string) error {
// Get the latest version of the class so that we can avoid conflicts
class, err := c.controlMachineClient.AWSMachineClasses(class.Namespace).Get(class.Name, metav1.GetOptions{})
if err != nil {
return
return err
}

clone := class.DeepCopy()
clone.Finalizers = finalizers
_, err = c.controlMachineClient.AWSMachineClasses(class.Namespace).Update(clone)
if err != nil {
// Keep retrying until update goes through
glog.Warning("Updated failed, retrying: ", err)
c.updateAWSMachineClassFinalizers(class, finalizers)
glog.Warning("Updating AWSMachineClass failed, retrying. ", class.Name, err)
return err
}
glog.V(3).Infof("Successfully added/removed finalizer on the awsmachineclass %q", class.Name)
return err
}

func (c *controller) enqueueAwsMachineClassAfter(obj interface{}, after time.Duration) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
return
}
c.awsMachineClassQueue.AddAfter(key, after)
}
47 changes: 34 additions & 13 deletions pkg/controller/azuremachineclass.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ limitations under the License.
package controller

import (
"time"

"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
Expand Down Expand Up @@ -108,6 +110,12 @@ func (c *controller) reconcileClusterAzureMachineClassKey(key string) error {
}

func (c *controller) reconcileClusterAzureMachineClass(class *v1alpha1.AzureMachineClass) error {
glog.V(4).Info("Start Reconciling azuremachineclass: ", class.Name)
defer func() {
c.enqueueAzureMachineClassAfter(class, 10*time.Minute)
glog.V(4).Info("Stop Reconciling azuremachineclass: ", class.Name)
}()

internalClass := &machine.AzureMachineClass{}
err := c.internalExternalScheme.Convert(class, internalClass, nil)
if err != nil {
Expand All @@ -122,7 +130,10 @@ func (c *controller) reconcileClusterAzureMachineClass(class *v1alpha1.AzureMach

// Manipulate finalizers
if class.DeletionTimestamp == nil {
c.addAzureMachineClassFinalizers(class)
err = c.addAzureMachineClassFinalizers(class)
if err != nil {
return err
}
}

machines, err := c.findMachinesForClass(AzureMachineClassKind, class.Name)
Expand All @@ -144,11 +155,10 @@ func (c *controller) reconcileClusterAzureMachineClass(class *v1alpha1.AzureMach
return err
}
if len(machineDeployments) == 0 && len(machineSets) == 0 && len(machines) == 0 {
c.deleteAzureMachineClassFinalizers(class)
return nil
return c.deleteAzureMachineClassFinalizers(class)
}

glog.V(4).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
glog.V(3).Infof("Cannot remove finalizer of %s because still Machine[s|Sets|Deployments] are referencing it", class.Name)
return nil
}

Expand All @@ -163,37 +173,48 @@ func (c *controller) reconcileClusterAzureMachineClass(class *v1alpha1.AzureMach
Manipulate Finalizers
*/

func (c *controller) addAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass) {
func (c *controller) addAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) {
finalizers.Insert(DeleteFinalizerName)
c.updateAzureMachineClassFinalizers(clone, finalizers.List())
return c.updateAzureMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) deleteAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass) {
func (c *controller) deleteAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass) error {
clone := class.DeepCopy()

if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) {
finalizers.Delete(DeleteFinalizerName)
c.updateAzureMachineClassFinalizers(clone, finalizers.List())
return c.updateAzureMachineClassFinalizers(clone, finalizers.List())
}
return nil
}

func (c *controller) updateAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass, finalizers []string) {
func (c *controller) updateAzureMachineClassFinalizers(class *v1alpha1.AzureMachineClass, finalizers []string) error {
// Get the latest version of the class so that we can avoid conflicts
class, err := c.controlMachineClient.AzureMachineClasses(class.Namespace).Get(class.Name, metav1.GetOptions{})
if err != nil {
return
return err
}

clone := class.DeepCopy()
clone.Finalizers = finalizers
_, err = c.controlMachineClient.AzureMachineClasses(class.Namespace).Update(clone)
if err != nil {
// Keep retrying until update goes through
glog.Warning("Updated failed, retrying")
c.updateAzureMachineClassFinalizers(class, finalizers)
glog.Warning("Updating AzureMachineClass failed, retrying. ", class.Name, err)
return err
}
glog.V(3).Infof("Successfully added/removed finalizer on the azuremachineclass %q", class.Name)
return err
}

func (c *controller) enqueueAzureMachineClassAfter(obj interface{}, after time.Duration) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
return
}
c.azureMachineClassQueue.AddAfter(key, after)
}
Loading

0 comments on commit 373b416

Please sign in to comment.