Skip to content
This repository has been archived by the owner on Aug 12, 2024. It is now read-only.

Commit

Permalink
use k8s cli-utils health checks for bundledeployment resources
Browse files Browse the repository at this point in the history 
Signed-off-by: Ankita Thomas <[email protected]>
  • Loading branch information
@ankitathomas @tmshort
ankitathomas authored and tmshort committed Oct 20, 2023
1 parent 61432e2 commit 24a4e98
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 195 deletions.
9 changes: 5 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ require (
k8s.io/client-go v0.26.1
k8s.io/component-base v0.26.1
k8s.io/kube-aggregator v0.26.1
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
k8s.io/utils v0.0.0-20230115233650-391b47cb4029
sigs.k8s.io/cli-utils v0.35.0
sigs.k8s.io/controller-runtime v0.14.4
sigs.k8s.io/yaml v1.3.0
)
Expand Down Expand Up @@ -76,14 +77,14 @@ require (
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-gorp/gorp/v3 v3.0.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-logr/zapr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down Expand Up @@ -115,7 +116,7 @@ require (
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.7 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
Expand Down
18 changes: 10 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,8 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
Expand Down Expand Up @@ -438,8 +438,8 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
Expand Down Expand Up @@ -704,8 +704,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=
github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc=
github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=
Expand Down Expand Up @@ -1706,8 +1706,8 @@ k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s=
k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo=
k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
k8s.io/utils v0.0.0-20230115233650-391b47cb4029 h1:L8zDtT4jrxj+TaQYD0k8KNlr556WaVQylDXswKmX+dE=
k8s.io/utils v0.0.0-20230115233650-391b47cb4029/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
oras.land/oras-go v1.2.2 h1:0E9tOHUfrNH7TCDk5KU0jVBEzCqbfdyuVfGmJ7ZeRPE=
oras.land/oras-go v1.2.2/go.mod h1:Apa81sKoZPpP7CDciE006tSZ0x3Q3+dOoBcMZ/aNxvw=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
Expand All @@ -1719,6 +1719,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyz
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo=
sigs.k8s.io/cli-utils v0.35.0 h1:dfSJaF1W0frW74PtjwiyoB4cwdRygbHnC7qe7HF0g/Y=
sigs.k8s.io/cli-utils v0.35.0/go.mod h1:ITitykCJxP1vaj1Cew/FZEaVJ2YsTN9Q71m02jebkoE=
sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M=
sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
Expand Down
153 changes: 23 additions & 130 deletions internal/healthchecks/builtin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@ import (
"errors"
"fmt"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
"sigs.k8s.io/cli-utils/pkg/kstatus/status"
"sigs.k8s.io/controller-runtime/pkg/client"
)

Expand All @@ -27,6 +25,10 @@ import (
// - Pods
// - APIServices
// - CustomResourceDefinitions
// - Jobs
// - Services
// - PersistentVolumeClaims
// - PodDisruptionBudgets
//
// If the resource is not supported, it is assumed to be healthy.
func AreObjectsHealthy(ctx context.Context, client client.Client, objects []client.Object) error {
Expand All @@ -45,147 +47,38 @@ func AreObjectsHealthy(ctx context.Context, client client.Client, objects []clie
continue
}

switch u.GroupVersionKind() {
case appsv1.SchemeGroupVersion.WithKind("Deployment"):
// Check if the deployment is available.
obj := &appsv1.Deployment{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
conditionExists := false
for _, condition := range obj.Status.Conditions {
if condition.Type == appsv1.DeploymentAvailable {
if condition.Status != "True" {
gvkErrors = appendResourceError(gvkErrors, obj, condition.Message)
}
conditionExists = true
break
}
}
if conditionExists {
continue
}
gvkErrors = appendResourceError(gvkErrors, obj, "DeploymentAvailable condition not found")
case appsv1.SchemeGroupVersion.WithKind("StatefulSet"):
// This logic has been adapted from the helm codebase.
obj := &appsv1.StatefulSet{}
if object.GetObjectKind().GroupVersionKind() == apiregistrationv1.SchemeGroupVersion.WithKind("APIService") {
obj := &apiregistrationv1.APIService{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
// This logic has been adapted from the helm codebase.
// - https://github.com/helm/helm/blob/e7bb860d9a32e8739c944b8e7b7f7031d752411a/pkg/kube/ready.go#L357-L410

// If the statefulset is not using the RollingUpdate strategy, we assume it's healthy.
if obj.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
continue
}
if obj.Status.ObservedGeneration < obj.Generation {
gvkErrors = appendResourceError(gvkErrors, obj, "StatefulSet is not ready (update has not yet been observed)")
}

var partition int
var replicas = 1
if obj.Spec.UpdateStrategy.RollingUpdate != nil && obj.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
partition = int(*obj.Spec.UpdateStrategy.RollingUpdate.Partition)
}
if obj.Spec.Replicas != nil {
replicas = int(*obj.Spec.Replicas)
}
expectedReplicas := replicas - partition

if obj.Status.UpdatedReplicas < int32(expectedReplicas) {
gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", expectedReplicas, obj.Status.UpdatedReplicas))
continue
}
if int(obj.Status.ReadyReplicas) != replicas {
gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", replicas, obj.Status.ReadyReplicas))
continue
}
if partition == 0 && obj.Status.CurrentRevision != obj.Status.UpdateRevision {
gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected revision %s, got %s)", obj.Status.CurrentRevision, obj.Status.UpdateRevision))
continue
}
case appsv1.SchemeGroupVersion.WithKind("DaemonSet"):
// Check if the daemonset is ready.
obj := &appsv1.DaemonSet{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
if obj.Status.NumberAvailable != obj.Status.DesiredNumberScheduled {
gvkErrors = appendResourceError(gvkErrors, obj, "DaemonSet is not ready")
}
case appsv1.SchemeGroupVersion.WithKind("ReplicaSet"):
// Check if the replicaset is ready.
obj := &appsv1.ReplicaSet{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
if obj.Status.AvailableReplicas != obj.Status.Replicas {
gvkErrors = appendResourceError(gvkErrors, obj, "ReplicaSet is not ready")
}
case corev1.SchemeGroupVersion.WithKind("Pod"):
// Check if the pod is running or succeeded.
obj := &corev1.Pod{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
if obj.Status.Phase != corev1.PodRunning && obj.Status.Phase != corev1.PodSucceeded {
gvkErrors = appendResourceError(gvkErrors, obj, "Pod is not Running or Succeeded")
}
case apiregistrationv1.SchemeGroupVersion.WithKind("APIService"):
// Check if the APIService is available.
obj := &apiregistrationv1.APIService{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
conditionExists := false
for _, condition := range obj.Status.Conditions {
var isAvailable *apiregistrationv1.APIServiceCondition
for i, condition := range obj.Status.Conditions {
if condition.Type == apiregistrationv1.Available {
if condition.Status != "True" {
gvkErrors = appendResourceError(gvkErrors, obj, condition.Message)
}
conditionExists = true
isAvailable = &obj.Status.Conditions[i]
break
}
}
if conditionExists {
continue
if isAvailable == nil {
gvkErrors = appendResourceError(gvkErrors, obj, "Available condition not found")
} else if isAvailable.Status == apiregistrationv1.ConditionFalse {
gvkErrors = appendResourceError(gvkErrors, obj, isAvailable.Message)
}
// If we are here we didn't find the "Available" condition, so we assume the APIService is non healthy.
gvkErrors = appendResourceError(gvkErrors, obj, "Available condition not found")
case apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"):
// Check if the CRD is established.
obj := &apiextensionsv1.CustomResourceDefinition{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil {
gvkErrors = appendResourceError(gvkErrors, obj, err.Error())
continue
}
conditionExists := false
for _, condition := range obj.Status.Conditions {
if condition.Type == apiextensionsv1.Established {
if condition.Status != "True" {
gvkErrors = appendResourceError(gvkErrors, obj, condition.Message)
}
conditionExists = true
break
}
}
if conditionExists {
continue
}
gvkErrors = appendResourceError(gvkErrors, obj, "Established condition not found")
default:
// If we don't know how to check the health of the object, we assume it's healthy.
continue
}
}

result, err := status.Compute(u)
if err != nil {
gvkErrors = appendResourceError(gvkErrors, object, err.Error())
}

if result.Status != status.CurrentStatus {
gvkErrors = appendResourceError(gvkErrors, object, fmt.Sprintf("object %s: %s", result.Status, result.Message))
}
}
return errors.Join(gvkErrors...)
}

Expand Down
Loading

0 comments on commit 24a4e98

Please sign in to comment.