feat: set property listener to configure certificate expiry grace period

.github/workflows/test.yml

@@ -6,12 +6,17 @@ on:
   pull_request:
 jobs:
   test:
-    runs-on: ubuntu-latest 
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        go-version:
+          - 1.23.x
+    runs-on: ${{ matrix.platform }}
     steps:
       - name: Install Go
         uses: actions/setup-go@v5
         with:
-          go-version: v1.22.x 
+          go-version: v1.23.x 
 
       - name: Checkout code
         uses: actions/checkout@v2

Makefile

@@ -4,7 +4,7 @@ test:
 
 .PHONY: tidy
 tidy:
-	go mod tidy -go=1.22.0 -compat=1.22
+	go mod tidy
 
 .PHONY: lint
 lint:

go.mod

@@ -1,41 +1,43 @@
 module github.com/flanksource/is-healthy
 
-go 1.22.0
-
-toolchain go1.22.9
+go 1.23
 
 require (
 	github.com/bmatcuk/doublestar/v4 v4.7.1
 	github.com/cert-manager/cert-manager v1.9.0
+	github.com/flanksource/commons v1.31.2
 	github.com/gobwas/glob v0.2.3
 	github.com/samber/lo v1.44.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/yuin/gopher-lua v1.1.0
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.24.2
-	k8s.io/apimachinery v0.24.2
+	k8s.io/api v0.28.2
+	k8s.io/apimachinery v0.28.2
 	layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/net v0.29.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	k8s.io/apiextensions-apiserver v0.24.2 // indirect
-	k8s.io/klog/v2 v2.70.0 // indirect
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
 	sigs.k8s.io/gateway-api v0.4.3 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

pkg/health/health.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/flanksource/commons/properties"
 	"github.com/samber/lo"
 	"golang.org/x/exp/slices"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -451,3 +452,15 @@ func GetHealthCheckFunc(gvk schema.GroupVersionKind) func(obj *unstructured.Unst
 	}
 	return nil
 }
+
+func init() {
+	properties.RegisterListener(func(p *properties.Properties) {
+		if v := p.Duration(defaultCertExpiryWarningPeriod, "health.cert-manager.expiryGracePeriod"); v != 0 {
+			certExpiryWarningPeriod = v
+		}
+
+		if v := p.Duration(defaultCrtRenewalWarningPeriod, "health.cert-manager.renewalGracePeriod"); v != 0 {
+			certRenewalWarningPeriod = v
+		}
+	})
+}

pkg/health/health_cert_manager.go

@@ -10,11 +10,74 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
-var defaultCertExpiryWarningPeriod = time.Hour * 24 * 2
+const (
+	defaultCertExpiryWarningPeriod = time.Hour * 24 * 2
+	defaultCrtRenewalWarningPeriod = time.Minute * 30
+)
 
-func SetDefaultCertificateExpiryWarningPeriod(p time.Duration) {
-	defaultCertExpiryWarningPeriod = p
-}
+var (
+	certExpiryWarningPeriod  = defaultCertExpiryWarningPeriod
+	certRenewalWarningPeriod = defaultCrtRenewalWarningPeriod
+)
+
+// https://github.com/cert-manager/cert-manager/blob/cb20920fcf80c73ab6310470d5464d40e22db492/internal/controller/certificates/policies/constants.go
+const (
+	// DoesNotExist is a policy violation reason for a scenario where
+	// Certificate's spec.secretName secret does not exist.
+	DoesNotExist string = "DoesNotExist"
+	// MissingData is a policy violation reason for a scenario where
+	// Certificate's spec.secretName secret has missing data.
+	MissingData string = "MissingData"
+	// InvalidKeyPair is a policy violation reason for a scenario where public
+	// key of certificate does not match private key.
+	InvalidKeyPair string = "InvalidKeyPair"
+	// InvalidCertificate is a policy violation whereby the signed certificate in
+	// the Input Secret could not be parsed or decoded.
+	InvalidCertificate string = "InvalidCertificate"
+	// InvalidCertificateRequest is a policy violation whereby the CSR in
+	// the Input CertificateRequest could not be parsed or decoded.
+	InvalidCertificateRequest string = "InvalidCertificateRequest"
+
+	// SecretMismatch is a policy violation reason for a scenario where Secret's
+	// private key does not match spec.
+	SecretMismatch string = "SecretMismatch"
+	// IncorrectIssuer is a policy violation reason for a scenario where
+	// Certificate has been issued by incorrect Issuer.
+	IncorrectIssuer string = "IncorrectIssuer"
+	// IncorrectCertificate is a policy violation reason for a scenario where
+	// the Secret referred to by this Certificate's spec.secretName,
+	// already has a `cert-manager.io/certificate-name` annotation
+	// with the name of another Certificate.
+	IncorrectCertificate string = "IncorrectCertificate"
+	// RequestChanged is a policy violation reason for a scenario where
+	// CertificateRequest not valid for Certificate's spec.
+	RequestChanged string = "RequestChanged"
+	// Renewing is a policy violation reason for a scenario where
+	// Certificate's renewal time is now or in the past.
+	Renewing string = "Renewing"
+	// Expired is a policy violation reason for a scenario where Certificate has
+	// expired.
+	Expired string = "Expired"
+	// SecretTemplateMisMatch is a policy violation whereby the Certificate's
+	// SecretTemplate is not reflected on the target Secret, either by having
+	// extra, missing, or wrong Annotations or Labels.
+	SecretTemplateMismatch string = "SecretTemplateMismatch"
+	// SecretManagedMetadataMismatch is a policy violation whereby the Secret is
+	// missing labels that should have been added by cert-manager
+	SecretManagedMetadataMismatch string = "SecretManagedMetadataMismatch"
+
+	// AdditionalOutputFormatsMismatch is a policy violation whereby the
+	// Certificate's AdditionalOutputFormats is not reflected on the target
+	// Secret, either by having extra, missing, or wrong values.
+	AdditionalOutputFormatsMismatch string = "AdditionalOutputFormatsMismatch"
+	// ManagedFieldsParseError is a policy violation whereby cert-manager was
+	// unable to decode the managed fields on a resource.
+	ManagedFieldsParseError string = "ManagedFieldsParseError"
+	// SecretOwnerRefMismatch is a policy violation whereby the Secret either has
+	// a missing owner reference to the Certificate, or has an owner reference it
+	// shouldn't have.
+	SecretOwnerRefMismatch string = "SecretOwnerRefMismatch"
+)
 
 func GetCertificateRequestHealth(obj *unstructured.Unstructured) (*HealthStatus, error) {
 	var certReq certmanagerv1.CertificateRequest
@@ -113,48 +176,84 @@ func GetCertificateHealth(obj *unstructured.Unstructured) (*HealthStatus, error)
 			continue
 		}
 
-		if c.Type == "Issuing" && cert.Status.NotBefore != nil {
+		if c.Type == certmanagerv1.CertificateConditionIssuing {
 			hs := &HealthStatus{
 				Status:  HealthStatusCode(c.Reason),
 				Ready:   false,
 				Message: c.Message,
 			}
 
-			if overdue := time.Since(cert.Status.NotBefore.Time); overdue > time.Hour {
-				hs.Health = HealthUnhealthy
-				return hs, nil
-			} else if overdue > time.Minute*15 {
-				hs.Health = HealthWarning
+			switch c.Reason {
+			case "ManuallyTriggered", DoesNotExist:
+				// We check for expiry below
+				hs.Status = "Issuing"
+				hs.Health = HealthUnknown
+
+			case Renewing:
+				renewalTime := cert.Status.RenewalTime.Time
+
+				if time.Since(renewalTime) > certRenewalWarningPeriod {
+					hs.Health = HealthWarning
+					hs.Message = fmt.Sprintf(
+						"Certificate has been in renewal state for > %s",
+						time.Since(renewalTime).Truncate(time.Minute),
+					)
+				} else {
+					hs.Health = HealthHealthy
+				}
+
+			default:
+				unhealthyReasons := map[string]string{
+					MissingData:                     "Certificate secret has missing data",
+					InvalidKeyPair:                  "Public key of certificate does not match private key",
+					InvalidCertificate:              "Signed certificate could not be parsed or decoded",
+					InvalidCertificateRequest:       "CSR could not be parsed or decoded",
+					SecretMismatch:                  "Secret's private key does not match spec",
+					IncorrectIssuer:                 "Certificate has been issued by incorrect Issuer",
+					IncorrectCertificate:            "Certificate's secretName already has an annotation with another Certificate",
+					Expired:                         "Certificate has expired",
+					SecretTemplateMismatch:          "SecretTemplate is not reflected on the target Secret",
+					SecretManagedMetadataMismatch:   "Secret is missing labels that should have been added by cert-manager",
+					AdditionalOutputFormatsMismatch: "Certificate's AdditionalOutputFormats are not reflected on the target Secret",
+					ManagedFieldsParseError:         "cert-manager was unable to decode the managed fields on a resource",
+					SecretOwnerRefMismatch:          "Secret has an incorrect owner reference",
+				}
+
+				if msg, exists := unhealthyReasons[string(c.Reason)]; exists {
+					return &HealthStatus{
+						Health:  HealthUnhealthy,
+						Status:  HealthStatusCode(c.Reason),
+						Message: msg,
+						Ready:   true,
+					}, nil
+				} else if cert.Status.NotBefore != nil {
+					if overdue := time.Since(cert.Status.NotBefore.Time); overdue > time.Hour {
+						hs.Health = HealthUnhealthy
+						return hs, nil
+					} else if overdue > time.Minute*15 {
+						hs.Health = HealthWarning
+						return hs, nil
+					}
+				}
+			}
+
+			// If we're issuing a new cert, at least ensure the existing cert hasn't expired
+			if expiryHealth := certExpiryCheck(cert); expiryHealth != nil {
+				return expiryHealth, nil
+			} else {
 				return hs, nil
 			}
 		}
 	}
 
-	if cert.Status.NotAfter != nil {
-		notAfterTime := cert.Status.NotAfter.Time
-		if notAfterTime.Before(time.Now()) {
-			return &HealthStatus{
-				Health:  HealthUnhealthy,
-				Status:  "Expired",
-				Message: "Certificate has expired",
-				Ready:   true,
-			}, nil
-		}
-
-		if time.Until(notAfterTime) < defaultCertExpiryWarningPeriod {
-			return &HealthStatus{
-				Health:  HealthWarning,
-				Status:  HealthStatusWarning,
-				Message: fmt.Sprintf("Certificate is expiring soon (%s)", notAfterTime),
-				Ready:   true,
-			}, nil
-		}
+	if expiryHealth := certExpiryCheck(cert); expiryHealth != nil {
+		return expiryHealth, nil
 	}
 
 	if cert.Status.RenewalTime != nil {
 		renewalTime := cert.Status.RenewalTime.Time
 
-		if time.Since(renewalTime) > time.Minute*5 {
+		if time.Since(renewalTime) > certRenewalWarningPeriod {
 			return &HealthStatus{
 				Health:  HealthWarning,
 				Status:  HealthStatusWarning,
@@ -171,3 +270,30 @@ func GetCertificateHealth(obj *unstructured.Unstructured) (*HealthStatus, error)
 
 	return status, nil
 }
+
+func certExpiryCheck(cert certmanagerv1.Certificate) *HealthStatus {
+	if cert.Status.NotAfter == nil {
+		return nil
+	}
+
+	notAfterTime := cert.Status.NotAfter.Time
+	if notAfterTime.Before(time.Now()) {
+		return &HealthStatus{
+			Health:  HealthUnhealthy,
+			Status:  "Expired",
+			Message: "Certificate has expired",
+			Ready:   true,
+		}
+	}
+
+	if time.Until(notAfterTime) < certExpiryWarningPeriod {
+		return &HealthStatus{
+			Health:  HealthWarning,
+			Status:  HealthStatusWarning,
+			Message: fmt.Sprintf("Certificate is expiring soon (%s)", notAfterTime),
+			Ready:   true,
+		}
+	}
+
+	return nil
+}

pkg/health/health_test.go

@@ -307,10 +307,41 @@ func TestCertificate(t *testing.T) {
 	// 	"2024-06-26T12:25:46Z": time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05Z"),
 	// }, health.HealthStatusWarning, health.HealthWarning, true)
 
+	assertAppHealthWithOverwriteMsg(t, "./testdata/certificate-renewal.yaml", map[string]string{
+		"2025-01-16T14:04:53Z": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339),        // not Before
+		"2025-01-16T14:09:52Z": time.Now().Add(-time.Minute * 10).UTC().Format(time.RFC3339), // renewal time
+	}, "Renewing", health.HealthHealthy, false, "Renewing certificate as renewal was scheduled at 2025-01-16 14:09:47 +0000 UTC")
+
+	assertAppHealthWithOverwriteMsg(t, "./testdata/certificate-renewal.yaml", map[string]string{
+		"2025-01-16T14:04:53Z": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339), // not Before
+		"2025-01-16T14:09:52Z": time.Now().
+			Add(-time.Minute * 40).
+			UTC().
+			Format(time.RFC3339),
+		// renewal time over the grace period
+	}, "Renewing", health.HealthWarning, false, "Certificate has been in renewal state for > 40m0s")
+
+	assertAppHealthMsg(t, "./testdata/certificate-issuing-first-time.yaml", "Issuing", health.HealthUnknown, false)
+
+	assertAppHealthMsg(
+		t,
+		"./testdata/certificate-issuing-manually-triggered.yaml",
+		"Issuing",
+		health.HealthUnknown,
+		false,
+	)
 	assertAppHealthMsg(t, "./testdata/certificate-healthy.yaml", "Issued", health.HealthHealthy, true)
+
 	b := "../resource_customizations/cert-manager.io/Certificate/testdata/"
 	assertAppHealthMsg(t, b+"degraded_configError.yaml", "ConfigError", health.HealthUnhealthy, true)
-	assertAppHealthMsg(t, b+"progressing_issuing.yaml", "Issuing", health.HealthUnknown, false)
+	assertAppHealthMsg(
+		t,
+		b+"progressing_issuing.yaml",
+		"Issuing",
+		health.HealthUnknown,
+		false,
+		"Issuing certificate as Secret does not exist",
+	)
 }
 
 func TestExternalSecrets(t *testing.T) {