From d235aeadbc1991d18174ef45077f7612037f8788 Mon Sep 17 00:00:00 2001 From: Ramon Quitales Date: Fri, 4 Oct 2024 12:53:53 -0700 Subject: [PATCH] add program metrics test --- .../controller/pulumi/metrics_program.go | 7 ++- .../controller/pulumi/metrics_program_test.go | 52 +++++++++++++++++++ .../controller/pulumi/metrics_stack.go | 6 ++- .../controller/pulumi/metrics_stack_test.go | 29 ++++++++++- operator/internal/controller/pulumi/utils.go | 13 +++++ 5 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 operator/internal/controller/pulumi/metrics_program_test.go diff --git a/operator/internal/controller/pulumi/metrics_program.go b/operator/internal/controller/pulumi/metrics_program.go index 6557a549..c338d3d8 100644 --- a/operator/internal/controller/pulumi/metrics_program.go +++ b/operator/internal/controller/pulumi/metrics_program.go @@ -26,5 +26,10 @@ func newProgramCallback(_ any) { // updateProgramCallback is a callback that is called when a Program object is updated. func deleteProgramCallback(_ any) { - numStacks.Dec() + numPrograms.Dec() + + val, err := getGaugeValue(numPrograms) + if err == nil && val < 0 { + numPrograms.Set(0) + } } diff --git a/operator/internal/controller/pulumi/metrics_program_test.go b/operator/internal/controller/pulumi/metrics_program_test.go new file mode 100644 index 00000000..24b4e301 --- /dev/null +++ b/operator/internal/controller/pulumi/metrics_program_test.go @@ -0,0 +1,52 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pulumi + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +var _ = Describe("Program Metrics", Ordered, func() { + BeforeAll(func() { + numPrograms.Set(0) + }) + + It("should increment the programs_active metric when a new Program is created", func() { + newProgramCallback(nil) + Expect(testutil.ToFloat64(numPrograms)).To(Equal(1.0)) + }) + + It("should increment the programs_active metric when another Program is created", func() { + newProgramCallback(nil) + Expect(testutil.ToFloat64(numPrograms)).To(Equal(2.0)) + }) + + It("should decrement the programs_active metric when a Program is deleted", func() { + deleteProgramCallback(nil) + Expect(testutil.ToFloat64(numPrograms)).To(Equal(1.0)) + }) + + It("should decrement the programs_active metric when another Program is deleted", func() { + deleteProgramCallback(nil) + Expect(testutil.ToFloat64(numPrograms)).To(Equal(0.0)) + }) + + It("should not decrement the programs_active metric when a Program is deleted and the metric is already at 0", func() { + deleteProgramCallback(nil) + Expect(testutil.ToFloat64(numPrograms)).To(Equal(0.0)) + }) +}) diff --git a/operator/internal/controller/pulumi/metrics_stack.go b/operator/internal/controller/pulumi/metrics_stack.go index 811b5917..771e0daa 100644 --- a/operator/internal/controller/pulumi/metrics_stack.go +++ b/operator/internal/controller/pulumi/metrics_stack.go @@ -81,6 +81,11 @@ func updateStackReconcilingMetrics(newStack *pulumiv1.Stack) { // deleteStackCallback is a callback that is called when a Stack object is deleted. func deleteStackCallback(oldObj any) { numStacks.Dec() + val, err := getGaugeValue(numStacks) + if err == nil && val < 0 { + numStacks.Set(0) + } + oldStack, ok := oldObj.(*pulumiv1.Stack) if !ok { return @@ -89,5 +94,4 @@ func deleteStackCallback(oldObj any) { // Reset any gauge metrics associated with the old stack. numStacksFailing.With(prometheus.Labels{"namespace": oldStack.Namespace, "name": oldStack.Name}).Set(0) numStacksReconciling.With(prometheus.Labels{"namespace": oldStack.Namespace, "name": oldStack.Name}).Set(0) - } diff --git a/operator/internal/controller/pulumi/metrics_stack_test.go b/operator/internal/controller/pulumi/metrics_stack_test.go index ea9f49d6..b97646ab 100644 --- a/operator/internal/controller/pulumi/metrics_stack_test.go +++ b/operator/internal/controller/pulumi/metrics_stack_test.go @@ -43,7 +43,6 @@ var _ = Describe("Stack Metrics", func() { }) It("should increment the numStacks metric", func() { - // Call the newStackCallback function newStackCallback(newStack) // Check if the numStacks metric has been incremented @@ -53,7 +52,6 @@ var _ = Describe("Stack Metrics", func() { }) It("should increment the numStacks metric again if another stack is created", func() { - // Call the newStackCallback function newStackCallback(newStack) // Check if the numStacks metric has been incremented @@ -61,6 +59,33 @@ var _ = Describe("Stack Metrics", func() { actual := testutil.ToFloat64(numStacks) Expect(actual).To(Equal(expected)) }) + + It("should decrement the numStacks metric if a stack is deleted", func() { + deleteStackCallback(newStack) + + // Check if the numStacks metric has been decremented + expected := 1.0 + actual := testutil.ToFloat64(numStacks) + Expect(actual).To(Equal(expected)) + }) + + It("should decrement the numStacks metric again if another stack is deleted", func() { + deleteStackCallback(newStack) + + // Check if the numStacks metric has been decremented + expected := 0.0 + actual := testutil.ToFloat64(numStacks) + Expect(actual).To(Equal(expected)) + }) + + It("should not decrement the numStacks metric if a stack is deleted and the metric is already at 0", func() { + deleteStackCallback(newStack) + + // Check if the numStacks metric has been decremented + expected := 0.0 + actual := testutil.ToFloat64(numStacks) + Expect(actual).To(Equal(expected)) + }) }) Context("when a stack is updated", func() { diff --git a/operator/internal/controller/pulumi/utils.go b/operator/internal/controller/pulumi/utils.go index ed59aeb4..db6bf04b 100644 --- a/operator/internal/controller/pulumi/utils.go +++ b/operator/internal/controller/pulumi/utils.go @@ -18,6 +18,9 @@ package pulumi import ( "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" ) func min(a, b time.Duration) time.Duration { @@ -44,3 +47,13 @@ func exactlyOneOf(these ...bool) bool { } return found } + +// getGaugeValue returns the value of a gauge metric. This is useful to check that a gauge +// does not go into negative values. +func getGaugeValue(metric prometheus.Gauge) (float64, error) { + var m = &dto.Metric{} + if err := metric.Write(m); err != nil { + return 0, err + } + return m.Gauge.GetValue(), nil +}