Skip to content

Commit

Permalink
Add metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
jichenjc committed May 12, 2021
1 parent c3779d3 commit 740f78d
Show file tree
Hide file tree
Showing 9 changed files with 147 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/onsi/ginkgo v1.16.0
github.com/onsi/gomega v1.11.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.9.0
github.com/spf13/pflag v1.0.5
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
gopkg.in/ini.v1 v1.62.0
Expand Down
3 changes: 3 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/controllers"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
"sigs.k8s.io/cluster-api-provider-openstack/version"
)
Expand Down Expand Up @@ -72,6 +73,8 @@ func init() {
_ = clusterv1.AddToScheme(scheme)
_ = infrav1.AddToScheme(scheme)
// +kubebuilder:scaffold:scheme

metrics.RegisterAPIPrometheusMetrics()
}

// InitFlags initializes the flags.
Expand Down
12 changes: 12 additions & 0 deletions pkg/cloud/services/compute/instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import (

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
capoerrors "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/errors"
)
Expand Down Expand Up @@ -265,10 +266,17 @@ func createInstance(is *Service, clusterName string, i *infrav1.Instance) (*infr

serverCreateOpts = applyServerGroupID(serverCreateOpts, i.ServerGroupID)

mc := metrics.NewMetricPrometheusContext("server", "create")

server, err := servers.Create(is.computeClient, keypairs.CreateOptsExt{
CreateOptsBuilder: serverCreateOpts,
KeyName: i.SSHKeyName,
}).Extract()

if mc.ObserveRequest(err) != nil {
return nil, err
}

if err != nil {
if errd := deletePorts(is, portsList); errd != nil {
return nil, fmt.Errorf("error recover creating Openstack instance: error cleaning up ports: %v", errd)
Expand Down Expand Up @@ -504,7 +512,11 @@ func createPort(is *Service, clusterName string, name string, net *infrav1.Netwo
if net.Subnet.ID != "" {
portCreateOpts.FixedIPs = []ports.IP{{SubnetID: net.Subnet.ID}}
}
mc := metrics.NewMetricPrometheusContext("port", "create")
newPort, err := ports.Create(is.networkClient, portCreateOpts).Extract()
if mc.ObserveRequest(err) != nil {
return ports.Port{}, err
}
if err != nil {
return ports.Port{}, fmt.Errorf("create port for server: %v", err)
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/cloud/services/loadbalancer/loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"sigs.k8s.io/cluster-api/util"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
)

Expand All @@ -55,8 +56,12 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust
Name: loadBalancerName,
VipSubnetID: openStackCluster.Status.Network.Subnet.ID,
}
mc := metrics.NewMetricPrometheusContext("server", "create")

lb, err = loadbalancers.Create(s.loadbalancerClient, lbCreateOpts).Extract()
if mc.ObserveRequest(err) != nil {
return err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateLoadBalancer", "Failed to create load balancer %s: %v", loadBalancerName, err)
return err
Expand Down
5 changes: 5 additions & 0 deletions pkg/cloud/services/networking/floatingip.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
)

Expand All @@ -46,7 +47,11 @@ func (s *Service) GetOrCreateFloatingIP(openStackCluster *infrav1.OpenStackClust

fpCreateOpts.FloatingNetworkID = openStackCluster.Status.ExternalNetwork.ID

mc := metrics.NewMetricPrometheusContext("floatingip", "create")
fp, err = floatingips.Create(s.client, fpCreateOpts).Extract()
if mc.ObserveRequest(err) != nil {
return nil, err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateFloatingIP", "Failed to create floating IP %s: %v", ip, err)
return nil, err
Expand Down
10 changes: 10 additions & 0 deletions pkg/cloud/services/networking/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/gophercloud/gophercloud/pagination"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
)

Expand Down Expand Up @@ -122,7 +123,12 @@ func (s *Service) ReconcileNetwork(openStackCluster *infrav1.OpenStackCluster, c
Name: networkName,
}
}
mc := metrics.NewMetricPrometheusContext("network", "create")

network, err := networks.Create(s.client, opts).Extract()
if mc.ObserveRequest(err) != nil {
return err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateNetwork", "Failed to create network %s: %v", networkName, err)
return err
Expand Down Expand Up @@ -220,7 +226,11 @@ func createSubnet(client *gophercloud.ServiceClient, openStackCluster *infrav1.O
CIDR: openStackCluster.Spec.NodeCIDR,
DNSNameservers: openStackCluster.Spec.DNSNameservers,
}
mc := metrics.NewMetricPrometheusContext("subnet", "create")
subnet, err := subnets.Create(client, opts).Extract()
if mc.ObserveRequest(err) != nil {
return nil, err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateSubnet", "Failed to create subnet %s: %v", name, err)
return nil, err
Expand Down
5 changes: 5 additions & 0 deletions pkg/cloud/services/networking/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
capoerrors "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/errors"
)
Expand Down Expand Up @@ -131,7 +132,11 @@ func createRouter(client *gophercloud.ServiceClient, openStackCluster *infrav1.O
NetworkID: openStackCluster.Status.ExternalNetwork.ID,
}
}
mc := metrics.NewMetricPrometheusContext("router", "create")
router, err := routers.Create(client, opts).Extract()
if mc.ObserveRequest(err) != nil {
return nil, err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateRouter", "Failed to create router %s: %v", name, err)
return nil, err
Expand Down
8 changes: 8 additions & 0 deletions pkg/cloud/services/networking/securitygroups.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"

infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
)

Expand Down Expand Up @@ -455,7 +456,14 @@ func (s *Service) createSecurityGroupIfNotExists(openStackCluster *infrav1.OpenS
Description: "Cluster API managed group",
}
s.logger.V(6).Info("Creating group", "name", groupName)

mc := metrics.NewMetricPrometheusContext("securitygroup", "create")

group, err := groups.Create(s.client, createOpts).Extract()

if mc.ObserveRequest(err) != nil {
return err
}
if err != nil {
record.Warnf(openStackCluster, "FailedCreateSecurityGroup", "Failed to create security group %s: %v", groupName, err)
return err
Expand Down
98 changes: 98 additions & 0 deletions pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

type OpenstackPrometheusMetrics struct {
Duration *prometheus.HistogramVec
Total *prometheus.CounterVec
Errors *prometheus.CounterVec
}

// MetricContext indicates the context for OpenStack metrics.
type MetricPrometheusContext struct {
Start time.Time
Attributes []string
Metrics *OpenstackPrometheusMetrics
}

// NewMetricContext creates a new MetricContext.
func NewMetricPrometheusContext(resource string, request string) *MetricPrometheusContext {
return &MetricPrometheusContext{
Start: time.Now(),
Attributes: []string{resource + "_" + request},
}
}

// ObserveRequest records the request latency and counts the errors.
func (mc *MetricPrometheusContext) ObserveRequest(err error) error {
return mc.Observe(APIRequestPrometheusMetrics, err)
}

// ObserveRequest records the request latency and counts the errors.
func (mc *MetricPrometheusContext) Observe(om *OpenstackPrometheusMetrics, err error) error {
if om == nil {
// mc.RequestMetrics not set, ignore this request
return nil
}

om.Duration.WithLabelValues(mc.Attributes...).Observe(
time.Since(mc.Start).Seconds())
om.Total.WithLabelValues(mc.Attributes...).Inc()
if err != nil {
om.Errors.WithLabelValues(mc.Attributes...).Inc()
}
return nil
}

var APIRequestPrometheusMetrics = &OpenstackPrometheusMetrics{
Duration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "capo",
Name: "openstack_api_request_duration_seconds",
Help: "Latency of an OpenStack API call",
}, []string{"request"}),
Total: prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "capo",
Name: "openstack_api_requests_total",
Help: "Total number of OpenStack API calls",
}, []string{"request"}),
Errors: prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "capo",
Name: "openstack_api_request_errors_total",
Help: "Total number of errors for an OpenStack API call",
}, []string{"request"}),
}

var registerAPIPrometheusMetrics sync.Once

func RegisterAPIPrometheusMetrics() {
registerAPIPrometheusMetrics.Do(func() {
metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Duration)
metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Total)
metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Errors)
})
}

0 comments on commit 740f78d

Please sign in to comment.