diff --git a/controllers/openstackcluster_controller.go b/controllers/openstackcluster_controller.go index 8dcb6133e0..2df9d2a711 100644 --- a/controllers/openstackcluster_controller.go +++ b/controllers/openstackcluster_controller.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" ) // OpenStackClusterReconciler reconciles a OpenStackCluster object. @@ -66,9 +67,14 @@ type OpenStackClusterReconciler struct { func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { log := ctrl.LoggerFrom(ctx) + mc := metrics.NewMetricPrometheusContext("snapshot", "create") + // Fetch the OpenStackCluster instance openStackCluster := &infrav1.OpenStackCluster{} err := r.Client.Get(ctx, req.NamespacedName, openStackCluster) + if mc.ObserveRequest(err) != nil { + return reconcile.Result{}, err + } if err != nil { if apierrors.IsNotFound(err) { return reconcile.Result{}, nil diff --git a/go.mod b/go.mod index a0a5bc6a56..2cf0a18ee5 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/onsi/ginkgo v1.16.0 github.com/onsi/gomega v1.11.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.9.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 gopkg.in/ini.v1 v1.62.0 diff --git a/main.go b/main.go index 4151600274..ffc67751af 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-openstack/controllers" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/version" ) @@ -72,6 +73,8 @@ func init() { _ = clusterv1.AddToScheme(scheme) _ = infrav1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme + + metrics.RegisterAPIPrometheusMetrics() } // InitFlags initializes the flags. diff --git a/pkg/cloud/services/compute/instance.go b/pkg/cloud/services/compute/instance.go index c5c6668a23..bd69508aa1 100644 --- a/pkg/cloud/services/compute/instance.go +++ b/pkg/cloud/services/compute/instance.go @@ -47,6 +47,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/networking" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" capoerrors "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/errors" ) @@ -265,10 +266,17 @@ func createInstance(is *Service, clusterName string, i *infrav1.Instance) (*infr serverCreateOpts = applyServerGroupID(serverCreateOpts, i.ServerGroupID) + mc := metrics.NewMetricPrometheusContext("server", "create") + server, err := servers.Create(is.computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: serverCreateOpts, KeyName: i.SSHKeyName, }).Extract() + + if mc.ObserveRequest(err) != nil { + return nil, err + } + if err != nil { if errd := deletePorts(is, portsList); errd != nil { return nil, fmt.Errorf("error recover creating Openstack instance: error cleaning up ports: %v", errd) @@ -504,7 +512,11 @@ func createPort(is *Service, clusterName string, name string, net *infrav1.Netwo if net.Subnet.ID != "" { portCreateOpts.FixedIPs = []ports.IP{{SubnetID: net.Subnet.ID}} } + mc := metrics.NewMetricPrometheusContext("port", "create") newPort, err := ports.Create(is.networkClient, portCreateOpts).Extract() + if mc.ObserveRequest(err) != nil { + return ports.Port{}, err + } if err != nil { return ports.Port{}, fmt.Errorf("create port for server: %v", err) } diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index 59b3f12ee4..35456f785f 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/cluster-api/util" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" ) @@ -55,8 +56,12 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust Name: loadBalancerName, VipSubnetID: openStackCluster.Status.Network.Subnet.ID, } + mc := metrics.NewMetricPrometheusContext("server", "create") lb, err = loadbalancers.Create(s.loadbalancerClient, lbCreateOpts).Extract() + if mc.ObserveRequest(err) != nil { + return err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateLoadBalancer", "Failed to create load balancer %s: %v", loadBalancerName, err) return err diff --git a/pkg/cloud/services/networking/floatingip.go b/pkg/cloud/services/networking/floatingip.go index b947bc3c05..1f37aaa83b 100644 --- a/pkg/cloud/services/networking/floatingip.go +++ b/pkg/cloud/services/networking/floatingip.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" ) @@ -46,7 +47,11 @@ func (s *Service) GetOrCreateFloatingIP(openStackCluster *infrav1.OpenStackClust fpCreateOpts.FloatingNetworkID = openStackCluster.Status.ExternalNetwork.ID + mc := metrics.NewMetricPrometheusContext("floatingip", "create") fp, err = floatingips.Create(s.client, fpCreateOpts).Extract() + if mc.ObserveRequest(err) != nil { + return nil, err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateFloatingIP", "Failed to create floating IP %s: %v", ip, err) return nil, err diff --git a/pkg/cloud/services/networking/network.go b/pkg/cloud/services/networking/network.go index 7ac0a88dd4..ade0efa850 100644 --- a/pkg/cloud/services/networking/network.go +++ b/pkg/cloud/services/networking/network.go @@ -27,6 +27,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" ) @@ -122,7 +123,12 @@ func (s *Service) ReconcileNetwork(openStackCluster *infrav1.OpenStackCluster, c Name: networkName, } } + mc := metrics.NewMetricPrometheusContext("network", "create") + network, err := networks.Create(s.client, opts).Extract() + if mc.ObserveRequest(err) != nil { + return err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateNetwork", "Failed to create network %s: %v", networkName, err) return err @@ -220,7 +226,11 @@ func createSubnet(client *gophercloud.ServiceClient, openStackCluster *infrav1.O CIDR: openStackCluster.Spec.NodeCIDR, DNSNameservers: openStackCluster.Spec.DNSNameservers, } + mc := metrics.NewMetricPrometheusContext("subnet", "create") subnet, err := subnets.Create(client, opts).Extract() + if mc.ObserveRequest(err) != nil { + return nil, err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateSubnet", "Failed to create subnet %s: %v", name, err) return nil, err diff --git a/pkg/cloud/services/networking/router.go b/pkg/cloud/services/networking/router.go index 5686416f35..2c74e08622 100644 --- a/pkg/cloud/services/networking/router.go +++ b/pkg/cloud/services/networking/router.go @@ -26,6 +26,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" capoerrors "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/errors" ) @@ -131,7 +132,11 @@ func createRouter(client *gophercloud.ServiceClient, openStackCluster *infrav1.O NetworkID: openStackCluster.Status.ExternalNetwork.ID, } } + mc := metrics.NewMetricPrometheusContext("router", "create") router, err := routers.Create(client, opts).Extract() + if mc.ObserveRequest(err) != nil { + return nil, err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateRouter", "Failed to create router %s: %v", name, err) return nil, err diff --git a/pkg/cloud/services/networking/securitygroups.go b/pkg/cloud/services/networking/securitygroups.go index 09c77fa52f..987f344559 100644 --- a/pkg/cloud/services/networking/securitygroups.go +++ b/pkg/cloud/services/networking/securitygroups.go @@ -23,6 +23,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" ) @@ -455,7 +456,14 @@ func (s *Service) createSecurityGroupIfNotExists(openStackCluster *infrav1.OpenS Description: "Cluster API managed group", } s.logger.V(6).Info("Creating group", "name", groupName) + + mc := metrics.NewMetricPrometheusContext("securitygroup", "create") + group, err := groups.Create(s.client, createOpts).Extract() + + if mc.ObserveRequest(err) != nil { + return err + } if err != nil { record.Warnf(openStackCluster, "FailedCreateSecurityGroup", "Failed to create security group %s: %v", groupName, err) return err diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000000..930698d21e --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,97 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +type OpenstackPrometheusMetrics struct { + Duration *prometheus.HistogramVec + Total *prometheus.CounterVec + Errors *prometheus.CounterVec +} + +// MetricContext indicates the context for OpenStack metrics. +type MetricPrometheusContext struct { + Start time.Time + Attributes []string + Metrics *OpenstackPrometheusMetrics +} + +// NewMetricContext creates a new MetricContext. +func NewMetricPrometheusContext(resource string, request string) *MetricPrometheusContext { + return &MetricPrometheusContext{ + Start: time.Now(), + Attributes: []string{resource + "_" + request}, + } +} + +// ObserveRequest records the request latency and counts the errors. +func (mc *MetricPrometheusContext) ObserveRequest(err error) error { + return mc.Observe(APIRequestPrometheusMetrics, err) +} + +// ObserveRequest records the request latency and counts the errors. +func (mc *MetricPrometheusContext) Observe(om *OpenstackPrometheusMetrics, err error) error { + if om == nil { + // mc.RequestMetrics not set, ignore this request + return nil + } + + om.Duration.WithLabelValues(mc.Attributes...).Observe( + time.Since(mc.Start).Seconds()) + om.Total.WithLabelValues(mc.Attributes...).Inc() + if err != nil { + om.Errors.WithLabelValues(mc.Attributes...).Inc() + } + return err +} + +var ( + APIRequestPrometheusMetrics = &OpenstackPrometheusMetrics{ + Duration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "capo", + Name: "openstack_api_request_duration_seconds", + Help: "Latency of an OpenStack API call", + }, []string{"request"}), + Total: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "capo", + Name: "openstack_api_requests_total", + Help: "Total number of OpenStack API calls", + }, []string{"request"}), + Errors: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "capo", + Name: "openstack_api_request_errors_total", + Help: "Total number of errors for an OpenStack API call", + }, []string{"request"}), + } +) + +var registerAPIPrometheusMetrics sync.Once + +func RegisterAPIPrometheusMetrics() { + registerAPIPrometheusMetrics.Do(func() { + metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Duration) + metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Total) + metrics.Registry.MustRegister(APIRequestPrometheusMetrics.Errors) + }) +}