From 216d0a0b9c35256f6b5fab6b5670f497a7e5900d Mon Sep 17 00:00:00 2001 From: Sean Schneeweiss Date: Mon, 31 Aug 2020 14:45:00 +0200 Subject: [PATCH 1/3] [occm] move metrics to seperate folder By moving the metrics definition to a seperate folder it can be used by pkg/cloudprovider/providers/openstack and pkg/util/openstack. Signed-off-by: Sean Schneeweiss --- .../openstack/{openstack_metrics.go => metrics/metrics.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/cloudprovider/providers/openstack/{openstack_metrics.go => metrics/metrics.go} (100%) diff --git a/pkg/cloudprovider/providers/openstack/openstack_metrics.go b/pkg/cloudprovider/providers/openstack/metrics/metrics.go similarity index 100% rename from pkg/cloudprovider/providers/openstack/openstack_metrics.go rename to pkg/cloudprovider/providers/openstack/metrics/metrics.go From 91bb72649113366ae8d8eac5fb088e00765c128e Mon Sep 17 00:00:00 2001 From: Sean Schneeweiss Date: Mon, 31 Aug 2020 19:47:34 +0200 Subject: [PATCH 2/3] [occm] API request and reconcile metrics This commit implements metrics for OpenStack API calls and for loadbalancer reconciliations. Signed-off-by: Sean Schneeweiss --- go.mod | 1 - pkg/cloudprovider/.import-restrictions | 1 + .../providers/openstack/metrics/metrics.go | 127 ++++++++--- .../providers/openstack/openstack.go | 23 +- .../openstack/openstack_instances.go | 13 +- .../openstack/openstack_loadbalancer.go | 210 +++++++++++++----- .../providers/openstack/openstack_routes.go | 25 ++- pkg/util/openstack/keymanager.go | 13 +- pkg/util/openstack/loadbalancer.go | 33 ++- pkg/util/openstack/network.go | 4 +- 10 files changed, 328 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index c6ca3f433a..3461824b8f 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/onsi/gomega v1.9.0 github.com/pborman/uuid v1.2.0 github.com/pelletier/go-toml v1.4.0 // indirect - github.com/prometheus/client_golang v1.7.1 github.com/sirupsen/logrus v1.6.0 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 diff --git a/pkg/cloudprovider/.import-restrictions b/pkg/cloudprovider/.import-restrictions index 0cff927521..d928c34095 100644 --- a/pkg/cloudprovider/.import-restrictions +++ b/pkg/cloudprovider/.import-restrictions @@ -5,6 +5,7 @@ "k8s.io/api", "k8s.io/client-go", "k8s.io/cloud-provider", + "k8s.io/component-base", "k8s.io/klog", "k8s.io/utils" ], diff --git a/pkg/cloudprovider/providers/openstack/metrics/metrics.go b/pkg/cloudprovider/providers/openstack/metrics/metrics.go index e483875d46..c24b86f5ce 100644 --- a/pkg/cloudprovider/providers/openstack/metrics/metrics.go +++ b/pkg/cloudprovider/providers/openstack/metrics/metrics.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,44 +14,109 @@ See the License for the specific language governing permissions and limitations under the License. */ -package openstack +package metrics import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/klog/v2" -) + "sync" + "time" -const ( - openstackSubsystem = "openstack" - openstackOperationKey = "cloudprovider_openstack_api_request_duration_seconds" - openstackOperationErrorKey = "cloudprovider_openstack_api_request_errors" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) +type openstackMetrics struct { + duration *metrics.HistogramVec + total *metrics.CounterVec + errors *metrics.CounterVec +} + var ( - openstackOperationsLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: openstackSubsystem, - Name: openstackOperationKey, - Help: "Latency of openstack api call", - }, - []string{"request"}, - ) - - openstackAPIRequestErrors = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: openstackSubsystem, - Name: openstackOperationErrorKey, - Help: "Cumulative number of openstack Api call errors", - }, - []string{"request"}, - ) + reconcileMetrics = &openstackMetrics{ + duration: metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: "cloudprovider_openstack_reconcile_duration_seconds", + Help: "Time taken by various parts of OpenStack cloud controller manager reconciliation loops", + Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0, 7.5, 10.0, 12.5, 15.0, 17.5, 20.0, 22.5, 25.0, 27.5, 30.0, 50.0, 75.0, 100.0, 1000.0}, + }, []string{"operation"}), + total: metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "cloudprovider_openstack_reconcile_total", + Help: "Total number of OpenStack cloud controller manager reconciliations", + }, []string{"operation"}), + errors: metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "cloudprovider_openstack_reconcile_errors_total", + Help: "Total number of OpenStack cloud controller manager reconciliation errors", + }, []string{"operation"}), + } + requestMetrics = &openstackMetrics{ + duration: metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: "openstack_api_request_duration_seconds", + Help: "Latency of an OpenStack API call", + }, []string{"request"}), + total: metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "openstack_api_requests_total", + Help: "Total number of OpenStack API calls", + }, []string{"request"}), + errors: metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "openstack_api_request_errors_total", + Help: "Total number of errors for an OpenStack API call", + }, []string{"request"}), + } ) -func RegisterMetrics() { - if err := prometheus.Register(openstackOperationsLatency); err != nil { - klog.V(5).Infof("unable to register for latency metrics") +// MetricContext indicates the context for OpenStack metrics. +type MetricContext struct { + start time.Time + attributes []string +} + +// NewMetricContext creates a new MetricContext. +func NewMetricContext(resource string, request string) *MetricContext { + return &MetricContext{ + start: time.Now(), + attributes: []string{resource + "_" + request}, } - if err := prometheus.Register(openstackAPIRequestErrors); err != nil { - klog.V(5).Infof("unable to register for error metrics") +} + +// ObserveReconcile records reconciliation duration, +// frequency and number of errors. +func (mc *MetricContext) ObserveReconcile(err error) error { + reconcileMetrics.duration.WithLabelValues(mc.attributes...).Observe( + time.Since(mc.start).Seconds()) + reconcileMetrics.total.WithLabelValues(mc.attributes...).Inc() + if err != nil { + reconcileMetrics.errors.WithLabelValues(mc.attributes...).Inc() + } + return err +} + +// ObserveRequest records the request latency and counts the errors. +func (mc *MetricContext) ObserveRequest(err error) error { + requestMetrics.duration.WithLabelValues(mc.attributes...).Observe( + time.Since(mc.start).Seconds()) + requestMetrics.total.WithLabelValues(mc.attributes...).Inc() + if err != nil { + requestMetrics.errors.WithLabelValues(mc.attributes...).Inc() } + return err +} + +var registerMetrics sync.Once + +// RegisterMetrics registers OpenStack metrics. +func RegisterMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister( + reconcileMetrics.duration, + reconcileMetrics.total, + reconcileMetrics.errors, + requestMetrics.duration, + requestMetrics.total, + requestMetrics.errors, + ) + }) } diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 3406008495..7799b69e9e 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -54,6 +54,7 @@ import ( netutil "k8s.io/apimachinery/pkg/util/net" certutil "k8s.io/client-go/util/cert" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" "k8s.io/cloud-provider-openstack/pkg/util/metadata" "k8s.io/cloud-provider-openstack/pkg/version" "k8s.io/klog/v2" @@ -290,7 +291,7 @@ func (l Logger) Printf(format string, args ...interface{}) { } func init() { - RegisterMetrics() + metrics.RegisterMetrics() cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { cfg, err := ReadConfig(config) @@ -627,8 +628,9 @@ func (os *OpenStack) GetNodeNameByID(instanceID string) (types.NodeName, error) return nodeName, err } + mc := metrics.NewMetricContext("server", "get") server, err := servers.Get(client, instanceID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nodeName, err } nodeName = mapServerToNodeName(server) @@ -644,6 +646,7 @@ func mapServerToNodeName(server *servers.Server) types.NodeName { } func foreachServer(client *gophercloud.ServiceClient, opts servers.ListOptsBuilder, handler func(*servers.Server) (bool, error)) error { + mc := metrics.NewMetricContext("server", "list") pager := servers.List(client, opts) err := pager.EachPage(func(page pagination.Page) (bool, error) { @@ -659,7 +662,7 @@ func foreachServer(client *gophercloud.ServiceClient, opts servers.ListOptsBuild } return true, nil }) - return err + return mc.ObserveRequest(err) } func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*ServerAttributesExt, error) { @@ -667,11 +670,12 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S Name: fmt.Sprintf("^%s$", regexp.QuoteMeta(mapNodeNameToServerName(name))), } - pager := servers.List(client, opts) - var s []ServerAttributesExt serverList := make([]ServerAttributesExt, 0, 1) + mc := metrics.NewMetricContext("server", "list") + pager := servers.List(client, opts) + err := pager.EachPage(func(page pagination.Page) (bool, error) { if err := servers.ExtractServersInto(page, &s); err != nil { return false, err @@ -682,7 +686,7 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S } return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } @@ -854,6 +858,7 @@ func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName, ne func getAttachedInterfacesByID(client *gophercloud.ServiceClient, serviceID string) ([]attachinterfaces.Interface, error) { var interfaces []attachinterfaces.Interface + mc := metrics.NewMetricContext("server_os_interface", "list") pager := attachinterfaces.List(client, serviceID) err := pager.EachPage(func(page pagination.Page) (bool, error) { s, err := attachinterfaces.ExtractInterfaces(page) @@ -863,7 +868,7 @@ func getAttachedInterfacesByID(client *gophercloud.ServiceClient, serviceID stri interfaces = append(interfaces, s...) return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { return interfaces, err } @@ -956,7 +961,9 @@ func (os *OpenStack) GetZoneByProviderID(ctx context.Context, providerID string) } var serverWithAttributesExt ServerAttributesExt - if err := servers.Get(compute, instanceID).ExtractInto(&serverWithAttributesExt); err != nil { + mc := metrics.NewMetricContext("server", "get") + err = servers.Get(compute, instanceID).ExtractInto(&serverWithAttributesExt) + if mc.ObserveRequest(err) != nil { return cloudprovider.Zone{}, err } diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/pkg/cloudprovider/providers/openstack/openstack_instances.go index d060832f5f..bad6de134a 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_instances.go +++ b/pkg/cloudprovider/providers/openstack/openstack_instances.go @@ -29,6 +29,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" "k8s.io/cloud-provider-openstack/pkg/util/errors" "k8s.io/cloud-provider-openstack/pkg/util/metadata" ) @@ -111,9 +112,10 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st return []v1.NodeAddress{}, err } + mc := metrics.NewMetricContext("server", "get") server, err := servers.Get(i.compute, instanceID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return []v1.NodeAddress{}, err } @@ -143,8 +145,9 @@ func (i *Instances) InstanceExistsByProviderID(ctx context.Context, providerID s return false, err } + mc := metrics.NewMetricContext("server", "get") _, err = servers.Get(i.compute, instanceID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { if errors.IsNotFound(err) { return false, nil } @@ -168,8 +171,9 @@ func (i *Instances) InstanceShutdownByProviderID(ctx context.Context, providerID return false, err } + mc := metrics.NewMetricContext("server", "get") server, err := servers.Get(i.compute, instanceID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return false, err } @@ -249,9 +253,10 @@ func (i *Instances) InstanceTypeByProviderID(ctx context.Context, providerID str return "", err } + mc := metrics.NewMetricContext("server", "get") server, err := servers.Get(i.compute, instanceID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return "", err } diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index 2ecd5364da..1cba4d1372 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -49,6 +49,7 @@ import ( cloudprovider "k8s.io/cloud-provider" klog "k8s.io/klog/v2" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" cpoutil "k8s.io/cloud-provider-openstack/pkg/util" cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" netsets "k8s.io/cloud-provider-openstack/pkg/util/net/sets" @@ -137,6 +138,7 @@ type listenerKey struct { func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, error) { seen := make(map[string]bool) + mc := metrics.NewMetricContext("extension", "list") pager := extensions.List(client) err := pager.EachPage(func(page pagination.Page) (bool, error) { exts, err := extensions.ExtractExtensions(page) @@ -149,12 +151,13 @@ func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, erro return true, nil }) - return seen, err + return seen, mc.ObserveRequest(err) } func getLoadBalancers(client *gophercloud.ServiceClient, opts loadbalancers.ListOpts) ([]loadbalancers.LoadBalancer, error) { + mc := metrics.NewMetricContext("loadbalancer", "list") allPages, err := loadbalancers.List(client, opts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } allLoadbalancers, err := loadbalancers.ExtractLoadBalancers(allPages) @@ -290,11 +293,11 @@ func getSecurityGroupName(service *corev1.Service) string { } func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpts) ([]rules.SecGroupRule, error) { + var securityRules []rules.SecGroupRule + mc := metrics.NewMetricContext("security_group_rule", "list") pager := rules.List(client, opts) - var securityRules []rules.SecGroupRule - err := pager.EachPage(func(page pagination.Page) (bool, error) { ruleList, err := rules.ExtractRules(page) if err != nil { @@ -304,7 +307,7 @@ func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpt return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } @@ -320,8 +323,9 @@ func waitLoadbalancerActiveProvisioningStatus(client *gophercloud.ServiceClient, var provisioningStatus string err := wait.ExponentialBackoff(backoff, func() (bool, error) { + mc := metrics.NewMetricContext("loadbalancer", "get") loadbalancer, err := loadbalancers.Get(client, loadbalancerID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return false, err } provisioningStatus = loadbalancer.ProvisioningStatus @@ -348,14 +352,15 @@ func waitLoadbalancerDeleted(client *gophercloud.ServiceClient, loadbalancerID s Steps: loadbalancerDeleteSteps, } err := wait.ExponentialBackoff(backoff, func() (bool, error) { + mc := metrics.NewMetricContext("loadbalancer", "get") _, err := loadbalancers.Get(client, loadbalancerID).Extract() if err != nil { if cpoerrors.IsNotFound(err) { - return true, nil + return true, mc.ObserveRequest(nil) } - return false, err + return false, mc.ObserveRequest(err) } - return false, nil + return false, mc.ObserveRequest(nil) }) if err == wait.ErrWaitTimeout { @@ -406,15 +411,15 @@ func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGrou EtherType: rules.EtherType6, } + mc := metrics.NewMetricContext("security_group_rule", "create") _, err := rules.Create(client, v4NodeSecGroupRuleCreateOpts).Extract() - - if err != nil { + if mc.ObserveRequest(err) != nil { return err } + mc = metrics.NewMetricContext("security_group_rule", "create") _, err = rules.Create(client, v6NodeSecGroupRuleCreateOpts).Extract() - - if err != nil { + if mc.ObserveRequest(err) != nil { return err } return nil @@ -451,8 +456,9 @@ func (lbaas *LbaasV2) createLoadBalancer(service *corev1.Service, name, clusterN createOpts.VipAddress = loadBalancerIP } + mc := metrics.NewMetricContext("loadbalancer", "create") loadbalancer, err := loadbalancers.Create(lbaas.lb, createOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating loadbalancer %v: %v", createOpts, err) } @@ -502,8 +508,9 @@ func (lbaas *LbaasV2) createOctaviaLoadBalancer(service *corev1.Service, name, c createOpts.VipAddress = loadBalancerIP } + mc := metrics.NewMetricContext("loadbalancer", "create") loadbalancer, err := loadbalancers.Create(lbaas.lb, createOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating loadbalancer %v: %v", createOpts, err) } @@ -672,8 +679,9 @@ func getSubnetIDForLB(compute *gophercloud.ServiceClient, node corev1.Node) (str // getPorts gets all the filtered ports. func getPorts(network *gophercloud.ServiceClient, listOpts neutronports.ListOpts) ([]neutronports.Port, error) { + mc := metrics.NewMetricContext("port", "list") allPages, err := neutronports.List(network, listOpts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return []neutronports.Port{}, err } allPorts, err := neutronports.ExtractPorts(allPages) @@ -702,12 +710,15 @@ func applyNodeSecurityGroupIDForLB(compute *gophercloud.ServiceClient, network * for _, port := range allPorts { newSGs := append(port.SecurityGroups, sg) updateOpts := neutronports.UpdateOpts{SecurityGroups: &newSGs} + mc := metrics.NewMetricContext("port", "update") res := neutronports.Update(network, port.ID, updateOpts) - if res.Err != nil { + if mc.ObserveRequest(res.Err) != nil { return fmt.Errorf("failed to update security group for port %s: %v", port.ID, res.Err) } // Add the security group ID as a tag to the port in order to find all these ports when removing the security group. - if err := neutrontags.Add(network, "ports", port.ID, sg).ExtractErr(); err != nil { + mc = metrics.NewMetricContext("port_tag", "add") + err := neutrontags.Add(network, "ports", port.ID, sg).ExtractErr() + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to add tag %s to port %s: %v", sg, port.ID, res.Err) } } @@ -736,12 +747,15 @@ func disassociateSecurityGroupForLB(network *gophercloud.ServiceClient, sg strin // Update port security groups newSGs := existingSGs.List() updateOpts := neutronports.UpdateOpts{SecurityGroups: &newSGs} + mc := metrics.NewMetricContext("port", "update") res := neutronports.Update(network, port.ID, updateOpts) - if res.Err != nil { + if mc.ObserveRequest(res.Err) != nil { return fmt.Errorf("failed to update security group for port %s: %v", port.ID, res.Err) } // Remove the security group ID tag from the port. - if err := neutrontags.Delete(network, "ports", port.ID, sg).ExtractErr(); err != nil { + mc = metrics.NewMetricContext("port_tag", "delete") + err := neutrontags.Delete(network, "ports", port.ID, sg).ExtractErr() + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to remove tag %s to port %s: %v", sg, port.ID, res.Err) } } @@ -805,6 +819,7 @@ func getFloatingNetworkIDForLB(client *gophercloud.ServiceClient) (string, error external.NetworkExternalExt } + mc := metrics.NewMetricContext("network", "list") err := networks.List(client, networks.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { var externalNetwork []NetworkWithExternalExt err := networks.ExtractNetworksInto(page, &externalNetwork) @@ -825,21 +840,21 @@ func getFloatingNetworkIDForLB(client *gophercloud.ServiceClient) (string, error }) if err != nil { if cpoerrors.IsNotFound(err) { - return "", ErrNotFound + return "", mc.ObserveRequest(ErrNotFound) } if err == ErrMultipleResults { klog.V(4).Infof("find multiple external networks, pick the first one when there are no explicit configuration.") - return floatingNetworkIds[0], nil + return floatingNetworkIds[0], mc.ObserveRequest(nil) } - return "", err + return "", mc.ObserveRequest(err) } if len(floatingNetworkIds) == 0 { - return "", ErrNotFound + return "", mc.ObserveRequest(ErrNotFound) } - return floatingNetworkIds[0], nil + return floatingNetworkIds[0], mc.ObserveRequest(nil) } func (lbaas *LbaasV2) deleteListeners(lbID string, listenerList []listeners.Listener) error { @@ -854,20 +869,26 @@ func (lbaas *LbaasV2) deleteListeners(lbID string, listenerList []listeners.List klog.V(2).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) // Delete pool automatically deletes all its members. + mc := metrics.NewMetricContext("loadbalancer_pool", "delete") err = v2pools.Delete(lbaas.lb, pool.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return fmt.Errorf("error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) } } + mc := metrics.NewMetricContext("loadbalancer_listener", "delete") err = listeners.Delete(lbaas.lb, listener.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return fmt.Errorf("failed to delete obsolete listener %s: %v", listener.ID, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) @@ -916,8 +937,9 @@ func (lbaas *LbaasV2) getServiceAddress(clusterName string, service *corev1.Serv floatUpdateOpts := floatingips.UpdateOpts{ PortID: &portID, } + mc := metrics.NewMetricContext("floating_ip", "update") floatIP, err = floatingips.Update(lbaas.network, floatingip.ID, floatUpdateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return "", fmt.Errorf("error updating LB floatingip %+v: %v", floatUpdateOpts, err) } } else { @@ -945,8 +967,9 @@ func (lbaas *LbaasV2) getServiceAddress(clusterName string, service *corev1.Serv klog.V(4).Infof("Creating floating ip with opts %+v", floatIPOpts) + mc := metrics.NewMetricContext("floating_ip", "create") floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return "", fmt.Errorf("error creating LB floatingip: %v", err) } } else { @@ -971,6 +994,7 @@ func (lbaas *LbaasV2) ensureOctaviaHealthMonitor(lbID string, pool *v2pools.Pool if port.Protocol == corev1.ProtocolUDP { monitorProtocol = "UDP-CONNECT" } + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "create") monitor, err := v2monitors.Create(lbaas.lb, v2monitors.CreateOpts{ PoolID: pool.ID, Type: monitorProtocol, @@ -978,7 +1002,7 @@ func (lbaas *LbaasV2) ensureOctaviaHealthMonitor(lbID string, pool *v2pools.Pool Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lbaas.opts.MonitorMaxRetries), }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to create healthmonitor for pool %s: %v", pool.ID, err) } provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) @@ -989,8 +1013,9 @@ func (lbaas *LbaasV2) ensureOctaviaHealthMonitor(lbID string, pool *v2pools.Pool } else if monitorID != "" && !svcConf.enableMonitor { klog.Infof("Deleting health monitor %s for pool %s", monitorID, pool.ID) + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "delete") err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to delete health monitor %s for pool %s, error: %v", monitorID, pool.ID, err) } } @@ -1063,8 +1088,9 @@ func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, listener *listeners.Listene klog.V(2).Infof("Creating pool for listener %s using protocol %s", listener.ID, poolProto) + mc := metrics.NewMetricContext("loadbalancer_pool", "create") pool, err = v2pools.Create(lbaas.lb, createOpt).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating pool for listener %s: %v", listener.ID, err) } @@ -1296,8 +1322,9 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node // check subnets belongs to network if floatingNetworkID != "" && floatingSubnetID != "" { + mc := metrics.NewMetricContext("subnet", "get") subnet, err := subnets.Get(lbaas.network, floatingSubnetID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to find subnet %q: %v", floatingSubnetID, err) } @@ -1432,6 +1459,12 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName // EnsureLoadBalancer creates a new load balancer or updates the existing one. func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *corev1.Service, nodes []*corev1.Node) (*corev1.LoadBalancerStatus, error) { + mc := metrics.NewMetricContext("loadbalancer", "ensure") + status, err := lbaas.ensureLoadBalancer(ctx, clusterName, apiService, nodes) + return status, mc.ObserveReconcile(err) +} + +func (lbaas *LbaasV2) ensureLoadBalancer(ctx context.Context, clusterName string, apiService *corev1.Service, nodes []*corev1.Node) (*corev1.LoadBalancerStatus, error) { serviceName := fmt.Sprintf("%s/%s", apiService.Namespace, apiService.Name) klog.V(4).Infof("EnsureLoadBalancer(%s, %s)", clusterName, serviceName) @@ -1736,8 +1769,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string klog.V(4).Infof("Creating pool for listener %s using protocol %s", listener.ID, poolProto) + mc := metrics.NewMetricContext("loadbalancer_pool", "create") pool, err = v2pools.Create(lbaas.lb, createOpt).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating pool for listener %s: %v", listener.ID, err) } provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) @@ -1767,13 +1801,14 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if !memberExists(members, addr, int(port.NodePort)) { klog.V(4).Infof("Creating member for pool %s", pool.ID) + mc := metrics.NewMetricContext("loadbalancer_member", "create") _, err := v2pools.CreateMember(lbaas.lb, pool.ID, v2pools.CreateMemberOpts{ Name: cutString(fmt.Sprintf("member_%d_%s_%s", portIndex, node.Name, name)), ProtocolPort: int(port.NodePort), Address: addr, SubnetID: lbaas.opts.SubnetID, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating LB pool member for node: %s, %v", node.Name, err) } @@ -1792,10 +1827,13 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string // Delete obsolete members for this pool for _, member := range members { klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) + mc := metrics.NewMetricContext("loadbalancer_member", "delete") err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) @@ -1813,6 +1851,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if port.Protocol == corev1.ProtocolUDP { monitorProtocol = "UDP-CONNECT" } + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "create") monitor, err := v2monitors.Create(lbaas.lb, v2monitors.CreateOpts{ Name: cutString(fmt.Sprintf("monitor_%d_%s)", portIndex, name)), PoolID: pool.ID, @@ -1821,7 +1860,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lbaas.opts.MonitorMaxRetries), }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating LB pool healthmonitor: %v", err) } provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) @@ -1831,8 +1870,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string monitorID = monitor.ID } else if monitorID != "" && !enableHealthMonitor { klog.Infof("Deleting health monitor %s for pool %s", monitorID, pool.ID) + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "delete") err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("failed to delete health monitor %s for pool %s, error: %v", monitorID, pool.ID, err) } } @@ -1851,10 +1891,13 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string monitorID := pool.MonitorID if monitorID != "" { klog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "delete") err = v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return nil, fmt.Errorf("error deleting obsolete monitor %s for pool %s: %v", monitorID, pool.ID, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting monitor, current provisioning status %s", provisioningStatus) @@ -1868,10 +1911,13 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string if members != nil { for _, member := range members { klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) + mc := metrics.NewMetricContext("loadbalancer_member", "delete") err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) @@ -1880,20 +1926,26 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } klog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) // delete pool + mc := metrics.NewMetricContext("loadbalancer_pool", "delete") err = v2pools.Delete(lbaas.lb, pool.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return nil, fmt.Errorf("error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) } } // delete listener + mc := metrics.NewMetricContext("loadbalancer_listener", "delete") err = listeners.Delete(lbaas.lb, listener.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return nil, fmt.Errorf("error deleteting obsolete listener: %v", err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) @@ -1934,8 +1986,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string floatUpdateOpts := floatingips.UpdateOpts{ PortID: &portID, } + mc := metrics.NewMetricContext("floating_ip", "update") floatIP, err = floatingips.Update(lbaas.network, floatingip.ID, floatUpdateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error updating LB floatingip %+v: %v", floatUpdateOpts, err) } } else { @@ -1963,8 +2016,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string } klog.V(4).Infof("creating floating ip with opts %+v", floatIPOpts) + mc := metrics.NewMetricContext("floating_ip", "create") floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error creating LB floatingip %+v: %v", floatIPOpts, err) } } else { @@ -1997,8 +2051,9 @@ func (lbaas *LbaasV2) getSubnet(subnet string) (*subnets.Subnet, error) { return nil, nil } + mc := metrics.NewMetricContext("subnet", "list") allPages, err := subnets.List(lbaas.network, subnets.ListOpts{Name: subnet}).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, fmt.Errorf("error listing subnets: %v", err) } subs, err := subnets.ExtractSubnets(allPages) @@ -2059,8 +2114,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 Description: fmt.Sprintf("Security Group for %s/%s Service LoadBalancer in cluster %s", apiService.Namespace, apiService.Name, clusterName), } + mc := metrics.NewMetricContext("security_group", "create") lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to create Security Group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) } lbSecGroupID = lbSecGroup.ID @@ -2090,9 +2146,10 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 EtherType: ethertype, } + mc := metrics.NewMetricContext("security_group_rule", "create") _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) } } @@ -2108,9 +2165,10 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 EtherType: rules.EtherType4, } + mc := metrics.NewMetricContext("security_group_rule", "create") _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) } @@ -2124,8 +2182,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 EtherType: rules.EtherType6, } + mc = metrics.NewMetricContext("security_group_rule", "create") _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) } @@ -2149,8 +2208,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 if !found { port.SecurityGroups = append(port.SecurityGroups, lbSecGroup.ID) updateOpts := neutronports.UpdateOpts{SecurityGroups: &port.SecurityGroups} + mc := metrics.NewMetricContext("port", "update") res := neutronports.Update(lbaas.network, portID, updateOpts) - if res.Err != nil { + if mc.ObserveRequest(res.Err) != nil { msg := fmt.Sprintf("Error occurred updating port %s for loadbalancer service %s/%s: %v", portID, apiService.Namespace, apiService.Name, res.Err) return fmt.Errorf(msg) } @@ -2163,8 +2223,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 // If Octavia is used, the VIP port security group is already taken good care of, we only need to allow ingress // traffic from Octavia amphorae to the node port on the worker nodes. if lbaas.opts.UseOctavia { + mc := metrics.NewMetricContext("subnet", "get") subnet, err := subnets.Get(lbaas.network, lbaas.opts.SubnetID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to find subnet %s from openstack: %v", lbaas.opts.SubnetID, err) } @@ -2195,7 +2256,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1 SecGroupID: lbSecGroupID, EtherType: rules.EtherType4, } - if _, err = rules.Create(lbaas.network, sgRuleCreateOpts).Extract(); err != nil { + mc = metrics.NewMetricContext("security_group_rule", "create") + _, err = rules.Create(lbaas.network, sgRuleCreateOpts).Extract() + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to create rule for security group %s: %v", lbSecGroupID, err) } @@ -2332,6 +2395,12 @@ func (lbaas *LbaasV2) updateOctaviaLoadBalancer(ctx context.Context, clusterName // UpdateLoadBalancer updates hosts under the specified load balancer. func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { + mc := metrics.NewMetricContext("loadbalancer", "update") + err := lbaas.updateLoadBalancer(ctx, clusterName, service, nodes) + return mc.ObserveReconcile(err) +} + +func (lbaas *LbaasV2) updateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { if lbaas.opts.UseOctavia { return lbaas.updateOctaviaLoadBalancer(ctx, clusterName, service, nodes) } @@ -2440,13 +2509,14 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string // Already exists, do not create member continue } + mc := metrics.NewMetricContext("loadbalancer_member", "create") _, err := v2pools.CreateMember(lbaas.lb, pool.ID, v2pools.CreateMemberOpts{ Name: cutString(fmt.Sprintf("member_%d_%s_%s_", portIndex, node.Name, loadbalancer.Name)), Address: addr, ProtocolPort: int(port.NodePort), SubnetID: lbaas.opts.SubnetID, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return err } provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) @@ -2461,10 +2531,12 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string // Still present, do not delete member continue } + mc := metrics.NewMetricContext("loadbalancer_member", "delete") err = v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) @@ -2526,10 +2598,13 @@ func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *corev1 } for _, rule := range secGroupRules { + mc := metrics.NewMetricContext("security_group_rule", "delete") res := rules.Delete(lbaas.network, rule.ID) if res.Err != nil && !cpoerrors.IsNotFound(res.Err) { + mc.ObserveRequest(err) return fmt.Errorf("error occurred deleting security group rule: %s: %v", rule.ID, res.Err) } + mc.ObserveRequest(nil) } } @@ -2564,6 +2639,12 @@ func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *corev1 // EnsureLoadBalancerDeleted deletes the specified load balancer func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error { + mc := metrics.NewMetricContext("loadbalancer", "delete") + err := lbaas.ensureLoadBalancerDeleted(ctx, clusterName, service) + return mc.ObserveReconcile(err) +} + +func (lbaas *LbaasV2) ensureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error { serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) klog.V(4).Infof("EnsureLoadBalancerDeleted(%s, %s)", clusterName, serviceName) @@ -2597,7 +2678,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName return err } if matched { - if err := floatingips.Delete(lbaas.network, fip.ID).ExtractErr(); err != nil { + mc := metrics.NewMetricContext("floating_ip", "delete") + err := floatingips.Delete(lbaas.network, fip.ID).ExtractErr() + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to delete floating IP %s for loadbalancer VIP port %s: %v", fip.FloatingIP, portID, err) } } @@ -2608,7 +2691,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName // delete the loadbalancer and all its sub-resources. if lbaas.opts.UseOctavia && lbaas.opts.CascadeDelete { deleteOpts := loadbalancers.DeleteOpts{Cascade: true} - if err := loadbalancers.Delete(lbaas.lb, loadbalancer.ID, deleteOpts).ExtractErr(); err != nil { + mc := metrics.NewMetricContext("loadbalancer", "delete") + err := loadbalancers.Delete(lbaas.lb, loadbalancer.ID, deleteOpts).ExtractErr() + if mc.ObserveRequest(err) != nil { return fmt.Errorf("failed to delete loadbalancer %s: %v", loadbalancer.ID, err) } } else { @@ -2637,10 +2722,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName // delete all monitors for _, monitorID := range monitorIDs { + mc := metrics.NewMetricContext("loadbalancer_healthmonitor", "delete") err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting monitor, current provisioning status %s", provisioningStatus) @@ -2656,10 +2743,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName } // delete all members for this pool for _, member := range membersList { + mc := metrics.NewMetricContext("loadbalancer_member", "delete") err := v2pools.DeleteMember(lbaas.lb, poolID, member.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) @@ -2667,10 +2756,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName } // delete pool + mc := metrics.NewMetricContext("loadbalancer_pool", "delete") err = v2pools.Delete(lbaas.lb, poolID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) @@ -2679,10 +2770,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName // delete all listeners for _, listener := range listenerList { + mc := metrics.NewMetricContext("loadbalancer_listener", "delete") err := listeners.Delete(lbaas.lb, listener.ID).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) @@ -2690,10 +2783,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName } // delete loadbalancer + mc := metrics.NewMetricContext("loadbalancer", "delete") err = loadbalancers.Delete(lbaas.lb, loadbalancer.ID, loadbalancers.DeleteOpts{}).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { - return err + return mc.ObserveRequest(err) } + mc.ObserveRequest(nil) err = waitLoadbalancerDeleted(lbaas.lb, loadbalancer.ID) if err != nil { return fmt.Errorf("failed to delete loadbalancer: %v", err) @@ -2731,10 +2826,12 @@ func (lbaas *LbaasV2) EnsureSecurityGroupDeleted(clusterName string, service *co } } + mc := metrics.NewMetricContext("security_group", "delete") lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID) if lbSecGroup.Err != nil && !cpoerrors.IsNotFound(lbSecGroup.Err) { - return lbSecGroup.Err + return mc.ObserveRequest(lbSecGroup.Err) } + mc.ObserveRequest(nil) if len(lbaas.opts.NodeSecurityGroupIDs) == 0 { // Just happen when nodes have not Security Group, or should not happen @@ -2757,10 +2854,13 @@ func (lbaas *LbaasV2) EnsureSecurityGroupDeleted(clusterName string, service *co } for _, rule := range secGroupRules { + mc := metrics.NewMetricContext("security_group_rule", "delete") res := rules.Delete(lbaas.network, rule.ID) if res.Err != nil && !cpoerrors.IsNotFound(res.Err) { + mc.ObserveRequest(res.Err) return fmt.Errorf("error occurred deleting security group rule: %s: %v", rule.ID, res.Err) } + mc.ObserveRequest(nil) } } } diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/pkg/cloudprovider/providers/openstack/openstack_routes.go index a566c5487f..8d223f4063 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_routes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_routes.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" "k8s.io/klog/v2" ) @@ -82,8 +83,9 @@ func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr return nil, err } + mc := metrics.NewMetricContext("router", "get") router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } @@ -108,19 +110,21 @@ func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) { origRoutes := router.Routes // shallow copy + mc := metrics.NewMetricContext("router", "update") _, err := routers.Update(network, router.ID, routers.UpdateOpts{ Routes: newRoutes, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } unwinder := func() { klog.V(4).Infof("Reverting routes change to router %v", router.ID) + mc := metrics.NewMetricContext("router", "update") _, err := routers.Update(network, router.ID, routers.UpdateOpts{ Routes: origRoutes, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { klog.Warningf("Unable to reset routes during error unwind: %v", err) } } @@ -131,19 +135,21 @@ func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, ne func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) { origPairs := port.AllowedAddressPairs // shallow copy + mc := metrics.NewMetricContext("port", "update") _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ AllowedAddressPairs: &newPairs, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } unwinder := func() { klog.V(4).Infof("Reverting allowed-address-pairs change to port %v", port.ID) + mc := metrics.NewMetricContext("port", "update") _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ AllowedAddressPairs: &origPairs, }).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { klog.Warningf("Unable to reset allowed-address-pairs during error unwind: %v", err) } } @@ -167,8 +173,9 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s klog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode) + mc := metrics.NewMetricContext("router", "get") router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return err } @@ -246,8 +253,9 @@ func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *clo } } + mc := metrics.NewMetricContext("router", "get") router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return err } @@ -335,8 +343,9 @@ func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName } func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) { + mc := metrics.NewMetricContext("port", "get") targetPort, err := neutronports.Get(client, portID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } diff --git a/pkg/util/openstack/keymanager.go b/pkg/util/openstack/keymanager.go index 8e027b237e..9e3b34105b 100644 --- a/pkg/util/openstack/keymanager.go +++ b/pkg/util/openstack/keymanager.go @@ -22,6 +22,7 @@ import ( "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/keymanager/v1/secrets" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" ) // EnsureSecret creates a secret if it doesn't exist. @@ -44,8 +45,9 @@ func GetSecret(client *gophercloud.ServiceClient, name string) (*secrets.Secret, listOpts := secrets.ListOpts{ Name: name, } + mc := metrics.NewMetricContext("secret", "list") allPages, err := secrets.List(client, listOpts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } allSecrets, err := secrets.ExtractSecrets(allPages) @@ -75,8 +77,9 @@ func CreateSecret(client *gophercloud.ServiceClient, name string, secretType str Payload: payload, SecretType: secrets.OpaqueSecret, } + mc := metrics.NewMetricContext("secret", "create") secret, err := secrets.Create(client, createOpts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return "", err } return secret.SecretRef, nil @@ -97,8 +100,9 @@ func DeleteSecrets(client *gophercloud.ServiceClient, partName string) error { listOpts := secrets.ListOpts{ SecretType: secrets.OpaqueSecret, } + mc := metrics.NewMetricContext("secret", "list") allPages, err := secrets.List(client, listOpts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return err } allSecrets, err := secrets.ExtractSecrets(allPages) @@ -112,8 +116,9 @@ func DeleteSecrets(client *gophercloud.ServiceClient, partName string) error { if err != nil { return err } + mc := metrics.NewMetricContext("secret", "delete") err = secrets.Delete(client, secretID).ExtractErr() - if err != nil { + if mc.ObserveRequest(err) != nil { return err } } diff --git a/pkg/util/openstack/loadbalancer.go b/pkg/util/openstack/loadbalancer.go index 529e1d4f3a..b9d678a497 100644 --- a/pkg/util/openstack/loadbalancer.go +++ b/pkg/util/openstack/loadbalancer.go @@ -29,6 +29,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" version "github.com/hashicorp/go-version" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" klog "k8s.io/klog/v2" cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" @@ -64,8 +65,9 @@ func getOctaviaVersion(client *gophercloud.ServiceClient) (string, error) { } var defaultVer = "0.0" + mc := metrics.NewMetricContext("version", "list") allPages, err := apiversions.List(client).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return defaultVer, err } versions, err := apiversions.ExtractAPIVersions(allPages) @@ -126,8 +128,9 @@ func waitLoadbalancerActive(client *gophercloud.ServiceClient, loadbalancerID st } err := wait.ExponentialBackoff(backoff, func() (bool, error) { + mc := metrics.NewMetricContext("loadbalancer", "get") loadbalancer, err := loadbalancers.Get(client, loadbalancerID).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return false, err } if loadbalancer.ProvisioningStatus == activeStatus { @@ -145,7 +148,9 @@ func waitLoadbalancerActive(client *gophercloud.ServiceClient, loadbalancerID st // UpdateListener updates a listener and wait for the lb active func UpdateListener(client *gophercloud.ServiceClient, lbID string, listenerID string, opts listeners.UpdateOpts) error { - if _, err := listeners.Update(client, listenerID, opts).Extract(); err != nil { + mc := metrics.NewMetricContext("loadbalancer_listener", "update") + _, err := listeners.Update(client, listenerID, opts).Extract() + if mc.ObserveRequest(err) != nil { return err } @@ -158,8 +163,9 @@ func UpdateListener(client *gophercloud.ServiceClient, lbID string, listenerID s // CreateListener creates a new listener func CreateListener(client *gophercloud.ServiceClient, lbID string, opts listeners.CreateOpts) (*listeners.Listener, error) { + mc := metrics.NewMetricContext("loadbalancer_listener", "create") listener, err := listeners.Create(client, opts).Extract() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } @@ -175,8 +181,9 @@ func GetLoadbalancerByName(client *gophercloud.ServiceClient, name string) (*loa opts := loadbalancers.ListOpts{ Name: name, } + mc := metrics.NewMetricContext("loadbalancer", "list") allPages, err := loadbalancers.List(client, opts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } loadbalancerList, err := loadbalancers.ExtractLoadBalancers(allPages) @@ -200,6 +207,7 @@ func GetListenerByName(client *gophercloud.ServiceClient, name string, lbID stri Name: name, LoadbalancerID: lbID, } + mc := metrics.NewMetricContext("loadbalancer_listener", "list") pager := listeners.List(client, opts) var listenerList []listeners.Listener @@ -214,7 +222,7 @@ func GetListenerByName(client *gophercloud.ServiceClient, name string, lbID stri } return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { if cpoerrors.IsNotFound(err) { return nil, ErrNotFound } @@ -236,6 +244,7 @@ func GetPoolByName(client *gophercloud.ServiceClient, name string, lbID string) Name: name, LoadbalancerID: lbID, } + mc := metrics.NewMetricContext("loadbalancer_pool", "list") err := pools.List(client, opts).EachPage(func(page pagination.Page) (bool, error) { v, err := pools.ExtractPools(page) if err != nil { @@ -247,7 +256,7 @@ func GetPoolByName(client *gophercloud.ServiceClient, name string, lbID string) } return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { if cpoerrors.IsNotFound(err) { return nil, ErrNotFound } @@ -266,6 +275,7 @@ func GetPoolByName(client *gophercloud.ServiceClient, name string, lbID string) // GetPoolsByListener finds pool for a listener. A listener always has exactly one pool. func GetPoolByListener(client *gophercloud.ServiceClient, lbID, listenerID string) (*pools.Pool, error) { listenerPools := make([]pools.Pool, 0, 1) + mc := metrics.NewMetricContext("loadbalancer_pool", "list") err := pools.List(client, pools.ListOpts{LoadbalancerID: lbID}).EachPage(func(page pagination.Page) (bool, error) { poolsList, err := pools.ExtractPools(page) if err != nil { @@ -283,7 +293,7 @@ func GetPoolByListener(client *gophercloud.ServiceClient, lbID, listenerID strin } return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { if cpoerrors.IsNotFound(err) { return nil, ErrNotFound } @@ -299,18 +309,21 @@ func GetPoolByListener(client *gophercloud.ServiceClient, lbID, listenerID strin // DeleteLoadbalancer deletes a loadbalancer with all its child objects. func DeleteLoadbalancer(client *gophercloud.ServiceClient, lbID string) error { + mc := metrics.NewMetricContext("loadbalancer", "delete") err := loadbalancers.Delete(client, lbID, loadbalancers.DeleteOpts{Cascade: true}).ExtractErr() if err != nil && !cpoerrors.IsNotFound(err) { + mc.ObserveRequest(err) return fmt.Errorf("error deleting loadbalancer %s: %v", lbID, err) } - return nil + return mc.ObserveRequest(nil) } // GetMembersbyPool get all the members in the pool. func GetMembersbyPool(client *gophercloud.ServiceClient, poolID string) ([]pools.Member, error) { var members []pools.Member + mc := metrics.NewMetricContext("loadbalancer_member", "list") err := pools.ListMembers(client, poolID, pools.ListMembersOpts{}).EachPage(func(page pagination.Page) (bool, error) { membersList, err := pools.ExtractMembers(page) if err != nil { @@ -320,7 +333,7 @@ func GetMembersbyPool(client *gophercloud.ServiceClient, poolID string) ([]pools return true, nil }) - if err != nil { + if mc.ObserveRequest(err) != nil { return nil, err } diff --git a/pkg/util/openstack/network.go b/pkg/util/openstack/network.go index baab4f2778..5f91a0abcb 100644 --- a/pkg/util/openstack/network.go +++ b/pkg/util/openstack/network.go @@ -19,14 +19,16 @@ package openstack import ( "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" + "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack/metrics" ) // GetFloatingIPs returns all the filtered floating IPs func GetFloatingIPs(client *gophercloud.ServiceClient, opts floatingips.ListOpts) ([]floatingips.FloatingIP, error) { var floatingIPList []floatingips.FloatingIP + mc := metrics.NewMetricContext("floating_ip", "list") allPages, err := floatingips.List(client, opts).AllPages() - if err != nil { + if mc.ObserveRequest(err) != nil { return floatingIPList, err } floatingIPList, err = floatingips.ExtractFloatingIPs(allPages) From b80a4c7acf446c9b180d9c8d3535238fa1e814d8 Mon Sep 17 00:00:00 2001 From: Sean Schneeweiss Date: Fri, 18 Sep 2020 13:55:38 +0200 Subject: [PATCH 3/3] [occm] rename extension metric to network_extension --- pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index 1cba4d1372..258350001f 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -138,7 +138,7 @@ type listenerKey struct { func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, error) { seen := make(map[string]bool) - mc := metrics.NewMetricContext("extension", "list") + mc := metrics.NewMetricContext("network_extension", "list") pager := extensions.List(client) err := pager.EachPage(func(page pagination.Page) (bool, error) { exts, err := extensions.ExtractExtensions(page)