-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
FS-1259; Add proc collector and custom metrics. Remove references to …
…serverservice where possible. Small refactor of inventory task. (#40)
- Loading branch information
1 parent
72432c9
commit 0034a40
Showing
6 changed files
with
125 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,61 +1,66 @@ | ||
| package client | ||
|
|
||
| import ( | ||
| "github.com/metal-toolbox/fleet-scheduler/internal/metrics" | ||
| fleetdbapi "github.com/metal-toolbox/fleetdb/pkg/api/v1" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| func (c *Client) CreateConditionInventoryForAllServers(pageSize int) error { | ||
| // First page, use the response from it to figure out how many pages we have to loop through | ||
| // Dont change page size | ||
| servers, response, err := c.getServerPage(pageSize, 1) | ||
| response, err := c.getServerPageAndCreateInventory(1, pageSize, 0) | ||
| if err != nil { | ||
| c.log.WithFields(logrus.Fields{ | ||
| "pageSize": pageSize, | ||
| "pageIndex": 1, | ||
| }).Logger.Errorf("Failed to get list of servers: %s", err.Error()) | ||
| return err | ||
| } | ||
| totalPages := response.TotalPages | ||
|
|
||
| // send first page of servers to the channel | ||
| for i := range servers { | ||
| err = c.CreateConditionInventory(servers[i].UUID) | ||
| // Now that we know how many pages to expect, lets loop through the rest of the pages | ||
| for i := 2; i <= totalPages; i++ { | ||
| _, err := c.getServerPageAndCreateInventory(i, pageSize, totalPages) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (c *Client) getServerPageAndCreateInventory(pageIndex, pageSize, totalPages int) (*fleetdbapi.ServerResponse, error) { | ||
| servers, response, err := c.getServerPage(pageSize, pageIndex) | ||
| if err != nil { | ||
| c.log.WithFields(logrus.Fields{ | ||
| "pageIndex": pageIndex, | ||
| "pageSize": pageSize, | ||
| "totalPages": totalPages, | ||
| }).Logger.Errorf("Failed to get page of servers, attempting to continue: %s", err.Error()) | ||
|
|
||
| metrics.FleetdbErrorCounter.With( | ||
| prometheus.Labels{"errors": err.Error()}, | ||
| ).Inc() | ||
|
|
||
| return response, err | ||
| } | ||
|
|
||
| c.log.WithFields(logrus.Fields{ | ||
| "index": 1, | ||
| "iterations": totalPages, | ||
| "got": len(servers), | ||
| "pageIndex": pageIndex, | ||
| "pageSize": pageSize, | ||
| "totalPages": totalPages, | ||
| }).Trace("Got server page") | ||
|
|
||
| // Start the second page, and loop through rest the pages | ||
| for i := 2; i <= totalPages; i++ { | ||
| servers, _, err = c.getServerPage(pageSize, i) | ||
| for i := range servers { | ||
| err = c.CreateConditionInventory(servers[i].UUID) | ||
| if err != nil { | ||
| c.log.WithFields(logrus.Fields{ | ||
| "pageSize": pageSize, | ||
| "pageIndex": i, | ||
| }).Logger.Errorf("Failed to get page of servers, attempting to continue: %s", err.Error()) | ||
|
|
||
| continue | ||
| metrics.ConditionorcErrorCounter.With( | ||
| prometheus.Labels{"errors": err.Error()}, | ||
| ).Inc() | ||
| return response, err | ||
| } | ||
|
|
||
| c.log.WithFields(logrus.Fields{ | ||
| "index": i, | ||
| "iterations": totalPages, | ||
| "got": len(servers), | ||
| }).Trace("Got server page") | ||
|
|
||
| for i := range servers { | ||
| err = c.CreateConditionInventory(servers[i].UUID) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| metrics.InventoryCounter.With( | ||
| prometheus.Labels{}, | ||
| ).Inc() | ||
| } | ||
|
|
||
| return nil | ||
| return response, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package metrics | ||
|
|
||
| import "github.com/prometheus/client_golang/prometheus" | ||
|
|
||
| var ( | ||
| ConditionorcErrorCounter *prometheus.CounterVec | ||
| FleetdbErrorCounter *prometheus.CounterVec | ||
|
|
||
| InventoryCounter *prometheus.CounterVec | ||
| ) | ||
|
|
||
| func init() { | ||
| ConditionorcErrorCounter = prometheus.NewCounterVec( | ||
| prometheus.CounterOpts{ | ||
| Namespace: "fleet-scheduler", | ||
| Subsystem: "conditionorc", | ||
| Name: "errors_total", | ||
| Help: "a count of all errors attempting to reach conditionorc", | ||
| }, []string{ | ||
| "errors", | ||
| }, | ||
| ) | ||
|
|
||
| FleetdbErrorCounter = prometheus.NewCounterVec( | ||
| prometheus.CounterOpts{ | ||
| Namespace: "fleet-scheduler", | ||
| Subsystem: "fleetdb", | ||
| Name: "errors_total", | ||
| Help: "a count of all errors attempting to reach fleetdb", | ||
| }, []string{ | ||
| "errors", | ||
| }, | ||
| ) | ||
|
|
||
| InventoryCounter = prometheus.NewCounterVec( | ||
| prometheus.CounterOpts{ | ||
| Namespace: "fleet-scheduler", | ||
| Subsystem: "core", | ||
| Name: "errors_total", | ||
| Help: "a count of all errors attempting to reach fleet-scheduler dependencies", | ||
| }, []string{}, | ||
| ) | ||
| } | ||
|
|
||
| func AddCustomMetrics(pusher *Pusher) { | ||
| pusher.AddCollector(ConditionorcErrorCounter) | ||
| pusher.AddCollector(FleetdbErrorCounter) | ||
| pusher.AddCollector(InventoryCounter) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters