From 69b1cdc733fe3993ffc0bd416eb895c22b2adbae Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:44:14 -0400 Subject: [PATCH 01/33] Update exporter.go to include new FirmwareInventory logic --- exporter/exporter.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/exporter/exporter.go b/exporter/exporter.go index 9fd0cb5..d0a4bf3 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -64,6 +64,8 @@ const ( STORAGEBATTERY = "StorBatteryMetrics" // ILOSELFTEST represents the processor metric endpoints ILOSELFTEST = "iloSelfTestMetrics" + // FIRMWAREINVENTORY represents the component firmware metric endpoints + FIRMWAREINVENTORY = "FirmwareInventoryMetrics" // OK is a string representation of the float 1.0 for device status OK = 1.0 // BAD is a string representation of the float 0.0 for device status @@ -344,6 +346,18 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud pool.NewTask(common.Fetch(exp.url+dimm.URL, target, profile, retryClient), exp.url+dimm.URL, handle(&exp, MEMORY))) } + // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs + firmwareInventoryEndpoints, err := getFirmwareInventoryUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) + if err != nil { + log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) + return nil, err + } + + // Firmware Inventory + for _, url := range firmwareInventoryEndpoints { + tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) + } + // call /redfish/v1/Managers/XXX/ for firmware version and ilo self test metrics tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+mgrEndpointFinal, target, profile, retryClient), From ebb72d1b5fdc6e1eaec088eba3ff70a6bacd2dce Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:44:50 -0400 Subject: [PATCH 02/33] Update handlers.go to include new functions for Firmware Inventory Metrics --- exporter/handlers.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/exporter/handlers.go b/exporter/handlers.go index 36cff69..59b71ab 100644 --- a/exporter/handlers.go +++ b/exporter/handlers.go @@ -56,7 +56,10 @@ func handle(exp *Exporter, metricType ...string) []common.Handler { handlers = append(handlers, exp.exportStorageBattery) } else if m == ILOSELFTEST { handlers = append(handlers, exp.exportIloSelfTest) + } else if m == FIRMWAREINVENTORY { + handlers = append(handlers, exp.exportFirmwareInventoryMetrics) } + } return handlers @@ -576,6 +579,20 @@ func (e *Exporter) exportProcessorMetrics(body []byte) error { return nil } +// exportFirmwareInventoryMetrics collects the inventory component's firmware metrics in json format and sets the prometheus guages +func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { + + var fwcomponent oem.FirmwareComponent + var component = (*e.DeviceMetrics)["firmwareInventoryMetrics"] + err := json.Unmarshal(body, &fwcomponent) + if err != nil { + return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) + } + + (*component)["componentFirmware"].WithLabelValues(fwcomponent.Name, fwcomponent.Description, fwcomponent.Version) + return nil +} + // exportIloSelfTest collects the iLO Self Test Results metrics in json format and sets the prometheus guage func (e *Exporter) exportIloSelfTest(body []byte) error { From 04952efaa1b9ffbcbb8acb52353a6ee4836d4f5b Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:45:12 -0400 Subject: [PATCH 03/33] Update helpers.go to include new functions for Firmware Inventory Metrics --- exporter/helpers.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/exporter/helpers.go b/exporter/helpers.go index c2b2002..007b40a 100644 --- a/exporter/helpers.go +++ b/exporter/helpers.go @@ -438,6 +438,42 @@ func getProcessorEndpoints(url, host string, client *retryablehttp.Client) (oem. return processors, nil } +// Get Component Firmware URLS +func getFirmwareInventoryUrls(url, host string, client *retryablehttp.Client) ([]string, error) { + var components oem.FirmwareComponents + var firmwareInventoryEndpoints []string + req := common.BuildRequest(url, host) + + resp, err := client.Do(req) + if err != nil { + return firmwareInventoryEndpoints, err + } + defer resp.Body.Close() + if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { + if resp.StatusCode == http.StatusUnauthorized { + return firmwareInventoryEndpoints, common.ErrInvalidCredential + } else { + return firmwareInventoryEndpoints, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return firmwareInventoryEndpoints, fmt.Errorf("Error reading Response Body - " + err.Error()) + } + + err = json.Unmarshal(body, &components) + if err != nil { + return firmwareInventoryEndpoints, fmt.Errorf("Error Unmarshalling Firmware Component struct - " + err.Error()) + } + + for _, member := range components.Members { + firmwareInventoryEndpoints = append(firmwareInventoryEndpoints, appendSlash(member.URL)) + } + + return firmwareInventoryEndpoints, nil +} + // appendSlash appends a slash to the end of a URL if it does not already have one func appendSlash(url string) string { if url[len(url)-1] != '/' { From cbabe050bd18e1c98593cc69bf281acccbefbc67 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:45:42 -0400 Subject: [PATCH 04/33] Update metrics.go to include new metrics for Firmware Inventory export --- exporter/metrics.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index b311988..525ca67 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -73,7 +73,7 @@ func NewDeviceMetrics() *map[string]*metrics { "nvmeDriveStatus": newServerMetric("redfish_nvme_drive_status", "Current NVME status 1 = OK, 0 = BAD, -1 = DISABLED", nil, []string{"chassisSerialNumber", "chassisModel", "protocol", "id", "serviceLabel"}), } - // Phyiscal Storage Disk Drive Metrics + // Physical Storage Disk Drive Metrics DiskDriveMetrics = &metrics{ "driveStatus": newServerMetric("redfish_disk_drive_status", "Current Disk Drive status 1 = OK, 0 = BAD, -1 = DISABLED", nil, []string{"name", "chassisSerialNumber", "chassisModel", "id", "location", "serialnumber", "capacityMiB"}), } @@ -92,23 +92,29 @@ func NewDeviceMetrics() *map[string]*metrics { "memoryDimmStatus": newServerMetric("redfish_memory_dimm_status", "Current dimm status 1 = OK, 0 = BAD", nil, []string{"name", "chassisSerialNumber", "chassisModel", "capacityMiB", "manufacturer", "partNumber"}), } + // Component Firmware Metrics + FirmwareInventoryMetrics = &metrics{ + "componentFirmware": newServerMetric("redfish_component_firmware", "Component Firmware Details", nil, []string{"name", "description", "version"}), + } + DeviceMetrics = &metrics{ "deviceInfo": newServerMetric("redfish_device_info", "Current snapshot of device firmware information", nil, []string{"name", "chassisSerialNumber", "chassisModel", "firmwareVersion", "biosVersion"}), } Metrics = &map[string]*metrics{ - "up": UpMetric, - "thermalMetrics": ThermalMetrics, - "powerMetrics": PowerMetrics, - "processorMetrics": ProcessorMetrics, - "nvmeMetrics": NVMeDriveMetrics, - "diskDriveMetrics": DiskDriveMetrics, - "logicalDriveMetrics": LogicalDriveMetrics, - "storBatteryMetrics": StorageBatteryMetrics, - "storageCtrlMetrics": StorageControllerMetrics, - "iloSelfTestMetrics": IloSelfTestMetrics, - "memoryMetrics": MemoryMetrics, - "deviceInfo": DeviceMetrics, + "up": UpMetric, + "thermalMetrics": ThermalMetrics, + "powerMetrics": PowerMetrics, + "processorMetrics": ProcessorMetrics, + "nvmeMetrics": NVMeDriveMetrics, + "diskDriveMetrics": DiskDriveMetrics, + "logicalDriveMetrics": LogicalDriveMetrics, + "storBatteryMetrics": StorageBatteryMetrics, + "storageCtrlMetrics": StorageControllerMetrics, + "iloSelfTestMetrics": IloSelfTestMetrics, + "firmwareInventoryMetrics": FirmwareInventoryMetrics, + "memoryMetrics": MemoryMetrics, + "deviceInfo": DeviceMetrics, } ) From 98db50657241fc7bd610c0b557128be9f8c48c6d Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:46:24 -0400 Subject: [PATCH 05/33] Create firmware.go to include new Firmware Component structs --- oem/firmware.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 oem/firmware.go diff --git a/oem/firmware.go b/oem/firmware.go new file mode 100644 index 0000000..d5ddc09 --- /dev/null +++ b/oem/firmware.go @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// /redfish/v1/UpdateService/FirmwareInventory/ + +package oem + +// Collection returns an array of the endpoints from the FirmwareInventory +type FirmwareComponents struct { + Members []struct { + URL string `json:"@odata.id"` + } `json:"Members"` + MembersCount int `json:"Members@odata.count"` +} + +// Collection returns the component details + +// /redfish/v1/UpdateService/FirmwareInventory/XXXX/ +type FirmwareComponent struct { + Name string `json:"Name"` + Description string `json:"Description"` + Version string `json:"Version"` +} From f6bb7127de2f6cb0252c1605dabc11614dec4a0e Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:00:02 -0400 Subject: [PATCH 06/33] Update exporter.go to reuse existing functions and structs --- exporter/exporter.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index d0a4bf3..fc06743 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -311,6 +311,13 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud zap.Strings("physical_drive_endpoints", driveEndpointsResp.physicalDriveURLs), zap.Any("trace_id", ctx.Value("traceID"))) + // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs + firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) + if err != nil { + log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) + return nil, err + } + // Loop through logicalDriveURLs, physicalDriveURLs, and nvmeDriveURLs and append each URL to the tasks pool for _, url := range driveEndpointsResp.logicalDriveURLs { tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, LOGICALDRIVE))) @@ -346,13 +353,6 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud pool.NewTask(common.Fetch(exp.url+dimm.URL, target, profile, retryClient), exp.url+dimm.URL, handle(&exp, MEMORY))) } - // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs - firmwareInventoryEndpoints, err := getFirmwareInventoryUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) - if err != nil { - log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) - return nil, err - } - // Firmware Inventory for _, url := range firmwareInventoryEndpoints { tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) From 945f80f1f1df29c02464069f524824fa2ba392c6 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:00:43 -0400 Subject: [PATCH 07/33] Update handlers.go to include health status in firmware componoent export --- exporter/handlers.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/exporter/handlers.go b/exporter/handlers.go index 59b71ab..b0ae055 100644 --- a/exporter/handlers.go +++ b/exporter/handlers.go @@ -582,6 +582,7 @@ func (e *Exporter) exportProcessorMetrics(body []byte) error { // exportFirmwareInventoryMetrics collects the inventory component's firmware metrics in json format and sets the prometheus guages func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { + var state float64 var fwcomponent oem.FirmwareComponent var component = (*e.DeviceMetrics)["firmwareInventoryMetrics"] err := json.Unmarshal(body, &fwcomponent) @@ -589,7 +590,17 @@ func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) } - (*component)["componentFirmware"].WithLabelValues(fwcomponent.Name, fwcomponent.Description, fwcomponent.Version) + if fwcomponent.Status.State == "Enabled" || fwcomponent.Status.State != "" { + if fwcomponent.Status.Health == "OK" { + state = OK + } else { + state = BAD + } + } else if fwcomponent.Status.Health == "" && fwcomponent.Status.State == "" { + state = OK + } + + (*component)["componentFirmware"].WithLabelValues(fwcomponent.Id, fwcomponent.Name, fwcomponent.Description, fwcomponent.Version).Set(state) return nil } From 3b12e963fd7bcf66eaf3b87085919209bd6217e8 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:01:25 -0400 Subject: [PATCH 08/33] Update metrics.go to include id in firmware component status --- exporter/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index 525ca67..0c8b850 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -94,7 +94,7 @@ func NewDeviceMetrics() *map[string]*metrics { // Component Firmware Metrics FirmwareInventoryMetrics = &metrics{ - "componentFirmware": newServerMetric("redfish_component_firmware", "Component Firmware Details", nil, []string{"name", "description", "version"}), + "componentFirmware": newServerMetric("redfish_component_firmware", "Current firmware component status 1 = OK, 0 = BAD", nil, []string{"id", "name", "description", "version"}), } DeviceMetrics = &metrics{ From ee4411fbe118f8b8224d8cdf176c3d14c59f3c4e Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:02:03 -0400 Subject: [PATCH 09/33] Update chassis.go to incorporate firmware component --- oem/chassis.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/oem/chassis.go b/oem/chassis.go index cb7d54b..f62918d 100644 --- a/oem/chassis.go +++ b/oem/chassis.go @@ -104,3 +104,13 @@ type HRef struct { type Link struct { URL string `json:"@odata.id"` } + +// Collection returns the component details +// /redfish/v1/UpdateService/FirmwareInventory/XXXX/ +type FirmwareComponent struct { + Name string `json:"Name"` + Description string `json:"Description"` + Version string `json:"Version"` + Id string `json:"Id"` + Status Status +} From f931a729d06840e043851a0b1e46890906d62f38 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:02:48 -0400 Subject: [PATCH 10/33] Delete oem/firmware.go, incorporated FirmwareComponent in chassis.go --- oem/firmware.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 oem/firmware.go diff --git a/oem/firmware.go b/oem/firmware.go deleted file mode 100644 index d5ddc09..0000000 --- a/oem/firmware.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2024 Comcast Cable Communications Management, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// /redfish/v1/UpdateService/FirmwareInventory/ - -package oem - -// Collection returns an array of the endpoints from the FirmwareInventory -type FirmwareComponents struct { - Members []struct { - URL string `json:"@odata.id"` - } `json:"Members"` - MembersCount int `json:"Members@odata.count"` -} - -// Collection returns the component details - -// /redfish/v1/UpdateService/FirmwareInventory/XXXX/ -type FirmwareComponent struct { - Name string `json:"Name"` - Description string `json:"Description"` - Version string `json:"Version"` -} From 900f4e53223bade5a59b121f5854b4f0e7e934d5 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:15:53 -0400 Subject: [PATCH 11/33] Update exporter.go to include graceful fail for iLO4 c220 missing firmware endpoint --- exporter/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index fc06743..e14f8c0 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -315,7 +315,7 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) if err != nil { log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) - return nil, err + firmwareInventoryEndpoints = []string{} } // Loop through logicalDriveURLs, physicalDriveURLs, and nvmeDriveURLs and append each URL to the tasks pool From e8c5a2ec92d498fbe5a74e8b8337b66c5669c0ca Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:16:54 -0400 Subject: [PATCH 12/33] Update helpers.go to reuse function instead of redundant function --- exporter/helpers.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/exporter/helpers.go b/exporter/helpers.go index 007b40a..c2b2002 100644 --- a/exporter/helpers.go +++ b/exporter/helpers.go @@ -438,42 +438,6 @@ func getProcessorEndpoints(url, host string, client *retryablehttp.Client) (oem. return processors, nil } -// Get Component Firmware URLS -func getFirmwareInventoryUrls(url, host string, client *retryablehttp.Client) ([]string, error) { - var components oem.FirmwareComponents - var firmwareInventoryEndpoints []string - req := common.BuildRequest(url, host) - - resp, err := client.Do(req) - if err != nil { - return firmwareInventoryEndpoints, err - } - defer resp.Body.Close() - if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { - if resp.StatusCode == http.StatusUnauthorized { - return firmwareInventoryEndpoints, common.ErrInvalidCredential - } else { - return firmwareInventoryEndpoints, fmt.Errorf("HTTP status %d", resp.StatusCode) - } - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return firmwareInventoryEndpoints, fmt.Errorf("Error reading Response Body - " + err.Error()) - } - - err = json.Unmarshal(body, &components) - if err != nil { - return firmwareInventoryEndpoints, fmt.Errorf("Error Unmarshalling Firmware Component struct - " + err.Error()) - } - - for _, member := range components.Members { - firmwareInventoryEndpoints = append(firmwareInventoryEndpoints, appendSlash(member.URL)) - } - - return firmwareInventoryEndpoints, nil -} - // appendSlash appends a slash to the end of a URL if it does not already have one func appendSlash(url string) string { if url[len(url)-1] != '/' { From 7e55385ab1f3629cd23208960a0fc84701889581 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:41:31 -0400 Subject: [PATCH 13/33] Update exporter.go to include work in progress loop for iLO4 firmware --- exporter/exporter.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index e14f8c0..27d2fb9 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -314,13 +314,18 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) if err != nil { - log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) - firmwareInventoryEndpoints = []string{} - } + // Try the iLo 4 firmware inventory endpoint + firmwareInventoryResp, err := getFirmwareComponents(exp.url+uri+"/Systems/1/FirmwareInventory/", target, retryClient) + if err != nil { + log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) + return nil, err + } + firmwareInventory := firmwareInventoryResp.Current - // Loop through logicalDriveURLs, physicalDriveURLs, and nvmeDriveURLs and append each URL to the tasks pool - for _, url := range driveEndpointsResp.logicalDriveURLs { - tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, LOGICALDRIVE))) + // Loop through firmwareInventory and set metrics + for _, Component := range firmwareInventory { + // TODO FINISH STREAMING METRICS HERE + } } for _, url := range driveEndpointsResp.physicalDriveURLs { From a7b2489e2c957cbcc7883e3dd4d0bb0d89c55c72 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:41:59 -0400 Subject: [PATCH 14/33] Update handlers.go to include work in progress loop for ilo4 firmware --- exporter/handlers.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/exporter/handlers.go b/exporter/handlers.go index b0ae055..68d0a63 100644 --- a/exporter/handlers.go +++ b/exporter/handlers.go @@ -584,8 +584,11 @@ func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { var state float64 var fwcomponent oem.FirmwareComponent + var ilo4fwcomponent oem.SystemFirmwareInventory var component = (*e.DeviceMetrics)["firmwareInventoryMetrics"] err := json.Unmarshal(body, &fwcomponent) + ilo4err := json.Unmarshal(body, &ilo4fwcomponent) + if err != nil { return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) } @@ -600,7 +603,15 @@ func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { state = OK } - (*component)["componentFirmware"].WithLabelValues(fwcomponent.Id, fwcomponent.Name, fwcomponent.Description, fwcomponent.Version).Set(state) + // non iLO 4 + if err != nil { + (*component)["componentFirmware"].WithLabelValues(fwcomponent.Id, fwcomponent.Name, fwcomponent.Description, fwcomponent.Version).Set(state) + } + + // iLO 4 export + if ilo4err != nil { + (*component)["ilo4ComponentFirmware"].WithLabelValues(ilo4fwcomponent.Current.Component.Details.Item.Name, ilo4fwcomponent.Current.Component.Details.Item.Key, ilo4fwcomponent.Current.Component.Details.Item.Location, ilo4fwcomponent.Current.Component.Details.VersionString) + } return nil } From a80b873161c20f8e55d6fcf8b7a158786ebc8231 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:43:14 -0400 Subject: [PATCH 15/33] Update helpers.go to include work in progress getFirmwareComponents function for ilo4 --- exporter/helpers.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/exporter/helpers.go b/exporter/helpers.go index c2b2002..4cb428c 100644 --- a/exporter/helpers.go +++ b/exporter/helpers.go @@ -438,6 +438,51 @@ func getProcessorEndpoints(url, host string, client *retryablehttp.Client) (oem. return processors, nil } +// Getting firmware components for iLO 4 hosts +func getFirmwareComponents(url, host string, client *retryablehttp.Client) (oem.SystemFirmwareInventory, error) { + var fc oem.SystemFirmwareInventory + var resp *http.Response + var err error + retryCount := 0 + req := common.BuildRequest(url, host) + + resp, err = common.DoRequest(client, req) + if err != nil { + return fc, err + } + defer resp.Body.Close() + if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { + if resp.StatusCode == http.StatusNotFound { + for retryCount < 3 && resp.StatusCode == http.StatusNotFound { + time.Sleep(client.RetryWaitMin) + resp, err = common.DoRequest(client, req) + retryCount = retryCount + 1 + } + if err != nil { + return fc, err + } else if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { + return fc, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + } else if resp.StatusCode == http.StatusUnauthorized { + return fc, common.ErrInvalidCredential + } else { + return fc, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fc, fmt.Errorf("Error reading Response Body - " + err.Error()) + } + + err = json.Unmarshal(body, &fc) + if err != nil { + return fc, fmt.Errorf("Error Unmarshalling SystemFirmwareInventory Collection struct - " + err.Error()) + } + + return fc, nil +} + // appendSlash appends a slash to the end of a URL if it does not already have one func appendSlash(url string) string { if url[len(url)-1] != '/' { From e721c44be4392d151190c141792abc0e420922b0 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:43:40 -0400 Subject: [PATCH 16/33] Update system.go to include SystemFirmwareInventory struct --- oem/system.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/oem/system.go b/oem/system.go index dc4207b..b8710e9 100644 --- a/oem/system.go +++ b/oem/system.go @@ -75,3 +75,19 @@ type MemorySummary struct { type StatusMemory struct { HealthRollup string `json:"HealthRollup"` } + +// SystemFirmwareInventory is the json object for SystemFirwmareInventory metadata +type SystemFirmwareInventory struct { + Current []struct { + Component []struct { + Details []struct { + Item []struct { + Name string `json:"Name,omitempty"` + Key string `json:"Key,omitempty"` + Location string `json:"Location,omitempty"` + } + VersionString string `json:"VersionString,omitempty"` + } + } + } +} From 0cce13950e631c975aaf285ff33f252bc41193a5 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:02:11 -0400 Subject: [PATCH 17/33] exporter.go 6/24 changes --- exporter/exporter.go | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index 27d2fb9..1fd17d2 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "errors" + "fmt" "net" "net/http" "net/url" @@ -311,21 +312,35 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud zap.Strings("physical_drive_endpoints", driveEndpointsResp.physicalDriveURLs), zap.Any("trace_id", ctx.Value("traceID"))) + // // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs + // firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) + // if err != nil { + // log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) + // firmwareInventoryEndpoints = []string{} + // } + // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs + // Call /redfish/v1/Managers/XXXX/UpdateService/FirmwareInventory/ for firmware inventory firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) if err != nil { // Try the iLo 4 firmware inventory endpoint - firmwareInventoryResp, err := getFirmwareComponents(exp.url+uri+"/Systems/1/FirmwareInventory/", target, retryClient) - if err != nil { - log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) - return nil, err + // Use the collected sysEndpoints.systems to build url(s) + if len(sysEndpoints.systems) > 0 { + // call /redfish/v1/Systems/XXXX/FirmwareInventory/ + for _, system := range sysEndpoints.systems { + firmwareInventoryEndpoints = append(firmwareInventoryEndpoints, system+"FirmwareInventory/") + } + // Ensure we have at least one firmware inventory endpoint + if len(firmwareInventoryEndpoints) == 0 { + log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) + return nil, err + } } - firmwareInventory := firmwareInventoryResp.Current + } - // Loop through firmwareInventory and set metrics - for _, Component := range firmwareInventory { - // TODO FINISH STREAMING METRICS HERE - } + // Loop through logicalDriveURLs, physicalDriveURLs, and nvmeDriveURLs and append each URL to the tasks pool + for _, url := range driveEndpointsResp.logicalDriveURLs { + tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, LOGICALDRIVE))) } for _, url := range driveEndpointsResp.physicalDriveURLs { @@ -360,6 +375,9 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud // Firmware Inventory for _, url := range firmwareInventoryEndpoints { + // DEBUG PRINT + fmt.Printf("task url to be appended: %s\n", exp.url+url) + // END DEBUG PRINT tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) } From a39a8f574e46bde158fd876467e5e48d879f98e5 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:02:39 -0400 Subject: [PATCH 18/33] handlers.go 6/24 changes --- exporter/handlers.go | 56 ++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/exporter/handlers.go b/exporter/handlers.go index 68d0a63..475022c 100644 --- a/exporter/handlers.go +++ b/exporter/handlers.go @@ -584,34 +584,60 @@ func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { var state float64 var fwcomponent oem.FirmwareComponent - var ilo4fwcomponent oem.SystemFirmwareInventory var component = (*e.DeviceMetrics)["firmwareInventoryMetrics"] + // var fcomponent = (*e.DeviceMetrics)["firmwareInventoryMetrics"] err := json.Unmarshal(body, &fwcomponent) - ilo4err := json.Unmarshal(body, &ilo4fwcomponent) - if err != nil { return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) } - if fwcomponent.Status.State == "Enabled" || fwcomponent.Status.State != "" { - if fwcomponent.Status.Health == "OK" { + if fwcomponent.GenericFirmware.Status.State == "Enabled" || fwcomponent.GenericFirmware.Status.State != "" { + if fwcomponent.GenericFirmware.Status.Health == "OK" { state = OK } else { state = BAD } - } else if fwcomponent.Status.Health == "" && fwcomponent.Status.State == "" { + } else if fwcomponent.GenericFirmware.Status.Health == "" && fwcomponent.GenericFirmware.Status.State == "" { state = OK } - // non iLO 4 - if err != nil { - (*component)["componentFirmware"].WithLabelValues(fwcomponent.Id, fwcomponent.Name, fwcomponent.Description, fwcomponent.Version).Set(state) - } - - // iLO 4 export - if ilo4err != nil { - (*component)["ilo4ComponentFirmware"].WithLabelValues(ilo4fwcomponent.Current.Component.Details.Item.Name, ilo4fwcomponent.Current.Component.Details.Item.Key, ilo4fwcomponent.Current.Component.Details.Item.Location, ilo4fwcomponent.Current.Component.Details.VersionString) - } + fmt.Printf("Component: %+v\n", fwcomponent.FirmwareSystemInventory.Current.Component) + // fmt.Printf("Component: %+v\n", fwcomponent.FirmwareSystemInventory.Current.Component[0].Details) + // // Loop through the components + // for _, component := range fwcomponent.FirmwareSystemInventory.Current.Component { + // // Access the component properties + // componentName := component.Details.FName + // componentVersion := component.Details.FVersionString + // componentLocation := component.Details.FLocation + + // // Process the component properties as needed + // // ... + + // // // Set the state for the component + // // componentState := OK // Assuming OK as the default state + // // if component.Status.State == "Enabled" || component.Status.State != "" { + // // if component.Status.Health == "OK" { + // // componentState = OK + // // } else { + // // componentState = BAD + // // } + // // } else if component.Status.Health == "" && component.Status.State == "" { + // // componentState = OK + // // } + + // fmt.Println("Component Name: ", componentName) + // fmt.Println("Component Version: ", componentVersion) + // fmt.Println("Component Location: ", componentLocation) + + // // Set the prometheus gauge for the component + // (*fcomponent)["componentFirmware"].WithLabelValues(componentName, componentVersion, componentLocation).Set(state) + // } + // DEBUG PRINT + fmt.Printf("Firmware Inventory Metrics: %+v\n", fwcomponent) + fmt.Println("Firmware Inventory Metrics: ", fwcomponent.GenericFirmware.Name, fwcomponent.GenericFirmware.Description, fwcomponent.GenericFirmware.Version, fwcomponent.GenericFirmware.Location) + // fmt.Println("Firmware Inventory Metrics: ", fwcomponent.FirmwareSystemInventory.Current.Component.Details.FName, fwcomponent.FirmwareSystemInventory.Current.Component.Details.FVersionString, fwcomponent.FirmwareSystemInventory.Current.Component.Details.FLocation) + // END DEBUG PRINT + (*component)["componentFirmware"].WithLabelValues(fwcomponent.GenericFirmware.Name, fwcomponent.GenericFirmware.Description, fwcomponent.GenericFirmware.Version, fwcomponent.GenericFirmware.Location).Set(state) return nil } From 622404339edd7e6b54212c3e5a9226526189f6f2 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:02:56 -0400 Subject: [PATCH 19/33] metrics.go 6/24 changes --- exporter/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index 0c8b850..af86a85 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -94,7 +94,7 @@ func NewDeviceMetrics() *map[string]*metrics { // Component Firmware Metrics FirmwareInventoryMetrics = &metrics{ - "componentFirmware": newServerMetric("redfish_component_firmware", "Current firmware component status 1 = OK, 0 = BAD", nil, []string{"id", "name", "description", "version"}), + "componentFirmware": newServerMetric("redfish_component_firmware", "Current firmware component status 1 = OK, 0 = BAD", nil, []string{"name", "description", "version", "location"}), } DeviceMetrics = &metrics{ From 94745ad78347fa401c0abdc198473be632e42cad Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:03:25 -0400 Subject: [PATCH 20/33] chassis.go 6/24 changes --- oem/chassis.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/oem/chassis.go b/oem/chassis.go index f62918d..cb7d54b 100644 --- a/oem/chassis.go +++ b/oem/chassis.go @@ -104,13 +104,3 @@ type HRef struct { type Link struct { URL string `json:"@odata.id"` } - -// Collection returns the component details -// /redfish/v1/UpdateService/FirmwareInventory/XXXX/ -type FirmwareComponent struct { - Name string `json:"Name"` - Description string `json:"Description"` - Version string `json:"Version"` - Id string `json:"Id"` - Status Status -} From e92b2dc92961882829d5241aa7b3940ba52b090b Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 24 Jun 2024 21:04:00 -0400 Subject: [PATCH 21/33] Create firmware.go 6/24 changes --- oem/firmware.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 oem/firmware.go diff --git a/oem/firmware.go b/oem/firmware.go new file mode 100644 index 0000000..815c1a3 --- /dev/null +++ b/oem/firmware.go @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package oem + +type GenericFirmware struct { + Name string `json:"Name,omitempty"` + Version string `json:"Version,VersionString,omitempty"` + Location string `json:"Location,omitempty"` + Description string `json:"Description,omitempty"` + Status Status +} + +// Collection returns the component details +// /redfish/v1/UpdateService/FirmwareInventory/XXXX/ +// type FirmwareComponent struct { +// Name string `json:"Name,omitempty"` +// Description string `json:"Description,omitempty"` +// Version string `json:"Version,omitempty"` +// Id string `json:"Id,omitempty"` +// Status Status +// } + +type FirmwareComponent struct { + GenericFirmware + FirmwareSystemInventory +} + +type FirmwareInventory struct { + Components []FirmwareComponent `json:"Components,omitempty"` +} + +// /redfish/v1/Systems/XXXX/FirmwareInventory +type FirmwareSystemInventory struct { + Current Current `json:"Current,omitempty"` + Status Status +} + +type Current struct { + Component Component `json:"Component,omitempty"` +} + +type Component struct { + Details Details `json:"Details,omitempty"` +} + +// type Details struct { +// Location string `json:"Location,omitempty"` +// Name string `json:"Name,omitempty"` +// VersionString string `json:"VersionString,omitempty"` +// } + +type Details struct { + GenericFirmware +} From d17e60ac0ea110262006f5af945467d869dcb0b0 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:29:20 -0400 Subject: [PATCH 22/33] Clean up debug prints and comments --- exporter/exporter.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index 1fd17d2..57a1c3e 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -20,7 +20,6 @@ import ( "context" "crypto/tls" "errors" - "fmt" "net" "net/http" "net/url" @@ -375,9 +374,6 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud // Firmware Inventory for _, url := range firmwareInventoryEndpoints { - // DEBUG PRINT - fmt.Printf("task url to be appended: %s\n", exp.url+url) - // END DEBUG PRINT tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) } From dccfcea6ca734174c516224bf89e98fd3d4fbf5f Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:29:49 -0400 Subject: [PATCH 23/33] Update handlers.go to include refactored exportFirmwareInventoryMetrics func --- exporter/handlers.go | 65 ++++++++++---------------------------------- 1 file changed, 15 insertions(+), 50 deletions(-) diff --git a/exporter/handlers.go b/exporter/handlers.go index 475022c..3df3ad8 100644 --- a/exporter/handlers.go +++ b/exporter/handlers.go @@ -581,63 +581,28 @@ func (e *Exporter) exportProcessorMetrics(body []byte) error { // exportFirmwareInventoryMetrics collects the inventory component's firmware metrics in json format and sets the prometheus guages func (e *Exporter) exportFirmwareInventoryMetrics(body []byte) error { - - var state float64 - var fwcomponent oem.FirmwareComponent + var fwcomponent oem.ILO4Firmware var component = (*e.DeviceMetrics)["firmwareInventoryMetrics"] - // var fcomponent = (*e.DeviceMetrics)["firmwareInventoryMetrics"] + err := json.Unmarshal(body, &fwcomponent) if err != nil { return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) } - - if fwcomponent.GenericFirmware.Status.State == "Enabled" || fwcomponent.GenericFirmware.Status.State != "" { - if fwcomponent.GenericFirmware.Status.Health == "OK" { - state = OK - } else { - state = BAD + // Export for iLO4 since it has a different structure + if len(fwcomponent.Current.Firmware) > 0 { + for _, firmware := range fwcomponent.Current.Firmware { + (*component)["componentFirmware"].WithLabelValues(firmware.Id, firmware.Name, firmware.Location, firmware.VersionString).Set(1.0) + } + } else { + // Export for iLO5, since it's structure matches the GenericFirmware struct + var fwcomponent oem.GenericFirmware + err := json.Unmarshal(body, &fwcomponent) + if err != nil { + return fmt.Errorf("Error Unmarshalling FirmwareInventoryMetrics - " + err.Error()) } - } else if fwcomponent.GenericFirmware.Status.Health == "" && fwcomponent.GenericFirmware.Status.State == "" { - state = OK - } - fmt.Printf("Component: %+v\n", fwcomponent.FirmwareSystemInventory.Current.Component) - // fmt.Printf("Component: %+v\n", fwcomponent.FirmwareSystemInventory.Current.Component[0].Details) - // // Loop through the components - // for _, component := range fwcomponent.FirmwareSystemInventory.Current.Component { - // // Access the component properties - // componentName := component.Details.FName - // componentVersion := component.Details.FVersionString - // componentLocation := component.Details.FLocation - - // // Process the component properties as needed - // // ... - - // // // Set the state for the component - // // componentState := OK // Assuming OK as the default state - // // if component.Status.State == "Enabled" || component.Status.State != "" { - // // if component.Status.Health == "OK" { - // // componentState = OK - // // } else { - // // componentState = BAD - // // } - // // } else if component.Status.Health == "" && component.Status.State == "" { - // // componentState = OK - // // } - - // fmt.Println("Component Name: ", componentName) - // fmt.Println("Component Version: ", componentVersion) - // fmt.Println("Component Location: ", componentLocation) - - // // Set the prometheus gauge for the component - // (*fcomponent)["componentFirmware"].WithLabelValues(componentName, componentVersion, componentLocation).Set(state) - // } - // DEBUG PRINT - fmt.Printf("Firmware Inventory Metrics: %+v\n", fwcomponent) - fmt.Println("Firmware Inventory Metrics: ", fwcomponent.GenericFirmware.Name, fwcomponent.GenericFirmware.Description, fwcomponent.GenericFirmware.Version, fwcomponent.GenericFirmware.Location) - // fmt.Println("Firmware Inventory Metrics: ", fwcomponent.FirmwareSystemInventory.Current.Component.Details.FName, fwcomponent.FirmwareSystemInventory.Current.Component.Details.FVersionString, fwcomponent.FirmwareSystemInventory.Current.Component.Details.FLocation) - // END DEBUG PRINT - (*component)["componentFirmware"].WithLabelValues(fwcomponent.GenericFirmware.Name, fwcomponent.GenericFirmware.Description, fwcomponent.GenericFirmware.Version, fwcomponent.GenericFirmware.Location).Set(state) + (*component)["componentFirmware"].WithLabelValues(fwcomponent.Id, fwcomponent.Name, fwcomponent.Description, fwcomponent.Version).Set(1.0) + } return nil } From 78d034a8264800c9f2f4e1aca062fda26362ce0b Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:31:54 -0400 Subject: [PATCH 24/33] Update metrics.go with new export firmware metric fields --- exporter/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index af86a85..0c8b850 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -94,7 +94,7 @@ func NewDeviceMetrics() *map[string]*metrics { // Component Firmware Metrics FirmwareInventoryMetrics = &metrics{ - "componentFirmware": newServerMetric("redfish_component_firmware", "Current firmware component status 1 = OK, 0 = BAD", nil, []string{"name", "description", "version", "location"}), + "componentFirmware": newServerMetric("redfish_component_firmware", "Current firmware component status 1 = OK, 0 = BAD", nil, []string{"id", "name", "description", "version"}), } DeviceMetrics = &metrics{ From c8f0ff6a0077d0b985eb292972c8e1946c60bc10 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:32:49 -0400 Subject: [PATCH 25/33] Update firmware.go to include finalized universal structs --- oem/firmware.go | 72 ++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/oem/firmware.go b/oem/firmware.go index 815c1a3..3e0f4ef 100644 --- a/oem/firmware.go +++ b/oem/firmware.go @@ -16,53 +16,51 @@ package oem -type GenericFirmware struct { - Name string `json:"Name,omitempty"` - Version string `json:"Version,VersionString,omitempty"` - Location string `json:"Location,omitempty"` - Description string `json:"Description,omitempty"` - Status Status -} +import "encoding/json" -// Collection returns the component details // /redfish/v1/UpdateService/FirmwareInventory/XXXX/ -// type FirmwareComponent struct { -// Name string `json:"Name,omitempty"` -// Description string `json:"Description,omitempty"` -// Version string `json:"Version,omitempty"` -// Id string `json:"Id,omitempty"` -// Status Status -// } - -type FirmwareComponent struct { - GenericFirmware - FirmwareSystemInventory +type GenericFirmware struct { + Id string `json:"Id,omitempty"` + Name string `json:"Name"` + Version string `json:"Version,omitempty"` + VersionString string `json:"VersionString,omitempty"` + Location string `json:"Location,omitempty"` + Description string `json:"Description,omitempty"` + Status Status `json:"Status,omitempty"` } -type FirmwareInventory struct { - Components []FirmwareComponent `json:"Components,omitempty"` +// /redfish/v1/Systems/1/FirmwareInventory/ +type ILO4Firmware struct { + Current FirmwareWrapper `json:"Current"` } -// /redfish/v1/Systems/XXXX/FirmwareInventory -type FirmwareSystemInventory struct { - Current Current `json:"Current,omitempty"` - Status Status +type FirmwareWrapper struct { + FirmwareSlice } -type Current struct { - Component Component `json:"Component,omitempty"` +type FirmwareSlice struct { + Firmware []GenericFirmware } -type Component struct { - Details Details `json:"Details,omitempty"` -} +// Function to unmarshal the FirmwareWrapper struct +func (w *FirmwareWrapper) UnmarshalJSON(data []byte) error { + var fw GenericFirmware + var jsonData map[string]interface{} + err := json.Unmarshal(data, &jsonData) -// type Details struct { -// Location string `json:"Location,omitempty"` -// Name string `json:"Name,omitempty"` -// VersionString string `json:"VersionString,omitempty"` -// } + if err == nil { + for _, items := range jsonData { + for _, item := range items.([]interface{}) { + component, _ := json.Marshal(item) + err = json.Unmarshal(component, &fw) + if err == nil { + w.Firmware = append(w.Firmware, fw) + } + } + } + } else { + return json.Unmarshal(data, &w.Firmware) + } -type Details struct { - GenericFirmware + return nil } From a62374328ee0937f58d8f278e46a7ebf9f026d3d Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:44:51 -0400 Subject: [PATCH 26/33] Remove comments from exporter.go --- exporter/exporter.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index 57a1c3e..d403540 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -311,13 +311,6 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud zap.Strings("physical_drive_endpoints", driveEndpointsResp.physicalDriveURLs), zap.Any("trace_id", ctx.Value("traceID"))) - // // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs - // firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) - // if err != nil { - // log.Error("error when getting FirmwareInventory url", zap.Error(err), zap.Any("trace_id", ctx.Value("traceID"))) - // firmwareInventoryEndpoints = []string{} - // } - // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs // Call /redfish/v1/Managers/XXXX/UpdateService/FirmwareInventory/ for firmware inventory firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) From 6e6b9d7b146f1310cee0edddde40602c281c7757 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:46:45 -0400 Subject: [PATCH 27/33] Remove comment from exporter.go --- exporter/exporter.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index d403540..7ca2b56 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -311,7 +311,6 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud zap.Strings("physical_drive_endpoints", driveEndpointsResp.physicalDriveURLs), zap.Any("trace_id", ctx.Value("traceID"))) - // Using initial /UpdateService/FirmwareInventory endpoint, grab all of the Firmware Inventory URLs // Call /redfish/v1/Managers/XXXX/UpdateService/FirmwareInventory/ for firmware inventory firmwareInventoryEndpoints, err := getMemberUrls(exp.url+uri+"/UpdateService/FirmwareInventory/", target, retryClient) if err != nil { From 672da07460ee62be044a88e0c35b0a636cfd67f6 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:26:41 -0400 Subject: [PATCH 28/33] Remove unused function from helpers.go --- exporter/helpers.go | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/exporter/helpers.go b/exporter/helpers.go index 42e1af0..7fb8146 100644 --- a/exporter/helpers.go +++ b/exporter/helpers.go @@ -475,51 +475,6 @@ func getProcessorEndpoints(url, host string, client *retryablehttp.Client) (oem. return processors, nil } -// Getting firmware components for iLO 4 hosts -func getFirmwareComponents(url, host string, client *retryablehttp.Client) (oem.SystemFirmwareInventory, error) { - var fc oem.SystemFirmwareInventory - var resp *http.Response - var err error - retryCount := 0 - req := common.BuildRequest(url, host) - - resp, err = common.DoRequest(client, req) - if err != nil { - return fc, err - } - defer resp.Body.Close() - if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { - if resp.StatusCode == http.StatusNotFound { - for retryCount < 3 && resp.StatusCode == http.StatusNotFound { - time.Sleep(client.RetryWaitMin) - resp, err = common.DoRequest(client, req) - retryCount = retryCount + 1 - } - if err != nil { - return fc, err - } else if !(resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices) { - return fc, fmt.Errorf("HTTP status %d", resp.StatusCode) - } - } else if resp.StatusCode == http.StatusUnauthorized { - return fc, common.ErrInvalidCredential - } else { - return fc, fmt.Errorf("HTTP status %d", resp.StatusCode) - } - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fc, fmt.Errorf("Error reading Response Body - " + err.Error()) - } - - err = json.Unmarshal(body, &fc) - if err != nil { - return fc, fmt.Errorf("Error Unmarshalling SystemFirmwareInventory Collection struct - " + err.Error()) - } - - return fc, nil -} - // appendSlash appends a slash to the end of a URL if it does not already have one func appendSlash(url string) string { if url[len(url)-1] != '/' { From 942eab100e189601a593c42ec77a16a3e1558edd Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:27:09 -0400 Subject: [PATCH 29/33] Remove unused struct from system.go --- oem/system.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/oem/system.go b/oem/system.go index 7882753..44bd5ff 100644 --- a/oem/system.go +++ b/oem/system.go @@ -76,19 +76,3 @@ type MemorySummary struct { type StatusMemory struct { HealthRollup string `json:"HealthRollup"` } - -// SystemFirmwareInventory is the json object for SystemFirwmareInventory metadata -type SystemFirmwareInventory struct { - Current []struct { - Component []struct { - Details []struct { - Item []struct { - Name string `json:"Name,omitempty"` - Key string `json:"Key,omitempty"` - Location string `json:"Location,omitempty"` - } - VersionString string `json:"VersionString,omitempty"` - } - } - } -} From 6358911a1fc4e870107a171dd8c9d495e029661d Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:05:00 -0400 Subject: [PATCH 30/33] Add firmware modules exclude flag --- cmd/fishymetrics/main.go | 42 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/cmd/fishymetrics/main.go b/cmd/fishymetrics/main.go index d4d2c05..1f1a8ba 100644 --- a/cmd/fishymetrics/main.go +++ b/cmd/fishymetrics/main.go @@ -56,24 +56,25 @@ const ( ) var ( - a = kingpin.New(app, "redfish api exporter with all the bells and whistles") - username = a.Flag("user", "BMC static username").Default("").Envar("BMC_USERNAME").String() - password = a.Flag("password", "BMC static password").Default("").Envar("BMC_PASSWORD").String() - bmcTimeout = a.Flag("timeout", "BMC scrape timeout").Default("15s").Envar("BMC_TIMEOUT").Duration() - bmcScheme = a.Flag("scheme", "BMC Scheme to use").Default("https").Envar("BMC_SCHEME").String() - logLevel = a.Flag("log.level", "log level verbosity").PlaceHolder("[debug|info|warn|error]").Default("info").Envar("LOG_LEVEL").String() - logMethod = a.Flag("log.method", "alternative method for logging in addition to stdout").PlaceHolder("[file|vector]").Default("").Envar("LOG_METHOD").String() - logFilePath = a.Flag("log.file-path", "directory path where log files are written if log-method is file").Default("/var/log/fishymetrics").Envar("LOG_FILE_PATH").String() - logFileMaxSize = a.Flag("log.file-max-size", "max file size in megabytes if log-method is file").Default("256").Envar("LOG_FILE_MAX_SIZE").String() - logFileMaxBackups = a.Flag("log.file-max-backups", "max file backups before they are rotated if log-method is file").Default("1").Envar("LOG_FILE_MAX_BACKUPS").String() - logFileMaxAge = a.Flag("log.file-max-age", "max file age in days before they are rotated if log-method is file").Default("1").Envar("LOG_FILE_MAX_AGE").String() - vectorEndpoint = a.Flag("vector.endpoint", "vector endpoint to send structured json logs to").Default("http://0.0.0.0:4444").Envar("VECTOR_ENDPOINT").String() - exporterPort = a.Flag("port", "exporter port").Default("9533").Envar("EXPORTER_PORT").String() - vaultAddr = a.Flag("vault.addr", "Vault instance address to get chassis credentials from").Default("https://vault.com").Envar("VAULT_ADDRESS").String() - vaultRoleId = a.Flag("vault.role-id", "Vault Role ID for AppRole").Default("").Envar("VAULT_ROLE_ID").String() - vaultSecretId = a.Flag("vault.secret-id", "Vault Secret ID for AppRole").Default("").Envar("VAULT_SECRET_ID").String() - driveModExclude = a.Flag("collector.drives.modules-exclude", "regex of drive module(s) to exclude from the scrape").Default("").Envar("COLLECTOR_DRIVES_MODULE_EXCLUDE").String() - credProfiles = common.CredentialProf(a.Flag("credentials.profiles", + a = kingpin.New(app, "redfish api exporter with all the bells and whistles") + username = a.Flag("user", "BMC static username").Default("").Envar("BMC_USERNAME").String() + password = a.Flag("password", "BMC static password").Default("").Envar("BMC_PASSWORD").String() + bmcTimeout = a.Flag("timeout", "BMC scrape timeout").Default("15s").Envar("BMC_TIMEOUT").Duration() + bmcScheme = a.Flag("scheme", "BMC Scheme to use").Default("https").Envar("BMC_SCHEME").String() + logLevel = a.Flag("log.level", "log level verbosity").PlaceHolder("[debug|info|warn|error]").Default("info").Envar("LOG_LEVEL").String() + logMethod = a.Flag("log.method", "alternative method for logging in addition to stdout").PlaceHolder("[file|vector]").Default("").Envar("LOG_METHOD").String() + logFilePath = a.Flag("log.file-path", "directory path where log files are written if log-method is file").Default("/var/log/fishymetrics").Envar("LOG_FILE_PATH").String() + logFileMaxSize = a.Flag("log.file-max-size", "max file size in megabytes if log-method is file").Default("256").Envar("LOG_FILE_MAX_SIZE").String() + logFileMaxBackups = a.Flag("log.file-max-backups", "max file backups before they are rotated if log-method is file").Default("1").Envar("LOG_FILE_MAX_BACKUPS").String() + logFileMaxAge = a.Flag("log.file-max-age", "max file age in days before they are rotated if log-method is file").Default("1").Envar("LOG_FILE_MAX_AGE").String() + vectorEndpoint = a.Flag("vector.endpoint", "vector endpoint to send structured json logs to").Default("http://0.0.0.0:4444").Envar("VECTOR_ENDPOINT").String() + exporterPort = a.Flag("port", "exporter port").Default("9533").Envar("EXPORTER_PORT").String() + vaultAddr = a.Flag("vault.addr", "Vault instance address to get chassis credentials from").Default("https://vault.com").Envar("VAULT_ADDRESS").String() + vaultRoleId = a.Flag("vault.role-id", "Vault Role ID for AppRole").Default("").Envar("VAULT_ROLE_ID").String() + vaultSecretId = a.Flag("vault.secret-id", "Vault Secret ID for AppRole").Default("").Envar("VAULT_SECRET_ID").String() + driveModExclude = a.Flag("collector.drives.modules-exclude", "regex of drive module(s) to exclude from the scrape").Default("").Envar("COLLECTOR_DRIVES_MODULE_EXCLUDE").String() + firmwareModExclude = a.Flag("collector.firmware.modules-exclude", "regex of firmware module(s) to exclude from the scrape").Default("").Envar("COLLECTOR_FIRMWARE_MODULE_EXCLUDE").String() + credProfiles = common.CredentialProf(a.Flag("credentials.profiles", `profile(s) with all necessary parameters to obtain BMC credential from secrets backend, i.e. --credentials.profiles=" profiles: @@ -204,6 +205,11 @@ func main() { excludes["drive"] = driveModPattern } + if *firmwareModExclude != "" { + firmwareModPattern := regexp.MustCompile(*firmwareModExclude) + excludes["firmware"] = firmwareModPattern + } + // validate logFilePath exists and is a directory if *logMethod == "file" { fd, err := os.Stat(*logFilePath) From 173294e856f8800e0775a653c683155e16d820af Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:05:41 -0400 Subject: [PATCH 31/33] Update exporter.go to include use of firmware.modules-exclude flag --- exporter/exporter.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index 7d22a28..c72a90d 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -23,6 +23,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strconv" "strings" "sync" @@ -426,7 +427,13 @@ func NewExporter(ctx context.Context, target, uri, profile, model string, exclud // Firmware Inventory for _, url := range firmwareInventoryEndpoints { - tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) + // this list can potentially be large and cause scrapes to take a long time please + // see the '--collector.firmware.modules-exclude' config in the README for more information + if reg, ok := excludes["firmware"]; ok { + if !reg.(*regexp.Regexp).MatchString(url) { + tasks = append(tasks, pool.NewTask(common.Fetch(exp.url+url, target, profile, retryClient), exp.url+url, handle(&exp, FIRMWAREINVENTORY))) + } + } } // call /redfish/v1/Managers/XXX/ for firmware version and ilo self test metrics From 26ad359ec8fb3e5b42a31c0a615cc05fedf0051a Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:06:44 -0400 Subject: [PATCH 32/33] Update README.md to include firmware.modules-exclude flag --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7310358..05631c1 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Flags: --vault.secret-id="" Vault Secret ID for AppRole --collector.drives.modules-exclude="" regex of drive module(s) to exclude from the scrape + --collector.firmware.modules-exclude="" + regex of firmware module to exclude from the scrape --credentials.profiles=CREDENTIALS.PROFILES profile(s) with all necessary parameters to obtain BMC credential from secrets backend, i.e. @@ -77,7 +79,7 @@ VAULT_SECRET_ID= ### Exclude flags -Since some hosts can contain many dozens of drives, this can cause a scrape to take a very long time and may not be entirely necessary. Because of this we've included an exclude flag specifically for the `drives.module` scope. +Since some hosts can contain many dozens of drives, this can cause a scrape to take a very long time and may not be entirely necessary. Because of this we've included an exclude flag specifically for the `drives.module` and `firmware.module` scopes. Example: @@ -88,6 +90,7 @@ Example: | Collector | Scope | Include Flag | Exclude Flag | | --------- | ------ | ------------ | -------------- | | drives | module | N/A | module-exclude | +| firmware | module | N/A | module-exclude | ## Usage From b57eb090ddd2bfe88e8ac34aa6673a42fe70b638 Mon Sep 17 00:00:00 2001 From: Jennifer Kaiser <65861760+jenniferKaiser21@users.noreply.github.com> Date: Fri, 5 Jul 2024 12:10:00 -0400 Subject: [PATCH 33/33] Update CHANGELOG.md for firmware metrics gathering addition --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6832a6..caf2b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ log is based on the [Keep a CHANGELOG](http://keepachangelog.com/) project. - Storage controller status metric for HP servers [#79](https://github.com/Comcast/fishymetrics/issues/79) - Ignore CPU metrics if Processor is Absent [#79](https://github.com/Comcast/fishymetrics/issues/79) - Added support for metrics collection from Dell servers [#77](https://github.com/Comcast/fishymetrics/issues/77) +- Added support for firmware metrics collection from all supported servers and iLO versions from a single universal exporter [#83](https://github.com/Comcast/fishymetrics/issues/83) ## Fixed