diff --git a/pkg/api/api_default_service.go b/pkg/api/api_default_service.go index 5d489e7..e58031f 100644 --- a/pkg/api/api_default_service.go +++ b/pkg/api/api_default_service.go @@ -17,7 +17,6 @@ import ( "sort" "cfm/pkg/common" - "cfm/pkg/common/datastore" "cfm/pkg/manager" "cfm/pkg/openapi" ) @@ -42,25 +41,25 @@ func NewCfmApiService(version string) openapi.DefaultAPIServicer { // AppliancesDeleteById - func (cfm *CfmApiService) AppliancesDeleteById(ctx context.Context, applianceId string) (openapi.ImplResponse, error) { + var a openapi.Appliance + appliance, err := manager.DeleteApplianceById(ctx, applianceId) if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } + a = openapi.Appliance{Id: applianceId} - // Update DataStore. - datastore.DStore().GetDataStore().DeleteAppliance(appliance.Id) - datastore.DStore().Store() + } else { - a := openapi.Appliance{ - Id: appliance.Id, - IpAddress: "", // Unused (May need for POC4) - Port: 0, // Unused (May need for POC4) - Status: "", // Unused - Blades: openapi.MemberItem{ - Uri: manager.GetCfmUriBlades(appliance.Id), - }, - TotalMemoryAvailableMiB: -1, // Not Implemented - TotalMemoryAllocatedMiB: -1, // Not Implemented + a = openapi.Appliance{ + Id: appliance.Id, + IpAddress: "", // Unused (May need for POC4) + Port: 0, // Unused (May need for POC4) + Status: "", // Unused + Blades: openapi.MemberItem{ + Uri: manager.GetCfmUriBlades(appliance.Id), + }, + TotalMemoryAvailableMiB: -1, // Not Implemented + TotalMemoryAllocatedMiB: -1, // Not Implemented + } } return openapi.Response(http.StatusOK, a), nil @@ -129,12 +128,6 @@ func (cfm *CfmApiService) AppliancesPost(ctx context.Context, credentials openap return formatErrorResp(ctx, err.(*common.RequestError)) } - // The user's purpose for "credentials" object is complete. Now, reusing this object for DataStore. - // Update credentials ID with final appliance ID. - credentials.CustomId = appliance.Id - datastore.DStore().GetDataStore().AddAppliance(&credentials) - datastore.DStore().Store() - a := openapi.Appliance{ Id: appliance.Id, IpAddress: "", // Unused @@ -152,13 +145,7 @@ func (cfm *CfmApiService) AppliancesPost(ctx context.Context, credentials openap // AppliancesResync - func (cfm *CfmApiService) AppliancesResyncById(ctx context.Context, applianceId string) (openapi.ImplResponse, error) { - appliance, failedBladeIds, err := manager.ResyncApplianceById(ctx, applianceId) - - for _, id := range *failedBladeIds { - // Update DataStore. - datastore.DStore().GetDataStore().DeleteBlade(id, applianceId) - datastore.DStore().Store() - } + appliance, err := manager.ResyncApplianceById(ctx, applianceId) if err != nil { if appliance != nil { @@ -268,8 +255,10 @@ func (cfm *CfmApiService) BladesComposeMemoryByResource(ctx context.Context, app return openapi.Response(http.StatusCreated, memory), nil } -// BladesDeleteById - +// BladesDeleteById - As long as the appliance id is valid, guarenteed blade deletion from service func (cfm *CfmApiService) BladesDeleteById(ctx context.Context, applianceId string, bladeId string) (openapi.ImplResponse, error) { + var b openapi.Blade + appliance, err := manager.GetApplianceById(ctx, applianceId) if err != nil { return formatErrorResp(ctx, err.(*common.RequestError)) @@ -277,29 +266,27 @@ func (cfm *CfmApiService) BladesDeleteById(ctx context.Context, applianceId stri blade, err := appliance.DeleteBladeById(ctx, bladeId) if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } - - // Update DataStore. - datastore.DStore().GetDataStore().DeleteBlade(blade.Id, applianceId) - datastore.DStore().Store() - - b := openapi.Blade{ - Id: blade.Id, - IpAddress: blade.GetNetIp(), - Port: int32(blade.GetNetPort()), - Status: "", // Unused - Ports: openapi.MemberItem{ - Uri: manager.GetCfmUriBladePorts(appliance.Id, blade.Id), - }, - Resources: openapi.MemberItem{ - Uri: manager.GetCfmUriBladeResources(appliance.Id, blade.Id), - }, - Memory: openapi.MemberItem{ - Uri: manager.GetCfmUriBladeMemory(appliance.Id, blade.Id), - }, - TotalMemoryAvailableMiB: -1, // Not Implemented - TotalMemoryAllocatedMiB: -1, // Not Implemented + b = openapi.Blade{Id: bladeId} + + } else { + + b = openapi.Blade{ + Id: blade.Id, + IpAddress: blade.GetNetIp(), + Port: int32(blade.GetNetPort()), + Status: string(blade.Status), + Ports: openapi.MemberItem{ + Uri: manager.GetCfmUriBladePorts(appliance.Id, blade.Id), + }, + Resources: openapi.MemberItem{ + Uri: manager.GetCfmUriBladeResources(appliance.Id, blade.Id), + }, + Memory: openapi.MemberItem{ + Uri: manager.GetCfmUriBladeMemory(appliance.Id, blade.Id), + }, + TotalMemoryAvailableMiB: -1, // Not Implemented + TotalMemoryAllocatedMiB: -1, // Not Implemented + } } return openapi.Response(http.StatusOK, b), nil @@ -371,7 +358,7 @@ func (cfm *CfmApiService) BladesGetById(ctx context.Context, applianceId string, Id: blade.Id, IpAddress: blade.GetNetIp(), Port: int32(blade.GetNetPort()), - Status: "", // Unused + Status: string(blade.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriBladePorts(appliance.Id, blade.Id), }, @@ -401,7 +388,7 @@ func (cfm *CfmApiService) BladesGetMemory(ctx context.Context, applianceId strin } // order returned uris by memory id - memoryIds := blade.GetAllMemoryIds() + memoryIds := blade.GetAllMemoryIds(ctx) sort.Strings(memoryIds) memory := blade.GetMemory(ctx) @@ -435,12 +422,17 @@ func (cfm *CfmApiService) BladesGetMemoryById(ctx context.Context, applianceId s return formatErrorResp(ctx, err.(*common.RequestError)) } - details, err := memory.GetDetails(ctx) - if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } + if memory != nil { + details, err := memory.GetDetails(ctx) + if err != nil { + return formatErrorResp(ctx, err.(*common.RequestError)) + } - return openapi.Response(http.StatusOK, details), nil + return openapi.Response(http.StatusOK, details), nil + + } else { + return openapi.Response(http.StatusOK, openapi.MemoryRegion{}), nil + } } // BladesGetPortById - @@ -460,12 +452,17 @@ func (cfm *CfmApiService) BladesGetPortById(ctx context.Context, applianceId str return formatErrorResp(ctx, err.(*common.RequestError)) } - details, err := port.GetDetails(ctx) - if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } + if port != nil { + details, err := port.GetDetails(ctx) + if err != nil { + return formatErrorResp(ctx, err.(*common.RequestError)) + } - return openapi.Response(http.StatusOK, details), nil + return openapi.Response(http.StatusOK, details), nil + + } else { + return openapi.Response(http.StatusOK, openapi.PortInformation{}), nil + } } // BladesGetPorts - @@ -481,7 +478,7 @@ func (cfm *CfmApiService) BladesGetPorts(ctx context.Context, applianceId string } // order returned uris by port id - portIds := blade.GetAllPortIds() + portIds := blade.GetAllPortIds(ctx) sort.Strings(portIds) ports := blade.GetPorts(ctx) @@ -515,12 +512,17 @@ func (cfm *CfmApiService) BladesGetResourceById(ctx context.Context, applianceId return formatErrorResp(ctx, err.(*common.RequestError)) } - details, err := resource.GetDetails(ctx) - if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } + if resource != nil { + details, err := resource.GetDetails(ctx) + if err != nil { + return formatErrorResp(ctx, err.(*common.RequestError)) + } - return openapi.Response(http.StatusOK, details), nil + return openapi.Response(http.StatusOK, details), nil + + } else { + return openapi.Response(http.StatusOK, openapi.MemoryResourceBlock{}), nil + } } // BladesGetResources - @@ -536,7 +538,7 @@ func (cfm *CfmApiService) BladesGetResources(ctx context.Context, applianceId st } // order returned uris by resourse id - resourceIds := blade.GetAllResourceIds() + resourceIds := blade.GetAllResourceIds(ctx) sort.Strings(resourceIds) resources := blade.GetResources(ctx) @@ -560,11 +562,10 @@ func (cfm *CfmApiService) BladesPost(ctx context.Context, applianceId string, cr return formatErrorResp(ctx, err.(*common.RequestError)) } - // Appliance can be empty but may include up to 8 blades if len(appliance.Blades) == MAX_COUNT_BLADES { err := common.RequestError{ StatusCode: common.StatusBladesExceedMaximum, - Err: fmt.Errorf("no more blades can be associated with this appliance (%s)", applianceId), + Err: fmt.Errorf("cfm-service at maximum blade capacity (%d) for this appliance (%s)", MAX_COUNT_BLADES, applianceId), } return formatErrorResp(ctx, &err) } @@ -574,12 +575,6 @@ func (cfm *CfmApiService) BladesPost(ctx context.Context, applianceId string, cr return formatErrorResp(ctx, err.(*common.RequestError)) } - // The user's purpose for "credentials" object is complete. Now, reusing this object for DataStore. - // Update credentials ID with final blade ID. - credentials.CustomId = blade.Id - datastore.DStore().GetDataStore().AddBlade(&credentials, appliance.Id) - datastore.DStore().Store() - totals, err := blade.GetResourceTotals(ctx) if err != nil { return formatErrorResp(ctx, err.(*common.RequestError)) @@ -589,7 +584,7 @@ func (cfm *CfmApiService) BladesPost(ctx context.Context, applianceId string, cr Id: blade.Id, IpAddress: blade.GetNetIp(), Port: int32(blade.GetNetPort()), - Status: "", // Unused + Status: string(blade.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriBladePorts(appliance.Id, blade.Id), }, @@ -615,10 +610,6 @@ func (cfm *CfmApiService) BladesResyncById(ctx context.Context, applianceId stri blade, err := appliance.ResyncBladeById(ctx, bladeId) if err != nil { - // Update DataStore. - datastore.DStore().GetDataStore().DeleteBlade(blade.Id, applianceId) - datastore.DStore().Store() - return formatErrorResp(ctx, err.(*common.RequestError)) } @@ -631,7 +622,7 @@ func (cfm *CfmApiService) BladesResyncById(ctx context.Context, applianceId stri Id: blade.Id, IpAddress: blade.GetNetIp(), Port: int32(blade.GetNetPort()), - Status: "", // Unused + Status: string(blade.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriBladePorts(appliance.Id, blade.Id), }, @@ -656,7 +647,7 @@ func (cfm *CfmApiService) HostGetMemory(ctx context.Context, hostId string) (ope } // order returned uris by memory id - memoryIds := host.GetAllMemoryIds() + memoryIds := host.GetAllMemoryIds(ctx) sort.Strings(memoryIds) memory := host.GetMemory(ctx) @@ -694,33 +685,33 @@ func (cfm *CfmApiService) HostsComposeMemory(ctx context.Context, hostId string, return openapi.Response(http.StatusCreated, memory), nil } -// HostsDeleteById - +// HostsDeleteById - Guarenteed host deletion from service. func (cfm *CfmApiService) HostsDeleteById(ctx context.Context, hostId string) (openapi.ImplResponse, error) { - host, err := manager.DeleteHostById(ctx, hostId) - if err != nil { - return formatErrorResp(ctx, err.(*common.RequestError)) - } + var h openapi.Host - // Update DataStore. - datastore.DStore().GetDataStore().DeleteHost(host.Id) - datastore.DStore().Store() - - h := openapi.Host{ - Id: host.Id, - IpAddress: host.GetNetIp(), - Port: int32(host.GetNetPort()), - Status: "", // Unused - Ports: openapi.MemberItem{ - Uri: manager.GetCfmUriHostPorts(host.Id), - }, - Memory: openapi.MemberItem{ - Uri: manager.GetCfmUriHostMemory(host.Id), - }, - MemoryDevices: openapi.MemberItem{ - Uri: manager.GetCfmUriHostMemoryDevices(host.Id), - }, - LocalMemoryMiB: -1, // Not implemented - RemoteMemoryMiB: -1, // Not implemented + host, err := manager.DeleteHostById(ctx, hostId) + if err != nil && host == nil { + h = openapi.Host{Id: hostId} + + } else { + + h = openapi.Host{ + Id: host.Id, + IpAddress: host.GetNetIp(), + Port: int32(host.GetNetPort()), + Status: string(host.Status), + Ports: openapi.MemberItem{ + Uri: manager.GetCfmUriHostPorts(host.Id), + }, + Memory: openapi.MemberItem{ + Uri: manager.GetCfmUriHostMemory(host.Id), + }, + MemoryDevices: openapi.MemberItem{ + Uri: manager.GetCfmUriHostMemoryDevices(host.Id), + }, + LocalMemoryMiB: -1, // Not implemented + RemoteMemoryMiB: -1, // Not implemented + } } return openapi.Response(http.StatusOK, h), nil @@ -764,7 +755,7 @@ func (cfm *CfmApiService) HostsGet(ctx context.Context) (openapi.ImplResponse, e // HostsGetById - Get information for a single CXL Host. func (cfm *CfmApiService) HostsGetById(ctx context.Context, hostId string) (openapi.ImplResponse, error) { host, err := manager.GetHostById(ctx, hostId) - if err != nil { + if err != nil || host == nil { return formatErrorResp(ctx, err.(*common.RequestError)) } @@ -777,7 +768,7 @@ func (cfm *CfmApiService) HostsGetById(ctx context.Context, hostId string) (open Id: host.Id, IpAddress: host.GetNetIp(), Port: int32(host.GetNetPort()), - Status: "", // Unused + Status: string(host.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriHostPorts(host.Id), }, @@ -842,7 +833,7 @@ func (cfm *CfmApiService) HostsGetMemoryDevices(ctx context.Context, hostId stri } // order returned uris by memory device id - memdevIds := host.GetAllMemoryDeviceIds() + memdevIds := host.GetAllMemoryDeviceIds(ctx) sort.Strings(memdevIds) memdevs := host.GetMemoryDevices(ctx) @@ -887,7 +878,7 @@ func (cfm *CfmApiService) HostsGetPorts(ctx context.Context, hostId string) (ope } // order returned uris by port id - portIds := host.GetAllPortIds() + portIds := host.GetAllPortIds(ctx) sort.Strings(portIds) ports := host.GetPorts(ctx) @@ -920,17 +911,11 @@ func (cfm *CfmApiService) HostsPost(ctx context.Context, credentials openapi.Cre return formatErrorResp(ctx, err.(*common.RequestError)) } - // The user's purpose for "credentials" object is complete. Now, reusing this object for DataStore. - // Update credentials ID with final host ID. - credentials.CustomId = host.Id - datastore.DStore().GetDataStore().AddHost(&credentials) - datastore.DStore().Store() - h := openapi.Host{ Id: host.Id, IpAddress: host.GetNetIp(), Port: int32(host.GetNetPort()), - Status: "", // Unused + Status: string(host.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriHostPorts(host.Id), }, @@ -951,10 +936,6 @@ func (cfm *CfmApiService) HostsPost(ctx context.Context, credentials openapi.Cre func (cfm *CfmApiService) HostsResyncById(ctx context.Context, hostId string) (openapi.ImplResponse, error) { host, err := manager.ResyncHostById(ctx, hostId) if err != nil { - // Update DataStore. - datastore.DStore().GetDataStore().DeleteHost(host.Id) - datastore.DStore().Store() - return formatErrorResp(ctx, err.(*common.RequestError)) } @@ -962,7 +943,7 @@ func (cfm *CfmApiService) HostsResyncById(ctx context.Context, hostId string) (o Id: host.Id, IpAddress: host.GetNetIp(), Port: int32(host.GetNetPort()), - Status: "", // Unused + Status: string(host.Status), Ports: openapi.MemberItem{ Uri: manager.GetCfmUriHostPorts(host.Id), }, diff --git a/pkg/api/api_redfish_service.go b/pkg/api/api_redfish_service.go index ccce4b1..7a74513 100644 --- a/pkg/api/api_redfish_service.go +++ b/pkg/api/api_redfish_service.go @@ -287,7 +287,7 @@ func (cfm *CfmApiService) RedfishV1ChassisChassisIdPCIeDevicesGet(ctx context.Co } // order returned uris by memory device id - memdevIds := host.GetAllMemoryDeviceIds() + memdevIds := host.GetAllMemoryDeviceIds(ctx) sort.Strings(memdevIds) members := []redfishapi.OdataV4IdRef{} @@ -588,7 +588,7 @@ func (cfm *CfmApiService) RedfishV1FabricsFabricIdConnectionsGet(ctx context.Con if err != nil { return formatRedfishErrorResp(ctx, err.(*common.RequestError)) } - memoryIds := blade.GetAllMemoryIds() + memoryIds := blade.GetAllMemoryIds(ctx) sort.Strings(memoryIds) for _, memId := range memoryIds { @@ -652,7 +652,7 @@ func (cfm *CfmApiService) RedfishV1FabricsFabricIdConnectionsPost(ctx context.Co endpointOdataId := connectionV131Connection.Links.TargetEndpoints[0].OdataId for bladeId, blade := range appliance.GetBlades(ctx) { - for _, memoryId := range blade.GetAllMemoryIds() { + for _, memoryId := range blade.GetAllMemoryIds(ctx) { if fmt.Sprintf("/redfish/v1/Systems/%s/MemoryDomains/%s/MemoryChunks/%s", fabricId, bladeId, memoryId) == memoryChunkOdataId { targetBladeId = bladeId r.MemoryId = memoryId @@ -662,7 +662,7 @@ func (cfm *CfmApiService) RedfishV1FabricsFabricIdConnectionsPost(ctx context.Co if r.MemoryId == "" { continue } - for _, portid := range blade.GetAllPortIds() { + for _, portid := range blade.GetAllPortIds(ctx) { if fmt.Sprintf("/redfish/v1/Fabrics/%s/Endpoints/%s", fabricId, bladeId+strings.Replace(portid, "port", "up", 1)) == endpointOdataId { r.PortId = portid break @@ -784,7 +784,7 @@ func (cfm *CfmApiService) RedfishV1FabricsFabricIdEndpointsGet(ctx context.Conte members := []redfishapi.OdataV4IdRef{} for _, bladeId := range bladeIds { blade, _ := appliance.GetBladeById(ctx, bladeId) - for _, portid := range blade.GetAllPortIds() { + for _, portid := range blade.GetAllPortIds(ctx) { members = append(members, redfishapi.OdataV4IdRef{ OdataId: fmt.Sprintf("/redfish/v1/Fabrics/%s/Endpoints/%s", fabricId, bladeId+strings.Replace(portid, "port", "up", 1))}) } @@ -943,7 +943,7 @@ func (cfm *CfmApiService) RedfishV1FabricsFabricIdSwitchesSwitchIdPortsGet(ctx c } // order returned uris by port id - portIds := blade.GetAllPortIds() + portIds := blade.GetAllPortIds(ctx) sort.Strings(portIds) members := []redfishapi.OdataV4IdRef{} for _, member := range portIds { @@ -1377,7 +1377,7 @@ func (cfm *CfmApiService) RedfishV1SystemsComputerSystemIdMemoryDomainsMemoryDom } // order returned uris by memory id - memoryIds := blade.GetAllMemoryIds() + memoryIds := blade.GetAllMemoryIds(ctx) sort.Strings(memoryIds) members := []redfishapi.OdataV4IdRef{} for _, member := range memoryIds { diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 2732282..65b89dd 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -10,6 +10,7 @@ import ( type BackendOperations interface { CreateSession(context.Context, *ConfigurationSettings, *CreateSessionRequest) (*CreateSessionResponse, error) DeleteSession(context.Context, *ConfigurationSettings, *DeleteSessionRequest) (*DeleteSessionResponse, error) + GetRootService(context.Context, *ConfigurationSettings, *GetRootServiceRequest) (*GetRootServiceResponse, error) GetMemoryResourceBlocks(context.Context, *ConfigurationSettings, *MemoryResourceBlocksRequest) (*MemoryResourceBlocksResponse, error) GetMemoryResourceBlockById(context.Context, *ConfigurationSettings, *MemoryResourceBlockByIdRequest) (*MemoryResourceBlockByIdResponse, error) GetPorts(context.Context, *ConfigurationSettings, *GetPortsRequest) (*GetPortsResponse, error) diff --git a/pkg/backend/httpfish.go b/pkg/backend/httpfish.go index 0ce0ab5..45e3dff 100644 --- a/pkg/backend/httpfish.go +++ b/pkg/backend/httpfish.go @@ -490,6 +490,21 @@ func (session *Session) auth() error { return response.err } +// GetRootService: Retrieve root service from endpoint +func (service *httpfishService) GetRootService(ctx context.Context, settings *ConfigurationSettings, req *GetRootServiceRequest) (*GetRootServiceResponse, error) { + session := service.service.session.(*Session) + + response := session.query(HTTPOperation.GET, redfish_serviceroot) + if response.err != nil { + return nil, fmt.Errorf("failed to get root service: %w", response.err) + } + + name, _ := response.stringFromJSON("Name") + uuid, _ := response.stringFromJSON("UUID") + + return &GetRootServiceResponse{Name: name, Uuid: uuid}, nil +} + // CreateSession: Create a new session with an endpoint service func (service *httpfishService) CreateSession(ctx context.Context, settings *ConfigurationSettings, req *CreateSessionRequest) (*CreateSessionResponse, error) { logger := klog.FromContext(ctx) diff --git a/pkg/backend/ops.go b/pkg/backend/ops.go index e2b662f..86e6098 100644 --- a/pkg/backend/ops.go +++ b/pkg/backend/ops.go @@ -262,3 +262,10 @@ type GetBackendInfoResponse struct { Version string SessionId string } +type GetRootServiceRequest struct { +} + +type GetRootServiceResponse struct { + Name string + Uuid string +} diff --git a/pkg/common/common.go b/pkg/common/common.go new file mode 100644 index 0000000..cad91de --- /dev/null +++ b/pkg/common/common.go @@ -0,0 +1,9 @@ +package common + +type ConnectionStatus string + +const ( + ONLINE ConnectionStatus = "online" + OFFLINE ConnectionStatus = "offline" + NOT_APPLICABLE ConnectionStatus = "n\\a" +) diff --git a/pkg/common/datastore/datastore.go b/pkg/common/datastore/datastore.go index 3124926..18affdc 100644 --- a/pkg/common/datastore/datastore.go +++ b/pkg/common/datastore/datastore.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + "cfm/pkg/common" "cfm/pkg/openapi" "k8s.io/klog/v2" @@ -16,59 +17,55 @@ const ( ) type DataStore struct { - SavedAppliances map[string]*ApplianceDataStore `json:"saved-appliances"` - SavedHosts map[string]*HostDataStore `json:"saved-hosts"` + ApplianceData map[string]*ApplianceDatum `json:"appliance-data"` + HostData map[string]*HostDatum `json:"host-data"` } func NewDataStore() *DataStore { return &DataStore{ - SavedAppliances: make(map[string]*ApplianceDataStore), - SavedHosts: make(map[string]*HostDataStore), + ApplianceData: make(map[string]*ApplianceDatum), + HostData: make(map[string]*HostDatum), } } -// AddAppliance: Add a new appliance to the data store -func (c *DataStore) AddAppliance(creds *openapi.Credentials) { - c.SavedAppliances[creds.CustomId] = NewApplianceDataStore(creds) +// AddApplianceDatum: Add a new appliance datum to the data store +func (c *DataStore) AddApplianceDatum(creds *openapi.Credentials) { + c.ApplianceData[creds.CustomId] = NewApplianceDatum(creds) } -// AddBlade: Add a new blade to the data store -func (c *DataStore) AddBlade(creds *openapi.Credentials, applianceId string) error { - appliance, exists := c.SavedAppliances[applianceId] - if !exists { - return fmt.Errorf("appliance [%s] not found in data store add", applianceId) - } - - appliance.AddBlade(creds) - - return nil +// AddHostDatum: Add a new host datum to the data store +func (c *DataStore) AddHostDatum(creds *openapi.Credentials) { + c.HostData[creds.CustomId] = NewHostDatum(creds) } -// AddHost: Add a new host to the data store -func (c *DataStore) AddHost(creds *openapi.Credentials) { - c.SavedHosts[creds.CustomId] = NewHostDataStore(creds) +// DeleteApplianceDatumById: Delete an appliance from the data store +func (c *DataStore) DeleteApplianceDatumById(applianceId string) { + delete(c.ApplianceData, applianceId) } -// DeleteAppliance: Delete an appliance from the data store -func (c *DataStore) DeleteAppliance(applianceId string) { - delete(c.SavedAppliances, applianceId) +// DeleteHostDatumById: Delete a host datum from the data store +func (c *DataStore) DeleteHostDatumById(hostId string) { + delete(c.HostData, hostId) } -// DeleteBlade: Delete a blade from an appliance's data store -func (c *DataStore) DeleteBlade(bladeId, applianceId string) error { - appliance, exists := c.SavedAppliances[applianceId] +// GetApplianceDatumById: Retrieve an appliance datum from the data store +func (c *DataStore) GetApplianceDatumById(applianceId string) (*ApplianceDatum, error) { + datum, exists := c.ApplianceData[applianceId] if !exists { - return fmt.Errorf("appliance [%s] not found in data store delete", applianceId) + return nil, fmt.Errorf("appliance datum [%s] not found in data store", applianceId) } - appliance.DeleteBlade(bladeId) - - return nil + return datum, nil } -// DeleteHost: Delete an host from the data store -func (c *DataStore) DeleteHost(hostId string) { - delete(c.SavedHosts, hostId) +// GetHostDatumById: Retrieve a host datum from the data store +func (c *DataStore) GetHostDatumById(hostId string) (*HostDatum, error) { + datum, exists := c.HostData[hostId] + if !exists { + return nil, fmt.Errorf("host datum [%s] not found in data store", hostId) + } + + return datum, nil } // Init: initialize the data store using command line args, ENV, or a file @@ -79,46 +76,77 @@ func (c *DataStore) InitDataStore(ctx context.Context, args []string) error { return nil } -type ApplianceDataStore struct { - Credentials *openapi.Credentials `json:"credentials"` - SavedBlades map[string]*BladeDataStore `json:"saved-blades"` +type ApplianceDatum struct { + Credentials *openapi.Credentials `json:"credentials"` + BladeData map[string]*BladeDatum `json:"blade-data"` + ConnectionStatus common.ConnectionStatus `json:"connection-status"` } -func NewApplianceDataStore(creds *openapi.Credentials) *ApplianceDataStore { - return &ApplianceDataStore{ - Credentials: creds, - SavedBlades: make(map[string]*BladeDataStore), +func NewApplianceDatum(creds *openapi.Credentials) *ApplianceDatum { + return &ApplianceDatum{ + Credentials: creds, + BladeData: make(map[string]*BladeDatum), + ConnectionStatus: common.NOT_APPLICABLE, // Will use for single-BMC appliance } } -func (c *ApplianceDataStore) AddBlade(creds *openapi.Credentials) { - c.SavedBlades[creds.CustomId] = NewBladeDataStore(creds) +func (a *ApplianceDatum) AddBladeDatum(creds *openapi.Credentials) { + a.BladeData[creds.CustomId] = NewBladeDatum(creds) } -func (c *ApplianceDataStore) DeleteBlade(bladeId string) { - delete(c.SavedBlades, bladeId) +func (a *ApplianceDatum) DeleteBladeDatumById(bladeId string) { + delete(a.BladeData, bladeId) } -type BladeDataStore struct { - Credentials *openapi.Credentials `json:"credentials"` +func (a *ApplianceDatum) GetBladeDatumById(ctx context.Context, bladeId string) (*BladeDatum, error) { + logger := klog.FromContext(ctx) + + blade, exists := a.BladeData[bladeId] + if !exists { + err := fmt.Errorf("blade datum [%s] not found in appliance data [%s] in data store", bladeId, a.Credentials.CustomId) + logger.Error(err, "failure: update blade") + return nil, err + } + + return blade, nil +} + +func (a *ApplianceDatum) SetConnectionStatus(status common.ConnectionStatus) { + a.ConnectionStatus = status +} + +type BladeDatum struct { + Credentials *openapi.Credentials `json:"credentials"` + ConnectionStatus common.ConnectionStatus `json:"connection-status"` } -func NewBladeDataStore(creds *openapi.Credentials) *BladeDataStore { - return &BladeDataStore{ - Credentials: creds, +func NewBladeDatum(creds *openapi.Credentials) *BladeDatum { + return &BladeDatum{ + Credentials: creds, + ConnectionStatus: common.ONLINE, } } -type HostDataStore struct { - Credentials *openapi.Credentials `json:"credentials"` +func (b *BladeDatum) SetConnectionStatus(status *common.ConnectionStatus) { + b.ConnectionStatus = *status } -func NewHostDataStore(creds *openapi.Credentials) *HostDataStore { - return &HostDataStore{ - Credentials: creds, +type HostDatum struct { + Credentials *openapi.Credentials `json:"credentials"` + ConnectionStatus common.ConnectionStatus `json:"connection-status"` +} + +func NewHostDatum(creds *openapi.Credentials) *HostDatum { + return &HostDatum{ + Credentials: creds, + ConnectionStatus: common.ONLINE, } } +func (h *HostDatum) SetConnectionStatus(status *common.ConnectionStatus) { + h.ConnectionStatus = *status +} + //////////////////////////////////////// ///////////// Helpers ////////////////// //////////////////////////////////////// @@ -131,9 +159,8 @@ func ReloadDataStore(ctx context.Context, s openapi.DefaultAPIServicer, c *DataS logger.V(2).Info("cfm-service: restoring saved appliances") var appliancesToDelete []string - for applianceId, appliance := range c.SavedAppliances { - appliance.Credentials.CustomId = applianceId - _, err = s.AppliancesPost(ctx, *appliance.Credentials) + for applianceId, applianceDatum := range c.ApplianceData { + _, err = s.AppliancesPost(ctx, *applianceDatum.Credentials) if err != nil { logger.V(2).Info("cfm-service: appliance restore failure", "applianceId", applianceId) appliancesToDelete = append(appliancesToDelete, applianceId) @@ -141,9 +168,8 @@ func ReloadDataStore(ctx context.Context, s openapi.DefaultAPIServicer, c *DataS } bladesToDelete := make(map[string]string) - for bladeId, blade := range appliance.SavedBlades { - blade.Credentials.CustomId = bladeId - _, err = s.BladesPost(ctx, applianceId, *blade.Credentials) + for bladeId, bladeDatum := range applianceDatum.BladeData { + _, err = s.BladesPost(ctx, applianceId, *bladeDatum.Credentials) if err != nil { logger.V(2).Info("cfm-service: blade restore failure", "bladeId", bladeId, "applianceId", applianceId) bladesToDelete[applianceId] = bladeId @@ -151,27 +177,20 @@ func ReloadDataStore(ctx context.Context, s openapi.DefaultAPIServicer, c *DataS } for applianceId, bladeId := range bladesToDelete { - delete(c.SavedAppliances[applianceId].SavedBlades, bladeId) + delete(c.ApplianceData[applianceId].BladeData, bladeId) } } for _, applianceId := range appliancesToDelete { - delete(c.SavedAppliances, applianceId) + delete(c.ApplianceData, applianceId) } logger.V(2).Info("cfm-service: restoring saved hosts") - var hostsToDelete []string - for hostId, host := range c.SavedHosts { - host.Credentials.CustomId = hostId - _, err = s.HostsPost(ctx, *host.Credentials) + for hostId, hostDatum := range c.HostData { + _, err = s.HostsPost(ctx, *hostDatum.Credentials) if err != nil { - logger.V(2).Info("cfm-service: host restore failure", "hostId", hostId) - hostsToDelete = append(hostsToDelete, hostId) + logger.V(2).Info("cfm-service: host datum restore failure", "hostId", hostId) continue } } - - for _, hostId := range hostsToDelete { - delete(c.SavedHosts, hostId) - } } diff --git a/pkg/common/status.go b/pkg/common/status.go index a23b7dc..259308c 100644 --- a/pkg/common/status.go +++ b/pkg/common/status.go @@ -57,6 +57,8 @@ const ( StatusGetMemoryDevicesDetailsFailure //409 StatusApplianceResyncFailure //409 + StatusBladeResyncFailure //409 + StatusHostResyncFailure //409 StatusApplianceIdDuplicate //409 StatusBladeIdDuplicate //409 @@ -150,6 +152,10 @@ func (e StatusCodeType) String() string { return "Get Memory Devices Details Failure" case StatusApplianceResyncFailure: return "Appliance Resync Failure" + case StatusBladeResyncFailure: + return "Blade Resync Failure" + case StatusHostResyncFailure: + return "Host Resync Failure" case StatusBladeIdDoesNotExist: return "Blade Id Does Not Exist" case StatusAppliancesExceedMaximum: @@ -209,6 +215,8 @@ func (e StatusCodeType) HttpStatusCode() int { StatusGetMemoryDevicesFailure, StatusGetMemoryDevicesDetailsFailure, StatusApplianceResyncFailure, + StatusBladeResyncFailure, + StatusHostResyncFailure, StatusApplianceIdDuplicate, StatusBladeIdDuplicate, StatusPortIdDuplicate, diff --git a/pkg/manager/appliance.go b/pkg/manager/appliance.go index c1b256b..af5f991 100644 --- a/pkg/manager/appliance.go +++ b/pkg/manager/appliance.go @@ -8,6 +8,7 @@ import ( "cfm/pkg/backend" "cfm/pkg/common" + "cfm/pkg/common/datastore" "cfm/pkg/openapi" "github.com/google/uuid" @@ -24,15 +25,16 @@ type Appliance struct { } // NewAppliance - Creates a new Appliance object. -func NewAppliance(ctx context.Context, id string) (*Appliance, error) { +func NewAppliance(ctx context.Context, c *openapi.Credentials) (*Appliance, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> NewAppliance: ") - applianceId := id + applianceId := c.CustomId if applianceId == "" { // Generate uuid here and combine the last N digits with the prefix to be the appliance default id uuid := uuid.New().String() applianceId = fmt.Sprintf("%s-%s", ID_PREFIX_APPLIANCE_DFLT, uuid[(len(uuid)-common.NumUuidCharsForId):]) + c.CustomId = applianceId } // Check for duplicate ID @@ -104,6 +106,7 @@ func (a *Appliance) AddBlade(ctx context.Context, c *openapi.Credentials) (*Blad // Generate default id using last N digits of the session id combined with the default prefix bladeId = fmt.Sprintf("%s-%s", ID_PREFIX_BLADE_DFLT, response.SessionId[(len(response.SessionId)-common.NumUuidCharsForId):]) } + c.CustomId = bladeId } // Check for duplicate ID @@ -127,6 +130,7 @@ func (a *Appliance) AddBlade(ctx context.Context, c *openapi.Credentials) (*Blad BladeId: bladeId, ApplianceId: a.Id, Ip: c.IpAddress, + Status: common.ONLINE, Port: uint16(c.Port), BackendOps: ops, Creds: c, @@ -150,28 +154,28 @@ func (a *Appliance) AddBlade(ctx context.Context, c *openapi.Credentials) (*Blad // Add blade to appliance a.Blades[blade.Id] = blade + // Add host to datastore + applianceDatum, _ := datastore.DStore().GetDataStore().GetApplianceDatumById(a.Id) + applianceDatum.AddBladeDatum(c) + datastore.DStore().Store() + logger.V(2).Info("success: add blade", "bladeId", blade.Id, "applianceId", a.Id) return blade, nil } -func (a *Appliance) DeleteAllBlades(ctx context.Context) error { +func (a *Appliance) DeleteAllBlades(ctx context.Context) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> DeleteAllBlades: ", "applianceId", a.Id) for id := range a.Blades { - _, err := a.DeleteBladeById(ctx, id) - if err != nil { - return err - } + a.DeleteBladeById(ctx, id) // ignore any errors } logger.V(2).Info("success: delete all blades", "applianceId", a.Id) - - return nil } -// DeleteBladeById: Delete the blade backend session and the local blade cache +// DeleteBladeById: Delete the blade from: backend, deviceCache and datastore func (a *Appliance) DeleteBladeById(ctx context.Context, bladeId string) (*Blade, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> DeleteBladeById: ", "bladeId", bladeId, "applianceId", a.Id) @@ -181,6 +185,10 @@ func (a *Appliance) DeleteBladeById(ctx context.Context, bladeId string) (*Blade if !ok { logger.V(2).Info("blade not found during delete:", "bladeId", bladeId, "applianceId", a.Id) newErr := fmt.Errorf("blade [%s] not found during delete", bladeId) + + logger.V(2).Info("force complete appliance blade deletion after error", "bladeId", blade.Id, "applianceId", a.Id) + a.deleteBlade(bladeId) + return nil, &common.RequestError{StatusCode: common.StatusBladeIdDoesNotExist, Err: newErr} } @@ -196,15 +204,14 @@ func (a *Appliance) DeleteBladeById(ctx context.Context, bladeId string) (*Blade newErr := fmt.Errorf("failed to delete blade [%s] backend [%s] session [%s]: %w", blade.Id, ops.GetBackendInfo(ctx).BackendName, blade.Socket.String(), err) logger.Error(newErr, "failure: delete blade by id") - // Currently, backend ALWAYS deletes the blade session from the backend map. For now, need to delete blade from appliance map as well. - logger.V(2).Info("force blade deletion after backend session failure", "bladeId", blade.Id, "applianceId", a.Id) - delete(a.Blades, blade.Id) + // Currently, backend ALWAYS deletes the blade session from the backend map. Do the same in the this (manager) layer + logger.V(2).Info("force complete appliance blade deletion after backend session failure", "bladeId", blade.Id, "applianceId", a.Id) + a.deleteBlade(bladeId) return nil, &common.RequestError{StatusCode: common.StatusBladeDeleteSessionFailure, Err: newErr} } - // delete blade - delete(a.Blades, blade.Id) + a.deleteBlade(bladeId) logger.V(2).Info("success: delete blade by id", "bladeId", blade.Id, "applianceId", a.Id) @@ -232,7 +239,9 @@ func (a *Appliance) GetBladeById(ctx context.Context, bladeId string) (*Blade, e return nil, &common.RequestError{StatusCode: common.StatusBladeIdDoesNotExist, Err: newErr} } - logger.V(2).Info("success: get blade by id", "bladeId", blade.Id, "applianceId", a.Id) + blade.UpdateConnectionStatusBackend(ctx) + + logger.V(2).Info("success: get blade by id", "status", blade.Status, "bladeId", blade.Id, "applianceId", a.Id) return blade, nil } @@ -291,20 +300,31 @@ func (a *Appliance) ResyncBladeById(ctx context.Context, bladeId string) (*Blade logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> ResyncBladeById: ", "bladeId", bladeId, "applianceId", a.Id) - blade, err := a.DeleteBladeById(ctx, bladeId) - if err != nil { - newErr := fmt.Errorf("failed to resync blade(delete): appliance [%s] blade [%s]: %w", a.Id, bladeId, err) - logger.Error(newErr, "failure: resync blade: ignoring") + // query device cache + blade, ok := deviceCache.GetBladeByIdOk(a.Id, bladeId) + if !ok || blade == nil { + newErr := fmt.Errorf("failed to get blade [%s]", bladeId) + logger.Error(newErr, "failure: resync blade by id") + return nil, &common.RequestError{StatusCode: common.StatusHostIdDoesNotExist, Err: newErr} } - blade, err = a.AddBlade(ctx, blade.creds) - if err != nil { - newErr := fmt.Errorf("failed to resync blade(add): appliance [%s] blade [%s]: %w", a.Id, bladeId, err) - logger.Error(newErr, "failure: resync blade") - return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} - } + blade.UpdateConnectionStatusBackend(ctx) - logger.V(2).Info("success: resync blade", "bladeId", bladeId, "applianceId", a.Id) + logger.V(2).Info("success: resync blade", "status", blade.Status, "bladeId", bladeId, "applianceId", a.Id) return blade, nil } + +///////////////////////////////////// +//////// Private Functions ////////// +///////////////////////////////////// + +func (a *Appliance) deleteBlade(bladeId string) { + // delete blade from manager cache + delete(a.Blades, bladeId) + + // delete blade from datastore + applianceDatum, _ := datastore.DStore().GetDataStore().GetApplianceDatumById(a.Id) + applianceDatum.DeleteBladeDatumById(bladeId) + datastore.DStore().Store() +} diff --git a/pkg/manager/blade.go b/pkg/manager/blade.go index 32753ad..8b752a7 100644 --- a/pkg/manager/blade.go +++ b/pkg/manager/blade.go @@ -11,6 +11,7 @@ import ( "cfm/pkg/backend" "cfm/pkg/common" + "cfm/pkg/common/datastore" "cfm/pkg/openapi" "k8s.io/klog/v2" @@ -19,9 +20,9 @@ import ( const ID_PREFIX_BLADE_DFLT string = "blade" type Blade struct { - Id string - Uri string - // Status string // Meaningless without async update + Id string + Uri string + Status common.ConnectionStatus Socket SocketDetails ApplianceId string Memory map[string]*BladeMemory @@ -42,6 +43,7 @@ type RequestNewBlade struct { Ip string Port uint16 ApplianceId string + Status common.ConnectionStatus BackendOps backend.BackendOperations Creds *openapi.Credentials } @@ -55,6 +57,7 @@ func NewBlade(ctx context.Context, r *RequestNewBlade) (*Blade, error) { Uri: GetCfmUriBladeId(r.ApplianceId, r.BladeId), Socket: *NewSocketDetails(r.Ip, r.Port), ApplianceId: r.ApplianceId, + Status: r.Status, Ports: make(map[string]*CxlBladePort), Resources: make(map[string]*BladeResource), Memory: make(map[string]*BladeMemory), @@ -125,7 +128,7 @@ func (b *Blade) AssignMemory(ctx context.Context, r *RequestAssignMemory) (*open if len(memory.resourceIds) != 0 { resourcesToUpdate = memory.resourceIds } else { - resourcesToUpdate = b.GetAllResourceIds() + resourcesToUpdate = b.GetAllResourceIds(ctx) } for _, resourceId := range resourcesToUpdate { resource, err := b.GetResourceById(ctx, resourceId) @@ -330,7 +333,7 @@ func (b *Blade) FreeMemoryById(ctx context.Context, memoryId string) (*openapi.M if len(memory.resourceIds) != 0 { resourcesToUpdate = memory.resourceIds } else { - resourcesToUpdate = b.GetAllResourceIds() + resourcesToUpdate = b.GetAllResourceIds(ctx) } for _, resourceId := range resourcesToUpdate { resource, err := b.GetResourceById(ctx, resourceId) @@ -362,33 +365,30 @@ func (b *Blade) FreeMemoryById(ctx context.Context, memoryId string) (*openapi.M return &memoryRegion, nil } -func (b *Blade) GetAllMemoryIds() []string { +func (b *Blade) GetAllMemoryIds(ctx context.Context) []string { var ids []string - // CACHE: Get - for id := range b.Memory { + for id := range b.GetMemory(ctx) { ids = append(ids, id) } return ids } -func (b *Blade) GetAllPortIds() []string { +func (b *Blade) GetAllPortIds(ctx context.Context) []string { var ids []string - // CACHE: Get - for id := range b.Ports { + for id := range b.GetPorts(ctx) { ids = append(ids, id) } return ids } -func (b *Blade) GetAllResourceIds() []string { +func (b *Blade) GetAllResourceIds(ctx context.Context) []string { var ids []string - // CACHE: Get - for id := range b.Resources { + for id := range b.GetResources(ctx) { ids = append(ids, id) } @@ -399,6 +399,11 @@ func (b *Blade) GetMemoryById(ctx context.Context, memoryId string) (*BladeMemor logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryById: ", "memoryId", memoryId, "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return nil, nil + } + memory, ok := b.Memory[memoryId] if !ok { newErr := fmt.Errorf("memory [%s] not found on appliance [%s] blade [%s] ", memoryId, b.ApplianceId, b.Id) @@ -415,6 +420,11 @@ func (b *Blade) GetMemory(ctx context.Context) map[string]*BladeMemory { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemory: ", "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make(map[string]*BladeMemory) + } + memory := b.Memory logger.V(2).Info("success: get memory", "count", len(memory), "bladeId", b.Id, "applianceId", b.ApplianceId) @@ -427,9 +437,12 @@ func (b *Blade) GetMemoryBackend(ctx context.Context) ([]string, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryBackend: ", "bladeId", b.Id, "applianceId", b.ApplianceId) - // HARDWARE: Get + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make([]string, 0), nil + } + req := backend.GetMemoryRequest{} - // get memory ids from backend response, err := b.backendOps.GetMemory(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { newErr := fmt.Errorf("get memory (backend) [%s] failure on blade [%s]: %w", b.backendOps.GetBackendInfo(ctx).BackendName, b.Id, err) @@ -454,6 +467,11 @@ func (b *Blade) GetPortById(ctx context.Context, portId string) (*CxlBladePort, logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPortById: ", "portId", portId, "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return nil, nil + } + port, ok := b.Ports[portId] if !ok { newErr := fmt.Errorf("port [%s] not found on appliance [%s] blade [%s] ", portId, b.ApplianceId, b.Id) @@ -470,6 +488,11 @@ func (b *Blade) GetPorts(ctx context.Context) map[string]*CxlBladePort { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPorts: ", "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make(map[string]*CxlBladePort) + } + ports := b.Ports logger.V(2).Info("success: get ports(blade) (cache)", "count", len(ports), "bladeId", b.Id, "applianceId", b.ApplianceId) @@ -482,6 +505,11 @@ func (b *Blade) GetPortsBackend(ctx context.Context) ([]string, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPortsBackend: ", "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make([]string, 0), nil + } + req := backend.GetPortsRequest{} response, err := b.backendOps.GetPorts(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { @@ -499,6 +527,11 @@ func (b *Blade) GetResourceById(ctx context.Context, resourceId string) (*BladeR logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetResourceById: ", "resourceId", resourceId, "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return nil, nil + } + resource, ok := b.Resources[resourceId] if !ok { newErr := fmt.Errorf("resource [%s] not found on appliance [%s] blade [%s] ", resourceId, b.ApplianceId, b.Id) @@ -515,6 +548,11 @@ func (b *Blade) GetResources(ctx context.Context) map[string]*BladeResource { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetResources: ", "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make(map[string]*BladeResource) + } + resources := b.Resources logger.V(2).Info("success: get resources(cache)", "count", len(resources), "bladeId", b.Id, "applianceId", b.ApplianceId) @@ -527,6 +565,11 @@ func (b *Blade) GetResourcesBackend(ctx context.Context) ([]string, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetResourcesBackend: ", "bladeId", b.Id, "applianceId", b.ApplianceId) + if !b.IsOnline(ctx) { + // If blade offline, not an error. Just no information to return. + return make([]string, 0), nil + } + req := backend.MemoryResourceBlocksRequest{} response, err := b.backendOps.GetMemoryResourceBlocks(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { @@ -546,8 +589,12 @@ func (b *Blade) GetResourceTotals(ctx context.Context) (*ResponseResourceTotals, var totalAvail, totalAlloc int32 - resources := b.GetResources(ctx) - for _, resource := range resources { + response := ResponseResourceTotals{ + TotalMemoryAvailableMiB: 0, + TotalMemoryAllocatedMiB: 0, + } + + for _, resource := range b.GetResources(ctx) { totals, err := resource.GetTotals(ctx) if err != nil || totals == nil { newErr := fmt.Errorf("failed to get resource totals: appliance [%s] blade [%s] resource [%s]: %w", b.ApplianceId, b.Id, resource.Id, err) @@ -559,12 +606,10 @@ func (b *Blade) GetResourceTotals(ctx context.Context) (*ResponseResourceTotals, totalAlloc += totals.TotalMemoryAllocatedMiB } - response := ResponseResourceTotals{ - TotalMemoryAvailableMiB: totalAvail, - TotalMemoryAllocatedMiB: totalAlloc, - } + response.TotalMemoryAvailableMiB = totalAvail + response.TotalMemoryAllocatedMiB = totalAlloc - logger.V(2).Info("success: get resource totals", "bladeId", b.Id, "applianceId", b.ApplianceId) + logger.V(2).Info("success: get resource totals", "totals", response, "bladeId", b.Id, "applianceId", b.ApplianceId) return &response, nil } @@ -583,6 +628,32 @@ func (b *Blade) InvalidateCache() { } } +func (b *Blade) IsOnline(ctx context.Context) bool { + return b.Status == common.ONLINE +} + +// UpdateConnectionStatusBackend - Query the blade root service to verify continued connection and update the object status accordingly. +func (b *Blade) UpdateConnectionStatusBackend(ctx context.Context) { + logger := klog.FromContext(ctx) + logger.V(4).Info(">>>>>> UpdateConnectionStatusBackend: ", "bladeId", b.Id) + + req := backend.GetRootServiceRequest{} + response, err := b.backendOps.GetRootService(ctx, &backend.ConfigurationSettings{}, &req) + if err != nil || response == nil { + b.Status = common.OFFLINE + } else { + b.Status = common.ONLINE + } + + // Update datastore status + applianceDatum, _ := datastore.DStore().GetDataStore().GetApplianceDatumById(b.ApplianceId) + bladeDatum, _ := applianceDatum.GetBladeDatumById(ctx, b.Id) + bladeDatum.SetConnectionStatus(&b.Status) + datastore.DStore().Store() + + logger.V(2).Info("update blade status(backend)", "status", b.Status, "bladeId", b.Id) +} + ///////////////////////////////////// //////// Private Functions ////////// ///////////////////////////////////// diff --git a/pkg/manager/host.go b/pkg/manager/host.go index 2913577..a2d650d 100644 --- a/pkg/manager/host.go +++ b/pkg/manager/host.go @@ -9,6 +9,7 @@ import ( "cfm/pkg/backend" "cfm/pkg/common" + "cfm/pkg/common/datastore" "cfm/pkg/openapi" "k8s.io/klog/v2" @@ -17,9 +18,9 @@ import ( const ID_PREFIX_HOST_DFLT string = "host" type Host struct { - Id string - Uri string - // Status string // Meaningless without async update + Id string + Uri string + Status common.ConnectionStatus Socket SocketDetails Ports map[string]*CxlHostPort MemoryDevices map[string]*HostMemoryDevice @@ -39,6 +40,7 @@ type RequestNewHost struct { HostId string Ip string Port uint16 + Status common.ConnectionStatus BackendOps backend.BackendOperations Creds *openapi.Credentials } @@ -51,6 +53,7 @@ func NewHost(ctx context.Context, r *RequestNewHost) (*Host, error) { Id: r.HostId, Uri: GetCfmUriHostId(r.HostId), Socket: *NewSocketDetails(r.Ip, r.Port), + Status: r.Status, Ports: make(map[string]*CxlHostPort), MemoryDevices: make(map[string]*HostMemoryDevice), Memory: make(map[string]*HostMemory), @@ -160,30 +163,30 @@ func (h *Host) FreeMemoryById(ctx context.Context, hostMemoryId string) (*openap return memory, nil } -func (h *Host) GetAllMemoryIds() []string { +func (h *Host) GetAllMemoryIds(ctx context.Context) []string { var ids []string - for id := range h.Memory { + for id := range h.GetMemory(ctx) { ids = append(ids, id) } return ids } -func (h *Host) GetAllMemoryDeviceIds() []string { +func (h *Host) GetAllMemoryDeviceIds(ctx context.Context) []string { var ids []string - for id := range h.MemoryDevices { + for id := range h.GetMemoryDevices(ctx) { ids = append(ids, id) } return ids } -func (h *Host) GetAllPortIds() []string { +func (h *Host) GetAllPortIds(ctx context.Context) []string { var ids []string - for id := range h.Ports { + for id := range h.GetPorts(ctx) { ids = append(ids, id) } @@ -194,6 +197,11 @@ func (h *Host) GetMemoryById(ctx context.Context, memoryId string) (*HostMemory, logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryById: ", "memoryId", memoryId, "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return nil, nil + } + memory, ok := h.Memory[memoryId] if !ok { newErr := fmt.Errorf("memory [%s] not found on host [%s]", memoryId, h.Id) @@ -210,6 +218,11 @@ func (h *Host) GetMemory(ctx context.Context) map[string]*HostMemory { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemory: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make(map[string]*HostMemory) + } + memory := h.Memory logger.V(2).Info("success: get memory", "count", len(memory), "hostId", h.Id) @@ -222,6 +235,11 @@ func (h *Host) GetMemoryBackend(ctx context.Context) ([]string, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryBackend: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make([]string, 0), nil + } + req := backend.GetMemoryRequest{} response, err := h.backendOps.GetMemory(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { @@ -237,6 +255,11 @@ func (h *Host) GetMemoryDeviceById(ctx context.Context, memdevId string) (*HostM logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryDeviceById: ", "memdevId", memdevId, "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return nil, nil + } + memdev, ok := h.MemoryDevices[memdevId] if !ok { newErr := fmt.Errorf("memory device [%s] not found on host [%s]", memdevId, h.Id) @@ -253,6 +276,11 @@ func (h *Host) GetMemoryDevices(ctx context.Context) map[string]*HostMemoryDevic logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryDevices: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make(map[string]*HostMemoryDevice) + } + memdevs := h.MemoryDevices logger.V(2).Info("success: get memory devices", "count", len(memdevs), "hostId", h.Id) @@ -283,7 +311,7 @@ func (h *Host) GetMemoryDomainAllMemoryIds(ctx context.Context, domain string) ( typeString, exist := HostMemoryDomain[domain] if exist { - for id, mem := range h.Memory { + for id, mem := range h.GetMemory(ctx) { details, err := mem.GetDetails(ctx) if err != nil { return nil, err @@ -308,6 +336,11 @@ func (h *Host) GetMemoryDevicesBackend(ctx context.Context) (map[string][]string logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryDevicesBackend: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make(map[string][]string, 0), nil + } + req := backend.GetMemoryDevicesRequest{} response, err := h.backendOps.GetMemoryDevices(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { @@ -321,13 +354,23 @@ func (h *Host) GetMemoryDevicesBackend(ctx context.Context) (map[string][]string return response.DeviceIdMap, nil } +type ResponseHostMemoryTotals struct { + LocalMemoryMib int32 + RemoteMemoryMib int32 +} + func (h *Host) GetMemoryTotals(ctx context.Context) (*ResponseHostMemoryTotals, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetMemoryTotals: ", "hostId", h.Id) var local, remote int32 - for _, memory := range h.Memory { + response := ResponseHostMemoryTotals{ + LocalMemoryMib: 0, + RemoteMemoryMib: 0, + } + + for _, memory := range h.GetMemory(ctx) { totals, err := memory.GetTotals(ctx) if err != nil || totals == nil { newErr := fmt.Errorf("failed to get memory totals: host [%s] memory [%s]: %w", h.Id, memory.Id, err) @@ -339,12 +382,10 @@ func (h *Host) GetMemoryTotals(ctx context.Context) (*ResponseHostMemoryTotals, remote += totals.RemoteMemoryMib } - response := ResponseHostMemoryTotals{ - LocalMemoryMib: local, - RemoteMemoryMib: remote, - } + response.LocalMemoryMib = local + response.RemoteMemoryMib = remote - logger.V(2).Info("success: get memory totals", "hostId", h.Id) + logger.V(2).Info("success: get memory totals", "totals", response, "hostId", h.Id) return &response, nil } @@ -353,6 +394,11 @@ func (h *Host) GetPortById(ctx context.Context, portId string) (*CxlHostPort, er logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPortById: ", "portId", portId, "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return nil, nil + } + port, ok := h.Ports[portId] if !ok { newErr := fmt.Errorf("port [%s] not found on host [%s]", portId, h.Id) @@ -369,6 +415,11 @@ func (h *Host) GetPorts(ctx context.Context) map[string]*CxlHostPort { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPorts: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make(map[string]*CxlHostPort) + } + ports := h.Ports logger.V(2).Info("success: get ports", "count", len(ports), "hostId", h.Id) @@ -381,6 +432,11 @@ func (h *Host) GetPortsBackend(ctx context.Context) ([]string, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> GetPortsBackend: ", "hostId", h.Id) + if !h.IsOnline(ctx) { + // If host offline, not an error. Just no information to return. + return make([]string, 0), nil + } + req := backend.GetPortsRequest{} response, err := h.backendOps.GetHostPortPcieDevices(ctx, &backend.ConfigurationSettings{}, &req) if err != nil || response == nil { @@ -416,9 +472,29 @@ func (h *Host) InvalidateCache() { } } -type ResponseHostMemoryTotals struct { - LocalMemoryMib int32 - RemoteMemoryMib int32 +func (h *Host) IsOnline(ctx context.Context) bool { + return h.Status == common.ONLINE +} + +// UpdateConnectionStatusBackend - Query the host root service to verify continued connection and update the object status accordingly. +func (h *Host) UpdateConnectionStatusBackend(ctx context.Context) { + logger := klog.FromContext(ctx) + logger.V(4).Info(">>>>>> UpdateConnectionStatusBackend: ", "hostId", h.Id) + + req := backend.GetRootServiceRequest{} + response, err := h.backendOps.GetRootService(ctx, &backend.ConfigurationSettings{}, &req) + if err != nil || response == nil { + h.Status = common.OFFLINE + } else { + h.Status = common.ONLINE + } + + // Update datastore status + hostDatum, _ := datastore.DStore().GetDataStore().GetHostDatumById(h.Id) + hostDatum.SetConnectionStatus(&h.Status) + datastore.DStore().Store() + + logger.V(2).Info("update host status(backend)", "status", h.Status, "hostId", h.Id) } ///////////////////////////////////// diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index e010b70..fc20617 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -8,6 +8,7 @@ import ( "cfm/pkg/backend" "cfm/pkg/common" + "cfm/pkg/common/datastore" "cfm/pkg/openapi" "k8s.io/klog/v2" @@ -42,20 +43,19 @@ func AddAppliance(ctx context.Context, c *openapi.Credentials) (*Appliance, erro logger.V(4).Info(">>>>>> AddAppliance: ") // Create a new cfm-service Appliance object - appliance, err := NewAppliance(ctx, c.CustomId) + appliance, err := NewAppliance(ctx, c) if err != nil || appliance == nil { newErr := fmt.Errorf("new appliance creation failure: %w", err) logger.Error(newErr, "failure: add appliance") return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} } - // Cache it - err = deviceCache.AddAppliance(appliance) - if err != nil { - newErr := fmt.Errorf("add appliance [%s] failure: %w", appliance.Id, err) - logger.Error(newErr, "failure: add appliance") - return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} - } + // Add appliance to device cache + deviceCache.AddAppliance(appliance) // ignore error, duplicate check done above + + // Add appliance to datastore + datastore.DStore().GetDataStore().AddApplianceDatum(c) + datastore.DStore().Store() logger.V(2).Info("success: add appliance", "applianceId", appliance.Id) @@ -74,24 +74,18 @@ func DeleteApplianceById(ctx context.Context, applianceId string) (*Appliance, e return nil, &common.RequestError{StatusCode: common.StatusApplianceIdDoesNotExist, Err: newErr} } - err := appliance.DeleteAllBlades(ctx) // cache and hardware interactions here - if err != nil { - newErr := fmt.Errorf("failed to delete all appliance [%s] blades: %w", appliance.Id, err) - logger.Error(newErr, "failure: delete appliance by id") - return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} - } + appliance.DeleteAllBlades(ctx) // delete appliance from cache - a := deviceCache.DeleteApplianceById(appliance.Id) - if a == nil { - newErr := fmt.Errorf("appliance [%s] cache delete failed", appliance.Id) - logger.Error(newErr, "failure: delete appliance by id") - return nil, &common.RequestError{StatusCode: common.StatusApplianceDeleteSessionFailure, Err: newErr} - } + deviceCache.DeleteApplianceById(appliance.Id) + + // delete appliance from datastore + datastore.DStore().GetDataStore().DeleteApplianceDatumById(appliance.Id) + datastore.DStore().Store() - logger.V(2).Info("success: delete appliance by id", "applianceId", a.Id) + logger.V(2).Info("success: delete appliance by id", "applianceId", appliance.Id) - return a, nil + return appliance, nil } func GetAllApplianceIds() []string { @@ -125,7 +119,7 @@ func GetAppliances(ctx context.Context) map[string]*Appliance { return appliances } -func ResyncApplianceById(ctx context.Context, applianceId string) (*Appliance, *[]string, error) { +func ResyncApplianceById(ctx context.Context, applianceId string) (*Appliance, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> ResyncApplianceById: ", "applianceId", applianceId) @@ -135,7 +129,7 @@ func ResyncApplianceById(ctx context.Context, applianceId string) (*Appliance, * if err != nil { newErr := fmt.Errorf("get appliance by id [%s] failure: %w", appliance.Id, err) logger.Error(newErr, "failure: resync appliance by id") - return nil, &failedBladeIds, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} + return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} } bladeIds := appliance.GetAllBladeIds() @@ -151,15 +145,15 @@ func ResyncApplianceById(ctx context.Context, applianceId string) (*Appliance, * if len(failedBladeIds) == 0 { logger.V(2).Info("success: resync appliance", "applianceId", applianceId, "bladeIds", bladeIds) - return appliance, &failedBladeIds, nil + return appliance, nil } else if len(failedBladeIds) < len(bladeIds) { newErr := fmt.Errorf("resync appliance by id [%s]: some failure(s): blade(s) [%s]: %w", appliance.Id, failedBladeIds, err) logger.Error(newErr, "partial success: resync appliance by id") - return appliance, &failedBladeIds, &common.RequestError{StatusCode: common.StatusApplianceResyncPartialSuccess, Err: newErr} + return appliance, &common.RequestError{StatusCode: common.StatusApplianceResyncPartialSuccess, Err: newErr} } else { newErr := fmt.Errorf("resync appliance by id [%s] failure: %w", appliance.Id, err) logger.Error(newErr, "failure: resync appliance by id") - return nil, &failedBladeIds, &common.RequestError{StatusCode: common.StatusApplianceResyncFailure, Err: newErr} + return nil, &common.RequestError{StatusCode: common.StatusApplianceResyncFailure, Err: newErr} } } @@ -217,6 +211,7 @@ func AddHost(ctx context.Context, c *openapi.Credentials) (*Host, error) { // Generate default id using last N digits of the session id combined with the default prefix hostId = fmt.Sprintf("%s-%s", ID_PREFIX_HOST_DFLT, response.SessionId[(len(response.SessionId)-common.NumUuidCharsForId):]) } + c.CustomId = hostId } // Check for duplicate ID. @@ -240,6 +235,7 @@ func AddHost(ctx context.Context, c *openapi.Credentials) (*Host, error) { HostId: hostId, Ip: c.IpAddress, Port: uint16(c.Port), + Status: common.ONLINE, BackendOps: ops, Creds: c, } @@ -259,8 +255,12 @@ func AddHost(ctx context.Context, c *openapi.Credentials) (*Host, error) { return nil, &common.RequestError{StatusCode: common.StatusManagerInitializationFailure, Err: newErr} } - // Cache it - deviceCache.AddHost(host) + // Add host to device cache + deviceCache.AddHost(host) // ignore error, duplicate check done above + + // Add host to datastore + datastore.DStore().GetDataStore().AddHostDatum(c) + datastore.DStore().Store() logger.V(2).Info("success: add host", "hostId", host.Id) @@ -290,24 +290,20 @@ func DeleteHostById(ctx context.Context, hostId string) (*Host, error) { newErr := fmt.Errorf("failed to delete host [%s] backend [%s] session [%s]: %w", host.Id, ops.GetBackendInfo(ctx).BackendName, host.Socket.String(), err) logger.Error(newErr, "failure: delete host by id") - // Currently, backend ALWAYS deletes the host session from the backend map. For now, need to delete host from manager map as well. - logger.V(2).Info("force host deletion after backend session failure", "hostId", host.Id) - deviceCache.DeleteHostById(host.Id) + // Currently, backend ALWAYS deletes the host session from the backend map. + // Delete host from manager cache and datastore as well + logger.V(2).Info("force host deletion after backend delete session failure", "hostId", host.Id) - return nil, &common.RequestError{StatusCode: common.StatusHostDeleteSessionFailure, Err: newErr} - } + deleteHost(host.Id) - // delete host from cache - h := deviceCache.DeleteHostById(host.Id) - if h == nil { - newErr := fmt.Errorf("host [%s] cache delete failed", host.Id) - logger.Error(newErr, "failure: delete host by id") - return nil, &common.RequestError{StatusCode: common.StatusHostDeleteSessionFailure, Err: newErr} + return host, &common.RequestError{StatusCode: common.StatusHostDeleteSessionFailure, Err: newErr} } - logger.V(2).Info("success: delete host by id", "hostId", h.Id) + deleteHost(host.Id) + + logger.V(2).Info("success: delete host by id", "hostId", host.Id) - return h, nil + return host, nil } func GetAllHostIds() []string { @@ -322,11 +318,13 @@ func GetHostById(ctx context.Context, hostId string) (*Host, error) { host, err := deviceCache.GetHostById(hostId) if err != nil { logger.Error(err, "failure: get host by id") - newErr := fmt.Errorf("failure: get host by id [%s]", hostId) + newErr := fmt.Errorf("failure: get host by id [%s]: %w", hostId, err) return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} } - logger.V(2).Info("success: get host by id", "hostId", hostId) + host.UpdateConnectionStatusBackend(ctx) + + logger.V(2).Info("success: get host by id", "status", host.Status, "hostId", host.Id) return host, nil } @@ -346,20 +344,32 @@ func ResyncHostById(ctx context.Context, hostId string) (*Host, error) { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> ResyncHostById: ", "hostId", hostId) - host, err := DeleteHostById(ctx, hostId) - if err != nil { - newErr := fmt.Errorf("failed to resync host(delete): host [%s]: %w", hostId, err) - logger.Error(newErr, "failure: resync host: ignoring") + // query device cache + host, ok := deviceCache.GetHostByIdOk(hostId) + if !ok || host == nil { + newErr := fmt.Errorf("failed to get host [%s]", hostId) + logger.Error(newErr, "failure: resync host by id") + return nil, &common.RequestError{StatusCode: common.StatusHostIdDoesNotExist, Err: newErr} } - host, err = AddHost(ctx, host.creds) - if err != nil { - newErr := fmt.Errorf("failed to resync host(add): host [%s]: %w", hostId, err) - logger.Error(newErr, "failure: resync host") - return nil, &common.RequestError{StatusCode: err.(*common.RequestError).StatusCode, Err: newErr} - } + host.UpdateConnectionStatusBackend(ctx) - logger.V(2).Info("success: resync host", "hostId", hostId) + logger.V(2).Info("success: resync host", "status", host.Status, "hostId", host.Id) return host, nil } + +//////////////////////////////////// +//////// Helper Functions ////////// +//////////////////////////////////// + +func deleteHost(hostId string) *Host { + // delete host from manager cache + h := deviceCache.DeleteHostById(hostId) + + // delete host from datastore + datastore.DStore().GetDataStore().DeleteHostDatumById(hostId) + datastore.DStore().Store() + + return h +} diff --git a/pkg/manager/port.go b/pkg/manager/port.go index c99bf65..1940110 100644 --- a/pkg/manager/port.go +++ b/pkg/manager/port.go @@ -35,7 +35,7 @@ type CxlBladePort struct { backendOps backend.BackendOperations } -func NewCxlBladePortById(ctx context.Context, applianceId, bladeId, portId string, ops backend.BackendOperations) (*CxlBladePort) { +func NewCxlBladePortById(ctx context.Context, applianceId, bladeId, portId string, ops backend.BackendOperations) *CxlBladePort { logger := klog.FromContext(ctx) logger.V(4).Info(">>>>>> NewCxlBladePortById: ", "portId", portId, "bladeId", bladeId, "applianceId", applianceId, "backend", ops.GetBackendInfo(ctx).BackendName)