Removing collection of FP64 metrics on GPUs that does not supports it (…

…#19) * Removing collection of FP64 metrics on GPUs that does not supports it For issue #18 * Adding support for NVIDIA RTX A6000 Co-authored-by: Bruno Travouillon <[email protected]> --------- Co-authored-by: Bruno Travouillon <[email protected]>
guilbaults · Nov 23, 2023 · 1814b0c · 1814b0c
1 parent 104cfb7
commit 1814b0c
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/slurm-job-exporter.py b/slurm-job-exporter.py
@@ -151,6 +151,16 @@ def __init__(self, dcgm_update_interval=10):
                         dcgm_fields.DCGM_FI_PROF_NVLINK_RX_BYTES: 'nvlink_rx_bytes',
                     }
 
+                    for gpu_id in pydcgm.DcgmSystemDiscovery(self.handle).GetAllSupportedGpuIds():
+                        device = pydcgm.dcgm_agent.dcgmGetDeviceAttributes(self.handle.handle, gpu_id)
+                        name = device.identifiers.deviceName
+                        print('Detected gpu {} with ID {}'.format(name, gpu_id))
+                        if name in ['NVIDIA RTX A6000']:
+                            # This GPU does not supports fp64, we don't support a mix of fp64 and non-fp64 GPUs in the same node
+                            print('Removing fp64 metrics since {} does not support fp64'.format(name))
+                            del self.fieldIds_dict[dcgm_fields.DCGM_FI_PROF_PIPE_FP64_ACTIVE]
+                            break
+
                     self.field_group = pydcgm.DcgmFieldGroup(self.handle, name="slurm-job-exporter-fg", fieldIds=list(self.fieldIds_dict.keys()))
                     self.group.samples.WatchFields(self.field_group, dcgm_update_interval * 1000 * 1000, dcgm_update_interval * 2.0, 0)
                     self.handle.GetSystem().UpdateAllFields(True)