Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Skip pyNVML to support Tegra devices #402

Open
wants to merge 3 commits into
base: branch-0.17
Choose a base branch
from
Open

Skip pyNVML to support Tegra devices #402

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c9f29a
Skip pyNVML to support Tegra devices
pentschev Sep 25, 2020
287406a
Support for Tegra in get_cpu_affinity
pentschev Oct 7, 2020
d2d5e30
Use CUDA driver via Numba to get memory info on Tegra
pentschev Oct 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 49 additions & 20 deletions dask_cuda/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,15 @@ def nvtx_annotate(message=None, color="blue", domain=None):
yield


@toolz.memoize
def _is_tegra():
import os

return os.path.isdir("/sys/class/tegra-firmware/") or os.path.isfile(
"/etc/nv_tegra_release"
)


class CPUAffinity:
def __init__(self, cores):
self.cores = cores
Expand Down Expand Up @@ -96,8 +105,13 @@ def get_cpu_count():

@toolz.memoize
def get_gpu_count():
pynvml.nvmlInit()
return pynvml.nvmlDeviceGetCount()
if _is_tegra():
import numba.cuda

return len(numba.cuda.gpus)
else:
pynvml.nvmlInit()
return pynvml.nvmlDeviceGetCount()


def get_cpu_affinity(device_index):
jakirkham marked this conversation as resolved.
Show resolved Hide resolved
Expand Down Expand Up @@ -125,21 +139,24 @@ def get_cpu_affinity(device_index):
40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
"""
pynvml.nvmlInit()

try:
# Result is a list of 64-bit integers, thus ceil(get_cpu_count() / 64)
affinity = pynvml.nvmlDeviceGetCpuAffinity(
pynvml.nvmlDeviceGetHandleByIndex(device_index),
math.ceil(get_cpu_count() / 64),
)
return unpack_bitmask(affinity)
except pynvml.NVMLError:
warnings.warn(
"Cannot get CPU affinity for device with index %d, setting default affinity"
% device_index
)
if _is_tegra():
return list(range(get_cpu_count()))
else:
pynvml.nvmlInit()

try:
# Result is a list of 64-bit integers, thus ceil(get_cpu_count() / 64)
affinity = pynvml.nvmlDeviceGetCpuAffinity(
pynvml.nvmlDeviceGetHandleByIndex(device_index),
math.ceil(get_cpu_count() / 64),
)
return unpack_bitmask(affinity)
except pynvml.NVMLError:
warnings.warn(
"Cannot get CPU affinity for device with index %d, setting default affinity"
% device_index
)
return list(range(get_cpu_count()))


def get_n_gpus():
Expand All @@ -153,10 +170,22 @@ def get_device_total_memory(index=0):
"""
Return total memory of CUDA device with index
"""
pynvml.nvmlInit()
return pynvml.nvmlDeviceGetMemoryInfo(
pynvml.nvmlDeviceGetHandleByIndex(index)
).total
if _is_tegra():
from ctypes import byref, c_size_t
import numba.cuda

driver = numba.cuda.driver.Driver()

numba.cuda.current_context()
free = c_size_t()
total = c_size_t()
driver.cuMemGetInfo(byref(free), byref(total))
return total.value
else:
pynvml.nvmlInit()
return pynvml.nvmlDeviceGetMemoryInfo(
pynvml.nvmlDeviceGetHandleByIndex(index)
).total


def get_ucx_net_devices(
Expand Down