From b54d727ef34d43f0d1f15c7350887065dffc4e01 Mon Sep 17 00:00:00 2001 From: Nick Petrovic Date: Thu, 18 Jan 2024 16:52:49 -0500 Subject: [PATCH] add custom k3s image for gpu support --- .dockerignore | 1 + Makefile | 1 + docker/Dockerfile.k3d | 37 ++++++++ docker/Dockerfile.worker | 89 ++++++++---------- docker/config.toml.tmpl | 118 ++++++++++++++++++++++++ hack/k3d.yaml | 2 +- internal/worker/metrics_linux.go | 4 +- manifests/k3d/nvidia-device-plugin.yaml | 1 + 8 files changed, 198 insertions(+), 55 deletions(-) create mode 100644 docker/Dockerfile.k3d create mode 100644 docker/config.toml.tmpl diff --git a/.dockerignore b/.dockerignore index bece379a2..1a23f7a2d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,6 @@ !go.mod !go.sum !sdk +!docker **/__pycache__ *.pyc diff --git a/Makefile b/Makefile index fda45a075..b380dd33f 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ setup-sdk: poetry install -C sdk k3d-up: + docker build . -f ./docker/Dockerfile.k3d -t localhost:5001/rancher/k3s:latest k3d cluster create --config hack/k3d.yaml kubectl config set contexts.k3d-beam.namespace beam okteto context use k3d-beam --namespace beam diff --git a/docker/Dockerfile.k3d b/docker/Dockerfile.k3d new file mode 100644 index 000000000..8c7510ced --- /dev/null +++ b/docker/Dockerfile.k3d @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1.6 +FROM rancher/k3s:v1.28.5-k3s1 as k3s +FROM nvidia/cuda:12.3.1-base-ubuntu20.04 + +ENV CRI_CONFIG_FILE=/var/lib/rancher/k3s/agent/etc/crictl.yaml +ENV PATH="$PATH:/bin/aux" + +RUN < /etc/nsswitch.conf +chmod 1777 /tmp +mkdir -vp /var/lib/rancher/k3s/agent/etc/containerd/ + +apt-get clean +apt-get autoremove -y +apt-get autopurge -y +rm -rf /var/lib/apt/lists/* /var/log/* +EOT + +COPY --from=k3s /bin /bin +COPY docker/config.toml.tmpl /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl + +VOLUME /var/lib/kubelet +VOLUME /var/lib/rancher/k3s +VOLUME /var/lib/cni +VOLUME /var/log + +ENTRYPOINT ["/bin/k3s"] +CMD ["agent"] diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index f4ad1f25e..4bd4e8852 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -3,14 +3,10 @@ ARG BASE_STAGE=dev FROM golang:1.21-bullseye AS golang -RUN < /etc/apt/sources.list.d/criu.list && \ + curl -fsSL https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add - && \ + curl -fsSL https://nvidia.github.io/nvidia-container-runtime/ubuntu20.04/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list && \ + apt-get update -COPY --from=golang /usr/local/go/ /usr/local/go/ -ENV PATH="/usr/local/go/bin:${PATH}" +RUN curl -sSL https://d.juicefs.com/install | sh - +RUN apt-get install -y --no-install-recommends criu nvidia-container-toolkit-base nvidia-container-toolkit -RUN < /etc/apt/sources.list.d/nvidia-container-toolkit.list -curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add - -curl -s -L https://nvidia.github.io/nvidia-container-runtime/ubuntu20.04/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list -apt-get update && apt-get install -y nvidia-container-runtime fuse3 libfuse3-dev - -# Build and install custom nvidia-container-toolkit -git clone https://github.com/beam-cloud/nvidia-container-toolkit.git -cd /workspace/nvidia-container-toolkit && make build && make binaries -cp /workspace/nvidia-container-toolkit/nvidia-container-runtime* /usr/bin/ - -# criu repo -curl -fsSL https://download.opensuse.org/repositories/devel:/tools:/criu/xUbuntu_20.04/Release.key | gpg --dearmor -o /usr/share/keyrings/criu.gpg -echo 'deb [signed-by=/usr/share/keyrings/criu.gpg] https://download.opensuse.org/repositories/devel:/tools:/criu/xUbuntu_20.04 /' > /etc/apt/sources.list.d/criu.list - -apt-get update -apt-get install -y --no-install-recommends nvidia-container-runtime criu -apt-get remove -y curl gpg -apt-get clean -apt-get autoremove -y -apt-get autopurge -y -rm -rf /var/lib/apt/lists/* /var/log/* -EOT +RUN apt-get remove -y curl gpg && \ + apt-get clean && apt-get autoremove -y && apt-get autopurge -y && \ + rm -rf /var/lib/apt/lists/* /var/log/* COPY --from=runc /usr/local/sbin/runc /usr/local/sbin/runc COPY --from=skopeo /usr/local/bin/skopeo /usr/local/bin/skopeo COPY --from=skopeo /workspace/default-policy.json /etc/containers/policy.json +COPY --from=nvidia-container-toolkit /workspace/nvidia-container-runtime* /usr/bin/ COPY --from=worker /usr/local/bin/worker /usr/local/bin/worker COPY ./sdk/src/beam /workspace/sdk -VOLUME ["/usr/lib/x86_64-linux-gnu", "/usr/lib/aarch64-linux-gnu"] +VOLUME "/usr/lib/x86_64-linux-gnu" +VOLUME "/usr/lib/aarch64-linux-gnu" diff --git a/docker/config.toml.tmpl b/docker/config.toml.tmpl new file mode 100644 index 000000000..97a73e691 --- /dev/null +++ b/docker/config.toml.tmpl @@ -0,0 +1,118 @@ +version = 2 + +[plugins."io.containerd.internal.v1.opt"] + path = "{{ .NodeConfig.Containerd.Opt }}" +[plugins."io.containerd.grpc.v1.cri"] + stream_server_address = "127.0.0.1" + stream_server_port = "10010" + enable_selinux = {{ .NodeConfig.SELinux }} + enable_unprivileged_ports = {{ .EnableUnprivileged }} + enable_unprivileged_icmp = {{ .EnableUnprivileged }} + +{{- if .DisableCgroup}} + disable_cgroup = true +{{end}} +{{- if .IsRunningInUserNS }} + disable_apparmor = true + restrict_oom_score_adj = true +{{end}} + +{{- if .NodeConfig.AgentConfig.PauseImage }} + sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}" +{{end}} + +{{- if .NodeConfig.AgentConfig.Snapshotter }} +[plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" + disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + default_runtime_name = "nvidia" +{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} +{{ if .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins."io.containerd.snapshotter.v1.stargz"] +cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" +[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain] +enable_keychain = true +{{end}} +{{ if .PrivateRegistryConfig }} +{{ if .PrivateRegistryConfig.Mirrors }} +[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors]{{end}} +{{range $k, $v := .PrivateRegistryConfig.Mirrors }} +[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}"] + endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] +{{if $v.Rewrites}} + [plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}".rewrite] +{{range $pattern, $replace := $v.Rewrites}} + "{{$pattern}}" = "{{$replace}}" +{{end}} +{{end}} +{{end}} +{{range $k, $v := .PrivateRegistryConfig.Configs }} +{{ if $v.Auth }} +[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth] + {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} + {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}} + {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} + {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} +{{end}} +{{ if $v.TLS }} +[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".tls] + {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} + {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} + {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} + {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} +{{end}} +{{end}} +{{end}} +{{end}} +{{end}} + +{{- if not .NodeConfig.NoFlannel }} +[plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}" + conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" +{{end}} + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = {{ .SystemdCgroup }} + +{{ if .PrivateRegistryConfig }} +{{ if .PrivateRegistryConfig.Mirrors }} +[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}} +{{range $k, $v := .PrivateRegistryConfig.Mirrors }} +[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"] + endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] +{{if $v.Rewrites}} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite] +{{range $pattern, $replace := $v.Rewrites}} + "{{$pattern}}" = "{{$replace}}" +{{end}} +{{end}} +{{end}} + +{{range $k, $v := .PrivateRegistryConfig.Configs }} +{{ if $v.Auth }} +[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth] + {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} + {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}} + {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} + {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} +{{end}} +{{ if $v.TLS }} +[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls] + {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} + {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} + {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} + {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} +{{end}} +{{end}} +{{end}} + +{{range $k, $v := .ExtraRuntimes}} +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"] + runtime_type = "{{$v.RuntimeType}}" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options] + BinaryName = "{{$v.BinaryName}}" +{{end}} diff --git a/hack/k3d.yaml b/hack/k3d.yaml index 0adc197f2..edee13471 100644 --- a/hack/k3d.yaml +++ b/hack/k3d.yaml @@ -3,7 +3,7 @@ apiVersion: k3d.io/v1alpha5 kind: Simple metadata: name: beam -image: docker.io/rancher/k3s:v1.28.5-k3s1 +image: localhost:5001/rancher/k3s:latest servers: 1 ports: diff --git a/internal/worker/metrics_linux.go b/internal/worker/metrics_linux.go index cd376c021..cfbcd45f6 100644 --- a/internal/worker/metrics_linux.go +++ b/internal/worker/metrics_linux.go @@ -9,7 +9,9 @@ import ( ) func (wm *WorkerMetrics) InitNvml() { - wm.nvmlActive = nvml.Init() == nvml.SUCCESS + // TODO: investigate segmentation violation + // wm.nvmlActive = nvml.Init() == nvml.SUCCESS + wm.nvmlActive = false } func (wm *WorkerMetrics) Shutdown() { diff --git a/manifests/k3d/nvidia-device-plugin.yaml b/manifests/k3d/nvidia-device-plugin.yaml index a3e24a667..f95d38514 100755 --- a/manifests/k3d/nvidia-device-plugin.yaml +++ b/manifests/k3d/nvidia-device-plugin.yaml @@ -21,6 +21,7 @@ spec: operator: Exists effect: NoSchedule priorityClassName: system-node-critical + runtimeClassName: nvidia containers: - image: nvcr.io/nvidia/k8s-device-plugin:v0.14.3 name: nvidia-device-plugin-ctr