Merge pull request #205 from whole-tale/k8s_jhu

fix: finalize support for K8S deployment @ JHU
whole-tale · Feb 11, 2025 · 9b3d867 · 9b3d867
2 parents fabf057 + 80d779f
commit 9b3d867
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 8 deletions.
diff --git a/gwvolman/r2d/remote.py b/gwvolman/r2d/remote.py
@@ -4,6 +4,7 @@
 import requests
 
 from .builder import ImageBuilderBase
+from ..utils import DOMAIN
 
 
 class RemoteImageBuilder(ImageBuilderBase):
@@ -22,7 +23,7 @@ def __init__(
         self.builder_url = builder_url or os.environ.get(
             "BUILDER_URL", "https://builder.local.xarthisius.xyz"
         )
-        self.registry_url = registry_url or "https://registry.local.xarthisius.xyz"
+        self.registry_url = registry_url or f"https://registry.{DOMAIN}"
         self.registry_user = registry_user or os.environ.get("REGISTRY_USER", "fido")
         self.registry_password = registry_password or os.environ.get("REGISTRY_PASS")
 
@@ -53,7 +54,7 @@ def push_image(self, image):
                 "image": image,
                 "registry_user": self.registry_user,
                 "registry_password": self.registry_password,
-                "registry_url": self.registry_url,
+                "registry_url": f"https://registry.{DOMAIN}",
             },
             stream=True,
         )
@@ -70,7 +71,7 @@ def run_r2d(self, tag, dry_run=False, task=None):
             f"{self.builder_url}/build",
             params={
                 "taleId": self.tale["_id"],
-                "apiUrl": self.gc.urlBase,
+                "apiUrl": f"https://girder.{DOMAIN}/api/v1",
                 "token": self.gc.token,
                 "registry_url": self.registry_url,
                 "dry_run": dry_run,

diff --git a/gwvolman/remote_builder/server.py b/gwvolman/remote_builder/server.py
@@ -99,9 +99,14 @@ async def build_stream():
                 },
             }
 
+            docker_group = os.environ.get("DOCKER_GROUP", "docker")
+            if docker_group.isdigit():
+                docker_group = int(docker_group)
+
             container = client.containers.run(
                 image=image_builder.container_config.repo2docker_version,
                 command=r2d_cmd,
+                group_add=[docker_group],
                 environment={"DOCKER_HOST": "unix:///var/run/docker.sock"},
                 privileged=True,
                 detach=True,

diff --git a/gwvolman/tasks_kubernetes.py b/gwvolman/tasks_kubernetes.py
@@ -169,7 +169,8 @@ def launch_container(self, task, payload):
             "domain": DOMAIN,
             "ingressClass": K8SDeployment.ingress_class,
             "deploymentNamespace": self.deployment.namespace,
-            "claimName": "girder-data",  # TODO pass from deployment
+            "claimName": K8SDeployment.existing_claim,
+            "claimSubPath": K8SDeployment.existing_claim_subpath,
             "girderApiUrl": girder_api_url,
             "mounterImage": self.deployment.mounter_image,
             "instanceId": instanceId,

diff --git a/gwvolman/utils.py b/gwvolman/utils.py
@@ -91,6 +91,8 @@ class K8SDeployment(object):
     girder_url = f"https://girder.{DOMAIN}"
     registry_url = f"https://registry.{DOMAIN}"
     builder_url = os.environ.get("BUILDER_URL", "https://builder.{DOMAIN}")
+    existing_claim = os.environ.get("GIRDER_EXISTING_CLAIM", "girder-data")
+    existing_claim_subpath = os.environ.get("GIRDER_EXISTING_CLAIM_SUBPATH", "")
     traefik_network = None
     tmpdir_mount = "/tmp"
     namespace = NAMESPACE

diff --git a/gwvolman/utils_k8s.py b/gwvolman/utils_k8s.py
@@ -113,7 +113,7 @@ def compose_volumes(params):
             client.V1Volume(
                 name=params["claimName"],
                 persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
-                    claim_name=params["claimName"]
+                    claim_name=params["claimName"],
                 ),
             )
         )
@@ -160,6 +160,7 @@ def tale_deployment(params):
             client.V1VolumeMount(
                 mount_path=VOLUMES_ROOT,
                 name=params["claimName"],
+                sub_path=params["claimSubPath"],
                 read_only=False,
             ),
         )
@@ -192,12 +193,12 @@ def tale_deployment(params):
             client.V1VolumeMount(
                 mount_path=os.path.join(params["mountPoint"], "workspace"),
                 name=params["claimName"],
-                sub_path=params["workspaceSubPath"],
+                sub_path=os.path.join(params["claimSubPath"], params["workspaceSubPath"]),
             ),
             client.V1VolumeMount(
                 mount_path=os.path.join(params["mountPoint"], "home"),
                 name=params["claimName"],
-                sub_path=params["homeSubPath"],
+                sub_path=os.path.join(params["claimSubPath"], params["homeSubPath"]),
             ),
         ]
     else:
@@ -255,6 +256,16 @@ def tale_deployment(params):
                                     )
                                 ],
                                 volume_mounts=instance_mounts,
+                                resources=client.V1ResourceRequirements(
+                                    limits={
+                                        "memory": "4Gi",
+                                        "cpu": "1",
+                                    },
+                                    requests={
+                                        "memory": "4Gi",
+                                        "cpu": "1",
+                                    },
+                                ),
                             ),
                             client.V1Container(
                                 name="mounter",
@@ -274,9 +285,13 @@ def tale_deployment(params):
                                 ),
                                 resources=client.V1ResourceRequirements(
                                     limits={
+                                        "memory": "1Gi",
+                                        "cpu": "1",
                                         "smarter-devices/fuse": 1,
                                     },
                                     requests={
+                                        "memory": "1Gi",
+                                        "cpu": "0.5",
                                         "smarter-devices/fuse": 1,
                                     },
                                 ),

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setup(
     name="gwvolman",
-    version="2.1.0",
+    version="2.1.1",
     description="WholeTale Girder Volume Manager",
     author="Kacper Kowalik",
     author_email="[email protected]",