diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1791a29b..e2549f99 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -145,7 +145,7 @@ jobs: # # ref: https://github.com/jupyterhub/action-k3s-helm/ # - - uses: jupyterhub/action-k3s-helm@v2 + - uses: jupyterhub/action-k3s-helm@v3 with: k3s-channel: ${{ matrix.k3s-channel }} metrics-enabled: false diff --git a/dask-gateway-server/dask_gateway_server/backends/kubernetes/backend.py b/dask-gateway-server/dask_gateway_server/backends/kubernetes/backend.py index 251453ae..37994275 100644 --- a/dask-gateway-server/dask_gateway_server/backends/kubernetes/backend.py +++ b/dask-gateway-server/dask_gateway_server/backends/kubernetes/backend.py @@ -352,6 +352,19 @@ async def setup(self, app): self.core_client = client.CoreV1Api(api_client=self.api_client) self.custom_client = client.CustomObjectsApi(api_client=self.api_client) + method = "list_cluster_custom_object" + method_kwargs = dict( + group="gateway.dask.org", + version=self.crd_version, + plural="daskclusters", + label_selector=self.label_selector, + ) + + self.watch_namespace = os.environ.get("WATCH_NAMESPACE") + if self.watch_namespace: + method = "list_namespaced_custom_object" + method_kwargs["namespace"] = self.watch_namespace + self.cluster_waiters = defaultdict(Flag) self.clusters = {} self.username_to_clusters = defaultdict(dict) @@ -360,13 +373,8 @@ async def setup(self, app): parent=self, name="cluster", client=self.custom_client, - method="list_cluster_custom_object", - method_kwargs=dict( - group="gateway.dask.org", - version=self.crd_version, - plural="daskclusters", - label_selector=self.label_selector, - ), + method=method, + method_kwargs=method_kwargs, on_update=self.on_cluster_event, on_delete=self.on_cluster_event, ) diff --git a/dask-gateway-server/dask_gateway_server/backends/kubernetes/controller.py b/dask-gateway-server/dask_gateway_server/backends/kubernetes/controller.py index 9606bc3c..31be413b 100644 --- a/dask-gateway-server/dask_gateway_server/backends/kubernetes/controller.py +++ b/dask-gateway-server/dask_gateway_server/backends/kubernetes/controller.py @@ -2,6 +2,7 @@ import collections import json import logging +import os import signal import sys import time @@ -378,18 +379,39 @@ async def setup(self): endpoints_selector = ( self.label_selector + ",app.kubernetes.io/component=dask-scheduler" ) + + method_clusters = "list_cluster_custom_object" + method_kwargs_clusters = dict( + group="gateway.dask.org", + version=self.crd_version, + plural="daskclusters", + label_selector=self.label_selector, + ) + + method_pod = "list_pod_for_all_namespaces" + method_kwargs_pod = dict(label_selector=self.label_selector) + + method_endpoints = "list_endpoints_for_all_namespaces" + method_kwargs_endpoints = dict(label_selector=endpoints_selector) + + self.watch_namespace = os.environ.get("WATCH_NAMESPACE") + if self.watch_namespace: + method_clusters = "list_namespaced_custom_object" + method_kwargs_clusters["namespace"] = self.watch_namespace + + method_pod = "list_namespaced_pod" + method_kwargs_pod["namespace"] = self.watch_namespace + + method_endpoints = "list_namespaced_endpoints" + method_kwargs_endpoints["namespace"] = self.watch_namespace + self.informers = { "cluster": Informer( parent=self, name="cluster", client=self.custom_client, - method="list_cluster_custom_object", - method_kwargs=dict( - group="gateway.dask.org", - version=self.crd_version, - plural="daskclusters", - label_selector=self.label_selector, - ), + method=method_clusters, + method_kwargs=method_kwargs_clusters, on_update=self.on_cluster_update, on_delete=self.on_cluster_delete, ), @@ -397,8 +419,8 @@ async def setup(self): parent=self, name="pod", client=self.core_client, - method="list_pod_for_all_namespaces", - method_kwargs=dict(label_selector=self.label_selector), + method=method_pod, + method_kwargs=method_kwargs_pod, on_update=self.on_pod_update, on_delete=self.on_pod_delete, ), @@ -406,8 +428,8 @@ async def setup(self): parent=self, name="endpoints", client=self.core_client, - method="list_endpoints_for_all_namespaces", - method_kwargs=dict(label_selector=endpoints_selector), + method=method_endpoints, + method_kwargs=method_kwargs_endpoints, on_update=self.on_endpoints_update, on_delete=self.on_endpoints_delete, ), diff --git a/resources/helm/dask-gateway/templates/controller/deployment.yaml b/resources/helm/dask-gateway/templates/controller/deployment.yaml index a9a89a13..db4d7ba0 100644 --- a/resources/helm/dask-gateway/templates/controller/deployment.yaml +++ b/resources/helm/dask-gateway/templates/controller/deployment.yaml @@ -55,6 +55,17 @@ spec: volumeMounts: - mountPath: /etc/dask-gateway/ name: configmap + env: + - name: WATCH_NAMESPACE + {{- if .Values.runInClusterScope }} + value: "" + {{- else if .Values.watchNamespace }} + value: {{ .Values.watchNamespace | quote }} + {{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} ports: - containerPort: 8000 name: api diff --git a/resources/helm/dask-gateway/templates/controller/rbac.yaml b/resources/helm/dask-gateway/templates/controller/rbac.yaml index ccb6ae84..7378e9c3 100644 --- a/resources/helm/dask-gateway/templates/controller/rbac.yaml +++ b/resources/helm/dask-gateway/templates/controller/rbac.yaml @@ -9,7 +9,11 @@ metadata: {{- include "dask-gateway.labels" . | nindent 4 }} --- apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.runInClusterScope }} kind: ClusterRole +{{- else }} +kind: Role +{{- end }} metadata: name: {{ include "dask-gateway.controllerName" . }} labels: @@ -31,7 +35,11 @@ rules: resources: ["secrets", "services"] verbs: ["create", "delete"] --- +{{- if .Values.runInClusterScope }} kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "dask-gateway.controllerName" . }} @@ -42,7 +50,11 @@ subjects: name: {{ include "dask-gateway.controllerName" . }} namespace: {{ .Release.Namespace }} roleRef: + {{- if .Values.runInClusterScope }} kind: ClusterRole + {{- else }} + kind: Role + {{- end }} name: {{ include "dask-gateway.controllerName" . }} apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/resources/helm/dask-gateway/templates/gateway/deployment.yaml b/resources/helm/dask-gateway/templates/gateway/deployment.yaml index 5480b732..d39b52ad 100644 --- a/resources/helm/dask-gateway/templates/gateway/deployment.yaml +++ b/resources/helm/dask-gateway/templates/gateway/deployment.yaml @@ -54,6 +54,16 @@ spec: - mountPath: /etc/dask-gateway/ name: configmap env: + - name: WATCH_NAMESPACE + {{- if .Values.runInClusterScope }} + value: "" + {{- else if .Values.watchNamespace }} + value: {{ .Values.watchNamespace | quote }} + {{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} {{- if (eq .Values.gateway.auth.type "jupyterhub") }} - name: JUPYTERHUB_API_TOKEN valueFrom: diff --git a/resources/helm/dask-gateway/templates/gateway/rbac.yaml b/resources/helm/dask-gateway/templates/gateway/rbac.yaml index 2cda2c40..b1b1362e 100644 --- a/resources/helm/dask-gateway/templates/gateway/rbac.yaml +++ b/resources/helm/dask-gateway/templates/gateway/rbac.yaml @@ -8,7 +8,11 @@ metadata: {{- include "dask-gateway.labels" . | nindent 4 }} --- apiVersion: rbac.authorization.k8s.io/v1 +{{- if .Values.runInClusterScope }} kind: ClusterRole +{{- else }} +kind: Role +{{- end }} metadata: name: {{ include "dask-gateway.apiName" . }} labels: @@ -21,7 +25,11 @@ rules: resources: ["daskclusters"] verbs: ["*"] --- +{{- if .Values.runInClusterScope }} kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "dask-gateway.apiName" . }} @@ -32,7 +40,11 @@ subjects: name: {{ include "dask-gateway.apiName" . }} namespace: {{ .Release.Namespace }} roleRef: + {{- if .Values.runInClusterScope }} kind: ClusterRole + {{- else }} + kind: Role + {{- end }} name: {{ include "dask-gateway.apiName" . }} apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/resources/helm/dask-gateway/templates/traefik/deployment.yaml b/resources/helm/dask-gateway/templates/traefik/deployment.yaml index 8ca1b70a..99a21e20 100644 --- a/resources/helm/dask-gateway/templates/traefik/deployment.yaml +++ b/resources/helm/dask-gateway/templates/traefik/deployment.yaml @@ -44,7 +44,9 @@ spec: - "--global.sendanonymoususage=False" - "--ping=true" - "--providers.kubernetescrd" + {{- if .Values.runInClusterScope }} - "--providers.kubernetescrd.allowCrossNamespace=true" + {{- end }} - '--providers.kubernetescrd.labelselector=gateway.dask.org/instance={{ include "dask-gateway.fullname" . }}' - "--providers.kubernetescrd.throttleduration=2" - "--log.level={{ .Values.traefik.loglevel }}" diff --git a/resources/helm/dask-gateway/templates/traefik/rbac.yaml b/resources/helm/dask-gateway/templates/traefik/rbac.yaml index 3e3e029f..d8fa29b2 100644 --- a/resources/helm/dask-gateway/templates/traefik/rbac.yaml +++ b/resources/helm/dask-gateway/templates/traefik/rbac.yaml @@ -5,7 +5,11 @@ apiVersion: v1 metadata: name: {{ include "dask-gateway.traefikName" . }} --- +{{- if .Values.runInClusterScope }} kind: ClusterRole +{{- else }} +kind: Role +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "dask-gateway.traefikName" . }} @@ -52,13 +56,21 @@ rules: - list - watch --- +{{- if .Values.runInClusterScope }} kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "dask-gateway.traefikName" . }} roleRef: apiGroup: rbac.authorization.k8s.io + {{- if .Values.runInClusterScope }} kind: ClusterRole + {{- else }} + kind: Role + {{- end }} name: {{ include "dask-gateway.traefikName" . }} subjects: - kind: ServiceAccount diff --git a/resources/helm/dask-gateway/values.schema.yaml b/resources/helm/dask-gateway/values.schema.yaml index d7950579..ec1d14e3 100644 --- a/resources/helm/dask-gateway/values.schema.yaml +++ b/resources/helm/dask-gateway/values.schema.yaml @@ -26,6 +26,16 @@ properties: description: | See the description of fullnameOverride. + runInClusterScope: + type: boolean + description: | + Run the whole stack either at the cluster scope or at the namespace scope + + watchNamespace: + type: [string, "null"] + description: | + Watch only a specific namespace + gateway: type: object additionalProperties: false diff --git a/resources/helm/dask-gateway/values.yaml b/resources/helm/dask-gateway/values.yaml index 46eee0ba..623799c9 100644 --- a/resources/helm/dask-gateway/values.yaml +++ b/resources/helm/dask-gateway/values.yaml @@ -6,6 +6,16 @@ nameOverride: "" ## fullnameOverride: "" +## Run the whole stack either at the cluster scope or at the namespace scope +## +runInClusterScope: true + +## Watch only a specific namespace +## If runInClusterScope is false and watchNamespace is empty, it will watch the release namespace only +## If runInClusterScope is true, it will watch all cluster namespaces +## +watchNamespace: "" + # gateway nested config relates to the api Pod and the dask-gateway-server # running within it, the k8s Service exposing it, as well as the schedulers # (gateway.backend.scheduler) and workers gateway.backend.worker) created by the