Transmit kubernetes kubectl tokens using juju secrets

.github/workflows/tests.yaml

@@ -6,12 +6,12 @@ on:
 jobs:
   call-inclusive-naming-check:
     name: Inclusive naming
-    uses: canonical-web-and-design/Inclusive-naming/.github/workflows/woke.yaml@main
+    uses: canonical/inclusive-naming/.github/workflows/woke.yaml@main
     with:
       fail-on-error: "true"
 
   lint-unit:
     name: Lint Unit
     uses: charmed-kubernetes/workflows/.github/workflows/lint-unit.yaml@main
     with:
-      python: "['3.8', '3.9', '3.10', '3.11']"
+      python: "['3.8', '3.10', '3.12']"

ops/ops/interface_kube_control/model.py

@@ -1,5 +1,6 @@
 from pydantic import Field, AnyHttpUrl, BaseModel, Json
-from typing import List, Dict, Optional
+import json
+from typing import List, Dict, Optional, Union
 import re
 
 
@@ -45,24 +46,77 @@ def effect(self) -> str:
         return self.groups[2]
 
 
-class Creds(BaseModel):
+class AuthRequest(BaseModel):
+    """Models the requests from the requirer side of the relation."""
+
+    unit: Optional[str]  # which unit name requested tokens
+    kubelet_user: Optional[str]  # name of the user the token is granted for
+    auth_group: Optional[str]  # kubernetes group associated with the token
+    schema_vers: Json[List[int]] = Field(
+        default_factory=list
+    )  # schemas versions supported by the requirer
+
+    def dict(self, **kwds):
+        d = super().dict(**kwds)
+        if self.schema_vers:
+            d["schema_vers"] = json.dumps(self.schema_vers)
+        return d
+
+    @property
+    def user(self) -> str:
+        return self.kubelet_user
+
+    @property
+    def group(self) -> str:
+        return self.auth_group
+
+    def __lt__(self, other):
+        return (self.unit, self.kubelet_user, self.auth_group) < (
+            other.unit,
+            other.kubelet_user,
+            other.auth_group,
+        )
+
+
+class CredsV0(BaseModel):
     client_token: str
     kubelet_token: str
     proxy_token: str
     scope: str
 
 
+class CredsV1(BaseModel):
+    secret_id: str = Field(alias="secret-id")
+    scope: str
+
+    def client_token(self, model) -> str:
+        secret = model.get_secret(self.secret_id)
+        content = secret.get_content(refresh=True)
+        return content["client-token"]
+
+    def kubelet_token(self, model) -> str:
+        secret = model.get_secret(self.secret_id)
+        content = secret.get_content(refresh=True)
+        return content["kubelet-token"]
+
+    def proxy_token(self, model) -> str:
+        secret = model.get_secret(self.secret_id)
+        content = secret.get_content(refresh=True)
+        return content["proxy-token"]
+
+
 class Data(BaseModel):
     api_endpoints: Json[List[AnyHttpUrl]] = Field(alias="api-endpoints")
     cluster_tag: str = Field(alias="cluster-tag")
     cohort_keys: Optional[Json[Dict[str, str]]] = Field(alias="cohort-keys")
-    creds: Json[Dict[str, Creds]] = Field(alias="creds")
+    creds: Json[Dict[str, Union[CredsV0, CredsV1]]]
     default_cni: Json[str] = Field(alias="default-cni")
     domain: str = Field(alias="domain")
     enable_kube_dns: bool = Field(alias="enable-kube-dns")
     has_xcp: Json[bool] = Field(alias="has-xcp")
     port: Json[int] = Field(alias="port")
     sdn_ip: Optional[str] = Field(default=None, alias="sdn-ip")
     registry_location: str = Field(alias="registry-location")
-    taints: Optional[Json[List[Taint]]] = Field(alias="taints")
-    labels: Optional[Json[List[Label]]] = Field(alias="labels")
+    taints: Optional[Json[List[Taint]]] = None
+    labels: Optional[Json[List[Label]]] = None
+    schema_ver: Optional[int] = Field(0, alias="schema-ver")

ops/ops/interface_kube_control/provides.py

@@ -1,29 +1,31 @@
 import json
-from collections import namedtuple
 
+from .model import AuthRequest, Label, Taint
 from ops import CharmBase, Relation, Unit
-from .model import Creds
 from typing import List
 
-AuthRequest = namedtuple("KubeControlAuthRequest", ["unit", "user", "group"])
-
 
 class KubeControlProvides:
     """Implements the Provides side of the kube-control interface."""
 
-    def __init__(self, charm: CharmBase, endpoint: str):
+    def __init__(self, charm: CharmBase, endpoint: str = "kube-control", schemas="0,1"):
         self.charm = charm
         self.endpoint = endpoint
+        # comma-separated set of schemas to advertise support
+        # schema 0 -- same as unschema'd
+        # schema 1 -- signals support for credentials in juju secrets
+        self.schema_ver = schemas.split(",")
 
     @property
     def auth_requests(self) -> List[AuthRequest]:
         """Return a list of authentication requests from related units."""
         requests = [
-            AuthRequest(unit=unit.name, user=user, group=group)
+            request
             for relation in self.relations
             for unit in relation.units
-            if (user := relation.data[unit].get("kubelet_user"))
-            and (group := relation.data[unit].get("auth_group"))
+            if (request := AuthRequest(unit=unit.name, **relation.data[unit]))
+            and request.user
+            and request.group
         ]
         requests.sort()
         return requests
@@ -112,30 +114,39 @@ def set_image_registry(self, image_registry) -> None:
 
     def set_labels(self, labels) -> None:
         """Send the Juju config labels of the control-plane."""
-        value = json.dumps(labels)
+        labels = [str(_) for _ in labels if Label.validate(_)]
+        dedup = sorted(set(labels))
+        value = json.dumps(dedup)
         for relation in self.relations:
             relation.data[self.unit]["labels"] = value
 
     def set_taints(self, taints) -> None:
         """Send the Juju config taints of the control-plane."""
-        value = json.dumps(taints)
+        taints = [str(_) for _ in taints if Taint.validate(_)]
+        dedup = sorted(set(taints))
+        value = json.dumps(dedup)
         for relation in self.relations:
             relation.data[self.unit]["taints"] = value
 
     def sign_auth_request(
-        self, request, client_token, kubelet_token, proxy_token
+        self, request: AuthRequest, client_token, kubelet_token, proxy_token
     ) -> None:
         """Send authorization tokens to the requesting unit."""
         creds = {}
         for relation in self.relations:
             creds.update(json.loads(relation.data[self.unit].get("creds", "{}")))
-        creds[request.user] = Creds(
-            client_token=client_token,
-            kubelet_token=kubelet_token,
-            proxy_token=proxy_token,
-            scope=request.unit,
-        ).dict()
-
+        tokens = {
+            "client_token": client_token,
+            "kubelet_token": kubelet_token,
+            "proxy_token": proxy_token,
+        }
+        if 1 in request.schema_vers:
+            # Requesting unit can use schema 1, use juju secrets
+            unit = self.charm.model.get_unit(request.unit)
+            secret = relation.app.set_secret(client_token, tokens)
+            secret.grant(relation, unit=unit)
+            tokens = {"secret-id": secret.id}
+        creds[request.user] = {"scope": request.unit, **tokens}
         value = json.dumps(creds)
         for relation in self.relations:
             relation.data[self.unit]["creds"] = value

ops/ops/interface_kube_control/requires.py

@@ -8,13 +8,14 @@
 
 import base64
 import logging
+from collections import defaultdict
 from os import PathLike
 from pathlib import Path
 from typing import Optional, Mapping, List
 
 import yaml
 from backports.cached_property import cached_property
-from .model import Data, Taint, Label
+from .model import AuthRequest, CredsV0, CredsV1, Data, Taint, Label
 from pydantic import ValidationError
 
 from ops.charm import CharmBase, RelationBrokenEvent
@@ -29,9 +30,13 @@ class KubeControlRequirer(Object):
     Implements the requirer side of the kube-control interface.
     """
 
-    def __init__(self, charm: CharmBase, endpoint: str = "kube-control"):
+    def __init__(self, charm: CharmBase, endpoint: str = "kube-control", schemas="0,1"):
         super().__init__(charm, f"relation-{endpoint}")
         self.endpoint = endpoint
+        # comma-separated set of schemas to advertise support
+        # schema 0 -- same as unschema'd
+        # schema 1 -- signals support for credentials in juju secrets
+        self.schema_ver = schemas.split(",")
 
     @cached_property
     def relation(self) -> Optional[Relation]:
@@ -41,10 +46,13 @@ def relation(self) -> Optional[Relation]:
     @cached_property
     def _data(self) -> Optional[Data]:
         if self.relation and self.relation.units:
-            rx = {}
+            schema_ver, rx = 0, defaultdict(dict)
             for unit in self.relation.units:
-                rx.update(self.relation.data[unit])
-            return Data(**rx)
+                unit_data = self.relation.data[unit]
+                unit_schema = int(unit_data.pop("schema-ver", 0))
+                schema_ver = max(schema_ver, unit_schema)
+                rx[unit_schema].update(**unit_data)
+            return Data(**rx[schema_ver])
         return None
 
     def evaluate_relation(self, event) -> Optional[str]:
@@ -122,15 +130,24 @@ def get_auth_credentials(self, user) -> Optional[Mapping[str, str]]:
         if not self.is_ready:
             return None
 
-        creds = self._data.creds
+        users = self._data.creds
 
-        if user in creds:
-            return {
-                "user": user,
-                "kubelet_token": creds[user].kubelet_token,
-                "proxy_token": creds[user].proxy_token,
-                "client_token": creds[user].client_token,
-            }
+        if user in users:
+            creds = users[user]
+            if isinstance(creds, CredsV0):
+                return {
+                    "user": user,
+                    "kubelet_token": creds.kubelet_token,
+                    "proxy_token": creds.proxy_token,
+                    "client_token": creds.client_token,
+                }
+            elif isinstance(creds, CredsV1):
+                return {
+                    "user": user,
+                    "kubelet_token": creds.kubelet_token(self.model),
+                    "proxy_token": creds.proxy_token(self.model),
+                    "client_token": creds.client_token(self.model),
+                }
         return None
 
     def get_dns(self) -> Mapping[str, str]:
@@ -156,21 +173,22 @@ def dns_ready(self) -> bool:
         )
 
     def set_auth_request(self, user, group="system:nodes") -> None:
-        """Notify contol-plane that we are requesting auth.
+        """Notify control-plane that we are requesting auth.
 
         Also, use this hostname for the kubelet system account.
 
         @params user   - user requesting authentication
-        @params groups - Determines the level of eleveted privileges of the
+        @params groups - Determines the level of elevated privileges of the
                          requested user.
                          Can be overridden to request sudo level access on the
                          cluster via changing to
-                         system:masters.  #wokeignore:rule=master
+                         system:masters.  # wokeignore:rule=master
         """
         if self.relation:
-            self.relation.data[self.model.unit].update(
-                dict(kubelet_user=user, auth_group=group)
+            req = AuthRequest(
+                kubelet_user=user, auth_group=group, schemas_ver=self.schema_ver
             )
+            self.relation.data[self.model.unit].update(req.dict())
 
     def set_gpu(self, enabled=True):
         """

ops/tests/data/kube_control_data.yaml

@@ -12,11 +12,15 @@ cohort-keys: |-
   }
 creds: |-
   {
-    "test/0": {
+    "system:node:node-1": {
       "client_token": "admin::redacted",
       "kubelet_token": "test/0::redacted",
       "proxy_token": "kube-proxy::redacted", 
-      "scope": "test/0"
+      "scope": "kubernetes-worker/0"
+    },
+    "system:node:node-2": {
+      "secret-id": "abcd::1234",
+      "scope": "kubernetes-worker/1"
     }
   }
 default-cni: '""'

ops/tests/data/kube_control_request.yaml

@@ -0,0 +1,12 @@
+---
+unit: requirer/0
+data:
+  kubelet_user: system:node:node-1
+  auth_group: system:nodes
+---
+unit: requirer/1
+data:
+  kubelet_user: system:node:node-2
+  auth_group: system:nodes
+  schema_vers: '[0, 1]'
+