From b5e933b09175fedc85f1b340dfd0d7f17de55af8 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Mon, 10 Jun 2024 11:14:21 -0500 Subject: [PATCH 01/10] Beginning notes on how to reconfigure for keystone in 1.29 --- assets/keystone.yaml | 44 ++++++---- pages/k8s/ldap.md | 9 +- pages/k8s/upgrade-notes.md | 165 +++++++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 19 deletions(-) diff --git a/assets/keystone.yaml b/assets/keystone.yaml index 3bc27591..a5326f12 100644 --- a/assets/keystone.yaml +++ b/assets/keystone.yaml @@ -1,28 +1,38 @@ -series: bionic +series: jammy applications: keystone: - charm: cs:keystone + charm: keystone + channel: yoga/stable num_units: 1 options: - openstack-origin: cloud:bionic-rocky worker-multiplier: 0.25 preferred-api-version: 3 - mysql: - charm: cs:percona-cluster - num_units: 1 - options: - innodb-buffer-pool-size: 256M - max-connections: 1000 openstack-dashboard: - charm: cs:openstack-dashboard + charm: openstack-dashboard + channel: yoga/stable num_units: 1 expose: true + mysql: + charm: mysql-innodb-cluster + channel: 8.0/stable + constraints: cores=2 mem=8G root-disk=64G + num_units: 3 options: - openstack-origin: cloud:bionic-rocky + enable-binlogs: true + innodb-buffer-pool-size: 256M + max-connections: 2000 + wait-timeout: 3600 + keystone-mysql-router: + channel: 8.0/stable + charm: mysql-router + openstack-dashboard-mysql-router: + channel: 8.0/stable + charm: mysql-router + relations: -- - keystone:shared-db - - mysql:shared-db -- - openstack-dashboard:identity-service - - keystone:identity-service -- - openstack-dashboard:shared-db - - mysql:shared-db +- [openstack-dashboard:identity-service, keystone:identity-service] +- [keystone-mysql-router:db-router, mysql:db-router] +- [keystone-mysql-router:shared-db, keystone:shared-db] +- [openstack-dashboard-mysql-router:db-router, mysql:db-router] +- [openstack-dashboard-mysql-router:shared-db, openstack-dashboard:shared-db] + diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 65ebf384..334d6290 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -24,14 +24,18 @@ or both authentication and authorisation. ## Requirements -* This document assumes you have already [installed][install] **Charmed Kubernetes**. +* This document assumes you have already [installed][install] **Charmed Kubernetes** + * Support for direct LDAP integration via keystone is dropped beginning in + **Charmed Kubernetes** 1.29, while, upgrades from 1.28 are partially supported. + See [upgrading to 1.29][upgrading] for more detail. * For LDAP authentication, this documentation assumes you already have a suitable LDAP server running. * You will need to install the Keystone client. This can be done by running: ```bash - sudo snap install client-keystone-auth --edge + sudo snap install client-keystone-auth ``` + ## Install Keystone Note: These instructions assume you are working with the `Queens` release of @@ -317,6 +321,7 @@ configuring Keystone/LDAP. [keystone-bundle]: https://raw.githubusercontent.com/juju-solutions/kubernetes-docs/master/assets/keystone.yaml [docs-ldap-keystone]: https://charmhub.io/keystone-ldap [trouble]: /kubernetes/docs/troubleshooting/#troubleshooting-keystoneldap-issues +[upgrading]: /kubernetes/docs/upgrade-notes [openstack-integrator]: /kubernetes/docs/openstack-integration diff --git a/pages/k8s/upgrade-notes.md b/pages/k8s/upgrade-notes.md index 4ae8d353..2ecb461c 100644 --- a/pages/k8s/upgrade-notes.md +++ b/pages/k8s/upgrade-notes.md @@ -24,6 +24,164 @@ any of the intervening steps. There is a known issue ([https://bugs.launchpad.net/juju/+bug/1904619](https://bugs.launchpad.net/juju/+bug/1904619)) with container profiles not surviving an upgrade in clouds running on LXD. If your container-based applications fail to work properly after an upgrade, please see this [topic on the troubleshooting page](/kubernetes/docs/troubleshooting#charms-deployed-to-lxd-containers-fail-after-upgradereboot) + + +## Upgrading to 1.29 + +There are several important changes starting in 1.29 that will effect all users: + + - `kubeapi-load-balancer`, `kubernetes-control-plane`, and `kubernetes-worker` charms + can be observed using the COS rather than LMA. + - Dropped specific relations and features which are outsourced to other charms + +### Observability Relations + +These represent relations which were slated to be removed in favour of observability with the COS. + +LMA Relations: +* `nrpe-external-master` (provides: `nrpe-external-master` on KCP and KW) +* `prometheus` (provides: `prometheus-manual` on KCP) +* `scrape` (provides: `prometheus` on KW) +* `grafana` (provides: `grafana-dashboard` ) + +In order to prepare for observability, see the [Integration with COS Lite docs][cos] which can be +performed following an upgrade of the charms but prior to upgrade of the kubernetes cluster. + +### kube-api-endpoint relation dropped + +The `kubernetes-control-plane:kube-api-endpoint` and `kubernetes-worker:kube-api-endpoint` relations +have been removed since these apis are are provided by the kube-control relation. Ensure these two +apps are linked by `kube-control` relation before removing this relation. + +``` +juju integrate kubernetes-control-plane:kube-control kubernetes-worker:kube-control +juju remove-relation kubernetes-control-plane:kube-api-endpoint kubernetes-worker:kube-api-endpoint +``` + +### loadbalancer relation dropped + +The `kubernetes-control-plane:loadbalancer` relation has been removed in favour of using +the `loadbalancer-internal` and `loadbalancer-external` relations. + +``` +juju integrate kubernetes-control-plane:loadbalancer-internal kubeapi-loadbalancer +juju integrate kubernetes-control-plane:loadbalancer-external kubeapi-loadbalancer +juju remove-relation kubernetes-control-plane:loadbalancer kubeapi-loadbalancer +``` + +### ceph-client relation deprecated + +The `kubernetes-control-plane:ceph-client` relation is being deprecated. + +Investment in ceph integration continues, but in the `ceph-csi` charm +which integrates ceph with kubernetes. + +After upgrading the kubernetes-control-plane charm, the charm +may enter `blocked` status with the message: +`ceph-client relation deprecated, use ceph-csi charm instead`. + +If you see this message, you can resolve it by removing the ceph-client +relation: + +``` +juju deploy ceph-csi +juju integrate ceph-csi kubernetes-control-plane +juju integrate ceph-csi ceph-mon +juju remove-relation kubernetes-control-plane:ceph-client ceph-mon +``` + +### Keystone/K8s Authentication management + +Charmed Kubernetes was installing and managing an older version of +keystone auth which attempts to manage authentication and authorization +through keystone. + +This service is better suited to be managed externally from the +kubernetes-control-plane charm. However, the charm will provides this +upgrade method to maintain the deployment of this service beyond 1.28. + +One can determine if keystone managements is applicable with +``` +juju status --relations | grep kubernetes-control-plane:keystone-credentials +``` +If this is empty, nothing regarding keystone management is required. + +If this states: +``` +juju status --relations | grep kubernetes-control-plane:keystone-credentials +keystone:identity-credentials kubernetes-control-plane:keystone-credentials keystone-credentials regular +``` +then you'll need to prepare a bit before the upgrade. + +#### Resources + +One should familiarize themself with k8s-keystone auth via the [upstream docs][keystone-auth] + +Keystone has two "Auth" options: +1) Authentication of users only called [keystone-authentication][] +2) Authentication and Authorization of users called [keystone-authorization][] + +Either option requires the deployment and management of the [k8s-keystone-auth webhook service][keystone-auth-webhook], +A deployment and service to act as an agent between keystone and the kubernetes-api-server. This deployment +provides a service endpoint for the kubernetes-api-server to use to interact with an external keystone service. + +#### Preparation +Starting in 1.29 the kubernetes-control-plane charm will drop the following: + +* `kubernetes-control-plane:keystone-credentials` relation +* `keystone-policy` config +* `enable-keystone-authorization` config +* `keystone-ssl-ca` config + +Before upgrading, it is important to capture the state of these config options + +``` +mkdir keystone-upgrade +juju config kubernetes-control-plane keystone-policy > keystone-upgrade/keystone-policy.yaml +juju config kubernetes-control-plane enable-keystone-authorization > keystone-upgrade/keystone-authorization +juju config kubernetes-control-plane keystone-ssl-ca | base64 -d > keystone-upgrade/keystone-webhook-ca.crt +juju exec kubernetes-control-plane/leader -- 'cat /root/cdk/keystone/webhook.yaml' > keystone-upgrade/webhook.yaml +``` + +After upgrading, the charm will enter a`blocked` state with the status message: +`Keystone credential relation is no longer managed`. +which will indicate that `k8s-keystone-auth` webhook service is still running, but is no longer managed. + +``` +# Add the keystone authorization webhook config and the `Webhook` authorization mode +juju config kubernetes-control-plane \ + authorization-webhook-config-file="$(cat keystone-upgrade/webhook.yaml)" \ + authorization-mode="Node,RBAC,Webhook" +# Acknowledge the charm no longer manages keystone +juju remove-relation kubernetes-control-plane:keystone-credentials keystone +``` + +### Administrative Actions missing +The `kubernetes-control-plane` and `kubernetes-worker` actions list was substantially reduced +during development of 1.29. The following are no longer present, but are slated to be reintroduced. +* `restart` +* `namespace-list` +* `namespace-create` +* `namespace-delete` +* `user-create` +* `user-delete` +* `user-list` +* `apply-manifest` + +### CIS-Benchmark Action missing + +The `kubernetes-control-plane` and `kubernetes-worker` action for cis-benchmark were +removed during the development of the 1.29 charms and an engineering decision to reintroduce +these actions are on-going, but development and testing incomplete. Details in [LP#2044219][] + +### Automatic labelling of GPU nodes + +While current worker nodes would remain unaffected as they would already be labeled, the worker +charm in 1.29 no longer labels the nodes with `gpu=true` and `cuda=true`. + +Parity with this feature has been attained by using the [nvidia-gpu-operator][] + + ## Upgrading to 1.24 @@ -382,6 +540,13 @@ You can now proceed with the rest of the upgrade. [dns-provider-config]: https://github.com/juju-solutions/kubernetes/blob/5f4868af82705a0636680a38d7f3ea760d35dadb/cluster/juju/layers/kubernetes-master/config.yaml#L58-L67 [docker-page]: https://jaas.ai/u/containers/docker#configuration [inclusive-naming]: /kubernetes/docs/inclusive-naming +[LP#2044219]: https://bugs.launchpad.net/charm-kubernetes-master/+bug/2044219 +[cos]: kubernetes/docs/how-to-cos-lite +[nvidia-gpu-operator]: https://charmhub.io/nvidia-gpu-operator +[keystone-auth]: https://github.com/kubernetes/cloud-provider-openstack/tree/master/docs/keystone-auth +[keystone-auth-webhook]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#k8s-keystone-auth +[keystone-authentication]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-auth-data-synchronization.md#full-example-using-keystone-for-authentication-and-kubernetes-rbac-for-authorization +[keystone-authorization]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#authorization-policy-definitionversion-2
From 1ff17b5a3beed3adbe30405d877e66be1c7e0695 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 11 Jun 2024 10:23:54 -0500 Subject: [PATCH 02/10] Correct command mistakes in keystone upgrade nodes, add Day2 Ops section --- pages/k8s/upgrade-notes.md | 42 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/pages/k8s/upgrade-notes.md b/pages/k8s/upgrade-notes.md index 2ecb461c..43c5f0f7 100644 --- a/pages/k8s/upgrade-notes.md +++ b/pages/k8s/upgrade-notes.md @@ -93,14 +93,14 @@ juju remove-relation kubernetes-control-plane:ceph-client ceph-mon ### Keystone/K8s Authentication management Charmed Kubernetes was installing and managing an older version of -keystone auth which attempts to manage authentication and authorization +keystone-auth which manages authentication and authorization through keystone. This service is better suited to be managed externally from the -kubernetes-control-plane charm. However, the charm will provides this +kubernetes-control-plane charm. However, the charm provides the following upgrade method to maintain the deployment of this service beyond 1.28. -One can determine if keystone managements is applicable with +One can determine if keystone management is applicable with ``` juju status --relations | grep kubernetes-control-plane:keystone-credentials ``` @@ -121,9 +121,9 @@ Keystone has two "Auth" options: 1) Authentication of users only called [keystone-authentication][] 2) Authentication and Authorization of users called [keystone-authorization][] -Either option requires the deployment and management of the [k8s-keystone-auth webhook service][keystone-auth-webhook], -A deployment and service to act as an agent between keystone and the kubernetes-api-server. This deployment -provides a service endpoint for the kubernetes-api-server to use to interact with an external keystone service. +Both options require the deployment and management of the [k8s-keystone-auth webhook service][keystone-auth-webhook], +a deployment which provides a service endpoint for the kubernetes-api-server to use +as an intermediate to interact with an external keystone service. #### Preparation Starting in 1.29 the kubernetes-control-plane charm will drop the following: @@ -140,22 +140,44 @@ mkdir keystone-upgrade juju config kubernetes-control-plane keystone-policy > keystone-upgrade/keystone-policy.yaml juju config kubernetes-control-plane enable-keystone-authorization > keystone-upgrade/keystone-authorization juju config kubernetes-control-plane keystone-ssl-ca | base64 -d > keystone-upgrade/keystone-webhook-ca.crt -juju exec kubernetes-control-plane/leader -- 'cat /root/cdk/keystone/webhook.yaml' > keystone-upgrade/webhook.yaml +juju exec -u kubernetes-control-plane/leader -- 'cat /root/cdk/keystone/webhook.yaml' > keystone-upgrade/webhook.yaml ``` -After upgrading, the charm will enter a`blocked` state with the status message: +#### Migration + +After upgrading, the charm will enter a `blocked` state with the status message: `Keystone credential relation is no longer managed`. -which will indicate that `k8s-keystone-auth` webhook service is still running, but is no longer managed. +which indicates that the `k8s-keystone-auth` webhook service is still running, but is no longer managed. +if `keystone-upgrade/keystone-authorization` contains `true`, then after the upgrade one should +enable the webhook. as part of Keystone-Auth option 2. ``` # Add the keystone authorization webhook config and the `Webhook` authorization mode juju config kubernetes-control-plane \ authorization-webhook-config-file="$(cat keystone-upgrade/webhook.yaml)" \ authorization-mode="Node,RBAC,Webhook" -# Acknowledge the charm no longer manages keystone +``` + +Finally, acknowledge the charm no longer manages keystone. +``` juju remove-relation kubernetes-control-plane:keystone-credentials keystone ``` +#### Day 2 Operations + +After migration, the deployment, service, secrets, and policies associated with +keystone-auth are no longer handled by the `kubernetes-control-plane` charm. + +The following components remain in the cluster unmanaged by the charm, and +should be considered managed by the cluster administrators. + +* `Deployment/kube-system/k8s-keystone-auth` +* `Service/kube-system/k8s-keystone-auth-service` +* `Secret/kube-system/keystone-auth-certs` +* `ConfigMap/kube-system/k8s-auth-policy` +* `ClusterRole/k8s-keystone-auth` + + ### Administrative Actions missing The `kubernetes-control-plane` and `kubernetes-worker` actions list was substantially reduced during development of 1.29. The following are no longer present, but are slated to be reintroduced. From f522270b9aa15b6068af722c2d4a5c9e33b6249d Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 13 Jun 2024 16:16:51 -0500 Subject: [PATCH 03/10] Add installation instructions to use LDAP or Keystone Auth for fresh installations --- pages/k8s/ldap.md | 188 ++++++++++++++++++++++++---------------------- 1 file changed, 100 insertions(+), 88 deletions(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 334d6290..1965df33 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -38,8 +38,8 @@ or both authentication and authorisation. ## Install Keystone -Note: These instructions assume you are working with the `Queens` release of -**OpenStack**, the default supported version for Ubuntu 18.04 (Bionic) +Note: These instructions assume you are working with the `Yoga` release of +**OpenStack**, the default supported version for Ubuntu 22.04 (Jammy) Keystone should be deployed using **Juju**. This is easily achieved by using a bundle, which will deploy and relate, Keystone, the OpenStack dashboard and a suitable @@ -51,13 +51,6 @@ Deploy the bundle with the following command: juju deploy ./keystone.yaml ``` -You should now add a relation for the kubernetes-control-plane nodes to accept Keystone -credentials: - -```bash -juju integrate keystone:identity-credentials kubernetes-control-plane:keystone-credentials -``` - You can check that the new applications have deployed and are running with: ```bash @@ -85,46 +78,7 @@ juju unexpose openstack-dashboard If you have an existing Keystone application deployed as part of OpenStack in a separate Juju model, it is possible to re-use it for authenticating and authorising users in Kubernetes. -To do so, first deploy the [openstack-integrator charm][openstack-integrator] - -```bash -juju deploy openstack-integrator -``` - -Use 'juju trust' to grant openstack-integrator a permission to access the OpenStack model, -or configure the credentials config parameter manually - -```bash -juju trust openstack-integrator -``` - -Finally add a relation between `kubernetes-control-plane` and `openstack-integrator` - -```bash -juju integrate kubernetes-control-plane:keystone-credentials openstack-integrator:credentials -``` - -## Fetch the Keystone script - -When related to Keystone directly (or to the `openstack-integrator:keystone-credentials` interface), -the Kubernetes master application will generate a utility script. -This should be copied to the local client with: - -```bash -juju scp kubernetes-control-plane/0:kube-keystone.sh ~/kube-keystone.sh -``` - -The file will need to be edited to replace the value for `OS_AUTH_URL`, which should -point at the public address for Keystone, and the username if different. At this point the -file should be sourced: - -```bash -source ~/kube-keystone.sh -``` - -The script should prompt you to enter an additional command to retrieve the token to -login to the OpenStack Dashboard. If this step fails, check that the details in the -`kube-keystone.sh` file are correct. +No extra steps are needed, other than the credentials to access that openstack deployment ## Access the OpenStack dashboard @@ -182,50 +136,104 @@ Now ensure the user is added to the project created above. ![dashboard image](https://assets.ubuntu.com/v1/d6149d7c-ldap5.png) +## Deploying the Keystone-Auth Webhook for Kubernetes + +### Understanding the Resources + +Following the upstream docs for [keystone-auth][], the admin should deploy the keystone-auth. +The follow components are key for authentication and authorization. + +* `Secret/keystone-auth-certs` + * provides the TLS cert/key pair for serving the keystone-auth webhook service + * provides the TLS ca cert for contacting keystone (if necessary) +* `ConfigMap/k8s-auth-policy` or `ConfigMap/keystone-sync-policy` + * Configuration for the deployment which translates keystone users/roles into kubernetes users/roles +* `Deployment/k8s-keystone-auth` + * defines the PODs backing this service + * defines the image used in the service + * defines the secrets for the service + * defines the configuration for the service + * the `sync-configmap-name` for keystone auth, and kubernetes-rbac for authorization + * the `policy-configmap-name` for keystone auth and keystone roles +* `ServiceAccount/k8s-keystone`, `ClusterRole/k8s-keystone-auth` and `ClusterRoleBinding/k8s-keystone-auth` + * RBAC rules applied to the deployment to access the cluster `ConfigMap` +* `Service/k8s-keystone-auth-service` + * Service mapping for the above `Deployment/k8s-keystone-auth`. + +### Setting up the Resources + +The follow adjustments are required to deploy the service + +* `Secret/keystone-auth-certs` + * requires the admin to generate a server cert/key pair for the service + * requires the admin to provide the ca cert for the Keystone TLS endpoint (if required) +* `ConfigMap/k8s-auth-policy` (Optional) + * Definitions for mapping keystone user/project/domain/roles to kubernetes endpoints + * See [keystone-authz-policy][] for details +* `ConfigMap/keystone-sync-policy` (Optional) + * Definitions for mapping keystone user/project/domain/roles to kubernetes endpoints + * See [keystone-authn-policy][] for details +* `Deployment/k8s-keystone-auth` + * Requires arg `keystone-ca-file` if `keystone-url` is `https` + * Requires arg `policy-configmap-name` or `sync-configmap-name` + * Requires secret volume mapping for the `tls.crt` and `tls.key` + +The following adjustments are required to prepare the api server +to use the authn endpoint (for authn and authz) and the authz webhook endpoint (for authz). + +* `authn-webhook-endpoint` + **Required** for Authentication and Authorization + + The api server requires the service endpoint to use as a custom authn endpoint. Once + applied to the cluster, the `Service/k8s-keystone-auth-service` should have a `ClusterIP` + which will be used as the `authn-webhook-endpoint`. + + ``` + SVC_IP=$(kubectl get svc -n kube-system k8s-keystone-auth-service -o json | jq -r '.spec.clusterIP') + juju config kubernetes-control-plane authn-webhook-endpoint="https://${SVC_IP}:8443/webhook" + ``` +* `authz-webhook-endpoint` + **Required** only for Authorization + + The api server requires the service endpoint in the `authorization-webhook-config-file`. + Also, to use this config, the `authorization-mode` must add the `Webhook` mode. + + The crafting of this `webhook-config.yaml` is defined at in the [keystone examples][keystone-webhook-config] + based on the format defined in the [kubernetes reference docs][webhook-config] + + ``` + # prepare webhook-config.yaml using the SVC_IP from above + juju config kubernetes-control-plane authorization-webhook-config-file=$(cat webhook-config.yaml) + juju config kubernetes-control-plane authorization-mode="Node,RBAC,Webhook" + ``` + ## Using kubectl with Keystone At this point, Keystone is set up and we have a domain, project, and user -created in Keystone. With the updated config file copied above in -`~/.kube/config`, we can use `kubectl` to authenticate with the api server -via a token from Keystone. The `client-keystone-auth` snap will automate -retrieving a token for us using the environment variables common to -OpenStack such as `OS_USERNAME`. These environment variables are exported in -the `kube-keystone.sh` script we downloaded earlier. To use it, update the -variables in `kube-keystone.sh` to match valid user credentials. Pay -special attention to the `OS_AUTH_URL` variable and ensure it is using an -IP address that is reachable from the client. Source that file into -your environment with `source ./kube-keystone.sh`. Any credentials that -are not supplied via environment variable are queried at run-time for -each invocation of kubectl. - -## Using Keystone with the kubernetes-dashboard - -When using Keystone with Kubernetes, the Kubernetes dashboard is -updated by the charms to use token authentication. This means that a token -from Keystone is required to log in to the Kubernetes dashboard. There is -currently no way to automate this, but the `kube-keystone.sh` file includes -a function called `get_keystone_token`, which uses the `OS_` environment -variables in order to retrieve a token from Keystone. +created in Keystone. + +The authenticating user will need an updated kubeconfig in order to +authenticate with the cluster. One can use `kubectl` to authenticate +with the api server via a token from Keystone. The `client-keystone-auth` +snap automates retrieving a token. + +See the [Client configuration][keystone-client-config] to in order to create +the kubeconfig to use against the keystone server. + +The client will require the `client-keystone-auth` binary to use this config, +which can be installed using -```bash -source ~/bin/kube-keystone.sh -``` -``` -Function get_keystone_token created. Type get_keystone_token in order to -generate a login token for the Kubernetes dashboard. -``` -Enter the command... -```bash -get_keystone_token -``` -...and a token will be generated: ``` -ccf9b218845f4d67835f8c6a7c2d1cd4 +snap install client-keystone-auth ``` -This token can then be used to log in to the Kubernetes dashboard. - -![dashboard image](https://assets.ubuntu.com/v1/4b79b35c-token-login.png) +One will require the following variables +* `OS_USERNAME` +* `OS_PASSWORD` +* `OS_PROJECT_NAME` +* `OS_DOMAIN_NAME` +* `keystone-url` +* `keystone-ca-file` if `keystone-url` is `https` ## LDAP via Keystone @@ -322,8 +330,12 @@ configuring Keystone/LDAP. [docs-ldap-keystone]: https://charmhub.io/keystone-ldap [trouble]: /kubernetes/docs/troubleshooting/#troubleshooting-keystoneldap-issues [upgrading]: /kubernetes/docs/upgrade-notes -[openstack-integrator]: /kubernetes/docs/openstack-integration - +[keystone-auth]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-client-keystone-auth.md +[keystone-authz-policy]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#prepare-the-authorization-policy-optional +[keystone-authn-policy]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-auth-data-synchronization.md +[keystone-client-config]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#clientkubectl-configuration +[keystone-webhook-config]: https://github.com/kubernetes/cloud-provider-openstack/blob/release-1.30/examples/webhook/keystone-apiserver-webhook.yaml +[webhook-config]: https://kubernetes.io/docs/reference/access-authn-authz/webhook/
From 19fb0c9419f5a2c2bb63f12c9a998e823796f286 Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Thu, 13 Jun 2024 22:43:58 +0100 Subject: [PATCH 04/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 1965df33..5d22bcca 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -39,7 +39,7 @@ or both authentication and authorisation. ## Install Keystone Note: These instructions assume you are working with the `Yoga` release of -**OpenStack**, the default supported version for Ubuntu 22.04 (Jammy) +**OpenStack**, the default supported version for Ubuntu 22.04 LTS (Jammy) Keystone should be deployed using **Juju**. This is easily achieved by using a bundle, which will deploy and relate, Keystone, the OpenStack dashboard and a suitable From abf3c8cda3b4b297cfc2dc3083ae61136cc2e7fb Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Thu, 13 Jun 2024 22:44:45 +0100 Subject: [PATCH 05/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 5d22bcca..bf8c2a4b 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -78,7 +78,7 @@ juju unexpose openstack-dashboard If you have an existing Keystone application deployed as part of OpenStack in a separate Juju model, it is possible to re-use it for authenticating and authorising users in Kubernetes. -No extra steps are needed, other than the credentials to access that openstack deployment +No extra steps are needed, other than the credentials to access that OpenStack deployment ## Access the OpenStack dashboard From d80a4c9128eeaf87b64db93d00c4e1cdc3fbbbab Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Fri, 14 Jun 2024 12:47:04 +0100 Subject: [PATCH 06/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index bf8c2a4b..bdaf69fb 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -25,7 +25,7 @@ or both authentication and authorisation. ## Requirements * This document assumes you have already [installed][install] **Charmed Kubernetes** - * Support for direct LDAP integration via keystone is dropped beginning in + * Support for direct LDAP integration via Keystone is dropped beginning in **Charmed Kubernetes** 1.29, while, upgrades from 1.28 are partially supported. See [upgrading to 1.29][upgrading] for more detail. * For LDAP authentication, this documentation assumes you already have a suitable LDAP From 811de6271730c4ba15545cd418da43500c5b2859 Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Fri, 14 Jun 2024 12:48:13 +0100 Subject: [PATCH 07/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index bdaf69fb..c04089cc 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -140,7 +140,7 @@ Now ensure the user is added to the project created above. ### Understanding the Resources -Following the upstream docs for [keystone-auth][], the admin should deploy the keystone-auth. +Following the upstream docs for [keystone-auth][], the admin should deploy `keystone-auth`. The follow components are key for authentication and authorization. * `Secret/keystone-auth-certs` From 61af02e991add68315e0f49121116d03edb70389 Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Fri, 14 Jun 2024 12:48:40 +0100 Subject: [PATCH 08/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index c04089cc..7d5d88a2 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -144,7 +144,7 @@ Following the upstream docs for [keystone-auth][], the admin should deploy `keys The follow components are key for authentication and authorization. * `Secret/keystone-auth-certs` - * provides the TLS cert/key pair for serving the keystone-auth webhook service + * provides the TLS cert/key pair for serving the `keystone-auth` webhook service * provides the TLS ca cert for contacting keystone (if necessary) * `ConfigMap/k8s-auth-policy` or `ConfigMap/keystone-sync-policy` * Configuration for the deployment which translates keystone users/roles into kubernetes users/roles From 17548d25d20addc91f6e9934cd6956da0ae136f1 Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Fri, 14 Jun 2024 12:49:07 +0100 Subject: [PATCH 09/10] Update pages/k8s/ldap.md --- pages/k8s/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 7d5d88a2..7a62c461 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -162,7 +162,7 @@ The follow components are key for authentication and authorization. ### Setting up the Resources -The follow adjustments are required to deploy the service +The following adjustments are required to deploy the service: * `Secret/keystone-auth-certs` * requires the admin to generate a server cert/key pair for the service From 91be1781ba6b59a6715f2cfd129592a1a1ecd63e Mon Sep 17 00:00:00 2001 From: Nick Veitch Date: Fri, 14 Jun 2024 13:17:44 +0100 Subject: [PATCH 10/10] docs update --- pages/k8s/ldap.md | 57 +++++++++-------- pages/k8s/upgrade-notes.md | 127 +++++++++++++++++++++---------------- 2 files changed, 101 insertions(+), 83 deletions(-) diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 7a62c461..de184080 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -141,20 +141,20 @@ Now ensure the user is added to the project created above. ### Understanding the Resources Following the upstream docs for [keystone-auth][], the admin should deploy `keystone-auth`. -The follow components are key for authentication and authorization. +The following components are key for authentication and authorisation. * `Secret/keystone-auth-certs` * provides the TLS cert/key pair for serving the `keystone-auth` webhook service * provides the TLS ca cert for contacting keystone (if necessary) * `ConfigMap/k8s-auth-policy` or `ConfigMap/keystone-sync-policy` - * Configuration for the deployment which translates keystone users/roles into kubernetes users/roles + * Configuration for the deployment which translates Keystone users/roles into Kubernetes users/roles * `Deployment/k8s-keystone-auth` * defines the PODs backing this service * defines the image used in the service * defines the secrets for the service * defines the configuration for the service - * the `sync-configmap-name` for keystone auth, and kubernetes-rbac for authorization - * the `policy-configmap-name` for keystone auth and keystone roles + * the `sync-configmap-name` for `keystone-auth`, and `kubernetes-rbac` for authorisation + * the `policy-configmap-name` for `keystone-auth` and Keystone roles * `ServiceAccount/k8s-keystone`, `ClusterRole/k8s-keystone-auth` and `ClusterRoleBinding/k8s-keystone-auth` * RBAC rules applied to the deployment to access the cluster `ConfigMap` * `Service/k8s-keystone-auth-service` @@ -168,41 +168,43 @@ The following adjustments are required to deploy the service: * requires the admin to generate a server cert/key pair for the service * requires the admin to provide the ca cert for the Keystone TLS endpoint (if required) * `ConfigMap/k8s-auth-policy` (Optional) - * Definitions for mapping keystone user/project/domain/roles to kubernetes endpoints + * Definitions for mapping keystone user/project/domain/roles to Kubernetes endpoints * See [keystone-authz-policy][] for details * `ConfigMap/keystone-sync-policy` (Optional) - * Definitions for mapping keystone user/project/domain/roles to kubernetes endpoints + * Definitions for mapping keystone user/project/domain/roles to Kubernetes endpoints * See [keystone-authn-policy][] for details * `Deployment/k8s-keystone-auth` * Requires arg `keystone-ca-file` if `keystone-url` is `https` * Requires arg `policy-configmap-name` or `sync-configmap-name` * Requires secret volume mapping for the `tls.crt` and `tls.key` -The following adjustments are required to prepare the api server -to use the authn endpoint (for authn and authz) and the authz webhook endpoint (for authz). +The following adjustments are required to prepare the API server to use the +authentication endpoint (for both authentication and authorisation) and the +authorisation webhook endpoint. * `authn-webhook-endpoint` - **Required** for Authentication and Authorization + **Required** for Authentication and Authorisation - The api server requires the service endpoint to use as a custom authn endpoint. Once - applied to the cluster, the `Service/k8s-keystone-auth-service` should have a `ClusterIP` - which will be used as the `authn-webhook-endpoint`. + The API server requires the service endpoint to use as a custom + authentication endpoint. Once applied to the cluster, the + `Service/k8s-keystone-auth-service` should have a `ClusterIP` which will be + used as the `authn-webhook-endpoint`. ``` SVC_IP=$(kubectl get svc -n kube-system k8s-keystone-auth-service -o json | jq -r '.spec.clusterIP') juju config kubernetes-control-plane authn-webhook-endpoint="https://${SVC_IP}:8443/webhook" ``` * `authz-webhook-endpoint` - **Required** only for Authorization + **Required** only for Authorisation - The api server requires the service endpoint in the `authorization-webhook-config-file`. + The API server requires the service endpoint in the `authorization-webhook-config-file`. Also, to use this config, the `authorization-mode` must add the `Webhook` mode. - The crafting of this `webhook-config.yaml` is defined at in the [keystone examples][keystone-webhook-config] - based on the format defined in the [kubernetes reference docs][webhook-config] + The crafting of this `webhook-config.yaml` is defined at in the [Keystone examples][keystone-webhook-config] + based on the format defined in the [Kubernetes reference docs][webhook-config] + First prepare `webhook-config.yaml` using the SVC_IP from above. Then: ``` - # prepare webhook-config.yaml using the SVC_IP from above juju config kubernetes-control-plane authorization-webhook-config-file=$(cat webhook-config.yaml) juju config kubernetes-control-plane authorization-mode="Node,RBAC,Webhook" ``` @@ -210,7 +212,7 @@ to use the authn endpoint (for authn and authz) and the authz webhook endpoint ( ## Using kubectl with Keystone At this point, Keystone is set up and we have a domain, project, and user -created in Keystone. +created in Keystone. The authenticating user will need an updated kubeconfig in order to authenticate with the cluster. One can use `kubectl` to authenticate @@ -218,7 +220,7 @@ with the api server via a token from Keystone. The `client-keystone-auth` snap automates retrieving a token. See the [Client configuration][keystone-client-config] to in order to create -the kubeconfig to use against the keystone server. +the kubeconfig to use against the Keystone server. The client will require the `client-keystone-auth` binary to use this config, which can be installed using @@ -227,13 +229,14 @@ which can be installed using snap install client-keystone-auth ``` -One will require the following variables -* `OS_USERNAME` -* `OS_PASSWORD` -* `OS_PROJECT_NAME` -* `OS_DOMAIN_NAME` -* `keystone-url` -* `keystone-ca-file` if `keystone-url` is `https` +The following variables will need to be set: + +- `OS_USERNAME` +- `OS_PASSWORD` +- `OS_PROJECT_NAME` +- `OS_DOMAIN_NAME` +- `keystone-url` +- `keystone-ca-file` if `keystone-url` is `https` ## LDAP via Keystone @@ -277,7 +280,7 @@ other methods such as RBAC for authorisation but using Keystone for authenticati usernames will come from Keystone, but what they can do in the cluster is controlled by another system. -In order to enable authorization feature in **Charmed Kubernetes** one should change the default config +In order to enable authorisation feature in **Charmed Kubernetes** , change the default config of the charm and switch to **RBAC** authorization mode as follows: ```bash diff --git a/pages/k8s/upgrade-notes.md b/pages/k8s/upgrade-notes.md index 43c5f0f7..54ffe3d7 100644 --- a/pages/k8s/upgrade-notes.md +++ b/pages/k8s/upgrade-notes.md @@ -39,19 +39,22 @@ There are several important changes starting in 1.29 that will effect all users: These represent relations which were slated to be removed in favour of observability with the COS. LMA Relations: + * `nrpe-external-master` (provides: `nrpe-external-master` on KCP and KW) * `prometheus` (provides: `prometheus-manual` on KCP) * `scrape` (provides: `prometheus` on KW) * `grafana` (provides: `grafana-dashboard` ) -In order to prepare for observability, see the [Integration with COS Lite docs][cos] which can be -performed following an upgrade of the charms but prior to upgrade of the kubernetes cluster. +In order to prepare for observability, see the [Integration with COS Lite +docs][cos] which can be performed following an upgrade of the charms but prior +to upgrade of the kubernetes cluster. ### kube-api-endpoint relation dropped -The `kubernetes-control-plane:kube-api-endpoint` and `kubernetes-worker:kube-api-endpoint` relations -have been removed since these apis are are provided by the kube-control relation. Ensure these two -apps are linked by `kube-control` relation before removing this relation. +The `kubernetes-control-plane:kube-api-endpoint` and +`kubernetes-worker:kube-api-endpoint` relations have been removed since these +APIs are are provided by the kube-control relation. Ensure these two apps are +linked by `kube-control` relation before removing this relation. ``` juju integrate kubernetes-control-plane:kube-control kubernetes-worker:kube-control @@ -60,8 +63,8 @@ juju remove-relation kubernetes-control-plane:kube-api-endpoint kubernetes-worke ### loadbalancer relation dropped -The `kubernetes-control-plane:loadbalancer` relation has been removed in favour of using -the `loadbalancer-internal` and `loadbalancer-external` relations. +The `kubernetes-control-plane:loadbalancer` relation has been removed in favour +of using the `loadbalancer-internal` and `loadbalancer-external` relations. ``` juju integrate kubernetes-control-plane:loadbalancer-internal kubeapi-loadbalancer @@ -73,10 +76,10 @@ juju remove-relation kubernetes-control-plane:loadbalancer kubeapi-loadbalancer The `kubernetes-control-plane:ceph-client` relation is being deprecated. -Investment in ceph integration continues, but in the `ceph-csi` charm -which integrates ceph with kubernetes. +Investment in Ceph integration continues, but in the `ceph-csi` charm +which integrates Ceph with Kubernetes. -After upgrading the kubernetes-control-plane charm, the charm +After upgrading the `kubernetes-control-plane` charm, the charm may enter `blocked` status with the message: `ceph-client relation deprecated, use ceph-csi charm instead`. @@ -93,47 +96,51 @@ juju remove-relation kubernetes-control-plane:ceph-client ceph-mon ### Keystone/K8s Authentication management Charmed Kubernetes was installing and managing an older version of -keystone-auth which manages authentication and authorization +keystone-auth which manages authentication and authorisation through keystone. This service is better suited to be managed externally from the -kubernetes-control-plane charm. However, the charm provides the following +`kubernetes-control-plane` charm. However, the charm provides the following upgrade method to maintain the deployment of this service beyond 1.28. -One can determine if keystone management is applicable with +One can determine if Keystone management is applicable with + ``` juju status --relations | grep kubernetes-control-plane:keystone-credentials ``` -If this is empty, nothing regarding keystone management is required. + +If this is empty, no steps regarding Keystone management are required. If this states: + ``` -juju status --relations | grep kubernetes-control-plane:keystone-credentials keystone:identity-credentials kubernetes-control-plane:keystone-credentials keystone-credentials regular ``` -then you'll need to prepare a bit before the upgrade. + +...then you'll need to prepare a bit before the upgrade. #### Resources -One should familiarize themself with k8s-keystone auth via the [upstream docs][keystone-auth] +One should familiarise themself with k8s-keystone auth via the [upstream docs][keystone-auth] Keystone has two "Auth" options: 1) Authentication of users only called [keystone-authentication][] -2) Authentication and Authorization of users called [keystone-authorization][] +2) Authentication and authorisation of users, called [keystone-authorization][] Both options require the deployment and management of the [k8s-keystone-auth webhook service][keystone-auth-webhook], -a deployment which provides a service endpoint for the kubernetes-api-server to use -as an intermediate to interact with an external keystone service. +a deployment which provides a service endpoint for the `kubernetes-api-server` to use +as an intermediate to interact with an external Keystone service. #### Preparation -Starting in 1.29 the kubernetes-control-plane charm will drop the following: -* `kubernetes-control-plane:keystone-credentials` relation -* `keystone-policy` config -* `enable-keystone-authorization` config -* `keystone-ssl-ca` config +Starting from version 1.29, the `kubernetes-control-plane` charm will drop the following: -Before upgrading, it is important to capture the state of these config options +- `kubernetes-control-plane:keystone-credentials` relation +- `keystone-policy` config +- `enable-keystone-authorization` config +- `keystone-ssl-ca` config + +Before upgrading, it is important to capture the state of these config options: ``` mkdir keystone-upgrade @@ -145,20 +152,23 @@ juju exec -u kubernetes-control-plane/leader -- 'cat /root/cdk/keystone/webhook. #### Migration -After upgrading, the charm will enter a `blocked` state with the status message: -`Keystone credential relation is no longer managed`. -which indicates that the `k8s-keystone-auth` webhook service is still running, but is no longer managed. +After upgrading, the charm will enter a `blocked` state with the status +message: `Keystone credential relation is no longer managed`. This indicates +that the `k8s-keystone-auth` webhook service is still running, but is no longer +managed. -if `keystone-upgrade/keystone-authorization` contains `true`, then after the upgrade one should -enable the webhook. as part of Keystone-Auth option 2. -``` -# Add the keystone authorization webhook config and the `Webhook` authorization mode +If `keystone-upgrade/keystone-authorization` contains `true`, then the webhook +should be enabled. This command adds the Keystone authorisation webhook config +and the `Webhook` authorisation mode: + +``` juju config kubernetes-control-plane \ authorization-webhook-config-file="$(cat keystone-upgrade/webhook.yaml)" \ authorization-mode="Node,RBAC,Webhook" ``` -Finally, acknowledge the charm no longer manages keystone. +Finally, acknowledge the charm no longer manages keystone by removing the relation: + ``` juju remove-relation kubernetes-control-plane:keystone-credentials keystone ``` @@ -166,40 +176,45 @@ juju remove-relation kubernetes-control-plane:keystone-credentials keystone #### Day 2 Operations After migration, the deployment, service, secrets, and policies associated with -keystone-auth are no longer handled by the `kubernetes-control-plane` charm. +`keystone-auth` are no longer handled by the `kubernetes-control-plane` charm. -The following components remain in the cluster unmanaged by the charm, and +The following components remain in the cluster, unmanaged by the charm, and should be considered managed by the cluster administrators. -* `Deployment/kube-system/k8s-keystone-auth` -* `Service/kube-system/k8s-keystone-auth-service` -* `Secret/kube-system/keystone-auth-certs` -* `ConfigMap/kube-system/k8s-auth-policy` -* `ClusterRole/k8s-keystone-auth` +- `Deployment/kube-system/k8s-keystone-auth` +- `Service/kube-system/k8s-keystone-auth-service` +- `Secret/kube-system/keystone-auth-certs` +- `ConfigMap/kube-system/k8s-auth-policy` +- `ClusterRole/k8s-keystone-auth` ### Administrative Actions missing -The `kubernetes-control-plane` and `kubernetes-worker` actions list was substantially reduced -during development of 1.29. The following are no longer present, but are slated to be reintroduced. -* `restart` -* `namespace-list` -* `namespace-create` -* `namespace-delete` -* `user-create` -* `user-delete` -* `user-list` -* `apply-manifest` + +The `kubernetes-control-plane` and `kubernetes-worker` actions list was +substantially reduced during development of 1.29. The following are no longer +present, but are slated to be reintroduced: + +- `restart` +- `namespace-list` +- `namespace-create` +- `namespace-delete` +- `user-create` +- `user-delete` +- `user-list` +- `apply-manifest` ### CIS-Benchmark Action missing -The `kubernetes-control-plane` and `kubernetes-worker` action for cis-benchmark were -removed during the development of the 1.29 charms and an engineering decision to reintroduce -these actions are on-going, but development and testing incomplete. Details in [LP#2044219][] +The `kubernetes-control-plane` and `kubernetes-worker` action for cis-benchmark +were removed during the development of the 1.29 charms and an engineering +decision to reintroduce these actions are on-going, but development and testing +incomplete. Details in [LP#2044219][] ### Automatic labelling of GPU nodes -While current worker nodes would remain unaffected as they would already be labeled, the worker -charm in 1.29 no longer labels the nodes with `gpu=true` and `cuda=true`. +While current worker nodes would remain unaffected as they would already be +labelled, the worker charm in 1.29 no longer labels the nodes with `gpu=true` +and `cuda=true`. Parity with this feature has been attained by using the [nvidia-gpu-operator][]